Retry backoff upstream

Packs/CybleEventsV2/Integrations/CybleEventsV2/CybleEventsV2.py

@@ -10,6 +10,7 @@
 from collections.abc import Sequence
 from datetime import datetime, timedelta
 import concurrent.futures
+import time
 from dateutil.parser import parse as parse_date
 
 
@@ -23,6 +24,8 @@
 MAX_ALERTS = None
 LIMIT_EVENT_ITEMS = 200
 MAX_RETRIES = 3
+# Fetch-incidents: backoff after each failed services or alerts API attempt (1 try + 5 retries).
+FETCH_INCIDENT_RETRY_BACKOFF_SECONDS = (5, 10, 20, 20, 20)
 MAX_THREADS = 1
 MIN_MINUTES_TO_FETCH = 10
 DEFAULT_REQUEST_TIMEOUT = 600
@@ -307,25 +310,39 @@ def get_data(self, service, input_params, is_update=False):
 
         if not url or not alerts_api_key:
             raise ValueError("Missing required URL or API key in input_params.")
-        try:
-            demisto.debug(f"[get_data] final payload is: {payload_json}")
-            response = self.make_request(url, alerts_api_key, "POST", payload_json)
-            demisto.debug(f"[get_data] Response status code: {response.status_code}")
 
-        except Exception as request_error:
-            raise Exception(f"HTTP request failed for service '{service}': {str(request_error)}")
+        backoffs = FETCH_INCIDENT_RETRY_BACKOFF_SECONDS
+        for attempt in range(len(backoffs) + 1):
+            try:
+                demisto.debug(f"[get_data] final payload is: {payload_json}")
+                response = self.make_request(url, alerts_api_key, "POST", payload_json)
+            except Exception as request_error:
+                if attempt < len(backoffs):
+                    time.sleep(backoffs[attempt])
+                    continue
+                raise Exception(f"HTTP request failed for service '{service}': {str(request_error)}") from request_error
 
-        if response.status_code != 200:
-            raise Exception(
-                f"Failed to fetch data from {service}. Status code: {response.status_code}, Response text: {response.text}"
-            )
+            demisto.debug(f"[get_data] Response status code: {response.status_code}")
+            if response.status_code != 200:
+                if attempt < len(backoffs):
+                    time.sleep(backoffs[attempt])
+                    continue
+                raise Exception(
+                    f"Failed to fetch data from {service}. Status code: {response.status_code}, Response text: {response.text}"
+                )
+
+            try:
+                json_response = response.json()
+            except ValueError as json_error:
+                if attempt < len(backoffs):
+                    time.sleep(backoffs[attempt])
+                    continue
+                raise Exception(f"Invalid JSON response from {service}: {str(json_error)}") from json_error
 
-        try:
-            json_response = response.json()
             demisto.debug(f"[get_data] JSON response received with keys: {list(json_response.keys())}")
             return json_response
-        except ValueError as json_error:
-            raise Exception(f"Invalid JSON response from {service}: {str(json_error)}")
+
+        return {}  # pragma: no cover
 
     def get_all_services(self, api_key, url):
         """
@@ -335,23 +352,41 @@ def get_all_services(self, api_key, url):
         :param ew: An event writer object for logging
         :return: A list of service dictionaries, or an empty list if the request fails
         """
-        try:
-            url = url + "/services"
-            demisto.debug(f"[get_all_services] url: {url}")
-            response = self.make_request(url, api_key)
+        services_url = url + "/services"
+        backoffs = FETCH_INCIDENT_RETRY_BACKOFF_SECONDS
+        for attempt in range(len(backoffs) + 1):
+            try:
+                demisto.debug(f"[get_all_services] url: {services_url}")
+                response = self.make_request(services_url, api_key)
+            except Exception as e:
+                if attempt < len(backoffs):
+                    time.sleep(backoffs[attempt])
+                    continue
+                raise Exception(f"Failed to get services: {str(e)}") from e
+
             demisto.debug(f"[get_all_services] status code: {response.status_code}")
             if response.status_code != 200:
-                raise Exception(f"Wrong status code: {response.status_code}")
-            response = response.json()
+                if attempt < len(backoffs):
+                    time.sleep(backoffs[attempt])
+                    continue
+                raise Exception(f"Failed to get services: Wrong status code: {response.status_code}")
 
-            if "data" in response and isinstance(response["data"], Sequence):
-                demisto.debug(f"Received services: {json.dumps(response['data'], indent=2)}")
-                return response["data"]
+            try:
+                parsed = response.json()
+            except Exception as e:
+                if attempt < len(backoffs):
+                    time.sleep(backoffs[attempt])
+                    continue
+                raise Exception(f"Failed to get services: {str(e)}") from e
 
-            else:
-                raise Exception("Wrong Format for services response")
-        except Exception as e:
-            raise Exception(f"Failed to get services: {str(e)}")
+            if "data" in parsed and isinstance(parsed["data"], Sequence):
+                demisto.debug(f"Received services: {json.dumps(parsed['data'], indent=2)}")
+                return parsed["data"]
+
+            # Valid HTTP + JSON but unexpected shape: do not retry (not a transient error).
+            raise Exception("Failed to get services: Wrong Format for services response")
+
+        return []  # pragma: no cover
 
     def insert_data_in_cortex(self, service, input_params, is_update):
         """

Packs/CybleEventsV2/Integrations/CybleEventsV2/CybleEventsV2_test.py

@@ -17,6 +17,7 @@
     set_request,
     DEFAULT_TAKE_LIMIT,
     ensure_aware,
+    FETCH_INCIDENT_RETRY_BACKOFF_SECONDS,
 )
 from CommonServerPython import GetModifiedRemoteDataResponse
 from CybleEventsV2 import check_response
@@ -1731,6 +1732,98 @@ def test_get_data_success(self, mock_demisto, mock_get_alert_payload):
             mock_make_request.assert_called_once_with(self.test_url, self.test_api_key, "POST", json.dumps({"dummy": "payload"}))
             assert mock_demisto.debug.called
 
+    @patch("CybleEventsV2.get_alert_payload", return_value={"dummy": "payload"})
+    def test_get_data_raises_when_missing_url_or_api_key(self, _mock_get_alert_payload):
+        with pytest.raises(ValueError, match="Missing required URL or API key"):
+            self.client.get_data(self.test_service, {"api_key": self.test_api_key})
+        with pytest.raises(ValueError, match="Missing required URL or API key"):
+            self.client.get_data(self.test_service, {"url": self.test_url})
+
+    @patch("CybleEventsV2.get_alert_payload", return_value={"dummy": "payload"})
+    @patch("CybleEventsV2.demisto")
+    def test_get_data_retries_on_non_200_then_success(self, mock_demisto, _mock_get_alert_payload):
+        input_params = {"url": self.test_url, "api_key": self.test_api_key}
+        fail_resp = Mock()
+        fail_resp.status_code = 503
+        fail_resp.text = "unavailable"
+        ok_resp = Mock()
+        ok_resp.status_code = 200
+        ok_resp.json.return_value = {"recovered": True}
+
+        with (
+            patch.object(self.client, "make_request", side_effect=[fail_resp, ok_resp]) as mock_make,
+            patch("CybleEventsV2.time.sleep") as mock_sleep,
+        ):
+            result = self.client.get_data(self.test_service, input_params)
+
+        assert result == {"recovered": True}
+        assert mock_make.call_count == 2
+        mock_sleep.assert_called()
+
+    @patch("CybleEventsV2.get_alert_payload", return_value={"dummy": "payload"})
+    @patch("CybleEventsV2.demisto")
+    def test_get_data_retries_on_invalid_json_then_success(self, mock_demisto, _mock_get_alert_payload):
+        input_params = {"url": self.test_url, "api_key": self.test_api_key}
+        bad_json = Mock()
+        bad_json.status_code = 200
+        bad_json.json.side_effect = ValueError("not json")
+        good_json = Mock()
+        good_json.status_code = 200
+        good_json.json.return_value = {"parsed": True}
+
+        with (
+            patch.object(self.client, "make_request", side_effect=[bad_json, good_json]),
+            patch("CybleEventsV2.time.sleep") as mock_sleep,
+        ):
+            result = self.client.get_data(self.test_service, input_params)
+
+        assert result == {"parsed": True}
+        mock_sleep.assert_called_once()
+        assert mock_sleep.call_args[0][0] == FETCH_INCIDENT_RETRY_BACKOFF_SECONDS[0]
+
+    @patch("CybleEventsV2.demisto")
+    def test_get_all_services_retries_on_request_error_then_success(self, mock_demisto):
+        fail_resp = Mock()
+        fail_resp.status_code = 200
+        fail_resp.json.return_value = {"data": ["a"]}
+        with (
+            patch.object(self.client, "make_request", side_effect=[ConnectionError("reset"), fail_resp]),
+            patch("CybleEventsV2.time.sleep") as mock_sleep,
+        ):
+            result = self.client.get_all_services(self.test_api_key, self.test_url)
+        assert result == ["a"]
+        mock_sleep.assert_called()
+
+    @patch("CybleEventsV2.demisto")
+    def test_get_all_services_exhausts_retries_raises(self, mock_demisto):
+        """After 1 initial try + len(backoffs) retries, the last request error is raised."""
+        attempts = len(FETCH_INCIDENT_RETRY_BACKOFF_SECONDS) + 1
+        with (
+            patch.object(self.client, "make_request", side_effect=[ConnectionError("reset")] * attempts),
+            patch("CybleEventsV2.time.sleep") as mock_sleep,
+            pytest.raises(Exception, match="Failed to get services: reset"),
+        ):
+            self.client.get_all_services(self.test_api_key, self.test_url)
+
+        assert mock_sleep.call_count == len(FETCH_INCIDENT_RETRY_BACKOFF_SECONDS)
+
+    @patch("CybleEventsV2.demisto")
+    def test_get_all_services_wrong_format_raises_without_backoff(self, mock_demisto):
+        """Wrong response shape after 200 + valid JSON must not use retry backoff."""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {"wrong_key": []}
+
+        with (
+            patch.object(self.client, "make_request", return_value=mock_response) as mock_make,
+            patch("CybleEventsV2.time.sleep") as mock_sleep,
+            pytest.raises(Exception, match="Failed to get services: Wrong Format for services response"),
+        ):
+            self.client.get_all_services(self.test_api_key, self.test_url)
+
+        mock_make.assert_called_once()
+        mock_sleep.assert_not_called()
+
     def test_insert_data_in_cortex_successful_processing(self):
         test_input_params = {"limit": "10", "hce": False}
 

Packs/CybleEventsV2/ReleaseNotes/1_1_7.md

@@ -0,0 +1,5 @@
+#### Integrations
+
+##### CybleEvents v2
+
+- Improved implementation of `fetch-incidents` by adding retry with backoff when calls to the services or alerts API fail (initial attempt plus retries with delays of 5, 10, 20, 20, and 20 seconds).

Packs/CybleEventsV2/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "CybleEventsV2",
     "description": "Cyble Events for Vision Users. Must have Vision API access to use the threat intelligence.",
     "support": "partner",
-    "currentVersion": "1.1.6",
+    "currentVersion": "1.1.7",
     "author": "Cyble Info Sec",
     "url": "https://cyble.com/",
     "email": "",