feat: Refine HTTP timeouts across numerous analyzers and ingestors. Closes #3495

api_app/analyzers_manager/classes.py

@@ -11,6 +11,7 @@
 import requests
 from django.conf import settings
 
+from api_app import http_utils
 from api_app.decorators import classproperty
 from certego_saas.apps.user.models import User
 
@@ -309,7 +310,7 @@ def __raise_in_case_bad_request(name, resp, params_to_check=None) -> bool:
     @staticmethod
     def __query_for_result(url: str, key: str) -> Tuple[int, dict]:
         headers = {"Accept": "application/json"}
-        resp = requests.get(f"{url}?key={key}", headers=headers)
+        resp = http_utils.get(f"{url}?key={key}", headers=headers)
         return resp.status_code, resp.json()
 
     def __polling(self, req_key: str, chance: int, re_poll_try: int = 0):
@@ -392,9 +393,9 @@ def _docker_run(
         try:
             if req_files:
                 form_data = {"request_json": json.dumps(req_data)}
-                resp1 = requests.post(self.url, files=req_files, data=form_data)
+                resp1 = http_utils.post(self.url, files=req_files, data=form_data)
             else:
-                resp1 = requests.post(self.url, json=req_data)
+                resp1 = http_utils.post(self.url, json=req_data)
         except requests.exceptions.ConnectionError:
             self._raise_container_not_running()
 
@@ -440,7 +441,7 @@ def _docker_get(self):
 
         # step #1: request new analysis
         try:
-            resp = requests.get(url=self.url)
+            resp = http_utils.get(url=self.url)
         except requests.exceptions.ConnectionError:
             self._raise_container_not_running()
 
@@ -454,7 +455,7 @@ def health_check(self, user: User = None) -> bool:
         basic health check: if instance is up or not (timeout - 10s)
         """
         try:
-            requests.head(self.url, timeout=10)
+            http_utils.head(self.url, timeout=10)
         except requests.exceptions.RequestException:
             health_status = False
         else:

api_app/analyzers_manager/file_analyzers/capa_info.py

@@ -10,9 +10,9 @@
 from pathlib import Path
 from shlex import quote
 
-import requests
 from django.conf import settings
 
+from api_app import http_utils
 from api_app.analyzers_manager.classes import FileAnalyzer
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
 from api_app.analyzers_manager.models import PythonModule
@@ -89,7 +89,7 @@ def _download_signatures(cls) -> None:
 
         signatures_url = "https://api.github.com/repos/mandiant/capa/contents/sigs"
         try:
-            response = requests.get(signatures_url)
+            response = http_utils.get(signatures_url)
             signatures_list = response.json()
 
             for signature in signatures_list:
@@ -98,7 +98,7 @@ def _download_signatures(cls) -> None:
 
                 signature_file_path = os.path.join(SIGNATURE_LOCATION, filename)
 
-                sig_content = requests.get(download_url, stream=True)
+                sig_content = http_utils.get(download_url, stream=True)
                 with open(signature_file_path, mode="wb") as file:
                     for chunk in sig_content.iter_content(chunk_size=10 * 1024):
                         file.write(chunk)
@@ -112,7 +112,7 @@ def _download_signatures(cls) -> None:
     def update(cls, analyzer_module: PythonModule) -> bool:
         try:
             logger.info("Updating capa rules")
-            response = requests.get("https://api.github.com/repos/mandiant/capa-rules/releases/latest")
+            response = http_utils.get("https://api.github.com/repos/mandiant/capa-rules/releases/latest")
             latest_version = response.json()["tag_name"]
             capa_rules_download_url = RULES_URL + latest_version + ".zip"
 
@@ -138,7 +138,7 @@ def update(cls, analyzer_module: PythonModule) -> bool:
     def run(self):
         cache_dir = self._ensure_cache_directory()
         try:
-            response = requests.get("https://api.github.com/repos/mandiant/capa-rules/releases/latest")
+            response = http_utils.get("https://api.github.com/repos/mandiant/capa-rules/releases/latest")
             latest_version = response.json()["tag_name"]
 
             capa_analyzer_module = self.python_module

api_app/analyzers_manager/file_analyzers/cape_sandbox.py

@@ -9,6 +9,7 @@
 import requests
 from billiard.exceptions import SoftTimeLimitExceeded
 
+from api_app import http_utils
 from api_app.analyzers_manager.classes import FileAnalyzer
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
 
@@ -73,7 +74,7 @@ def config(self, runtime_configuration: Dict):
         self.__cert_file = NamedTemporaryFile(mode="w")
         self.__cert_file.write(self._clean_certificate(self._certificate))
         self.__cert_file.flush()
-        self.__session = requests.Session()
+        self.__session = http_utils.Session()
         self.__session.verify = self.__cert_file.name
         self.__session.headers = {
             "Authorization": f"Token {self._api_key_name}",

api_app/analyzers_manager/file_analyzers/cuckoo_scan.py

@@ -6,8 +6,7 @@
 import time
 from typing import Dict
 
-import requests
-
+from api_app import http_utils
 from api_app.analyzers_manager.classes import FileAnalyzer
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
 
@@ -24,7 +23,7 @@ def config(self, runtime_configuration: Dict):
         super().config(runtime_configuration)
         # cuckoo installation can be with or without the api_token
         # it depends on version and configuration
-        self.session = requests.Session()
+        self.session = http_utils.Session()
         if not hasattr(self, "_api_key_name"):
             logger.info(f"{self.__repr__()}, (md5: {self.md5}) -> Continuing w/o API key..")
         else:

api_app/analyzers_manager/file_analyzers/docguard.py

@@ -3,8 +3,7 @@
 
 import logging
 
-import requests
-
+from api_app import http_utils
 from api_app.analyzers_manager.classes import FileAnalyzer
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
 
@@ -27,7 +26,7 @@ def run(self):
         binary = self.read_file_bytes()
         if not binary:
             raise AnalyzerRunException("File is empty")
-        response = requests.post(
+        response = http_utils.post(
             self.url + "/FileAnalyzing/AnalyzeFile",
             headers=headers,
             files={"file": (self.filename, binary)},

api_app/analyzers_manager/file_analyzers/filescan.py

@@ -4,8 +4,7 @@
 import logging
 import time
 
-import requests
-
+from api_app import http_utils
 from api_app.analyzers_manager.classes import FileAnalyzer
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
 
@@ -29,7 +28,7 @@ def __upload_file_for_scan(self) -> int:
         binary = self.read_file_bytes()
         if not binary:
             raise AnalyzerRunException("File is empty")
-        response = requests.post(
+        response = http_utils.post(
             self.url + "/scan/file",
             files={"file": (self.filename, binary)},
             headers={"X-Api-Key": self._api_key},
@@ -56,7 +55,7 @@ def __fetch_report(self, task_id: int) -> dict:
 
         for chance in range(self.max_tries):
             logger.info(f"[POLLING] {obj_repr} -> #{chance + 1}/{self.max_tries}")
-            response = requests.get(url, params=params, headers={"X-Api-Key": self._api_key})
+            response = http_utils.get(url, params=params, headers={"X-Api-Key": self._api_key})
             report = response.json()
             if report["allFinished"]:
                 break

api_app/analyzers_manager/file_analyzers/malpedia_scan.py

@@ -1,8 +1,7 @@
 # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
 # See the file 'LICENSE' for copying permission.
 
-import requests
-
+from api_app import http_utils
 from api_app.analyzers_manager.classes import FileAnalyzer
 
 
@@ -22,7 +21,7 @@ def run(self):
         # construct req
         headers = {"Authorization": f"APIToken {self._api_key_name}"}
         files = {"file": binary}
-        response = requests.post(self.binary_url, headers=headers, files=files)
+        response = http_utils.post(self.binary_url, headers=headers, files=files)
         response.raise_for_status()
 
         result = response.json()

api_app/analyzers_manager/file_analyzers/malprob.py

@@ -1,7 +1,6 @@
 import logging
 
-import requests
-
+from api_app import http_utils
 from api_app.analyzers_manager.classes import FileAnalyzer
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
 
@@ -24,7 +23,7 @@ def run(self):
 
         if self._job.tlp == self._job.TLP.CLEAR.value:
             logger.info(f"uploading {file_name}:{self.md5} to MalProb.io for analysis")
-            scan = requests.post(
+            scan = http_utils.post(
                 f"{self.url}/scan/",
                 files={"file": binary_file},
                 data={"name": file_name, "private": self.private},
@@ -41,7 +40,7 @@ def run(self):
                 return scan.json()
 
         logger.info(f"rescanning {file_name} using {self.md5} on MalProb.io")
-        rescan = requests.post(
+        rescan = http_utils.post(
             f"{self.url}/rescan/",
             data={"hashcode": self.md5},
             headers=headers,

api_app/analyzers_manager/file_analyzers/mobsf_service.py

@@ -1,8 +1,7 @@
 import logging
 import time
 
-import requests
-
+from api_app import http_utils
 from api_app.analyzers_manager.classes import FileAnalyzer
 
 logger = logging.getLogger(__name__)
@@ -37,7 +36,7 @@ def update(self) -> bool:
         pass
 
     def query_mobsf(self, endpoint, headers, data):
-        response = requests.post(
+        response = http_utils.post(
             url=self.mobsf_host + endpoint,
             data=data,
             headers=headers,
@@ -112,7 +111,7 @@ def run(self):
         logger.info(f"File bytes for file:{self.filename} read successfully. Initiating upload request")
 
         upload_url = self.mobsf_host + self.UPLOAD_ENDPOINT
-        upload_response = requests.post(
+        upload_response = http_utils.post(
             url=upload_url,
             files={"file": (self.filename, binary, "application/octet-stream")},
             headers=headers,

api_app/analyzers_manager/file_analyzers/phishing/phishing_form_compiler.py

@@ -3,12 +3,12 @@
 from typing import Dict
 from urllib.parse import urljoin
 
-import requests
 from faker import Faker  # skipcq: BAN-B410
 from lxml.etree import HTMLParser  # skipcq: BAN-B410
 from lxml.html import document_fromstring
 from requests import HTTPError, Response
 
+from api_app import http_utils
 from api_app.analyzers_manager.classes import FileAnalyzer
 from api_app.models import PythonConfig
 
@@ -183,7 +183,7 @@ def perform_request_to_form(self, form) -> Response:
         headers = {
             "User-Agent": self.user_agent,
         }
-        response = requests.post(
+        response = http_utils.post(
             url=dest_url,
             data=params,
             headers=headers,

api_app/analyzers_manager/file_analyzers/sublime.py

@@ -7,6 +7,7 @@
 import requests
 from django.utils.functional import cached_property
 
+from api_app import http_utils
 from api_app.analyzers_manager.classes import FileAnalyzer
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
 from api_app.analyzers_manager.models import MimeTypes
@@ -141,7 +142,7 @@ def _analysis(self, session: requests.Session, content: str):
 
     def run(self) -> Dict:
         self.headers["Authorization"] = f"Bearer {self._api_key}"
-        session = requests.Session()
+        session = http_utils.Session()
         session.headers = self.headers
         report = self._analysis(session, self.raw_message)
         if self.analyze_internal_eml_on_pec and self.file_mimetype == MimeTypes.EML.value and self.is_pec():

api_app/analyzers_manager/file_analyzers/unpac_me.py

@@ -7,6 +7,7 @@
 
 import requests
 
+from api_app import http_utils
 from api_app.analyzers_manager.classes import FileAnalyzer
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
 
@@ -52,10 +53,10 @@ def run(self):
     def _req_with_checks(self, url, files=None, post=False):
         try:
             if post:
-                r = requests.post(self.url + url, files=files, headers=self.headers)
+                r = http_utils.post(self.url + url, files=files, headers=self.headers)
             else:
                 headers = self.headers if self.private == "private" else {}
-                r = requests.get(self.url + url, files=files, headers=headers)
+                r = http_utils.get(self.url + url, files=files, headers=headers)
             r.raise_for_status()
         except requests.exceptions.HTTPError as e:
             logger.error(f"md5 {self.md5} job {self.job_id} url {url} has http error {str(e)}")

api_app/analyzers_manager/file_analyzers/virushee.py

@@ -5,8 +5,7 @@
 import time
 from typing import Dict, Optional
 
-import requests
-
+from api_app import http_utils
 from api_app.analyzers_manager.classes import FileAnalyzer
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
 
@@ -24,7 +23,7 @@ class VirusheeFileUpload(FileAnalyzer):
 
     def config(self, runtime_configuration: Dict):
         super().config(runtime_configuration)
-        self.__session = requests.Session()
+        self.__session = http_utils.Session()
         if not hasattr(self, "_api_key_name"):
             logger.info(f"{self.__repr__()} -> Continuing w/o API key..")
         else:

api_app/analyzers_manager/file_analyzers/yara_scan.py

@@ -11,11 +11,11 @@
 from urllib.parse import urlparse
 
 import git
-import requests
 import yara
 from django.conf import settings
 from django.utils.functional import cached_property
 
+from api_app import http_utils
 from api_app.analyzers_manager.classes import FileAnalyzer
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
 from api_app.models import Parameter, PluginConfig
@@ -94,7 +94,7 @@ def update(self):
 
     def _update_zip(self):
         logger.info(f"About to download zip file from {self.url} to {self.directory}")
-        response = requests.get(self.url, stream=True)
+        response = http_utils.get(self.url, stream=True)
         try:
             response.raise_for_status()
         except Exception as e:

api_app/analyzers_manager/file_analyzers/yaraify_file_scan.py

@@ -7,6 +7,7 @@
 
 import requests
 
+from api_app import http_utils
 from api_app.analyzers_manager.classes import FileAnalyzer
 from api_app.analyzers_manager.exceptions import (
     AnalyzerConfigurationException,
@@ -72,7 +73,7 @@ def run(self):
                 "file": (name_to_send, file),
             }
             logger.info(f"yara file scan md5 {self.md5} sending sample for analysis")
-            response = requests.post(self.url, files=files_, headers=self.authentication_header)
+            response = http_utils.post(self.url, files=files_, headers=self.authentication_header)
             response.raise_for_status()
             scan_response = response.json()
             scan_query_status = scan_response.get("query_status")
@@ -90,7 +91,7 @@ def run(self):
                             f"task_id: {task_id}"
                         )
                         data = {"query": "get_results", "task_id": task_id}
-                        response = requests.post(self.url, json=data, headers=self.authentication_header)
+                        response = http_utils.post(self.url, json=data, headers=self.authentication_header)
                         response.raise_for_status()
                         task_response = response.json()
                         logger.debug(task_response)