From 2f0ef543aedef2e9e61b342d9471bf01130eac0f Mon Sep 17 00:00:00 2001 From: chauhan-varun Date: Thu, 19 Mar 2026 22:24:50 +0530 Subject: [PATCH 1/7] feat: Introduce `http_utils.py` to centralize HTTP request handling across numerous analyzers and ingestors. --- api_app/analyzers_manager/classes.py | 11 ++- .../file_analyzers/capa_info.py | 10 +- .../file_analyzers/cape_sandbox.py | 3 +- .../file_analyzers/cuckoo_scan.py | 5 +- .../file_analyzers/docguard.py | 5 +- .../file_analyzers/filescan.py | 7 +- .../file_analyzers/malpedia_scan.py | 5 +- .../file_analyzers/malprob.py | 7 +- .../file_analyzers/mobsf_service.py | 7 +- .../phishing/phishing_form_compiler.py | 4 +- .../file_analyzers/sublime.py | 3 +- .../file_analyzers/unpac_me.py | 5 +- .../file_analyzers/virushee.py | 5 +- .../file_analyzers/yara_scan.py | 4 +- .../file_analyzers/yaraify_file_scan.py | 5 +- .../analyzers_manager/file_analyzers/yarax.py | 6 +- .../observable_analyzers/abuseipdb.py | 5 +- .../observable_analyzers/apivoid.py | 3 +- .../observable_analyzers/auth0.py | 5 +- .../basic_observable_analyzer.py | 3 +- .../observable_analyzers/bgp_ranking.py | 9 +- .../observable_analyzers/binaryedge.py | 7 +- .../observable_analyzers/bitcoinabuse.py | 5 +- .../observable_analyzers/censys.py | 5 +- .../observable_analyzers/checkphish.py | 7 +- .../criminalip/criminalip.py | 4 +- .../criminalip/criminalip_scan.py | 8 +- .../observable_analyzers/crowdsec.py | 4 +- .../observable_analyzers/crt_sh.py | 5 +- .../observable_analyzers/crxcavator.py | 3 +- .../observable_analyzers/cyberchef.py | 3 +- .../observable_analyzers/cycat.py | 7 +- .../observable_analyzers/dehashed.py | 4 +- .../dns/dns_malicious_detectors/adguard.py | 4 +- .../cleanbrowsing_malicious_detector.py | 4 +- .../cloudflare_malicious_detector.py | 3 +- .../quad9_malicious_detector.py | 5 +- .../dns_malicious_detectors/spamhaus_wqs.py | 5 +- .../dns_resolvers/cloudflare_dns_resolver.py | 3 +- .../dns/dns_resolvers/google_dns_resolver.py | 3 +- .../observable_analyzers/dnsdb.py | 4 +- .../observable_analyzers/docguard_get.py | 3 +- .../download_file_from_uri.py | 5 +- .../observable_analyzers/dshield.py | 3 +- .../observable_analyzers/emailrep.py | 5 +- .../observable_analyzers/expand_url.py | 3 +- .../observable_analyzers/feodo_tracker.py | 3 +- .../observable_analyzers/filescan_search.py | 3 +- .../observable_analyzers/firehol_iplist.py | 4 +- .../observable_analyzers/greedybear.py | 9 +- .../observable_analyzers/greynoise_labs.py | 6 +- .../observable_analyzers/ha_get.py | 7 +- .../observable_analyzers/haveibeenpwned.py | 5 +- .../observable_analyzers/hibp_utils.py | 5 +- .../observable_analyzers/honeydb.py | 5 +- .../observable_analyzers/hudsonrock.py | 9 +- .../observable_analyzers/hunter_how.py | 3 +- .../observable_analyzers/hunter_io.py | 5 +- .../observable_analyzers/hunting_abuse.py | 3 +- .../observable_analyzers/inquest.py | 5 +- .../observable_analyzers/intelx.py | 3 +- .../observable_analyzers/ip2location.py | 3 +- .../observable_analyzers/ip2whois.py | 3 +- .../observable_analyzers/ipapi.py | 7 +- .../observable_analyzers/ipinfo.py | 3 +- .../observable_analyzers/ipqs.py | 3 +- .../observable_analyzers/ipquery.py | 3 +- .../observable_analyzers/ja4_db.py | 4 +- .../observable_analyzers/koodous.py | 5 +- .../observable_analyzers/leakix.py | 5 +- .../observable_analyzers/malprob.py | 7 +- .../observable_analyzers/maxmind.py | 4 +- .../observable_analyzers/mb_get.py | 5 +- .../observable_analyzers/mmdb_server.py | 8 +- .../observable_analyzers/mnemonic_pdns.py | 5 +- .../observable_analyzers/nerd.py | 3 +- .../observable_analyzers/netlas.py | 3 +- .../observable_analyzers/nvd_cve.py | 3 +- .../observable_analyzers/onyphe.py | 3 +- .../observable_analyzers/orkl_search.py | 7 +- .../observable_analyzers/otx.py | 4 +- .../observable_analyzers/phishing_army.py | 4 +- .../observable_analyzers/phishstats.py | 5 +- .../observable_analyzers/phishtank.py | 3 +- .../observable_analyzers/phoneinfoga_scan.py | 3 +- .../observable_analyzers/pulsedive.py | 9 +- .../observable_analyzers/robtex.py | 5 +- .../observable_analyzers/securitytrails.py | 3 +- .../observable_analyzers/shodan.py | 3 +- .../observable_analyzers/spamhaus_drop.py | 4 +- .../observable_analyzers/spyse.py | 5 +- .../observable_analyzers/ss_api_net.py | 3 +- .../observable_analyzers/stalkphish.py | 3 +- .../observable_analyzers/stratosphere.py | 4 +- .../observable_analyzers/talos.py | 4 +- .../observable_analyzers/threatfox.py | 5 +- .../observable_analyzers/threatminer.py | 3 +- .../observable_analyzers/threatstream.py | 3 +- .../observable_analyzers/tor.py | 4 +- .../observable_analyzers/tor_nodes_danmeuk.py | 4 +- .../observable_analyzers/tranco.py | 5 +- .../triage/triage_base.py | 4 +- .../observable_analyzers/tweetfeeds.py | 5 +- .../observable_analyzers/urldna.py | 3 +- .../observable_analyzers/urlhaus.py | 5 +- .../observable_analyzers/urlscan.py | 3 +- .../observable_analyzers/validin.py | 5 +- .../observable_analyzers/virushee.py | 3 +- .../observable_analyzers/vulners.py | 7 +- .../observable_analyzers/whoisripe.py | 5 +- .../observable_analyzers/whoisxmlapi.py | 5 +- .../observable_analyzers/wigle.py | 5 +- .../observable_analyzers/xforce.py | 4 +- .../observable_analyzers/yaraify.py | 5 +- .../observable_analyzers/yeti.py | 5 +- .../observable_analyzers/zoomeye.py | 3 +- api_app/classes.py | 3 +- api_app/connectors_manager/connectors/yeti.py | 3 +- api_app/core/update_checker.py | 3 +- api_app/http_utils.py | 97 +++++++++++++++++++ .../ingestors_manager/ingestors/greedybear.py | 5 +- .../ingestors_manager/ingestors/malshare.py | 7 +- .../ingestors/malware_bazaar.py | 6 +- .../ingestors_manager/ingestors/threatfox.py | 5 +- api_app/mixins.py | 13 +-- intel_owl/settings/commons.py | 7 ++ tests/api_app/test_http_utils.py | 63 ++++++++++++ 127 files changed, 463 insertions(+), 290 deletions(-) create mode 100644 api_app/http_utils.py create mode 100644 tests/api_app/test_http_utils.py diff --git a/api_app/analyzers_manager/classes.py b/api_app/analyzers_manager/classes.py index 5f0d09097d..f79379df05 100644 --- a/api_app/analyzers_manager/classes.py +++ b/api_app/analyzers_manager/classes.py @@ -11,6 +11,7 @@ import requests from django.conf import settings +from api_app import http_utils from api_app.decorators import classproperty from certego_saas.apps.user.models import User @@ -309,7 +310,7 @@ def __raise_in_case_bad_request(name, resp, params_to_check=None) -> bool: @staticmethod def __query_for_result(url: str, key: str) -> Tuple[int, dict]: headers = {"Accept": "application/json"} - resp = requests.get(f"{url}?key={key}", headers=headers) + resp = http_utils.get(f"{url}?key={key}", headers=headers) return resp.status_code, resp.json() def __polling(self, req_key: str, chance: int, re_poll_try: int = 0): @@ -392,9 +393,9 @@ def _docker_run( try: if req_files: form_data = {"request_json": json.dumps(req_data)} - resp1 = requests.post(self.url, files=req_files, data=form_data) + resp1 = http_utils.post(self.url, files=req_files, data=form_data) else: - resp1 = requests.post(self.url, json=req_data) + resp1 = http_utils.post(self.url, json=req_data) except requests.exceptions.ConnectionError: self._raise_container_not_running() @@ -440,7 +441,7 @@ def _docker_get(self): # step #1: request new analysis try: - resp = requests.get(url=self.url) + resp = http_utils.get(url=self.url) except requests.exceptions.ConnectionError: self._raise_container_not_running() @@ -454,7 +455,7 @@ def health_check(self, user: User = None) -> bool: basic health check: if instance is up or not (timeout - 10s) """ try: - requests.head(self.url, timeout=10) + http_utils.head(self.url, timeout=settings.HTTP_TIMEOUT) except requests.exceptions.RequestException: health_status = False else: diff --git a/api_app/analyzers_manager/file_analyzers/capa_info.py b/api_app/analyzers_manager/file_analyzers/capa_info.py index 020c5ac252..b3ec5499a1 100644 --- a/api_app/analyzers_manager/file_analyzers/capa_info.py +++ b/api_app/analyzers_manager/file_analyzers/capa_info.py @@ -10,9 +10,9 @@ from pathlib import Path from shlex import quote -import requests from django.conf import settings +from api_app import http_utils from api_app.analyzers_manager.classes import FileAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.analyzers_manager.models import PythonModule @@ -89,7 +89,7 @@ def _download_signatures(cls) -> None: signatures_url = "https://api.github.com/repos/mandiant/capa/contents/sigs" try: - response = requests.get(signatures_url) + response = http_utils.get(signatures_url) signatures_list = response.json() for signature in signatures_list: @@ -98,7 +98,7 @@ def _download_signatures(cls) -> None: signature_file_path = os.path.join(SIGNATURE_LOCATION, filename) - sig_content = requests.get(download_url, stream=True) + sig_content = http_utils.get(download_url, stream=True) with open(signature_file_path, mode="wb") as file: for chunk in sig_content.iter_content(chunk_size=10 * 1024): file.write(chunk) @@ -112,7 +112,7 @@ def _download_signatures(cls) -> None: def update(cls, analyzer_module: PythonModule) -> bool: try: logger.info("Updating capa rules") - response = requests.get("https://api.github.com/repos/mandiant/capa-rules/releases/latest") + response = http_utils.get("https://api.github.com/repos/mandiant/capa-rules/releases/latest") latest_version = response.json()["tag_name"] capa_rules_download_url = RULES_URL + latest_version + ".zip" @@ -138,7 +138,7 @@ def update(cls, analyzer_module: PythonModule) -> bool: def run(self): cache_dir = self._ensure_cache_directory() try: - response = requests.get("https://api.github.com/repos/mandiant/capa-rules/releases/latest") + response = http_utils.get("https://api.github.com/repos/mandiant/capa-rules/releases/latest") latest_version = response.json()["tag_name"] capa_analyzer_module = self.python_module diff --git a/api_app/analyzers_manager/file_analyzers/cape_sandbox.py b/api_app/analyzers_manager/file_analyzers/cape_sandbox.py index df9aa85748..b660539d7b 100644 --- a/api_app/analyzers_manager/file_analyzers/cape_sandbox.py +++ b/api_app/analyzers_manager/file_analyzers/cape_sandbox.py @@ -9,6 +9,7 @@ import requests from billiard.exceptions import SoftTimeLimitExceeded +from api_app import http_utils from api_app.analyzers_manager.classes import FileAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -73,7 +74,7 @@ def config(self, runtime_configuration: Dict): self.__cert_file = NamedTemporaryFile(mode="w") self.__cert_file.write(self._clean_certificate(self._certificate)) self.__cert_file.flush() - self.__session = requests.Session() + self.__session = http_utils.Session() self.__session.verify = self.__cert_file.name self.__session.headers = { "Authorization": f"Token {self._api_key_name}", diff --git a/api_app/analyzers_manager/file_analyzers/cuckoo_scan.py b/api_app/analyzers_manager/file_analyzers/cuckoo_scan.py index db805d5e23..80168d3da3 100644 --- a/api_app/analyzers_manager/file_analyzers/cuckoo_scan.py +++ b/api_app/analyzers_manager/file_analyzers/cuckoo_scan.py @@ -6,8 +6,7 @@ import time from typing import Dict -import requests - +from api_app import http_utils from api_app.analyzers_manager.classes import FileAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -24,7 +23,7 @@ def config(self, runtime_configuration: Dict): super().config(runtime_configuration) # cuckoo installation can be with or without the api_token # it depends on version and configuration - self.session = requests.Session() + self.session = http_utils.Session() if not hasattr(self, "_api_key_name"): logger.info(f"{self.__repr__()}, (md5: {self.md5}) -> Continuing w/o API key..") else: diff --git a/api_app/analyzers_manager/file_analyzers/docguard.py b/api_app/analyzers_manager/file_analyzers/docguard.py index ab1022d9b9..16e1ff5c77 100644 --- a/api_app/analyzers_manager/file_analyzers/docguard.py +++ b/api_app/analyzers_manager/file_analyzers/docguard.py @@ -3,8 +3,7 @@ import logging -import requests - +from api_app import http_utils from api_app.analyzers_manager.classes import FileAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -27,7 +26,7 @@ def run(self): binary = self.read_file_bytes() if not binary: raise AnalyzerRunException("File is empty") - response = requests.post( + response = http_utils.post( self.url + "/FileAnalyzing/AnalyzeFile", headers=headers, files={"file": (self.filename, binary)}, diff --git a/api_app/analyzers_manager/file_analyzers/filescan.py b/api_app/analyzers_manager/file_analyzers/filescan.py index 344a36be99..5d4defd8b2 100644 --- a/api_app/analyzers_manager/file_analyzers/filescan.py +++ b/api_app/analyzers_manager/file_analyzers/filescan.py @@ -4,8 +4,7 @@ import logging import time -import requests - +from api_app import http_utils from api_app.analyzers_manager.classes import FileAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -29,7 +28,7 @@ def __upload_file_for_scan(self) -> int: binary = self.read_file_bytes() if not binary: raise AnalyzerRunException("File is empty") - response = requests.post( + response = http_utils.post( self.url + "/scan/file", files={"file": (self.filename, binary)}, headers={"X-Api-Key": self._api_key}, @@ -56,7 +55,7 @@ def __fetch_report(self, task_id: int) -> dict: for chance in range(self.max_tries): logger.info(f"[POLLING] {obj_repr} -> #{chance + 1}/{self.max_tries}") - response = requests.get(url, params=params, headers={"X-Api-Key": self._api_key}) + response = http_utils.get(url, params=params, headers={"X-Api-Key": self._api_key}) report = response.json() if report["allFinished"]: break diff --git a/api_app/analyzers_manager/file_analyzers/malpedia_scan.py b/api_app/analyzers_manager/file_analyzers/malpedia_scan.py index 5b16dace95..113e82c99c 100644 --- a/api_app/analyzers_manager/file_analyzers/malpedia_scan.py +++ b/api_app/analyzers_manager/file_analyzers/malpedia_scan.py @@ -1,8 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import requests - +from api_app import http_utils from api_app.analyzers_manager.classes import FileAnalyzer @@ -22,7 +21,7 @@ def run(self): # construct req headers = {"Authorization": f"APIToken {self._api_key_name}"} files = {"file": binary} - response = requests.post(self.binary_url, headers=headers, files=files) + response = http_utils.post(self.binary_url, headers=headers, files=files) response.raise_for_status() result = response.json() diff --git a/api_app/analyzers_manager/file_analyzers/malprob.py b/api_app/analyzers_manager/file_analyzers/malprob.py index 211a0e0245..fe558bf7dd 100644 --- a/api_app/analyzers_manager/file_analyzers/malprob.py +++ b/api_app/analyzers_manager/file_analyzers/malprob.py @@ -1,7 +1,6 @@ import logging -import requests - +from api_app import http_utils from api_app.analyzers_manager.classes import FileAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -24,7 +23,7 @@ def run(self): if self._job.tlp == self._job.TLP.CLEAR.value: logger.info(f"uploading {file_name}:{self.md5} to MalProb.io for analysis") - scan = requests.post( + scan = http_utils.post( f"{self.url}/scan/", files={"file": binary_file}, data={"name": file_name, "private": self.private}, @@ -41,7 +40,7 @@ def run(self): return scan.json() logger.info(f"rescanning {file_name} using {self.md5} on MalProb.io") - rescan = requests.post( + rescan = http_utils.post( f"{self.url}/rescan/", data={"hashcode": self.md5}, headers=headers, diff --git a/api_app/analyzers_manager/file_analyzers/mobsf_service.py b/api_app/analyzers_manager/file_analyzers/mobsf_service.py index f7143bb555..ab3671403b 100644 --- a/api_app/analyzers_manager/file_analyzers/mobsf_service.py +++ b/api_app/analyzers_manager/file_analyzers/mobsf_service.py @@ -1,8 +1,7 @@ import logging import time -import requests - +from api_app import http_utils from api_app.analyzers_manager.classes import FileAnalyzer logger = logging.getLogger(__name__) @@ -37,7 +36,7 @@ def update(self) -> bool: pass def query_mobsf(self, endpoint, headers, data): - response = requests.post( + response = http_utils.post( url=self.mobsf_host + endpoint, data=data, headers=headers, @@ -112,7 +111,7 @@ def run(self): logger.info(f"File bytes for file:{self.filename} read successfully. Initiating upload request") upload_url = self.mobsf_host + self.UPLOAD_ENDPOINT - upload_response = requests.post( + upload_response = http_utils.post( url=upload_url, files={"file": (self.filename, binary, "application/octet-stream")}, headers=headers, diff --git a/api_app/analyzers_manager/file_analyzers/phishing/phishing_form_compiler.py b/api_app/analyzers_manager/file_analyzers/phishing/phishing_form_compiler.py index 0bc2f50f6f..8964f196cc 100644 --- a/api_app/analyzers_manager/file_analyzers/phishing/phishing_form_compiler.py +++ b/api_app/analyzers_manager/file_analyzers/phishing/phishing_form_compiler.py @@ -3,12 +3,12 @@ from typing import Dict from urllib.parse import urljoin -import requests from faker import Faker # skipcq: BAN-B410 from lxml.etree import HTMLParser # skipcq: BAN-B410 from lxml.html import document_fromstring from requests import HTTPError, Response +from api_app import http_utils from api_app.analyzers_manager.classes import FileAnalyzer from api_app.models import PythonConfig @@ -183,7 +183,7 @@ def perform_request_to_form(self, form) -> Response: headers = { "User-Agent": self.user_agent, } - response = requests.post( + response = http_utils.post( url=dest_url, data=params, headers=headers, diff --git a/api_app/analyzers_manager/file_analyzers/sublime.py b/api_app/analyzers_manager/file_analyzers/sublime.py index 3dacc0a626..28e0fb6516 100644 --- a/api_app/analyzers_manager/file_analyzers/sublime.py +++ b/api_app/analyzers_manager/file_analyzers/sublime.py @@ -7,6 +7,7 @@ import requests from django.utils.functional import cached_property +from api_app import http_utils from api_app.analyzers_manager.classes import FileAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.analyzers_manager.models import MimeTypes @@ -141,7 +142,7 @@ def _analysis(self, session: requests.Session, content: str): def run(self) -> Dict: self.headers["Authorization"] = f"Bearer {self._api_key}" - session = requests.Session() + session = http_utils.Session() session.headers = self.headers report = self._analysis(session, self.raw_message) if self.analyze_internal_eml_on_pec and self.file_mimetype == MimeTypes.EML.value and self.is_pec(): diff --git a/api_app/analyzers_manager/file_analyzers/unpac_me.py b/api_app/analyzers_manager/file_analyzers/unpac_me.py index 7ac0b87e92..0a1b0ebfed 100644 --- a/api_app/analyzers_manager/file_analyzers/unpac_me.py +++ b/api_app/analyzers_manager/file_analyzers/unpac_me.py @@ -7,6 +7,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager.classes import FileAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -52,10 +53,10 @@ def run(self): def _req_with_checks(self, url, files=None, post=False): try: if post: - r = requests.post(self.url + url, files=files, headers=self.headers) + r = http_utils.post(self.url + url, files=files, headers=self.headers) else: headers = self.headers if self.private == "private" else {} - r = requests.get(self.url + url, files=files, headers=headers) + r = http_utils.get(self.url + url, files=files, headers=headers) r.raise_for_status() except requests.exceptions.HTTPError as e: logger.error(f"md5 {self.md5} job {self.job_id} url {url} has http error {str(e)}") diff --git a/api_app/analyzers_manager/file_analyzers/virushee.py b/api_app/analyzers_manager/file_analyzers/virushee.py index d31e77ebc6..f226da7759 100644 --- a/api_app/analyzers_manager/file_analyzers/virushee.py +++ b/api_app/analyzers_manager/file_analyzers/virushee.py @@ -5,8 +5,7 @@ import time from typing import Dict, Optional -import requests - +from api_app import http_utils from api_app.analyzers_manager.classes import FileAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -24,7 +23,7 @@ class VirusheeFileUpload(FileAnalyzer): def config(self, runtime_configuration: Dict): super().config(runtime_configuration) - self.__session = requests.Session() + self.__session = http_utils.Session() if not hasattr(self, "_api_key_name"): logger.info(f"{self.__repr__()} -> Continuing w/o API key..") else: diff --git a/api_app/analyzers_manager/file_analyzers/yara_scan.py b/api_app/analyzers_manager/file_analyzers/yara_scan.py index 0f200e2e6a..7a47fe0ab8 100644 --- a/api_app/analyzers_manager/file_analyzers/yara_scan.py +++ b/api_app/analyzers_manager/file_analyzers/yara_scan.py @@ -11,11 +11,11 @@ from urllib.parse import urlparse import git -import requests import yara from django.conf import settings from django.utils.functional import cached_property +from api_app import http_utils from api_app.analyzers_manager.classes import FileAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.models import Parameter, PluginConfig @@ -94,7 +94,7 @@ def update(self): def _update_zip(self): logger.info(f"About to download zip file from {self.url} to {self.directory}") - response = requests.get(self.url, stream=True) + response = http_utils.get(self.url, stream=True) try: response.raise_for_status() except Exception as e: diff --git a/api_app/analyzers_manager/file_analyzers/yaraify_file_scan.py b/api_app/analyzers_manager/file_analyzers/yaraify_file_scan.py index 2b6dbb7f68..52afa1aba3 100644 --- a/api_app/analyzers_manager/file_analyzers/yaraify_file_scan.py +++ b/api_app/analyzers_manager/file_analyzers/yaraify_file_scan.py @@ -7,6 +7,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager.classes import FileAnalyzer from api_app.analyzers_manager.exceptions import ( AnalyzerConfigurationException, @@ -72,7 +73,7 @@ def run(self): "file": (name_to_send, file), } logger.info(f"yara file scan md5 {self.md5} sending sample for analysis") - response = requests.post(self.url, files=files_, headers=self.authentication_header) + response = http_utils.post(self.url, files=files_, headers=self.authentication_header) response.raise_for_status() scan_response = response.json() scan_query_status = scan_response.get("query_status") @@ -90,7 +91,7 @@ def run(self): f"task_id: {task_id}" ) data = {"query": "get_results", "task_id": task_id} - response = requests.post(self.url, json=data, headers=self.authentication_header) + response = http_utils.post(self.url, json=data, headers=self.authentication_header) response.raise_for_status() task_response = response.json() logger.debug(task_response) diff --git a/api_app/analyzers_manager/file_analyzers/yarax.py b/api_app/analyzers_manager/file_analyzers/yarax.py index 937b3f21aa..4d75d6675f 100644 --- a/api_app/analyzers_manager/file_analyzers/yarax.py +++ b/api_app/analyzers_manager/file_analyzers/yarax.py @@ -2,10 +2,10 @@ import os import pathlib -import requests import yara_x from django.conf import settings +from api_app import http_utils from api_app.analyzers_manager.classes import FileAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.mixins import RulesUtiliyMixin @@ -37,7 +37,7 @@ def update(cls, rule_set, analyzer_module) -> bool: rule_set_download_url = "" filename = "" try: - response = requests.get(RULES_URL) + response = http_utils.get(RULES_URL) assets = response.json()["assets"] latest_version = response.json()["tag_name"] for asset in assets: @@ -79,7 +79,7 @@ def run(self): rule_dir = f"{BASE_RULES_LOCATION}/{self.rule_set}" - response = requests.get(RULES_URL) + response = http_utils.get(RULES_URL) latest_version = response.json()["tag_name"] diff --git a/api_app/analyzers_manager/observable_analyzers/abuseipdb.py b/api_app/analyzers_manager/observable_analyzers/abuseipdb.py index 8f60508e33..f36d9aed8e 100644 --- a/api_app/analyzers_manager/observable_analyzers/abuseipdb.py +++ b/api_app/analyzers_manager/observable_analyzers/abuseipdb.py @@ -1,8 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import requests - +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.models import AnalyzerReport @@ -26,7 +25,7 @@ def run(self): "maxAgeInDays": self.max_age, "verbose": self.verbose, } - response = requests.get(self.url, params=params_, headers=headers) + response = http_utils.get(self.url, params=params_, headers=headers) response.raise_for_status() result = response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/apivoid.py b/api_app/analyzers_manager/observable_analyzers/apivoid.py index 531af877ad..a153481fa5 100644 --- a/api_app/analyzers_manager/observable_analyzers/apivoid.py +++ b/api_app/analyzers_manager/observable_analyzers/apivoid.py @@ -4,6 +4,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerConfigurationException @@ -39,6 +40,6 @@ def run(self): payload = {parameter: self.observable_name} - r = requests.post(complete_url, headers=headers, json=payload) + r = http_utils.post(complete_url, headers=headers, json=payload) r.raise_for_status() return r.json() diff --git a/api_app/analyzers_manager/observable_analyzers/auth0.py b/api_app/analyzers_manager/observable_analyzers/auth0.py index 3cdb82cfb1..999706ee78 100644 --- a/api_app/analyzers_manager/observable_analyzers/auth0.py +++ b/api_app/analyzers_manager/observable_analyzers/auth0.py @@ -1,8 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes @@ -19,7 +18,7 @@ def update(cls) -> bool: def run(self): headers = {"X-Auth-Token": self._api_key_name} url = f"{self.url}/{self.observable_name}" - response = requests.get(url, headers=headers) + response = http_utils.get(url, headers=headers) response.raise_for_status() json_response = response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/basic_observable_analyzer.py b/api_app/analyzers_manager/observable_analyzers/basic_observable_analyzer.py index ed50b29125..a48b94a545 100644 --- a/api_app/analyzers_manager/observable_analyzers/basic_observable_analyzer.py +++ b/api_app/analyzers_manager/observable_analyzers/basic_observable_analyzer.py @@ -6,6 +6,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.constants import HTTPMethods from api_app.analyzers_manager.exceptions import ( @@ -74,7 +75,7 @@ def run(self): url = self.url if not self.params.keys(): url = self.url + self.observable_name - response = requests.get( + response = http_utils.get( url, params=self.params, headers=self.headers, diff --git a/api_app/analyzers_manager/observable_analyzers/bgp_ranking.py b/api_app/analyzers_manager/observable_analyzers/bgp_ranking.py index a7127a976a..bcb5eda6dc 100644 --- a/api_app/analyzers_manager/observable_analyzers/bgp_ranking.py +++ b/api_app/analyzers_manager/observable_analyzers/bgp_ranking.py @@ -1,8 +1,7 @@ import json import logging -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -27,7 +26,7 @@ def run(self): # get ASN from ip logger.info(f"Extracting ASN from IP: {self.observable_name}") - response = requests.get( + response = http_utils.get( self.url + "/ipasn_history/?ip=" + self.observable_name, timeout=self.timeout, ) @@ -40,7 +39,7 @@ def run(self): # get ASN rank from extracted ASN logger.info(f"Extracting ASN rank and position from ASN: {asn}") - response = requests.post( + response = http_utils.post( self.url + "/json/asn", data=json.dumps({"asn": asn}), timeout=self.timeout, @@ -62,7 +61,7 @@ def run(self): if self.period: # get ASN history from extracted ASN logger.info(f"Extracting ASN history for period: {self.period}") - response = requests.post( + response = http_utils.post( self.url + "/json/asn_history", data=json.dumps({"asn": asn, "period": self.period}), timeout=self.timeout, diff --git a/api_app/analyzers_manager/observable_analyzers/binaryedge.py b/api_app/analyzers_manager/observable_analyzers/binaryedge.py index dd896a9269..e465cdbf5b 100644 --- a/api_app/analyzers_manager/observable_analyzers/binaryedge.py +++ b/api_app/analyzers_manager/observable_analyzers/binaryedge.py @@ -4,6 +4,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -26,12 +27,12 @@ def run(self): results = {} if self.observable_classification == Classification.IP: try: - response_recent_ip_info = requests.get( + response_recent_ip_info = http_utils.get( self.url + "ip/" + self.observable_name, headers=self.headers ) response_recent_ip_info.raise_for_status() - response_query_ip = requests.get( + response_query_ip = http_utils.get( self.url + "search?query=ip:" + self.observable_name, headers=self.headers, ) @@ -46,7 +47,7 @@ def run(self): } elif self.observable_classification == Classification.DOMAIN: try: - response_domain_report = requests.get( + response_domain_report = http_utils.get( self.url + "domains/subdomain/" + self.observable_name, headers=self.headers, ) diff --git a/api_app/analyzers_manager/observable_analyzers/bitcoinabuse.py b/api_app/analyzers_manager/observable_analyzers/bitcoinabuse.py index 00cb21c222..67a45ac804 100644 --- a/api_app/analyzers_manager/observable_analyzers/bitcoinabuse.py +++ b/api_app/analyzers_manager/observable_analyzers/bitcoinabuse.py @@ -1,8 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes @@ -14,7 +13,7 @@ class BitcoinAbuseAPI(classes.ObservableAnalyzer): def run(self): params = {"address": self.observable_name, "api_token": self._api_key_name} - response = requests.get(self.url, params=params) + response = http_utils.get(self.url, params=params) response.raise_for_status() return response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/censys.py b/api_app/analyzers_manager/observable_analyzers/censys.py index ae3e65b8ed..0c1ffba9ab 100644 --- a/api_app/analyzers_manager/observable_analyzers/censys.py +++ b/api_app/analyzers_manager/observable_analyzers/censys.py @@ -1,8 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -30,7 +29,7 @@ def run(self): raise AnalyzerRunException( f"not supported observable type {self.observable_classification}. Supported is IP." ) - response = requests.get( + response = http_utils.get( self.url + uri, auth=(self._api_id_name, self._api_secret_name), headers={ diff --git a/api_app/analyzers_manager/observable_analyzers/checkphish.py b/api_app/analyzers_manager/observable_analyzers/checkphish.py index 5eb2d7a55b..66748d220f 100644 --- a/api_app/analyzers_manager/observable_analyzers/checkphish.py +++ b/api_app/analyzers_manager/observable_analyzers/checkphish.py @@ -3,8 +3,7 @@ import time -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -28,7 +27,7 @@ def run(self): "urlInfo": {"url": self.observable_name}, } - response = requests.post(CheckPhish.url, json=json_data) + response = http_utils.post(CheckPhish.url, json=json_data) response.raise_for_status() job_id = response.json().get("jobID") @@ -49,7 +48,7 @@ def __poll_analysis_status(self, job_id): for chance in range(self.polling_tries): if chance != 0: time.sleep(self.polling_time) - response = requests.post(CheckPhish.status_url, json=json_data) + response = http_utils.post(CheckPhish.status_url, json=json_data) response.raise_for_status() result = response.json() status_json = result.get("status", "") diff --git a/api_app/analyzers_manager/observable_analyzers/criminalip/criminalip.py b/api_app/analyzers_manager/observable_analyzers/criminalip/criminalip.py index 719a728957..41dfaf0d9b 100644 --- a/api_app/analyzers_manager/observable_analyzers/criminalip/criminalip.py +++ b/api_app/analyzers_manager/observable_analyzers/criminalip/criminalip.py @@ -1,9 +1,9 @@ import logging from typing import Dict -import requests from requests import HTTPError +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.choices import Classification @@ -22,7 +22,7 @@ class CriminalIp(classes.ObservableAnalyzer, CriminalIpBase): hash_view: bool = True # domain def make_request(self, url: str, params: Dict[str, str] = None) -> Dict: - resp = requests.get(url, headers=self.getHeaders(), params=params) + resp = http_utils.get(url, headers=self.getHeaders(), params=params) resp.raise_for_status() resp = resp.json() if resp.get("status", None) not in [None, 200]: diff --git a/api_app/analyzers_manager/observable_analyzers/criminalip/criminalip_scan.py b/api_app/analyzers_manager/observable_analyzers/criminalip/criminalip_scan.py index 59b213ee31..9e05559e24 100644 --- a/api_app/analyzers_manager/observable_analyzers/criminalip/criminalip_scan.py +++ b/api_app/analyzers_manager/observable_analyzers/criminalip/criminalip_scan.py @@ -1,9 +1,9 @@ import logging import time -import requests from requests import HTTPError +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -25,7 +25,7 @@ def update(self): def run(self): HEADER = self.getHeaders() poll_distance = 5 # seconds - resp = requests.post( + resp = http_utils.post( url=f"{self.url}{self.scan_endpoint}", headers=HEADER, data={"query": self.observable_name}, @@ -39,7 +39,7 @@ def run(self): logger.debug(f"{resp=}") scan_id = resp["data"]["scan_id"] while True: - resp = requests.get(url=f"{self.url}{self.status_endpoint}{scan_id}", headers=HEADER) + resp = http_utils.get(url=f"{self.url}{self.status_endpoint}{scan_id}", headers=HEADER) resp.raise_for_status() scan_percent = resp.json()["data"]["scan_percentage"] @@ -49,7 +49,7 @@ def run(self): self.timeout -= poll_distance if self.timeout <= 0: raise AnalyzerRunException(f"Timeout with scan percentage: {scan_percent}") - resp = requests.get(url=f"{self.url}{self.report_endpoint}{scan_id}", headers=HEADER) + resp = http_utils.get(url=f"{self.url}{self.report_endpoint}{scan_id}", headers=HEADER) resp.raise_for_status() resp = resp.json() logger.info(f"response from CriminalIp_scan for {self.observable_name} -> {resp}") diff --git a/api_app/analyzers_manager/observable_analyzers/crowdsec.py b/api_app/analyzers_manager/observable_analyzers/crowdsec.py index 142f30940d..b0c3f5b17a 100644 --- a/api_app/analyzers_manager/observable_analyzers/crowdsec.py +++ b/api_app/analyzers_manager/observable_analyzers/crowdsec.py @@ -2,9 +2,9 @@ # See the file 'LICENSE' for copying permission. import logging -import requests from django.conf import settings +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.data_model_manager.enums import DataModelTags @@ -25,7 +25,7 @@ def run(self): "User-Agent": f"crowdsec-intelowl/{settings.VERSION}", } url = f"{self.url}/v2/smoke/{self.observable_name}" - response = requests.get(url, headers=headers) + response = http_utils.get(url, headers=headers) if response.status_code == 404: result = {"not_found": True} else: diff --git a/api_app/analyzers_manager/observable_analyzers/crt_sh.py b/api_app/analyzers_manager/observable_analyzers/crt_sh.py index 46a1ba5b71..b60a9f8d3e 100644 --- a/api_app/analyzers_manager/observable_analyzers/crt_sh.py +++ b/api_app/analyzers_manager/observable_analyzers/crt_sh.py @@ -1,7 +1,6 @@ import logging -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes logger = logging.getLogger(__name__) @@ -19,7 +18,7 @@ def update(self): def run(self): headers = {"accept": "application/json"} - response = requests.get(f"{self.url}/?q={self.observable_name}", headers=headers) + response = http_utils.get(f"{self.url}/?q={self.observable_name}", headers=headers) response.raise_for_status() response = response.json() return response diff --git a/api_app/analyzers_manager/observable_analyzers/crxcavator.py b/api_app/analyzers_manager/observable_analyzers/crxcavator.py index 604cc8eac4..fb637cb900 100644 --- a/api_app/analyzers_manager/observable_analyzers/crxcavator.py +++ b/api_app/analyzers_manager/observable_analyzers/crxcavator.py @@ -3,6 +3,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -17,7 +18,7 @@ def update(cls) -> bool: def run(self): try: - response = requests.get(self.url + self.observable_name) + response = http_utils.get(self.url + self.observable_name) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) diff --git a/api_app/analyzers_manager/observable_analyzers/cyberchef.py b/api_app/analyzers_manager/observable_analyzers/cyberchef.py index f0a51f0ec6..e9eda11f73 100644 --- a/api_app/analyzers_manager/observable_analyzers/cyberchef.py +++ b/api_app/analyzers_manager/observable_analyzers/cyberchef.py @@ -6,6 +6,7 @@ import requests from django.conf import settings +from api_app import http_utils from api_app.analyzers_manager.classes import DockerBasedAnalyzer, ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -47,7 +48,7 @@ def run(self): request_payload = {"input": self.observable_name, "recipe": self.recipe} if self.output_type: request_payload["outputType"] = self.output_type - response = requests.post(self.url, json=request_payload) + response = http_utils.post(self.url, json=request_payload) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) diff --git a/api_app/analyzers_manager/observable_analyzers/cycat.py b/api_app/analyzers_manager/observable_analyzers/cycat.py index c61a6022e5..cb1fd9789d 100644 --- a/api_app/analyzers_manager/observable_analyzers/cycat.py +++ b/api_app/analyzers_manager/observable_analyzers/cycat.py @@ -1,8 +1,7 @@ import logging import re -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes logger = logging.getLogger(__name__) @@ -20,7 +19,7 @@ def update(self) -> bool: def uuid_lookup(self, uuid: str): logger.info(f"performing lookup on uuid: {uuid}, observable: {self.observable_name}") - response = requests.get( + response = http_utils.get( self.url + "/lookup/" + uuid, headers={"accept": "application/json"}, ) @@ -37,7 +36,7 @@ def run(self): final_response = self.uuid_lookup(self.observable_name) else: - response = requests.get( + response = http_utils.get( self.url + "/search/" + self.observable_name, headers={"accept": "application/json"}, ) diff --git a/api_app/analyzers_manager/observable_analyzers/dehashed.py b/api_app/analyzers_manager/observable_analyzers/dehashed.py index 59d82042ca..025cc87cf1 100644 --- a/api_app/analyzers_manager/observable_analyzers/dehashed.py +++ b/api_app/analyzers_manager/observable_analyzers/dehashed.py @@ -5,9 +5,9 @@ import logging import re -import requests from requests.structures import CaseInsensitiveDict +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.choices import Classification @@ -89,7 +89,7 @@ def __search(self, value: str) -> list: total_entries = [] for page_no in range(1, self.pages + 1): logger.info(f"{self.__repr__()} -> fetching search results for page #{page_no}") - resp = requests.get(f"{url}&page={page_no}", headers=headers) + resp = http_utils.get(f"{url}&page={page_no}", headers=headers) resp.raise_for_status() entries_fetched = resp.json().get("entries", None) if not entries_fetched: diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/adguard.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/adguard.py index a3ed4f0a5f..1b5030767b 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/adguard.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/adguard.py @@ -1,9 +1,9 @@ import logging import dns.message -import requests from dns.rrset import RRset +from api_app import http_utils from api_app.analyzers_manager import classes from ..dns_responses import malicious_detector_response @@ -22,7 +22,7 @@ def update(self) -> bool: def filter_query(self, observable: str) -> list[RRset]: logger.info(f"Sending filtered request to AdGuard DNS API for query: {observable}") - r_filtered = requests.get( + r_filtered = http_utils.get( url=self.build_query_url(observable), headers=self.headers, ) diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/cleanbrowsing_malicious_detector.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/cleanbrowsing_malicious_detector.py index 886d2e9ec1..8deafd6b4a 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/cleanbrowsing_malicious_detector.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/cleanbrowsing_malicious_detector.py @@ -7,8 +7,8 @@ import dns.message import dns.rdatatype -import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.choices import Classification @@ -32,7 +32,7 @@ def run(self): query = dns.message.make_query(observable, dns.rdatatype.A) query_wire = query.to_wire() - response = requests.post( + response = http_utils.post( self.url, data=query_wire, headers={ diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/cloudflare_malicious_detector.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/cloudflare_malicious_detector.py index ca4fa1997a..0a19e8e68f 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/cloudflare_malicious_detector.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/cloudflare_malicious_detector.py @@ -7,6 +7,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -32,7 +33,7 @@ def run(self): "type": "A", } headers = {"accept": "application/dns-json"} - response = requests.get( + response = http_utils.get( "https://security.cloudflare-dns.com/dns-query", params=params, headers=headers, diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/quad9_malicious_detector.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/quad9_malicious_detector.py index 2dae328059..1410138d98 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/quad9_malicious_detector.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/quad9_malicious_detector.py @@ -5,8 +5,7 @@ import logging -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes from ..dns_responses import malicious_detector_response @@ -66,7 +65,7 @@ def _google_dns_query(self, observable) -> bool: :rtype: bool """ params = {"name": observable} - google_response = requests.get(self.google_url, params=params) + google_response = http_utils.get(self.google_url, params=params) google_response.raise_for_status() data = google_response.json() # the DNS server encountered an internal error diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/spamhaus_wqs.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/spamhaus_wqs.py index 5202da3ab0..053bd2646e 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/spamhaus_wqs.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_malicious_detectors/spamhaus_wqs.py @@ -1,7 +1,6 @@ import logging -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -20,7 +19,7 @@ def update(self): def run(self): headers = {"Authorization": f"Bearer {self._api_key}"} - response = requests.get( + response = http_utils.get( url=f"""{self.url}/ {"DBL" if self.observable_classification == Classification.DOMAIN.value else "AUTHBL"} /{self.observable_name}""", diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/cloudflare_dns_resolver.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/cloudflare_dns_resolver.py index 91e16bc89e..a65b39ae46 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/cloudflare_dns_resolver.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/cloudflare_dns_resolver.py @@ -8,6 +8,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -34,7 +35,7 @@ def run(self): "type": self.query_type, } headers = {"accept": "application/dns-json"} - response = requests.get("https://cloudflare-dns.com/dns-query", params=params, headers=headers) + response = http_utils.get("https://cloudflare-dns.com/dns-query", params=params, headers=headers) response.raise_for_status() response_dict = response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/google_dns_resolver.py b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/google_dns_resolver.py index 01759cac6d..5512614f86 100644 --- a/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/google_dns_resolver.py +++ b/api_app/analyzers_manager/observable_analyzers/dns/dns_resolvers/google_dns_resolver.py @@ -8,6 +8,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -33,7 +34,7 @@ def run(self): "name": observable, "type": self.query_type, } - response = requests.get("https://dns.google.com/resolve", params=params) + response = http_utils.get("https://dns.google.com/resolve", params=params) response.raise_for_status() data = response.json() resolutions = data.get("Answer", None) diff --git a/api_app/analyzers_manager/observable_analyzers/dnsdb.py b/api_app/analyzers_manager/observable_analyzers/dnsdb.py index ce3d92d977..df300c7a13 100644 --- a/api_app/analyzers_manager/observable_analyzers/dnsdb.py +++ b/api_app/analyzers_manager/observable_analyzers/dnsdb.py @@ -6,8 +6,8 @@ from urllib.parse import urlparse import dateparser -import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -76,7 +76,7 @@ def run(self): params = self._create_params() # perform request - response = requests.get(url, params=params, headers=headers) + response = http_utils.get(url, params=params, headers=headers) # for API v1, 404 means no results found if self.api_version == 1 and response.status_code == 404: self.no_results_found = True diff --git a/api_app/analyzers_manager/observable_analyzers/docguard_get.py b/api_app/analyzers_manager/observable_analyzers/docguard_get.py index ce881946a0..2d02aa25ab 100644 --- a/api_app/analyzers_manager/observable_analyzers/docguard_get.py +++ b/api_app/analyzers_manager/observable_analyzers/docguard_get.py @@ -5,6 +5,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -44,7 +45,7 @@ def run(self): uri = f"{self.observable_name}" if self.observable_classification == Classification.HASH: try: - response = requests.get(self.url + uri, headers=headers) + response = http_utils.get(self.url + uri, headers=headers) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) diff --git a/api_app/analyzers_manager/observable_analyzers/download_file_from_uri.py b/api_app/analyzers_manager/observable_analyzers/download_file_from_uri.py index 47382efc61..e75802a02f 100644 --- a/api_app/analyzers_manager/observable_analyzers/download_file_from_uri.py +++ b/api_app/analyzers_manager/observable_analyzers/download_file_from_uri.py @@ -3,8 +3,7 @@ import base64 import logging -import requests - +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -35,7 +34,7 @@ def run(self): } try: - r = requests.get( + r = http_utils.get( self.observable_name, headers=headers, proxies=proxies, diff --git a/api_app/analyzers_manager/observable_analyzers/dshield.py b/api_app/analyzers_manager/observable_analyzers/dshield.py index dabc92646d..f32a638a69 100644 --- a/api_app/analyzers_manager/observable_analyzers/dshield.py +++ b/api_app/analyzers_manager/observable_analyzers/dshield.py @@ -5,6 +5,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer logger = logging.getLogger(__name__) @@ -26,7 +27,7 @@ def run(self): for query_type, values in result.items(): try: - response = requests.get(self.url + values["uri"], headers=headers) + response = http_utils.get(self.url + values["uri"], headers=headers) response.raise_for_status() except requests.RequestException as e: logger.warning(e, stack_info=True) diff --git a/api_app/analyzers_manager/observable_analyzers/emailrep.py b/api_app/analyzers_manager/observable_analyzers/emailrep.py index ba2faa3781..4781b0dda6 100644 --- a/api_app/analyzers_manager/observable_analyzers/emailrep.py +++ b/api_app/analyzers_manager/observable_analyzers/emailrep.py @@ -1,8 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -37,7 +36,7 @@ def run(self): url = self.url.format(self.observable_name) - response = requests.get(url, headers=headers) + response = http_utils.get(url, headers=headers) response.raise_for_status() return response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/expand_url.py b/api_app/analyzers_manager/observable_analyzers/expand_url.py index 31a4eb981d..fe07da4598 100644 --- a/api_app/analyzers_manager/observable_analyzers/expand_url.py +++ b/api_app/analyzers_manager/observable_analyzers/expand_url.py @@ -3,6 +3,7 @@ import requests from bs4 import BeautifulSoup +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -45,7 +46,7 @@ def expand_url(self, url) -> list[str]: logger.info(f"Expanding {url}") try: while no_more_redirects is False: - final_response = requests.get(url, headers=headers, allow_redirects=True) + final_response = http_utils.get(url, headers=headers, allow_redirects=True) final_response.raise_for_status() for response in final_response.history: redirection_chain.append(response.url) diff --git a/api_app/analyzers_manager/observable_analyzers/feodo_tracker.py b/api_app/analyzers_manager/observable_analyzers/feodo_tracker.py index ed713a79eb..faa32f1f4d 100644 --- a/api_app/analyzers_manager/observable_analyzers/feodo_tracker.py +++ b/api_app/analyzers_manager/observable_analyzers/feodo_tracker.py @@ -9,6 +9,7 @@ import requests from django.conf import settings +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.decorators import classproperty @@ -86,7 +87,7 @@ def update(cls) -> bool: logger.info(f"starting download of db from {db_url}") try: - r = requests.get(db_url, headers=cls.get_service_auth_headers()) + r = http_utils.get(db_url, headers=cls.get_service_auth_headers()) r.raise_for_status() except requests.RequestException: return False diff --git a/api_app/analyzers_manager/observable_analyzers/filescan_search.py b/api_app/analyzers_manager/observable_analyzers/filescan_search.py index 96331560ab..f8664cbc9b 100644 --- a/api_app/analyzers_manager/observable_analyzers/filescan_search.py +++ b/api_app/analyzers_manager/observable_analyzers/filescan_search.py @@ -5,6 +5,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -25,7 +26,7 @@ def run(self): endpoint = "?query={input}" url = f"{self.url}/{endpoint.format(input=observable_name_base64)}" try: - response = requests.get(url, headers={"X-Api-Key": self._api_key}) + response = http_utils.get(url, headers={"X-Api-Key": self._api_key}) response.raise_for_status() except requests.RequestException as error: raise AnalyzerRunException(error) diff --git a/api_app/analyzers_manager/observable_analyzers/firehol_iplist.py b/api_app/analyzers_manager/observable_analyzers/firehol_iplist.py index ac4afe9402..78777c8a1e 100644 --- a/api_app/analyzers_manager/observable_analyzers/firehol_iplist.py +++ b/api_app/analyzers_manager/observable_analyzers/firehol_iplist.py @@ -7,9 +7,9 @@ import traceback from datetime import datetime -import requests from django.conf import settings +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import ( AnalyzerConfigurationException, @@ -63,7 +63,7 @@ def download_iplist(list_name): logger.info(f"starting download of {list_name} from firehol iplist") url = f"https://iplists.firehol.org/files/{list_name}" - r = requests.get(url) + r = http_utils.get(url) r.raise_for_status() data_extracted = r.content.decode() diff --git a/api_app/analyzers_manager/observable_analyzers/greedybear.py b/api_app/analyzers_manager/observable_analyzers/greedybear.py index 96642f8f1e..5c56619460 100644 --- a/api_app/analyzers_manager/observable_analyzers/greedybear.py +++ b/api_app/analyzers_manager/observable_analyzers/greedybear.py @@ -3,8 +3,7 @@ import logging -import requests - +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.choices import Classification from api_app.helpers import get_hash_type @@ -40,19 +39,19 @@ def run(self): if self.same_cluster_commands: params_["include_similar"] = True logger.info(f"Fetching command sequence for SHA-256 hash: {self.observable_name}.") - command_sequence_response = requests.get( + command_sequence_response = http_utils.get( self.url + command_sequence_uri, params=params_, headers=headers ) result = {"command_sequence_results": command_sequence_response.json()} else: result = {"command_sequence_results": "Unsupported hash type. Only SHA-256 is supported."} else: - enrichment_response = requests.get(self.url + enrichment_uri, params=params_, headers=headers) + enrichment_response = http_utils.get(self.url + enrichment_uri, params=params_, headers=headers) if self.command_sequence_toggle: logger.info(f"Fetching command sequence for observable: {self.observable_name}.") if self.same_cluster_commands: params_["include_similar"] = True - command_sequence_response = requests.get( + command_sequence_response = http_utils.get( self.url + command_sequence_uri, params=params_, headers=headers ) result["command_sequence_results"] = command_sequence_response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/greynoise_labs.py b/api_app/analyzers_manager/observable_analyzers/greynoise_labs.py index bb850cec0c..73927d5935 100644 --- a/api_app/analyzers_manager/observable_analyzers/greynoise_labs.py +++ b/api_app/analyzers_manager/observable_analyzers/greynoise_labs.py @@ -4,9 +4,9 @@ import logging import os -import requests from django.conf import settings +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.models import PluginConfig @@ -74,7 +74,7 @@ def run(self): "query": value["query_string"], "variables": {"ip": f"{self.observable_name}"}, } - response = requests.post(headers=headers, json=json_body, url=url) + response = http_utils.post(headers=headers, json=json_body, url=url) response.raise_for_status() result[key] = response.json() @@ -100,7 +100,7 @@ def _update_db(cls, auth_token: str): try: logger.info("Fetching data from greynoise API (Greynoise_Labs).....") - response = requests.post( + response = http_utils.post( headers=headers, json={"query": queries["topc2s"]["query_string"]}, url=url, diff --git a/api_app/analyzers_manager/observable_analyzers/ha_get.py b/api_app/analyzers_manager/observable_analyzers/ha_get.py index 10ad0175c3..47ea1a7d80 100644 --- a/api_app/analyzers_manager/observable_analyzers/ha_get.py +++ b/api_app/analyzers_manager/observable_analyzers/ha_get.py @@ -7,6 +7,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -26,7 +27,7 @@ def update(cls) -> bool: def _fetch_sample_summary(self, sha256: str, headers: Dict[str, str]) -> Optional[Dict[str, Any]]: overview_uri = f"overview/{sha256}" try: - res = requests.get(self.api_url + overview_uri, headers=headers) + res = http_utils.get(self.api_url + overview_uri, headers=headers) res.raise_for_status() data = res.json() return data if isinstance(data, dict) else None @@ -34,14 +35,14 @@ def _fetch_sample_summary(self, sha256: str, headers: Dict[str, str]) -> Optiona return None def _search_terms(self, key: str, value: str, headers: Dict[str, str]): - return requests.post( + return http_utils.post( self.api_url + "search/terms", data={key: value}, headers=headers, ) def _search_hash(self, value: str, headers: Dict[str, str]): - return requests.get( + return http_utils.get( self.api_url + "search/hash", params={"hash": value}, headers=headers, diff --git a/api_app/analyzers_manager/observable_analyzers/haveibeenpwned.py b/api_app/analyzers_manager/observable_analyzers/haveibeenpwned.py index 5dd3e47cee..b9f3684ab7 100644 --- a/api_app/analyzers_manager/observable_analyzers/haveibeenpwned.py +++ b/api_app/analyzers_manager/observable_analyzers/haveibeenpwned.py @@ -1,8 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes @@ -28,7 +27,7 @@ def run(self): headers = {"hibp-api-key": self._api_key_name} - response = requests.get(self.url + self.observable_name, params=params, headers=headers) + response = http_utils.get(self.url + self.observable_name, params=params, headers=headers) response.raise_for_status() result = response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/hibp_utils.py b/api_app/analyzers_manager/observable_analyzers/hibp_utils.py index 4fb825ed8a..8982b2b15d 100644 --- a/api_app/analyzers_manager/observable_analyzers/hibp_utils.py +++ b/api_app/analyzers_manager/observable_analyzers/hibp_utils.py @@ -1,9 +1,10 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import requests from requests.exceptions import RequestException +from api_app import http_utils + # Constants BASE_URL = "https://haveibeenpwned.com/api/v3/" PWNED_PASSWORDS_URL = "https://api.pwnedpasswords.com/range/" @@ -37,7 +38,7 @@ def make_hibp_request( :raises AnalyzerRunException: On errors with details. """ try: - response = requests.get( + response = http_utils.get( url, params=params or {}, headers=get_headers(api_key), diff --git a/api_app/analyzers_manager/observable_analyzers/honeydb.py b/api_app/analyzers_manager/observable_analyzers/honeydb.py index 89de2a5f2e..39236c8b28 100644 --- a/api_app/analyzers_manager/observable_analyzers/honeydb.py +++ b/api_app/analyzers_manager/observable_analyzers/honeydb.py @@ -4,8 +4,7 @@ import logging from typing import Dict -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerConfigurationException @@ -64,7 +63,7 @@ def _request_analysis(self, endpoint): logger.error(f"endpoint {endpoint} not supported") return try: - response = requests.get(url, headers=self.headers) + response = http_utils.get(url, headers=self.headers) response.raise_for_status() except Exception as e: logger.exception(e) diff --git a/api_app/analyzers_manager/observable_analyzers/hudsonrock.py b/api_app/analyzers_manager/observable_analyzers/hudsonrock.py index 789d8f5f21..573e1b8e4e 100644 --- a/api_app/analyzers_manager/observable_analyzers/hudsonrock.py +++ b/api_app/analyzers_manager/observable_analyzers/hudsonrock.py @@ -1,8 +1,7 @@ import logging import re -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerConfigurationException from api_app.choices import Classification @@ -71,7 +70,7 @@ def run(self): ] ) ) - response = requests.post(url, headers=headers, json={"ip": self.observable_name}) + response = http_utils.post(url, headers=headers, json={"ip": self.observable_name}) elif self.observable_classification == Classification.DOMAIN: url = ( @@ -91,7 +90,7 @@ def run(self): ] ) ) - response = requests.post(url, headers=headers, json={"domains": [self.observable_name]}) + response = http_utils.post(url, headers=headers, json={"domains": [self.observable_name]}) elif self.observable_classification == Classification.GENERIC: # checking for email @@ -102,7 +101,7 @@ def run(self): + "/search-by-login" + self.get_param_url(["sortby", "page", "installed_software"]) ) - response = requests.post(url, headers=headers, json={"login": self.observable_name}) + response = http_utils.post(url, headers=headers, json={"login": self.observable_name}) else: raise AnalyzerConfigurationException( f"Invalid observable type {self.observable_classification}" diff --git a/api_app/analyzers_manager/observable_analyzers/hunter_how.py b/api_app/analyzers_manager/observable_analyzers/hunter_how.py index 4b21fa1ee8..d8a467d74f 100644 --- a/api_app/analyzers_manager/observable_analyzers/hunter_how.py +++ b/api_app/analyzers_manager/observable_analyzers/hunter_how.py @@ -6,6 +6,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -38,7 +39,7 @@ def config(self, runtime_configuration: Dict): def run(self): try: - response_ip = requests.get(self.url, params=self.parameters) + response_ip = http_utils.get(self.url, params=self.parameters) response_ip.raise_for_status() except requests.RequestException as e: diff --git a/api_app/analyzers_manager/observable_analyzers/hunter_io.py b/api_app/analyzers_manager/observable_analyzers/hunter_io.py index 1ead8acff1..5014f1c36b 100644 --- a/api_app/analyzers_manager/observable_analyzers/hunter_io.py +++ b/api_app/analyzers_manager/observable_analyzers/hunter_io.py @@ -1,8 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes @@ -17,7 +16,7 @@ def update(cls) -> bool: def run(self): url = f"{self.url}domain={self.observable_name}&api_key={self._api_key_name}" - response = requests.get(url) + response = http_utils.get(url) response.raise_for_status() return response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/hunting_abuse.py b/api_app/analyzers_manager/observable_analyzers/hunting_abuse.py index 4183f0bcfe..419f4a296a 100644 --- a/api_app/analyzers_manager/observable_analyzers/hunting_abuse.py +++ b/api_app/analyzers_manager/observable_analyzers/hunting_abuse.py @@ -9,6 +9,7 @@ import requests from django.conf import settings +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -44,7 +45,7 @@ def update(cls) -> bool: data = {"query": "get_fplist", "format": "json"} try: - response = requests.post(cls.url, json=data, headers=headers) + response = http_utils.post(cls.url, json=data, headers=headers) response.raise_for_status() with open(database_location, "w", encoding="utf-8") as f: diff --git a/api_app/analyzers_manager/observable_analyzers/inquest.py b/api_app/analyzers_manager/observable_analyzers/inquest.py index eaf57f531a..78c5abc1ae 100644 --- a/api_app/analyzers_manager/observable_analyzers/inquest.py +++ b/api_app/analyzers_manager/observable_analyzers/inquest.py @@ -5,8 +5,7 @@ import re from typing import Dict -import requests - +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerConfigurationException, AnalyzerRunException from api_app.choices import Classification @@ -138,7 +137,7 @@ def run(self): "Supported are: 'dfi_search', 'iocdb_search', 'repdb_search'." ) - response = requests.get(self.url + uri, headers=headers, timeout=30) + response = http_utils.get(self.url + uri, headers=headers, timeout=30) response.raise_for_status() result = response.json() if self.inquest_analysis == "dfi_search" and self.observable_classification == Classification.HASH: diff --git a/api_app/analyzers_manager/observable_analyzers/intelx.py b/api_app/analyzers_manager/observable_analyzers/intelx.py index ee7c567a33..62f7161abb 100644 --- a/api_app/analyzers_manager/observable_analyzers/intelx.py +++ b/api_app/analyzers_manager/observable_analyzers/intelx.py @@ -8,6 +8,7 @@ import requests from django.utils.functional import cached_property +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import ( AnalyzerConfigurationException, @@ -44,7 +45,7 @@ def config(self, runtime_configuration: Dict): @cached_property def _session(self): - session = requests.Session() + session = http_utils.Session() session.headers.update({"x-key": self._api_key_name, "User-Agent": "IntelOwl"}) return session diff --git a/api_app/analyzers_manager/observable_analyzers/ip2location.py b/api_app/analyzers_manager/observable_analyzers/ip2location.py index 53a9f3235d..2a7d63fd82 100644 --- a/api_app/analyzers_manager/observable_analyzers/ip2location.py +++ b/api_app/analyzers_manager/observable_analyzers/ip2location.py @@ -3,6 +3,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -17,7 +18,7 @@ def update(cls) -> bool: pass def get_response(self, payload): - return requests.get(self.url, params=payload) + return http_utils.get(self.url, params=payload) def run(self): try: diff --git a/api_app/analyzers_manager/observable_analyzers/ip2whois.py b/api_app/analyzers_manager/observable_analyzers/ip2whois.py index 0b74b19c22..d3e378eb1b 100644 --- a/api_app/analyzers_manager/observable_analyzers/ip2whois.py +++ b/api_app/analyzers_manager/observable_analyzers/ip2whois.py @@ -3,6 +3,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -15,7 +16,7 @@ def update(self): pass def get_response(self, payload): - return requests.get(self.url, params=payload) + return http_utils.get(self.url, params=payload) def run(self): try: diff --git a/api_app/analyzers_manager/observable_analyzers/ipapi.py b/api_app/analyzers_manager/observable_analyzers/ipapi.py index 47d3b4022d..f056cdeab8 100644 --- a/api_app/analyzers_manager/observable_analyzers/ipapi.py +++ b/api_app/analyzers_manager/observable_analyzers/ipapi.py @@ -2,8 +2,7 @@ # See the file 'LICENSE' for copying permission. from typing import Dict -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes @@ -25,10 +24,10 @@ def config(self, runtime_configuration: Dict): ] def run(self): - response_batch = requests.post(self.batch_url, json=self.IP) + response_batch = http_utils.post(self.batch_url, json=self.IP) response_batch.raise_for_status() - response_dns = requests.get(self.dns_url) + response_dns = http_utils.get(self.dns_url) response_dns.raise_for_status() response = {"ip_info": response_batch.json(), "dns_info": response_dns.json()} diff --git a/api_app/analyzers_manager/observable_analyzers/ipinfo.py b/api_app/analyzers_manager/observable_analyzers/ipinfo.py index 444d3919d3..d0191aa271 100644 --- a/api_app/analyzers_manager/observable_analyzers/ipinfo.py +++ b/api_app/analyzers_manager/observable_analyzers/ipinfo.py @@ -3,6 +3,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -18,7 +19,7 @@ def update(cls) -> bool: def run(self): try: - response = requests.get( + response = http_utils.get( self.url + self.observable_name, params={"token": self._api_key_name}, ) diff --git a/api_app/analyzers_manager/observable_analyzers/ipqs.py b/api_app/analyzers_manager/observable_analyzers/ipqs.py index d80ebb5a65..c81550eb0e 100644 --- a/api_app/analyzers_manager/observable_analyzers/ipqs.py +++ b/api_app/analyzers_manager/observable_analyzers/ipqs.py @@ -3,6 +3,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -127,7 +128,7 @@ def run(self): try: if calling_endpoint and payload is not None: - response = requests.get( + response = http_utils.get( calling_endpoint + self.observable_name, headers=ipqs_headers, params=payload, diff --git a/api_app/analyzers_manager/observable_analyzers/ipquery.py b/api_app/analyzers_manager/observable_analyzers/ipquery.py index 36ac7b2f7f..9ac683a5b0 100644 --- a/api_app/analyzers_manager/observable_analyzers/ipquery.py +++ b/api_app/analyzers_manager/observable_analyzers/ipquery.py @@ -5,6 +5,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -22,7 +23,7 @@ def run(self): logger.info(f"Running IPQuery Analyzer for {self.observable_name}") try: - response = requests.get(f"{self.url}{self.observable_name}?format=json") + response = http_utils.get(f"{self.url}{self.observable_name}?format=json") response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) diff --git a/api_app/analyzers_manager/observable_analyzers/ja4_db.py b/api_app/analyzers_manager/observable_analyzers/ja4_db.py index 6b67441936..5558c283fe 100644 --- a/api_app/analyzers_manager/observable_analyzers/ja4_db.py +++ b/api_app/analyzers_manager/observable_analyzers/ja4_db.py @@ -2,9 +2,9 @@ import logging import os -import requests from django.conf import settings +from api_app import http_utils from api_app.analyzers_manager import classes logger = logging.getLogger(__name__) @@ -62,7 +62,7 @@ def check_ja4_fingerprint(self, observable: str) -> str: @classmethod def update(cls): logger.info(f"Updating database from {cls.url}") - response = requests.get(url=cls.url) + response = http_utils.get(url=cls.url) response.raise_for_status() data = response.json() database_location = cls.location() diff --git a/api_app/analyzers_manager/observable_analyzers/koodous.py b/api_app/analyzers_manager/observable_analyzers/koodous.py index 5a97a5d4d3..ca9805166e 100644 --- a/api_app/analyzers_manager/observable_analyzers/koodous.py +++ b/api_app/analyzers_manager/observable_analyzers/koodous.py @@ -1,8 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes @@ -17,7 +16,7 @@ def update(cls) -> bool: pass def get_response(self, url): - return requests.get(url, headers={"Authorization": f"Token {self._api_key_name}"}) + return http_utils.get(url, headers={"Authorization": f"Token {self._api_key_name}"}) def run(self): common_url = self.url + self.observable_name diff --git a/api_app/analyzers_manager/observable_analyzers/leakix.py b/api_app/analyzers_manager/observable_analyzers/leakix.py index af4eb658bf..c4cc315962 100644 --- a/api_app/analyzers_manager/observable_analyzers/leakix.py +++ b/api_app/analyzers_manager/observable_analyzers/leakix.py @@ -1,7 +1,6 @@ import logging -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes logger = logging.getLogger(__name__) @@ -20,6 +19,6 @@ def update(self) -> bool: def run(self): headers = {"api-key": f"{self._api_key}", "Accept": "application/json"} - response = requests.get(url=self.url + f"/{self.observable_name}", headers=headers) + response = http_utils.get(url=self.url + f"/{self.observable_name}", headers=headers) response.raise_for_status() return response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/malprob.py b/api_app/analyzers_manager/observable_analyzers/malprob.py index 1695d9aa62..588a653162 100644 --- a/api_app/analyzers_manager/observable_analyzers/malprob.py +++ b/api_app/analyzers_manager/observable_analyzers/malprob.py @@ -1,5 +1,6 @@ -import requests +from urllib.parse import quote +from api_app import http_utils from api_app.analyzers_manager import classes @@ -10,8 +11,8 @@ def update(self): pass def run(self): - response = requests.get( - f"{self.url}/search/{self.observable_name}", + response = http_utils.get( + f"{self.url}/search/{quote(self.observable_name, safe='')}", timeout=10, ) response.raise_for_status() diff --git a/api_app/analyzers_manager/observable_analyzers/maxmind.py b/api_app/analyzers_manager/observable_analyzers/maxmind.py index 2b7a62a4e5..991fc5d215 100644 --- a/api_app/analyzers_manager/observable_analyzers/maxmind.py +++ b/api_app/analyzers_manager/observable_analyzers/maxmind.py @@ -8,12 +8,12 @@ import tarfile import maxminddb -import requests from django.conf import settings from geoip2.database import Reader from geoip2.errors import AddressNotFoundError, GeoIP2Error from geoip2.models import ASN, City, Country +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import ( AnalyzerConfigurationException, @@ -119,7 +119,7 @@ def _download_db(cls, db_name: str, api_key: str) -> str: "https://download.maxmind.com/app/geoip_download?edition_id=" f"{db_name}&license_key={api_key}&suffix=tar.gz" ) - response = requests.get(url) + response = http_utils.get(url) if response.status_code >= 300: raise AnalyzerRunException( f"failed request for new maxmind db {db_name}." diff --git a/api_app/analyzers_manager/observable_analyzers/mb_get.py b/api_app/analyzers_manager/observable_analyzers/mb_get.py index 915be55c1c..b1ff778002 100644 --- a/api_app/analyzers_manager/observable_analyzers/mb_get.py +++ b/api_app/analyzers_manager/observable_analyzers/mb_get.py @@ -2,8 +2,7 @@ # See the file 'LICENSE' for copying permission. import logging -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.mixins import AbuseCHMixin @@ -33,7 +32,7 @@ def query_mb_api(cls, observable_name: str, headers: dict = None) -> dict: if headers is None: headers = {} - response = requests.post(cls.url, data=post_data, headers=headers) + response = http_utils.post(cls.url, data=post_data, headers=headers) response.raise_for_status() result = response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/mmdb_server.py b/api_app/analyzers_manager/observable_analyzers/mmdb_server.py index a088884d27..7e3e5add1f 100644 --- a/api_app/analyzers_manager/observable_analyzers/mmdb_server.py +++ b/api_app/analyzers_manager/observable_analyzers/mmdb_server.py @@ -1,7 +1,7 @@ import logging +from urllib.parse import quote, urljoin -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes logger = logging.getLogger(__name__) @@ -19,6 +19,8 @@ def update(self) -> bool: observable_name: str def run(self): - response = requests.get(self.url + self.observable_name) + encoded_name = quote(self.observable_name, safe="") + url = urljoin(self.url, encoded_name) + response = http_utils.get(url) response.raise_for_status() return response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/mnemonic_pdns.py b/api_app/analyzers_manager/observable_analyzers/mnemonic_pdns.py index 0065eb03a2..c02872f0a2 100644 --- a/api_app/analyzers_manager/observable_analyzers/mnemonic_pdns.py +++ b/api_app/analyzers_manager/observable_analyzers/mnemonic_pdns.py @@ -3,8 +3,7 @@ import json -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes @@ -21,7 +20,7 @@ def update(cls) -> bool: def run(self): if self.cof_format: self.url += "cof/" - response = requests.get(self.url + self.observable_name, data={"limit": self.limit}) + response = http_utils.get(self.url + self.observable_name, data={"limit": self.limit}) response.raise_for_status() if self.cof_format: diff --git a/api_app/analyzers_manager/observable_analyzers/nerd.py b/api_app/analyzers_manager/observable_analyzers/nerd.py index 7a163a2a17..8a729f9b0d 100644 --- a/api_app/analyzers_manager/observable_analyzers/nerd.py +++ b/api_app/analyzers_manager/observable_analyzers/nerd.py @@ -4,6 +4,7 @@ import requests from requests.exceptions import HTTPError +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import ( AnalyzerConfigurationException, @@ -38,7 +39,7 @@ def run(self): ) try: - response = requests.get(self.url + uri, headers=headers) + response = http_utils.get(self.url + uri, headers=headers) response.raise_for_status() result = response.json() except requests.RequestException as e: diff --git a/api_app/analyzers_manager/observable_analyzers/netlas.py b/api_app/analyzers_manager/observable_analyzers/netlas.py index 57fe7b5ee9..2715999fcd 100644 --- a/api_app/analyzers_manager/observable_analyzers/netlas.py +++ b/api_app/analyzers_manager/observable_analyzers/netlas.py @@ -4,6 +4,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -27,7 +28,7 @@ def config(self, runtime_configuration: Dict): def run(self): try: - response = requests.get(self.url, params=self.parameters, headers=self.headers) + response = http_utils.get(self.url, params=self.parameters, headers=self.headers) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) diff --git a/api_app/analyzers_manager/observable_analyzers/nvd_cve.py b/api_app/analyzers_manager/observable_analyzers/nvd_cve.py index d7678f89a7..97d083d626 100644 --- a/api_app/analyzers_manager/observable_analyzers/nvd_cve.py +++ b/api_app/analyzers_manager/observable_analyzers/nvd_cve.py @@ -2,6 +2,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager.classes import AnalyzerRunException, ObservableAnalyzer @@ -25,7 +26,7 @@ def run(self): raise ValueError(f"Invalid CVE format: {self.observable_name}") params = {"cveId": self.observable_name.upper()} - response = requests.get(url=self.url, params=params, headers=headers) + response = http_utils.get(url=self.url, params=params, headers=headers) response.raise_for_status() except ValueError as e: diff --git a/api_app/analyzers_manager/observable_analyzers/onyphe.py b/api_app/analyzers_manager/observable_analyzers/onyphe.py index 84f690766e..1a2f4b8e50 100644 --- a/api_app/analyzers_manager/observable_analyzers/onyphe.py +++ b/api_app/analyzers_manager/observable_analyzers/onyphe.py @@ -3,6 +3,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -36,7 +37,7 @@ def run(self): ) try: - response = requests.get(self.url + uri, headers=headers) + response = http_utils.get(self.url + uri, headers=headers) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) diff --git a/api_app/analyzers_manager/observable_analyzers/orkl_search.py b/api_app/analyzers_manager/observable_analyzers/orkl_search.py index 8b4fd5b8fb..fbc2723b78 100644 --- a/api_app/analyzers_manager/observable_analyzers/orkl_search.py +++ b/api_app/analyzers_manager/observable_analyzers/orkl_search.py @@ -1,7 +1,6 @@ import logging -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.choices import Classification @@ -21,7 +20,7 @@ def run(self): "accept": "application/json", } if self.observable_classification == Classification.HASH.value: - response = requests.get( + response = http_utils.get( url=f"{self.url}/library/entry/sha1/{self.observable_name}", headers=headers, ) @@ -30,7 +29,7 @@ def run(self): "message": "No LibraryEntry found with SHA1 hash", } else: - response = requests.get( + response = http_utils.get( url=f"""{self.url}/library/search?query={self.observable_name} &full={self.full}&limit={self.limit}""", headers=headers, diff --git a/api_app/analyzers_manager/observable_analyzers/otx.py b/api_app/analyzers_manager/observable_analyzers/otx.py index 6a0a1951e0..fceb6a3fda 100644 --- a/api_app/analyzers_manager/observable_analyzers/otx.py +++ b/api_app/analyzers_manager/observable_analyzers/otx.py @@ -6,8 +6,8 @@ from urllib.parse import urlparse import OTXv2 -import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -32,7 +32,7 @@ def session(self): # this was needed because, otherwise, the analyzer could last too much time # and become the bottleneck of all the application if self.request_session is None: - self.request_session = requests.Session() + self.request_session = http_utils.Session() return self.request_session diff --git a/api_app/analyzers_manager/observable_analyzers/phishing_army.py b/api_app/analyzers_manager/observable_analyzers/phishing_army.py index bb7c492a8d..2178aa2c45 100644 --- a/api_app/analyzers_manager/observable_analyzers/phishing_army.py +++ b/api_app/analyzers_manager/observable_analyzers/phishing_army.py @@ -4,9 +4,9 @@ import logging from urllib.parse import urlparse -import requests from django.db import transaction +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.models import PhishingArmyDomain from api_app.choices import Classification @@ -33,7 +33,7 @@ def run(self): def update(cls) -> bool: try: logger.info("starting download of db from Phishing Army") - response = requests.get(cls.url) + response = http_utils.get(cls.url) response.raise_for_status() unique_domains = { diff --git a/api_app/analyzers_manager/observable_analyzers/phishstats.py b/api_app/analyzers_manager/observable_analyzers/phishstats.py index 9b9f949cb5..ab6d241cf3 100644 --- a/api_app/analyzers_manager/observable_analyzers/phishstats.py +++ b/api_app/analyzers_manager/observable_analyzers/phishstats.py @@ -4,8 +4,7 @@ from ipaddress import AddressValueError, IPv4Address from urllib.parse import urlparse -import requests - +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -46,7 +45,7 @@ def __build_phishstats_url(self) -> str: def run(self): api_url = self.__build_phishstats_url() - response = requests.get(api_url) + response = http_utils.get(api_url) response.raise_for_status() return {"api_url": api_url, "results": response.json()} diff --git a/api_app/analyzers_manager/observable_analyzers/phishtank.py b/api_app/analyzers_manager/observable_analyzers/phishtank.py index d942065cab..baf85fe793 100644 --- a/api_app/analyzers_manager/observable_analyzers/phishtank.py +++ b/api_app/analyzers_manager/observable_analyzers/phishtank.py @@ -7,6 +7,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -35,7 +36,7 @@ def run(self): else: data["app_key"] = self._api_key_name try: - resp = requests.post("https://checkurl.phishtank.com/checkurl/", data=data, headers=headers) + resp = http_utils.post("https://checkurl.phishtank.com/checkurl/", data=data, headers=headers) resp.raise_for_status() result = resp.json() except requests.RequestException as e: diff --git a/api_app/analyzers_manager/observable_analyzers/phoneinfoga_scan.py b/api_app/analyzers_manager/observable_analyzers/phoneinfoga_scan.py index 43d4299792..fa095b09ee 100644 --- a/api_app/analyzers_manager/observable_analyzers/phoneinfoga_scan.py +++ b/api_app/analyzers_manager/observable_analyzers/phoneinfoga_scan.py @@ -4,6 +4,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerConfigurationException @@ -68,7 +69,7 @@ def run(self): "GOOGLE_API_KEY": self._GOOGLE_API_KEY, "GOOGLECSE_MAX_RESULTS": self.googlecse_max_results, } - response = requests.post( + response = http_utils.post( url, headers={ "Content-Type": "application/json", diff --git a/api_app/analyzers_manager/observable_analyzers/pulsedive.py b/api_app/analyzers_manager/observable_analyzers/pulsedive.py index ff3a9dcaed..5c7b26cbfe 100644 --- a/api_app/analyzers_manager/observable_analyzers/pulsedive.py +++ b/api_app/analyzers_manager/observable_analyzers/pulsedive.py @@ -5,8 +5,7 @@ import time from typing import Dict -import requests - +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import ( AnalyzerConfigurationException, @@ -53,7 +52,7 @@ def run(self): params = f"indicator={self.observable_name}" if hasattr(self, "_api_key_name"): params += self.default_param - resp = requests.get(f"{self.url}/info.php?{params}") + resp = http_utils.get(f"{self.url}/info.php?{params}") # handle 404 case, submit for analysis if resp.status_code == 404 and self.scan_mode != "basic": @@ -70,7 +69,7 @@ def __submit_for_analysis(self) -> dict: if hasattr(self, "_api_key_name"): params += self.default_param headers = {"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8"} - resp = requests.post(f"{self.url}/analyze.php", data=params, headers=headers) + resp = http_utils.post(f"{self.url}/analyze.php", data=params, headers=headers) resp.raise_for_status() qid = resp.json().get("qid", None) # 3. retrieve result using qid after waiting for 10 seconds @@ -90,7 +89,7 @@ def __poll_for_result(self, params): for chance in range(self.max_tries): logger.info(f"polling request #{chance + 1} for observable: {self.observable_name} <- {obj_repr}") time.sleep(self.poll_distance) - resp = requests.get(url) + resp = http_utils.get(url) resp.raise_for_status() resp_json = resp.json() status = resp_json.get("status", None) diff --git a/api_app/analyzers_manager/observable_analyzers/robtex.py b/api_app/analyzers_manager/observable_analyzers/robtex.py index 1f95a45442..c8322dbf2f 100644 --- a/api_app/analyzers_manager/observable_analyzers/robtex.py +++ b/api_app/analyzers_manager/observable_analyzers/robtex.py @@ -4,8 +4,7 @@ import json from urllib.parse import urlparse -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -38,7 +37,7 @@ def run(self): loaded_results = [] for uri in uris: - response = requests.get(self.url + uri) + response = http_utils.get(self.url + uri) response.raise_for_status() result = response.text.split("\r\n") for item in result: diff --git a/api_app/analyzers_manager/observable_analyzers/securitytrails.py b/api_app/analyzers_manager/observable_analyzers/securitytrails.py index b37654ec66..1be5490cab 100644 --- a/api_app/analyzers_manager/observable_analyzers/securitytrails.py +++ b/api_app/analyzers_manager/observable_analyzers/securitytrails.py @@ -3,6 +3,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -52,7 +53,7 @@ def run(self): ) try: - response = requests.get(self.url + uri, headers=headers) + response = http_utils.get(self.url + uri, headers=headers) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) diff --git a/api_app/analyzers_manager/observable_analyzers/shodan.py b/api_app/analyzers_manager/observable_analyzers/shodan.py index 30d72c262e..00c365fe2d 100644 --- a/api_app/analyzers_manager/observable_analyzers/shodan.py +++ b/api_app/analyzers_manager/observable_analyzers/shodan.py @@ -3,6 +3,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import ( AnalyzerConfigurationException, @@ -36,7 +37,7 @@ def run(self): ) try: - response = requests.get(self.url + uri, params=params) + response = http_utils.get(self.url + uri, params=params) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) diff --git a/api_app/analyzers_manager/observable_analyzers/spamhaus_drop.py b/api_app/analyzers_manager/observable_analyzers/spamhaus_drop.py index 8b0e706ca5..a3030e03a8 100644 --- a/api_app/analyzers_manager/observable_analyzers/spamhaus_drop.py +++ b/api_app/analyzers_manager/observable_analyzers/spamhaus_drop.py @@ -4,9 +4,9 @@ import logging import os -import requests from django.conf import settings +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -94,7 +94,7 @@ def update(cls): db_url = cls.asn_url else: raise AnalyzerRunException(f"Invalid data_type provided to update: {data_type}") - response = requests.get(url=db_url) + response = http_utils.get(url=db_url) response.raise_for_status() data = cls.convert_to_json(response.text) database_location = cls.location(data_type) diff --git a/api_app/analyzers_manager/observable_analyzers/spyse.py b/api_app/analyzers_manager/observable_analyzers/spyse.py index 90dfce6174..5e9ff12658 100644 --- a/api_app/analyzers_manager/observable_analyzers/spyse.py +++ b/api_app/analyzers_manager/observable_analyzers/spyse.py @@ -3,8 +3,7 @@ import re -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -48,7 +47,7 @@ def run(self): "Authorization": f"Bearer {self._api_key_name}", } api_uri = self.__build_spyse_api_uri() - response = requests.get(api_uri, headers=headers) + response = http_utils.get(api_uri, headers=headers) response.raise_for_status() result = response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/ss_api_net.py b/api_app/analyzers_manager/observable_analyzers/ss_api_net.py index f286e633da..6666b46526 100644 --- a/api_app/analyzers_manager/observable_analyzers/ss_api_net.py +++ b/api_app/analyzers_manager/observable_analyzers/ss_api_net.py @@ -5,6 +5,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import ( AnalyzerConfigurationException, @@ -44,7 +45,7 @@ def run(self): if self.use_proxy: params["proxy"] = self.proxy - resp = requests.get(self.url, params=params) + resp = http_utils.get(self.url, params=params) resp.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) diff --git a/api_app/analyzers_manager/observable_analyzers/stalkphish.py b/api_app/analyzers_manager/observable_analyzers/stalkphish.py index 88462ba9cf..7521fb3f5b 100644 --- a/api_app/analyzers_manager/observable_analyzers/stalkphish.py +++ b/api_app/analyzers_manager/observable_analyzers/stalkphish.py @@ -3,6 +3,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -38,7 +39,7 @@ def run(self): ) try: - response = requests.get(self.url + uri, headers=headers) + response = http_utils.get(self.url + uri, headers=headers) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) diff --git a/api_app/analyzers_manager/observable_analyzers/stratosphere.py b/api_app/analyzers_manager/observable_analyzers/stratosphere.py index 920f40a203..467183b972 100644 --- a/api_app/analyzers_manager/observable_analyzers/stratosphere.py +++ b/api_app/analyzers_manager/observable_analyzers/stratosphere.py @@ -5,9 +5,9 @@ import os from datetime import date, datetime -import requests from django.conf import settings +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -63,7 +63,7 @@ def run(self): @staticmethod def download_dataset(url, db_loc): # Dataset website certificates are not correctly configured. - p = requests.get(url, verify=False) # lgtm [py/request-without-cert-validation] + p = http_utils.get(url, verify=False) # lgtm [py/request-without-cert-validation] p.raise_for_status() with open(db_loc, "w", encoding="utf-8") as f: diff --git a/api_app/analyzers_manager/observable_analyzers/talos.py b/api_app/analyzers_manager/observable_analyzers/talos.py index 5a21d5c299..235490b8c4 100644 --- a/api_app/analyzers_manager/observable_analyzers/talos.py +++ b/api_app/analyzers_manager/observable_analyzers/talos.py @@ -4,9 +4,9 @@ import logging import os -import requests from django.conf import settings +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -39,7 +39,7 @@ def update(cls) -> bool: try: logger.info("starting download of db from talos") url = "https://snort.org/downloads/ip-block-list" - r = requests.get(url) + r = http_utils.get(url) r.raise_for_status() with open(database_location, "w", encoding="utf-8") as f: diff --git a/api_app/analyzers_manager/observable_analyzers/threatfox.py b/api_app/analyzers_manager/observable_analyzers/threatfox.py index a6b89c2a2f..6f861adfb5 100644 --- a/api_app/analyzers_manager/observable_analyzers/threatfox.py +++ b/api_app/analyzers_manager/observable_analyzers/threatfox.py @@ -4,8 +4,7 @@ import json import logging -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.mixins import AbuseCHMixin @@ -25,7 +24,7 @@ def run(self): payload = {"query": "search_ioc", "search_term": self.observable_name} - response = requests.post( + response = http_utils.post( self.url, data=json.dumps(payload), headers=self.authentication_header, diff --git a/api_app/analyzers_manager/observable_analyzers/threatminer.py b/api_app/analyzers_manager/observable_analyzers/threatminer.py index 7881a96bbe..62d671be8f 100644 --- a/api_app/analyzers_manager/observable_analyzers/threatminer.py +++ b/api_app/analyzers_manager/observable_analyzers/threatminer.py @@ -3,6 +3,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -33,7 +34,7 @@ def run(self): ) try: - response = requests.get(self.url + uri, params=params, timeout=30) + response = http_utils.get(self.url + uri, params=params, timeout=30) response.raise_for_status() except requests.Timeout: diff --git a/api_app/analyzers_manager/observable_analyzers/threatstream.py b/api_app/analyzers_manager/observable_analyzers/threatstream.py index a8660cb888..dd93cd7ad5 100644 --- a/api_app/analyzers_manager/observable_analyzers/threatstream.py +++ b/api_app/analyzers_manager/observable_analyzers/threatstream.py @@ -3,6 +3,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import ( AnalyzerConfigurationException, @@ -62,7 +63,7 @@ def run(self): ) try: api_header = {"Authorization": f"apikey {self._api_user_name}:{self._api_key_name}"} - response = requests.get(self.url + uri, params=params, headers=api_header) + response = http_utils.get(self.url + uri, params=params, headers=api_header) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) diff --git a/api_app/analyzers_manager/observable_analyzers/tor.py b/api_app/analyzers_manager/observable_analyzers/tor.py index 5fe1d98ad2..36523f4f11 100644 --- a/api_app/analyzers_manager/observable_analyzers/tor.py +++ b/api_app/analyzers_manager/observable_analyzers/tor.py @@ -4,9 +4,9 @@ import logging import re -import requests from django.db import transaction +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.models import TorExitNode @@ -27,7 +27,7 @@ def run(self): def update(cls) -> bool: try: logger.info("starting download of db from tor project") - response = requests.get(cls.url) + response = http_utils.get(cls.url) response.raise_for_status() unique_ips = set( diff --git a/api_app/analyzers_manager/observable_analyzers/tor_nodes_danmeuk.py b/api_app/analyzers_manager/observable_analyzers/tor_nodes_danmeuk.py index 3de6c24fcb..193e82d48a 100644 --- a/api_app/analyzers_manager/observable_analyzers/tor_nodes_danmeuk.py +++ b/api_app/analyzers_manager/observable_analyzers/tor_nodes_danmeuk.py @@ -3,9 +3,9 @@ import logging -import requests from django.db import transaction +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.models import TorDanMeUKNode @@ -29,7 +29,7 @@ def run(self): def update(cls) -> bool: try: logger.info("starting download of tor nodes from https://dan.me.uk") - response = requests.get(cls.url) + response = http_utils.get(cls.url) response.raise_for_status() unique_ips = {ip for ip in response.content.decode().split("\n") if ip.strip()} diff --git a/api_app/analyzers_manager/observable_analyzers/tranco.py b/api_app/analyzers_manager/observable_analyzers/tranco.py index 00f8dbe0a5..2afa891791 100644 --- a/api_app/analyzers_manager/observable_analyzers/tranco.py +++ b/api_app/analyzers_manager/observable_analyzers/tranco.py @@ -3,8 +3,7 @@ from urllib.parse import urlparse -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.choices import Classification @@ -22,7 +21,7 @@ def run(self): observable_to_analyze = urlparse(self.observable_name).hostname url = self.url + observable_to_analyze - response = requests.get(url) + response = http_utils.get(url) response.raise_for_status() return response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/triage/triage_base.py b/api_app/analyzers_manager/observable_analyzers/triage/triage_base.py index 2ff575b0dc..8a6d9db273 100644 --- a/api_app/analyzers_manager/observable_analyzers/triage/triage_base.py +++ b/api_app/analyzers_manager/observable_analyzers/triage/triage_base.py @@ -6,9 +6,9 @@ from abc import ABCMeta from typing import Dict -import requests from requests.exceptions import ChunkedEncodingError +from api_app import http_utils from api_app.analyzers_manager.classes import BaseAnalyzerMixin from api_app.analyzers_manager.exceptions import ( AnalyzerConfigurationException, @@ -46,7 +46,7 @@ def config(self, runtime_configuration: Dict): @property def session(self): if not hasattr(self, "_session"): - session = requests.Session() + session = http_utils.Session() session.headers = { "Authorization": f"Bearer {self._api_key_name}", "User-Agent": "IntelOwl", diff --git a/api_app/analyzers_manager/observable_analyzers/tweetfeeds.py b/api_app/analyzers_manager/observable_analyzers/tweetfeeds.py index 64e4373844..44e1405e0a 100644 --- a/api_app/analyzers_manager/observable_analyzers/tweetfeeds.py +++ b/api_app/analyzers_manager/observable_analyzers/tweetfeeds.py @@ -6,6 +6,7 @@ import requests from django.conf import settings +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -61,7 +62,7 @@ def run(self): logger.info(f"TweetFeeds extending using {run_url}") # simply make api call and search for observable - response = requests.get(run_url) + response = http_utils.get(run_url) response.raise_for_status() db = response.json() for tweet in db: @@ -82,7 +83,7 @@ def update(cls) -> bool: logger.info(f"Updating TweetFeeds {db_url} at {db_location}") try: - response = requests.get(db_url) + response = http_utils.get(db_url) response.raise_for_status() except requests.RequestException as e: logger.error(f"TweetFeeds failed to update {db_url}: {e}") diff --git a/api_app/analyzers_manager/observable_analyzers/urldna.py b/api_app/analyzers_manager/observable_analyzers/urldna.py index 02a7c5b459..9beb14f77e 100644 --- a/api_app/analyzers_manager/observable_analyzers/urldna.py +++ b/api_app/analyzers_manager/observable_analyzers/urldna.py @@ -6,6 +6,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -42,7 +43,7 @@ def run(self): "Authorization": self._api_key_name, } - self.session = requests.Session() + self.session = http_utils.Session() self.session.headers = headers if self.urldna_analysis == "SEARCH": result = self.__urldna_search() diff --git a/api_app/analyzers_manager/observable_analyzers/urlhaus.py b/api_app/analyzers_manager/observable_analyzers/urlhaus.py index 09fd658dd2..1c6d606a33 100644 --- a/api_app/analyzers_manager/observable_analyzers/urlhaus.py +++ b/api_app/analyzers_manager/observable_analyzers/urlhaus.py @@ -2,8 +2,7 @@ # See the file 'LICENSE' for copying permission. import logging -import requests - +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -36,7 +35,7 @@ def run(self): else: raise AnalyzerRunException(f"not supported observable type {self.observable_classification}.") - response = requests.post( + response = http_utils.post( self.url + uri, data=post_data, headers=self.authentication_header | headers, diff --git a/api_app/analyzers_manager/observable_analyzers/urlscan.py b/api_app/analyzers_manager/observable_analyzers/urlscan.py index f33220d280..3b963abb64 100644 --- a/api_app/analyzers_manager/observable_analyzers/urlscan.py +++ b/api_app/analyzers_manager/observable_analyzers/urlscan.py @@ -6,6 +6,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -32,7 +33,7 @@ def run(self): else: headers["API-Key"] = self._api_key_name - self.session = requests.Session() + self.session = http_utils.Session() self.session.headers = headers if self.urlscan_analysis == "search": result = self.__urlscan_search() diff --git a/api_app/analyzers_manager/observable_analyzers/validin.py b/api_app/analyzers_manager/observable_analyzers/validin.py index 9dacad4ae8..17051f175b 100644 --- a/api_app/analyzers_manager/observable_analyzers/validin.py +++ b/api_app/analyzers_manager/observable_analyzers/validin.py @@ -2,6 +2,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import ( # AnalyzerConfigurationException AnalyzerRunException, @@ -31,7 +32,7 @@ def _run_all_queries(self, endpoints, headers): for query_name, query_url in (endpoints.get(self.observable_classification)).items(): logger.info(f"Executing query {query_name}") try: - response = requests.get(self.url + query_url, headers=headers) + response = http_utils.get(self.url + query_url, headers=headers) if response.status_code != 200: logger.error(f"Query {query_name} failed") @@ -47,7 +48,7 @@ def _run_specific_query(self, endpoints, headers): if self.observable_classification in endpoints: try: query_url = endpoints[self.observable_classification][self.scan_choice] - response = requests.get(self.url + query_url, headers=headers) + response = http_utils.get(self.url + query_url, headers=headers) return response.json() except KeyError: raise AnalyzerRunException( diff --git a/api_app/analyzers_manager/observable_analyzers/virushee.py b/api_app/analyzers_manager/observable_analyzers/virushee.py index 42535bae14..ea7589c3f0 100644 --- a/api_app/analyzers_manager/observable_analyzers/virushee.py +++ b/api_app/analyzers_manager/observable_analyzers/virushee.py @@ -3,6 +3,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.analyzers_manager.exceptions import AnalyzerRunException @@ -16,7 +17,7 @@ def update(cls) -> bool: pass def run(self): - self.__session = requests.Session() + self.__session = http_utils.Session() if hasattr(self, "_api_key_name"): self.__session.headers["X-API-Key"] = self._api_key_name url = self.url.format(input=self.observable_name) diff --git a/api_app/analyzers_manager/observable_analyzers/vulners.py b/api_app/analyzers_manager/observable_analyzers/vulners.py index 0780f5243d..fe10f935ba 100644 --- a/api_app/analyzers_manager/observable_analyzers/vulners.py +++ b/api_app/analyzers_manager/observable_analyzers/vulners.py @@ -1,7 +1,6 @@ import logging -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes logger = logging.getLogger(__name__) @@ -19,14 +18,14 @@ class Vulners(classes.ObservableAnalyzer): url = "https://vulners.com/api/v3" def search_ai(self): - return requests.post( + return http_utils.post( url=self.url + "/ai/scoretext/", headers={"Content-Type": "application/json"}, json={"text": self.observable_name, "apiKey": self._api_key_name}, ) def search_databse(self): - return requests.post( + return http_utils.post( url=self.url + "/search/lucene", headers={"Content-Type": "application/json"}, json={ diff --git a/api_app/analyzers_manager/observable_analyzers/whoisripe.py b/api_app/analyzers_manager/observable_analyzers/whoisripe.py index db0fdaa0f7..27d939585e 100644 --- a/api_app/analyzers_manager/observable_analyzers/whoisripe.py +++ b/api_app/analyzers_manager/observable_analyzers/whoisripe.py @@ -1,8 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes @@ -12,7 +11,7 @@ class WhoIsRipeAPI(classes.ObservableAnalyzer): def run(self): params = {"query-string": self.observable_name} - response = requests.get(self.url, params=params) + response = http_utils.get(self.url, params=params) response.raise_for_status() return response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/whoisxmlapi.py b/api_app/analyzers_manager/observable_analyzers/whoisxmlapi.py index 3891c00392..2d58ddd623 100644 --- a/api_app/analyzers_manager/observable_analyzers/whoisxmlapi.py +++ b/api_app/analyzers_manager/observable_analyzers/whoisxmlapi.py @@ -1,8 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes @@ -17,7 +16,7 @@ def run(self): "domainName": self.observable_name, "outputFormat": "JSON", } - response = requests.get(self.url, params=params) + response = http_utils.get(self.url, params=params) response.raise_for_status() return response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/wigle.py b/api_app/analyzers_manager/observable_analyzers/wigle.py index 5a9d97de37..e992ba6232 100644 --- a/api_app/analyzers_manager/observable_analyzers/wigle.py +++ b/api_app/analyzers_manager/observable_analyzers/wigle.py @@ -1,8 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerConfigurationException @@ -50,7 +49,7 @@ def run(self): "'Bluetooth Network', 'GSM/LTE/WCDMA Network'" ) - response = requests.get( + response = http_utils.get( self.url + uri, headers={"Authorization": "Basic " + self._api_key_name}, ) diff --git a/api_app/analyzers_manager/observable_analyzers/xforce.py b/api_app/analyzers_manager/observable_analyzers/xforce.py index c0f4e21307..b7d7b9c818 100644 --- a/api_app/analyzers_manager/observable_analyzers/xforce.py +++ b/api_app/analyzers_manager/observable_analyzers/xforce.py @@ -3,9 +3,9 @@ from urllib.parse import quote_plus -import requests from requests.auth import HTTPBasicAuth +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.choices import Classification @@ -36,7 +36,7 @@ def run(self): else: observable_to_check = self.observable_name url = f"{self.url}/{endpoint}/{observable_to_check}" - response = requests.get(url, auth=auth, headers=headers, timeout=self.timeout) + response = http_utils.get(url, auth=auth, headers=headers, timeout=self.timeout) if response.status_code == 404: result["found"] = False else: diff --git a/api_app/analyzers_manager/observable_analyzers/yaraify.py b/api_app/analyzers_manager/observable_analyzers/yaraify.py index 5d6dcaf2a1..27cde1febe 100644 --- a/api_app/analyzers_manager/observable_analyzers/yaraify.py +++ b/api_app/analyzers_manager/observable_analyzers/yaraify.py @@ -2,8 +2,7 @@ # See the file 'LICENSE' for copying permission. import logging -import requests - +from api_app import http_utils from api_app.analyzers_manager.classes import ObservableAnalyzer from api_app.choices import Classification from api_app.mixins import AbuseCHMixin @@ -30,7 +29,7 @@ def run(self): if getattr(self, "_api_key_name", None): data["malpedia-token"] = self._api_key_name - response = requests.post(self.url, json=data, headers=self.authentication_header) + response = http_utils.post(self.url, json=data, headers=self.authentication_header) response.raise_for_status() result = response.json() diff --git a/api_app/analyzers_manager/observable_analyzers/yeti.py b/api_app/analyzers_manager/observable_analyzers/yeti.py index 3c5d8b7185..edf40f926f 100644 --- a/api_app/analyzers_manager/observable_analyzers/yeti.py +++ b/api_app/analyzers_manager/observable_analyzers/yeti.py @@ -1,8 +1,7 @@ # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl # See the file 'LICENSE' for copying permission. -import requests - +from api_app import http_utils from api_app.analyzers_manager import classes @@ -25,7 +24,7 @@ def run(self): url = f"{self._url_key_name}/api/v2/observables/search/" # search for observables - resp = requests.post( + resp = http_utils.post( url=url, headers=headers, json=payload, diff --git a/api_app/analyzers_manager/observable_analyzers/zoomeye.py b/api_app/analyzers_manager/observable_analyzers/zoomeye.py index 111a1f994f..8cfff34a47 100644 --- a/api_app/analyzers_manager/observable_analyzers/zoomeye.py +++ b/api_app/analyzers_manager/observable_analyzers/zoomeye.py @@ -3,6 +3,7 @@ import requests +from api_app import http_utils from api_app.analyzers_manager import classes from api_app.analyzers_manager.exceptions import ( AnalyzerConfigurationException, @@ -56,7 +57,7 @@ def run(self): self.__build_zoomeye_url() try: - response = requests.get(self.final_url, headers={"API-KEY": self._api_key_name}) + response = http_utils.get(self.final_url, headers={"API-KEY": self._api_key_name}) response.raise_for_status() except requests.RequestException as e: raise AnalyzerRunException(e) diff --git a/api_app/classes.py b/api_app/classes.py index f347bcfd6d..25e821dd9c 100644 --- a/api_app/classes.py +++ b/api_app/classes.py @@ -13,6 +13,7 @@ from django.utils.functional import cached_property from requests import HTTPError +from api_app import http_utils from api_app.decorators import abstractclassproperty, classproperty from api_app.models import AbstractReport, Job, PythonConfig, PythonModule from certego_saas.apps.user.models import User @@ -373,7 +374,7 @@ def health_check(self, user: User = None) -> bool: try: # momentarily set this to False to # avoid fails for https services - response = requests.head(url, timeout=10, verify=False) + response = http_utils.head(url, timeout=settings.HTTP_TIMEOUT, verify=False) # This may happen when even the HEAD request is protected by authentication # We cannot create a generic health check that consider auth too # because every analyzer has its own way to authenticate diff --git a/api_app/connectors_manager/connectors/yeti.py b/api_app/connectors_manager/connectors/yeti.py index 44676f787b..20b29cfae7 100644 --- a/api_app/connectors_manager/connectors/yeti.py +++ b/api_app/connectors_manager/connectors/yeti.py @@ -4,6 +4,7 @@ import requests from django.conf import settings +from api_app import http_utils from api_app.connectors_manager import classes from api_app.connectors_manager.exceptions import ConnectorRunException from tests.mock_utils import MockUpResponse, if_mock_connections, patch @@ -53,7 +54,7 @@ def run(self): # create observable with `obs_value` if it doesn't exists # new context, tags, source are appended with existing ones try: - resp = requests.post( + resp = http_utils.post( url=url, headers=headers, json=payload, diff --git a/api_app/core/update_checker.py b/api_app/core/update_checker.py index 82871ca3d0..cb950e87a2 100644 --- a/api_app/core/update_checker.py +++ b/api_app/core/update_checker.py @@ -6,6 +6,7 @@ from django.db import IntegrityError, transaction from django.utils.timezone import now +from api_app import http_utils from api_app.models import UpdateCheckStatus logger = logging.getLogger(__name__) @@ -39,7 +40,7 @@ def fetch_latest_version() -> Tuple[Optional[str], Optional[str]]: return None, "UPDATE_CHECK_URL not configured" try: - resp = requests.get(url, headers={"User-Agent": "IntelOwl-Update-Checker"}, timeout=5) + resp = http_utils.get(url, headers={"User-Agent": "IntelOwl-Update-Checker"}, timeout=5) except requests.RequestException as exc: logger.error("Update check HTTP request failed: %s", exc) return None, "Failed to fetch release information" diff --git a/api_app/http_utils.py b/api_app/http_utils.py new file mode 100644 index 0000000000..230e77c9bb --- /dev/null +++ b/api_app/http_utils.py @@ -0,0 +1,97 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + +from functools import wraps + +import requests +from django.conf import settings + + +def _get_timeout(kwargs): + """ + Internal helper to extract timeout from kwargs or use the default. + """ + return kwargs.pop("timeout", settings.HTTP_TIMEOUT) + + +def get(url, params=None, **kwargs): + """ + Sends a GET request with a default timeout. + """ + timeout = _get_timeout(kwargs) + return requests.get(url, params=params, timeout=timeout, **kwargs) + + +def post(url, data=None, json=None, **kwargs): + """ + Sends a POST request with a default timeout. + """ + timeout = _get_timeout(kwargs) + return requests.post(url, data=data, json=json, timeout=timeout, **kwargs) + + +def put(url, data=None, **kwargs): + """ + Sends a PUT request with a default timeout. + """ + timeout = _get_timeout(kwargs) + return requests.put(url, data=data, timeout=timeout, **kwargs) + + +def patch(url, data=None, **kwargs): + """ + Sends a PATCH request with a default timeout. + """ + timeout = _get_timeout(kwargs) + return requests.patch(url, data=data, timeout=timeout, **kwargs) + + +def delete(url, **kwargs): + """ + Sends a DELETE request with a default timeout. + """ + timeout = _get_timeout(kwargs) + return requests.delete(url, timeout=timeout, **kwargs) + + +def head(url, **kwargs): + """ + Sends a HEAD request with a default timeout. + """ + timeout = _get_timeout(kwargs) + return requests.head(url, timeout=timeout, **kwargs) + + +def request(method, url, **kwargs): + """ + Constructs and sends a Request with a default timeout. + """ + timeout = _get_timeout(kwargs) + return requests.request(method, url, timeout=timeout, **kwargs) + + +class Session(requests.Session): + """ + A wrapper around requests.Session that automatically applies + the default HTTP_TIMEOUT to all requests. + """ + + def request(self, method, url, **kwargs): + if "timeout" not in kwargs: + kwargs["timeout"] = settings.HTTP_TIMEOUT + return super().request(method, url, **kwargs) + + +def verify_timeout(func): + """ + Decorator that ensures the decorated function (which should call a requests method) + is called with a 'timeout' argument. If not, it injects the default. + """ + + @wraps(func) + def wrapper(*args, **kwargs): + if "timeout" not in kwargs: + kwargs["timeout"] = settings.HTTP_TIMEOUT + return func(*args, **kwargs) + + return wrapper diff --git a/api_app/ingestors_manager/ingestors/greedybear.py b/api_app/ingestors_manager/ingestors/greedybear.py index 4fabb17fc1..86325cbfcb 100644 --- a/api_app/ingestors_manager/ingestors/greedybear.py +++ b/api_app/ingestors_manager/ingestors/greedybear.py @@ -3,8 +3,7 @@ from typing import Any, Iterable from unittest.mock import patch -import requests - +from api_app import http_utils from api_app.ingestors_manager.classes import Ingestor from api_app.ingestors_manager.exceptions import ( IngestorConfigurationException, @@ -42,7 +41,7 @@ def run(self) -> Iterable[Any]: raise IngestorConfigurationException(f"Invalid age: {self.age}. Must be one of {self.VALID_AGE}") req_url = f"{self.url}/api/feeds/{self.feed_type}/{self.attack_type}/{self.age}.json" - result = requests.get(req_url) + result = http_utils.get(req_url) result.raise_for_status() content = result.json() if not isinstance(content.get("iocs"), list): diff --git a/api_app/ingestors_manager/ingestors/malshare.py b/api_app/ingestors_manager/ingestors/malshare.py index d936160033..f947dc7d17 100644 --- a/api_app/ingestors_manager/ingestors/malshare.py +++ b/api_app/ingestors_manager/ingestors/malshare.py @@ -2,8 +2,7 @@ from typing import Any, Iterable from unittest.mock import patch -import requests - +from api_app import http_utils from api_app.ingestors_manager.classes import Ingestor from api_app.ingestors_manager.exceptions import IngestorRunException from tests.mock_utils import MockUpResponse, if_mock_connections @@ -33,7 +32,7 @@ def download_sample(self, sample_hash: str) -> bytes: "action": "getfile", "hash": sample_hash, } - response = requests.get(self.base_url, params=params) + response = http_utils.get(self.base_url, params=params) response.raise_for_status() if not isinstance(response.content, bytes): raise ValueError("The downloaded file is not instance of bytes") @@ -53,7 +52,7 @@ def run(self) -> Iterable[Any]: "api_key": self._api_key_name, "action": "getlist", } - result = requests.get(self.base_url, params=params) + result = http_utils.get(self.base_url, params=params) result.raise_for_status() content = result.json() if not isinstance(content, list): diff --git a/api_app/ingestors_manager/ingestors/malware_bazaar.py b/api_app/ingestors_manager/ingestors/malware_bazaar.py index 23025e0c25..4c81462483 100644 --- a/api_app/ingestors_manager/ingestors/malware_bazaar.py +++ b/api_app/ingestors_manager/ingestors/malware_bazaar.py @@ -5,9 +5,9 @@ from unittest.mock import patch import pyzipper -import requests from django.utils import timezone +from api_app import http_utils from api_app.ingestors_manager.classes import Ingestor from api_app.ingestors_manager.exceptions import IngestorRunException from api_app.mixins import AbuseCHMixin @@ -32,7 +32,7 @@ def update(cls) -> bool: # retrieve information about the given signature def get_signature_information(self, signature): - result = requests.post( + result = http_utils.post( self.url, data={"query": "get_siginfo", "signature": signature, "limit": self.limit}, headers=self.authentication_header, @@ -70,7 +70,7 @@ def get_recent_samples(self): def download_sample(self, h): logger.info(f"Downloading sample {h}") - sample_archive = requests.post( + sample_archive = http_utils.post( self.url, data={ "query": "get_file", diff --git a/api_app/ingestors_manager/ingestors/threatfox.py b/api_app/ingestors_manager/ingestors/threatfox.py index 87f0a5ab11..0faf45d7df 100644 --- a/api_app/ingestors_manager/ingestors/threatfox.py +++ b/api_app/ingestors_manager/ingestors/threatfox.py @@ -2,8 +2,7 @@ from typing import Any, Iterable from unittest.mock import patch -import requests - +from api_app import http_utils from api_app.ingestors_manager.classes import Ingestor from api_app.ingestors_manager.exceptions import IngestorRunException from api_app.mixins import AbuseCHMixin @@ -23,7 +22,7 @@ def update(cls) -> bool: pass def run(self) -> Iterable[Any]: - result = requests.post( + result = http_utils.post( self.url, json={"query": "get_iocs", "days": self.days}, headers=self.authentication_header, diff --git a/api_app/mixins.py b/api_app/mixins.py index 8b072552b8..7701a7a5f5 100644 --- a/api_app/mixins.py +++ b/api_app/mixins.py @@ -15,6 +15,7 @@ from jbxapi import ApiError, JoeSandbox from rest_framework.response import Response +from api_app import http_utils from api_app.analyzers_manager.classes import BaseAnalyzerMixin from api_app.analyzers_manager.exceptions import AnalyzerRunException from api_app.analyzers_manager.models import AnalyzerRulesFileVersion, PythonModule @@ -107,17 +108,17 @@ def _perform_request(self, uri: str, method: str, ignore_404: bool = False, **kw logger.debug( f"about to send get request to url {url} with headers {self.headers} and kwargs: {kwargs}" ) - response = requests.get(url, headers=self.headers, **kwargs) + response = http_utils.get(url, headers=self.headers, **kwargs) else: logger.debug( f"about to send get request to url {url} with headers {self.headers} and no kwargs" ) - response = requests.get(url, headers=self.headers) + response = http_utils.get(url, headers=self.headers) elif method == "POST": logger.debug( f"about to send post request to url {url} with headers {self.headers} and kwargs: {kwargs}" ) - response = requests.post(url, headers=self.headers, **kwargs) + response = http_utils.post(url, headers=self.headers, **kwargs) else: raise NotImplementedError() logger.info(f"requests done to: {response.request.url} ") @@ -250,7 +251,7 @@ def _vt_download_file(self, file_hash: str) -> bytes: try: endpoint = self.url + f"files/{file_hash}/download" logger.info(f"Requesting file from {endpoint}") - response = requests.get(endpoint, headers=self.headers) + response = http_utils.get(endpoint, headers=self.headers) if not isinstance(response.content, bytes): raise ValueError("VT downloaded file is not instance of bytes") except Exception as e: @@ -370,7 +371,7 @@ def _vt_get_relationships( uri + f"/{relationship}?limit={self._get_relationship_limit(relationship)}" ) logger.debug(f"requesting uri: {rel_uri}") - response = requests.get(self.url + rel_uri, headers=self.headers) + response = http_utils.get(self.url + rel_uri, headers=self.headers) result[relationship] = response.json() except Exception as e: logger.error( @@ -798,7 +799,7 @@ def _download_rules( os.makedirs(rule_set_directory) logger.info(f"Created fresh rules directory at {rule_set_directory}") - response = requests.get(rule_set_download_url, stream=True) + response = http_utils.get(rule_set_download_url, stream=True) logger.info(f"Started downloading rules with version: {latest_version} from {rule_set_download_url}") try: diff --git a/intel_owl/settings/commons.py b/intel_owl/settings/commons.py index 4fb22f258a..03114c715a 100644 --- a/intel_owl/settings/commons.py +++ b/intel_owl/settings/commons.py @@ -58,3 +58,10 @@ "https://api.github.com/repos/intelowlproject/IntelOwl/releases/latest", ) INTEL_OWL_VERSION = VERSION +try: + HTTP_TIMEOUT = int(get_secret("HTTP_TIMEOUT", 30)) + if HTTP_TIMEOUT <= 0: + HTTP_TIMEOUT = 30 +except (ValueError, TypeError): + # fallback to default if the environment variable is not a valid integer + HTTP_TIMEOUT = 30 diff --git a/tests/api_app/test_http_utils.py b/tests/api_app/test_http_utils.py new file mode 100644 index 0000000000..e999930f65 --- /dev/null +++ b/tests/api_app/test_http_utils.py @@ -0,0 +1,63 @@ +import importlib +from unittest.mock import patch + +from django.conf import settings +from django.test import SimpleTestCase + +from api_app import http_utils + + +class HttpUtilsTestCase(SimpleTestCase): + @patch("requests.get") + def test_get_default_timeout(self, mock_get): + http_utils.get("http://example.com") + mock_get.assert_called_with("http://example.com", params=None, timeout=settings.HTTP_TIMEOUT) + + @patch("requests.get") + def test_get_override_timeout(self, mock_get): + http_utils.get("http://example.com", timeout=10) + mock_get.assert_called_with("http://example.com", params=None, timeout=10) + + @patch("requests.post") + def test_post_override_timeout(self, mock_post): + http_utils.post("http://example.com", data={"key": "value"}, timeout=10) + mock_post.assert_called_with("http://example.com", data={"key": "value"}, json=None, timeout=10) + + def test_verify_timeout_decorator(self): + @http_utils.verify_timeout + def dummy_request(url, **kwargs): + """Dummy docstring""" + return kwargs.get("timeout") + + self.assertEqual(dummy_request("http://example.com"), settings.HTTP_TIMEOUT) + self.assertEqual(dummy_request("http://example.com", timeout=5), 5) + # Verify metadata preservation + self.assertEqual(dummy_request.__name__, "dummy_request") + self.assertEqual(dummy_request.__doc__, "Dummy docstring") + + def test_config_robustness(self): + import intel_owl.settings.commons as commons + + test_cases = [ + ("15", 15), + ("invalid", 30), + ("0", 30), + ("-5", 30), + (" 15 ", 15), + (None, 30), + ] + + try: + for env_value, expected in test_cases: + with self.subTest(env_value=env_value): + + def fake_get_secret(key, default=None): + if key == "HTTP_TIMEOUT": + return env_value + return default + + with patch("intel_owl.settings._util.get_secret", side_effect=fake_get_secret): + importlib.reload(commons) + self.assertEqual(commons.HTTP_TIMEOUT, expected) + finally: + importlib.reload(commons) From 81c9612966666820dbf80a794a344ff01001e0ce Mon Sep 17 00:00:00 2001 From: chauhan-varun Date: Fri, 20 Mar 2026 14:12:14 +0530 Subject: [PATCH 2/7] test: add env_value parameter to fake_get_secret helper function. --- tests/api_app/test_http_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api_app/test_http_utils.py b/tests/api_app/test_http_utils.py index e999930f65..d39f0fe814 100644 --- a/tests/api_app/test_http_utils.py +++ b/tests/api_app/test_http_utils.py @@ -51,7 +51,7 @@ def test_config_robustness(self): for env_value, expected in test_cases: with self.subTest(env_value=env_value): - def fake_get_secret(key, default=None): + def fake_get_secret(key, default=None, env_value=env_value): if key == "HTTP_TIMEOUT": return env_value return default From fbc27b6fc274e5897ff3ce50fc98394b3bf745fe Mon Sep 17 00:00:00 2001 From: chauhan-varun Date: Fri, 20 Mar 2026 14:30:16 +0530 Subject: [PATCH 3/7] Refactor: Improve test mocks by updating HTTP utility paths, module references, and function signatures. --- .../unit_tests/file_analyzers/test_capa_info.py | 2 +- .../unit_tests/file_analyzers/test_cuckoo_scan.py | 2 +- .../unit_tests/file_analyzers/test_hfinger.py | 2 +- .../unit_tests/observable_analyzers/test_greynoise_labs.py | 6 +++++- .../unit_tests/observable_analyzers/test_mnemonic_pdns.py | 2 +- .../unit_tests/observable_analyzers/test_spamhaus_wqs.py | 2 +- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/api_app/analyzers_manager/unit_tests/file_analyzers/test_capa_info.py b/tests/api_app/analyzers_manager/unit_tests/file_analyzers/test_capa_info.py index a4d8694b01..74e6241c73 100644 --- a/tests/api_app/analyzers_manager/unit_tests/file_analyzers/test_capa_info.py +++ b/tests/api_app/analyzers_manager/unit_tests/file_analyzers/test_capa_info.py @@ -39,7 +39,7 @@ def get_mocked_response(self): patch.object(CapaInfo, "update", return_value=True), patch("subprocess.run", return_value=response_from_command), patch( - "api_app.analyzers_manager.file_analyzers.capa_info.requests.get", + "api_app.analyzers_manager.file_analyzers.capa_info.http_utils.get", return_value=mock_requests_get, ), patch.object(CapaInfo, "_check_if_latest_version", return_value=True), diff --git a/tests/api_app/analyzers_manager/unit_tests/file_analyzers/test_cuckoo_scan.py b/tests/api_app/analyzers_manager/unit_tests/file_analyzers/test_cuckoo_scan.py index a23a87c8b9..bc5400348c 100644 --- a/tests/api_app/analyzers_manager/unit_tests/file_analyzers/test_cuckoo_scan.py +++ b/tests/api_app/analyzers_manager/unit_tests/file_analyzers/test_cuckoo_scan.py @@ -37,7 +37,7 @@ def get_mocked_response(self): # Patch requests.Session to return our mocked session return patch( - "api_app.analyzers_manager.file_analyzers.cuckoo_scan.requests.Session", + "api_app.analyzers_manager.file_analyzers.cuckoo_scan.http_utils.Session", return_value=mock_session, ) diff --git a/tests/api_app/analyzers_manager/unit_tests/file_analyzers/test_hfinger.py b/tests/api_app/analyzers_manager/unit_tests/file_analyzers/test_hfinger.py index 5a20b2f5f7..3cab57713a 100644 --- a/tests/api_app/analyzers_manager/unit_tests/file_analyzers/test_hfinger.py +++ b/tests/api_app/analyzers_manager/unit_tests/file_analyzers/test_hfinger.py @@ -11,7 +11,7 @@ class TestHfinger(BaseFileAnalyzerTest): def get_mocked_response(self): return [ patch( - "hfinger.analysis.hfinger_analyze", + "api_app.analyzers_manager.file_analyzers.hfinger.hfinger_analyze", return_value=[ { "epoch_time": "1388111476.787707000", diff --git a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_greynoise_labs.py b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_greynoise_labs.py index ae88ade954..581e2edee4 100644 --- a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_greynoise_labs.py +++ b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_greynoise_labs.py @@ -1,3 +1,4 @@ +from types import SimpleNamespace from unittest.mock import patch from api_app.analyzers_manager.observable_analyzers.greynoise_labs import GreynoiseLabs @@ -12,7 +13,10 @@ class GreynoiseLabsTestCase(BaseAnalyzerTest): @classmethod def get_extra_config(cls): - return {"_auth_token": "demo_token", "report": {"errors": []}} + return { + "_auth_token": "demo_token", + "report": SimpleNamespace(errors=[], save=lambda: None), + } @staticmethod def get_mocked_response(): diff --git a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_mnemonic_pdns.py b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_mnemonic_pdns.py index 175dc64f8d..da5e52cdca 100644 --- a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_mnemonic_pdns.py +++ b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_mnemonic_pdns.py @@ -31,7 +31,7 @@ def get_mocked_response(): '{"rrtype": "A", "rdata": "5.6.7.8", "time_first": "2023-02-01", "time_last": "2023-02-02"}' ) - def side_effect(url, data=None): + def side_effect(url, data=None, **kwargs): if "cof" in url: return MockUpResponse(cof_response, 200) return MockUpResponse(json_response, 200) diff --git a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_spamhaus_wqs.py b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_spamhaus_wqs.py index c2c4e982fa..acc1987c57 100644 --- a/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_spamhaus_wqs.py +++ b/tests/api_app/analyzers_manager/unit_tests/observable_analyzers/test_spamhaus_wqs.py @@ -14,7 +14,7 @@ class SpamhausWQSTestCase(BaseAnalyzerTest): @staticmethod def get_mocked_response(): - def mock_get(url, headers): + def mock_get(url, headers=None, **kwargs): if "example.com" in url: # simulate a positive detection return MockUpResponse({}, 200) From 6f4d4b485a51771e41e1eed2d9d1494a15f8358e Mon Sep 17 00:00:00 2001 From: chauhan-varun Date: Fri, 20 Mar 2026 22:21:58 +0530 Subject: [PATCH 4/7] feat: Set default HTTP_TIMEOUT to 90 seconds in environment files and application settings. --- docker/env_file_app_ci | 1 + docker/env_file_app_template | 2 ++ intel_owl/settings/commons.py | 6 +++--- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/env_file_app_ci b/docker/env_file_app_ci index 4f40487488..65cb95f09f 100644 --- a/docker/env_file_app_ci +++ b/docker/env_file_app_ci @@ -10,6 +10,7 @@ DB_NAME=intel_owl_db # Config variables OLD_JOBS_RETENTION_DAYS=2 INTELOWL_WEB_CLIENT_DOMAIN=localhost +HTTP_TIMEOUT=90 # Storage LOCAL_STORAGE=True diff --git a/docker/env_file_app_template b/docker/env_file_app_template index 650e96e3bc..5bad30576e 100644 --- a/docker/env_file_app_template +++ b/docker/env_file_app_template @@ -12,6 +12,8 @@ DB_NAME=intel_owl_db OLD_JOBS_RETENTION_DAYS=14 # used for generating links to web client e.g. job results page; Default: localhost INTELOWL_WEB_CLIENT_DOMAIN=localhost +# used for the default timeout of HTTP requests performed by analyzers; Default: 90 seconds +HTTP_TIMEOUT=90 # used for automated correspondence from the site manager DEFAULT_FROM_EMAIL= # used for correspondence with users diff --git a/intel_owl/settings/commons.py b/intel_owl/settings/commons.py index 03114c715a..0eb333707b 100644 --- a/intel_owl/settings/commons.py +++ b/intel_owl/settings/commons.py @@ -59,9 +59,9 @@ ) INTEL_OWL_VERSION = VERSION try: - HTTP_TIMEOUT = int(get_secret("HTTP_TIMEOUT", 30)) + HTTP_TIMEOUT = int(get_secret("HTTP_TIMEOUT", 90)) if HTTP_TIMEOUT <= 0: - HTTP_TIMEOUT = 30 + HTTP_TIMEOUT = 90 except (ValueError, TypeError): # fallback to default if the environment variable is not a valid integer - HTTP_TIMEOUT = 30 + HTTP_TIMEOUT = 90 From 06504ca68fa8ca274e3502028ad71f217a7663da Mon Sep 17 00:00:00 2001 From: chauhan-varun Date: Fri, 20 Mar 2026 22:40:38 +0530 Subject: [PATCH 5/7] test: update expected default timeout values from 30 to 90 in `test_get_timeout_from_request_header`. --- tests/api_app/test_http_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/api_app/test_http_utils.py b/tests/api_app/test_http_utils.py index d39f0fe814..19d13fc747 100644 --- a/tests/api_app/test_http_utils.py +++ b/tests/api_app/test_http_utils.py @@ -40,11 +40,11 @@ def test_config_robustness(self): test_cases = [ ("15", 15), - ("invalid", 30), - ("0", 30), - ("-5", 30), + ("invalid", 90), + ("0", 90), + ("-5", 90), (" 15 ", 15), - (None, 30), + (None, 90), ] try: From a3ee2db9678ffd7870bc8c808da52ffe8ec1a762 Mon Sep 17 00:00:00 2001 From: chauhan-varun Date: Sat, 21 Mar 2026 13:34:13 +0530 Subject: [PATCH 6/7] refactor: Set HTTP HEAD request timeouts to a fixed 10 seconds for health and availability checks. --- api_app/analyzers_manager/classes.py | 2 +- api_app/classes.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api_app/analyzers_manager/classes.py b/api_app/analyzers_manager/classes.py index f79379df05..580ff86d97 100644 --- a/api_app/analyzers_manager/classes.py +++ b/api_app/analyzers_manager/classes.py @@ -455,7 +455,7 @@ def health_check(self, user: User = None) -> bool: basic health check: if instance is up or not (timeout - 10s) """ try: - http_utils.head(self.url, timeout=settings.HTTP_TIMEOUT) + http_utils.head(self.url, timeout=10) except requests.exceptions.RequestException: health_status = False else: diff --git a/api_app/classes.py b/api_app/classes.py index 25e821dd9c..a112732673 100644 --- a/api_app/classes.py +++ b/api_app/classes.py @@ -374,7 +374,7 @@ def health_check(self, user: User = None) -> bool: try: # momentarily set this to False to # avoid fails for https services - response = http_utils.head(url, timeout=settings.HTTP_TIMEOUT, verify=False) + response = http_utils.head(url, timeout=10, verify=False) # This may happen when even the HEAD request is protected by authentication # We cannot create a generic health check that consider auth too # because every analyzer has its own way to authenticate From d23179d27349f30436c41afd57a2d7f60115d78c Mon Sep 17 00:00:00 2001 From: chauhan-varun Date: Thu, 26 Mar 2026 22:59:28 +0530 Subject: [PATCH 7/7] refactor: remove unused django.conf.settings import from observable analyzers --- api_app/analyzers_manager/observable_analyzers/firehol_iplist.py | 1 - api_app/analyzers_manager/observable_analyzers/spamhaus_drop.py | 1 - api_app/analyzers_manager/observable_analyzers/stratosphere.py | 1 - 3 files changed, 3 deletions(-) diff --git a/api_app/analyzers_manager/observable_analyzers/firehol_iplist.py b/api_app/analyzers_manager/observable_analyzers/firehol_iplist.py index dd1714b2e1..8c15065233 100644 --- a/api_app/analyzers_manager/observable_analyzers/firehol_iplist.py +++ b/api_app/analyzers_manager/observable_analyzers/firehol_iplist.py @@ -5,7 +5,6 @@ import logging import traceback -from django.conf import settings from django.db import transaction from django.utils import timezone diff --git a/api_app/analyzers_manager/observable_analyzers/spamhaus_drop.py b/api_app/analyzers_manager/observable_analyzers/spamhaus_drop.py index 498d41e2e6..f3cabb8070 100644 --- a/api_app/analyzers_manager/observable_analyzers/spamhaus_drop.py +++ b/api_app/analyzers_manager/observable_analyzers/spamhaus_drop.py @@ -3,7 +3,6 @@ import json import logging -from django.conf import settings from django.db import transaction from api_app import http_utils diff --git a/api_app/analyzers_manager/observable_analyzers/stratosphere.py b/api_app/analyzers_manager/observable_analyzers/stratosphere.py index 37bae20158..0c11c93177 100644 --- a/api_app/analyzers_manager/observable_analyzers/stratosphere.py +++ b/api_app/analyzers_manager/observable_analyzers/stratosphere.py @@ -3,7 +3,6 @@ import logging -from django.conf import settings from django.db import transaction from api_app import http_utils