diff --git a/README.md b/README.md index 69854d4..5631cff 100644 --- a/README.md +++ b/README.md @@ -206,13 +206,18 @@ Ensure you have **Python 3.12 or later** installed before proceeding with the in ```bash python3 --version -``` +``` + +Subdominator core modules doesn't include PDF generation dependency, if you fancy a PDF report, use PDF extra tags when installing: +```bash +pipx install 'subdominator[PDF]' +``` #### ✅ **Install Subdominator from PyPI** (Recommended) The easiest way to install Subdominator is via PyPI: ```bash -pip install --upgrade subdominator +pip install --upgrade 'subdominator[PDF]' ``` #### ✅ **Install the Latest Version from GitHub** @@ -226,13 +231,13 @@ pip install --upgrade git+https://github.com/RevoltSecurities/Subdominator To avoid dependency conflicts, you can install Subdominator using `pipx`: ```bash -pipx install subdominator +pipx install 'subdominator[PDF]' ``` To install the latest version from GitHub with `pipx`: ```bash -pipx install git+https://github.com/RevoltSecurities/Subdominator +pipx install 'subdominator[PDF] @ git+https://github.com/RevoltSecurities/Subdominator' ``` #### ✅ **Install from Git Source** (For Development) @@ -242,7 +247,7 @@ For users who want to contribute or modify the tool, clone and install directly git clone https://github.com/RevoltSecurities/Subdominator.git cd Subdominator pip install --upgrade pip -pip install -r requirements.txt +pip install -e . # or pip install -e ".[PDF]" to support PDF report generation ``` After installation, you can verify if Subdominator is installed correctly by running: diff --git a/requirements.txt b/requirements.txt index 77bbe78..77beba4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,16 +2,19 @@ aiofiles>=24.1.0 aiohttp>=3.10.11 appdirs>=1.4.4 art>=6.4 -beautifulsoup4>=4.13.3 -colorama>=0.4.6 -fake_useragent>=2.0.3 -httpx>=0.28.1 -Jinja2>=3.1.6 -prompt_toolkit>=3.0.50 -PyYAML>=6.0.2 -Requests>=2.32.3 -rich>=13.9.4 -setuptools>=75.6.0 -SQLAlchemy>=2.0.32 -tldextract>=5.1.2 -weasyprint>=65.0 + beautifulsoup4>=4.13.3 + colorama>=0.4.6 + fake_useragent>=2.0.3 + httpx>=0.28.1 + Jinja2>=3.1.6 + prompt_toolkit>=3.0.50 + PyYAML>=6.0.2 + Requests>=2.32.3 + rich>=13.9.4 + setuptools>=75.6.0 + SQLAlchemy>=2.0.32 + tldextract>=5.1.2 + aiosqlite>=0.21.0 + greenlet>=3 + socksio>=1 + \ No newline at end of file diff --git a/setup.py b/setup.py index 6b99a55..fd65a4a 100644 --- a/setup.py +++ b/setup.py @@ -30,9 +30,15 @@ 'setuptools>=75.6.0', 'SQLAlchemy>=2.0.32', 'tldextract>=5.1.2', - 'weasyprint>=65.0', 'aiosqlite>=0.21.0', + 'socksio>=1', + 'greenlet>=3', ], + extra_require={ + "PDF": [ + 'weasyprint>=65.0', + ], + }, entry_points={ 'console_scripts': [ 'subdominator = subdominator.subdominator:main' diff --git a/subdominator/modules/handler.py b/subdominator/modules/handler.py index f74f6dd..44e4887 100644 --- a/subdominator/modules/handler.py +++ b/subdominator/modules/handler.py @@ -5,11 +5,11 @@ from rich.console import Console from rich.markdown import Markdown import httpx -from colorama import Fore,Style +from colorama import Fore, Style import random import json -red = Fore.RED +red = Fore.RED green = Fore.GREEN magenta = Fore.MAGENTA cyan = Fore.CYAN @@ -19,19 +19,27 @@ white = Fore.WHITE reset = Style.RESET_ALL bold = Style.BRIGHT -colors = [ green, cyan, blue] +colors = [green, cyan, blue] random_color = random.choice(colors) try: from .banner.banner import banner from .cli.cli import cli - from .config.config import config, custompath, Username, cachedir,db_config + from .config.config import config, custompath, Username, cachedir, db_config from .help.help import help from .source.source import sources from .update.update import * from .version.version import version from .logger.logger import logger - from .utils.utils import filters,reader,split_to_list, check_directory_permission,check_file_permission, Exit + from .utils.utils import ( + filters, + reader, + split_to_list, + check_directory_permission, + check_file_permission, + Exit, + CleanExit, + ) from .subscraper.abuseipdb.abuseipdb import abuseipdb from .subscraper.alienvault.alientvault import alienvault from .subscraper.anubis.anubis import anubis @@ -88,128 +96,173 @@ from .save.save import file, dir, jsonsave from .notify.notify import notify except ImportError as e: - print(f"[{bold}{red}INFO{reset}]: {bold}{white}Import Error occured in Subdominator Module imports due to: {e}{reset}", file=sys.stderr) - print(f"[{bold}{blue}INFO{reset}]: {bold}{white}If you are encountering this issue more than a time please report the issues in Subdominator Github page.. {reset}", file=sys.stderr) + print( + f"[{bold}{red}INFO{reset}]: {bold}{white}Import Error occured in Subdominator Module imports due to: {e}{reset}", + file=sys.stderr, + ) + print( + f"[{bold}{blue}INFO{reset}]: {bold}{white}If you are encountering this issue more than a time please report the issues in Subdominator Github page.. {reset}", + file=sys.stderr, + ) sys.exit(1) - + args = cli() if not args.config_path: configpath = config() else: - - configpath = custompath(args.config_path,args) + configpath = custompath(args.config_path, args) if not configpath: - logger(f"Unable to load the custom provider config file, please check the file exist", "warn", args.no_color) + logger( + f"Unable to load the custom provider config file, please check the file exist", + "warn", + args.no_color, + ) Exit(1) - + if not args.config_db_path: dbpath = db_config() else: - dbpath = custompath(args.config_db_path,args) + dbpath = custompath(args.config_db_path, args) if not dbpath: - logger(f"Unable to load the custom subdominator DB , please check the DB exist", "warn", args.no_color) + logger( + f"Unable to load the custom subdominator DB , please check the DB exist", + "warn", + args.no_color, + ) Exit(1) - + from .models.models import AsyncSessionLocal -from .crud.crud import get_all_domains,get_domain,get_subdomains,add_or_update_domain +from .crud.crud import get_all_domains, get_domain, get_subdomains, add_or_update_domain from .shell.shell import SubDominatorShell banners = banner() username = Username() + async def __initiate__(domain): try: - async with httpx.AsyncClient(proxy=args.proxy, verify=False) as session: - tasks = [abuseipdb(domain, session, args), - alienvault(domain, session, args), - anubis(domain, session, args), - arpsyndicate(domain, session, configpath,args), - bevigil(domain, session, configpath, username, args), - binaryedge(domain, session, configpath, username, args), - bufferover(domain, session, configpath, username, args), - builtwith(domain, session, configpath, username, args), - c99(domain, session, configpath, username, args), - censys(domain, session, configpath, username, args), - certspotter(domain, session, configpath, args), - chaos(domain, session, configpath, args), - coderog(domain, session, configpath, args), - commoncrawl(domain, args), - crtsh(domain, session, args), - cyfare(domain, session, args), - digitorus(domain, session, args), - fofa(domain,session, configpath, username, args), - dnsdumpster(domain, session, configpath,args), - dnsrepo(domain,session, configpath, username,args), - facebook(domain, session, configpath, username, args), - fullhunt(domain, session, configpath, username, args), - google(domain, session, configpath, username, args), - hackertarget(domain, session, args), - huntermap(domain, session, configpath, username, args), - intelx(domain, session, configpath, username, args), - leakix(domain, session, configpath, username, args), - merklemap(domain, session,configpath,username,args), - myssl(domain, session, args), - netlas(domain, session, configpath, username, args), - quake(domain, session, configpath, username, args), - rapidapi(domain, session, configpath, args), - rapiddns(domain, session, args), - rapidfinder(domain, session, configpath, username, args), - rapidscan(domain, session, configpath, username, args), - redhuntlabs(domain, session, configpath, username, args) , - racent(domain, session, args) , - rsecloud(domain, session, configpath, username, args) , - securitytrails(domain, session, configpath, username, args), - shodan(domain, session, configpath, username, args), - shodanx(domain, session, args), - shrewdeye(domain, session, args), - sitedossier(domain, session, args), - trickest(domain, configpath, args), - urlscan(domain, session, args), - virustotal(domain, session, configpath, username, args), - waybackarchive(domain, args), - whoisxml(domain, session, configpath, username, args), - zoomeyeapi(domain, session, configpath, username, args), - digitalyama(domain, session, configpath, username, args), - odin(domain, session,configpath,username,args), - hudsonrock(domain, session, args), - threatcrowd(domain, session, args) - ] - results = await asyncio.gather(*tasks) - return results + limits = httpx.Limits(max_keepalive_connections=20, max_connections=50) + timeout = httpx.Timeout(args.timeout, connect=args.timeout) + async with httpx.AsyncClient( + proxy=args.proxy, verify=False, limits=limits, timeout=timeout + ) as session: + tasks = [ + abuseipdb(domain, session, args), + alienvault(domain, session, args), + anubis(domain, session, args), + arpsyndicate(domain, session, configpath, args), + bevigil(domain, session, configpath, username, args), + binaryedge(domain, session, configpath, username, args), + bufferover(domain, session, configpath, username, args), + builtwith(domain, session, configpath, username, args), + c99(domain, session, configpath, username, args), + censys(domain, session, configpath, username, args), + certspotter(domain, session, configpath, args), + chaos(domain, session, configpath, args), + coderog(domain, session, configpath, args), + commoncrawl(domain, args), + crtsh(domain, session, args), + cyfare(domain, session, args), + digitorus(domain, session, args), + fofa(domain, session, configpath, username, args), + dnsdumpster(domain, session, configpath, args), + dnsrepo(domain, session, configpath, username, args), + facebook(domain, session, configpath, username, args), + fullhunt(domain, session, configpath, username, args), + google(domain, session, configpath, username, args), + hackertarget(domain, session, args), + huntermap(domain, session, configpath, username, args), + intelx(domain, session, configpath, username, args), + leakix(domain, session, configpath, username, args), + merklemap(domain, session, configpath, username, args), + myssl(domain, session, args), + netlas(domain, session, configpath, username, args), + quake(domain, session, configpath, username, args), + rapidapi(domain, session, configpath, args), + rapiddns(domain, session, args), + rapidfinder(domain, session, configpath, username, args), + rapidscan(domain, session, configpath, username, args), + redhuntlabs(domain, session, configpath, username, args), + racent(domain, session, args), + rsecloud(domain, session, configpath, username, args), + securitytrails(domain, session, configpath, username, args), + shodan(domain, session, configpath, username, args), + shodanx(domain, session, args), + shrewdeye(domain, session, args), + sitedossier(domain, session, args), + trickest(domain, configpath, args), + urlscan(domain, session, args), + virustotal(domain, session, configpath, username, args), + waybackarchive(domain, args), + whoisxml(domain, session, configpath, username, args), + zoomeyeapi(domain, session, configpath, username, args), + digitalyama(domain, session, configpath, username, args), + odin(domain, session, configpath, username, args), + hudsonrock(domain, session, args), + threatcrowd(domain, session, args), + ] + results = await asyncio.wait_for( + asyncio.gather(*tasks, return_exceptions=True), + timeout=args.timeout * len(tasks), + ) + return results + except asyncio.TimeoutError: + if args.verbose: + logger( + f"Overall timeout reached after {args.timeout * len(tasks)} seconds", + "warn", + args.no_color, + ) + return None except Exception as e: if args.verbose: logger(f"Exception handler sources: {e}, {type(e)}", "warn", args.no_color) + def gitversion(): try: latest = version() current = "v2.1.1" if latest == current: - print(f"[{blue}{bold}version{reset}]:{bold}{white}subdominator current version {current} ({green}latest{reset}{bold}{white}){reset}", file=sys.stderr) + print( + f"[{blue}{bold}version{reset}]:{bold}{white}subdominator current version {current} ({green}latest{reset}{bold}{white}){reset}", + file=sys.stderr, + ) else: - print(f"[{blue}{bold}version{reset}]: {bold}{white}subdominator current version {current} ({red}outdated{reset}{bold}{white}){reset}", file=sys.stderr) + print( + f"[{blue}{bold}version{reset}]: {bold}{white}subdominator current version {current} ({red}outdated{reset}{bold}{white}){reset}", + file=sys.stderr, + ) except KeyboardInterrupt as e: Exit(1) - - + + def update_handler(): try: if args.show_updates: updatelog() Exit() - + current = "v2.1.1" pypiold = "2.1.1" git = version() - + if current == git: - logger(f"Hey {username} subdominator is already in new version", "info", args.no_color) + logger( + f"Hey {username} subdominator is already in new version", + "info", + args.no_color, + ) Exit() - + zipurl = getzip() if not zipurl: - logger(f"Hey {username} failed to update subdominator please try manually from github", "warn" , args.no_color) + logger( + f"Hey {username} failed to update subdominator please try manually from github", + "warn", + args.no_color, + ) Exit() cache = cachedir() if cache: @@ -217,73 +270,100 @@ def update_handler(): else: launch(zipurl, os.getcwd()) pypiversion = getverify("subdominator") - + if pypiold == pypiversion: - logger(f"Hey {username} failed to update subdominator please try manually from github", "warn" , args.no_color) + logger( + f"Hey {username} failed to update subdominator please try manually from github", + "warn", + args.no_color, + ) Exit() - logger(f"Verified the latest version of subdominator from pypi and updated from {current} --> {git}", "info" , args.no_color) + logger( + f"Verified the latest version of subdominator from pypi and updated from {current} --> {git}", + "info", + args.no_color, + ) updatelog() Exit() - + except KeyboardInterrupt as e: Exit() + def show_sources(): try: resources = sources() - logger(f"Current Available passive resources: [{len(resources)}]", "info" , args.no_color) - - logger(f"Sources marked with an * needs API key(s) or token(s) configuration to works", "info" , args.no_color) - - logger(f"Hey {username} you can config your api keys or token here {configpath} to work", "info" , args.no_color) + logger( + f"Current Available passive resources: [{len(resources)}]", + "info", + args.no_color, + ) + + logger( + f"Sources marked with an * needs API key(s) or token(s) configuration to works", + "info", + args.no_color, + ) + + logger( + f"Hey {username} you can config your api keys or token here {configpath} to work", + "info", + args.no_color, + ) console = Console() for sourced in resources: console.print(Markdown(sourced)) - Exit() + Exit() except KeyboardInterrupt as e: Exit(1) - + + async def _domain_handler_(domain): try: start = time.time() results = await __initiate__(domain) filtered = filters(results) - final= set() + final = set() for subdomain in filtered: if subdomain.endswith(f".{domain}") and subdomain not in final: if args.filter_wildcards: if subdomain.startswith("*."): - subdomain = subdomain[2:] + subdomain = subdomain[2:] final.add(subdomain) - + for subdomain in final: if args.json: - output = {"domain":f"{domain}", "subdomain": f"{subdomain}"} + output = {"domain": f"{domain}", "subdomain": f"{subdomain}"} output = json.dumps(output, indent=2) else: output = subdomain - + print(output) if args.output: file(output, domain, args) elif args.output_directory: dir(output, domain, args) - + async with AsyncSessionLocal() as db: await add_or_update_domain(db, domain, final) - + if args.notify: await notify(domain, final, configpath, username, args) - + end = time.time() total_time = end - start if not args.silent: - logger(f"Total {len(final)} subdomains found for {domain} in {total_time:.2f} seconds{reset}", "info" , args.no_color) - logger(f"Happy Hacking {username} ☠️ 🔥 🚀{reset}", "info" , args.no_color) + logger( + f"Total {len(final)} subdomains found for {domain} in {total_time:.2f} seconds{reset}", + "info", + args.no_color, + ) + logger(f"Happy Hacking {username} ☠️ 🔥 🚀{reset}", "info", args.no_color) except KeyboardInterrupt as e: - Exit(1) + raise CleanExit() + async def handler(): try: @@ -291,13 +371,13 @@ async def handler(): print(banners, file=sys.stderr) help(configpath, dbpath) Exit() - + if args.shell: print(banners, file=sys.stderr) Shell = SubDominatorShell() await Shell.cmdloop() Exit(0) - + if not args.silent: print(banners, file=sys.stderr) if not args.disable_update_check: @@ -306,62 +386,105 @@ async def handler(): show_sources() if args.update or args.show_updates: update_handler() - + if args.include_resources: args.include_resources = split_to_list(args.include_resources) - + if args.exclude_resources: args.exclude_resources = split_to_list(args.exclude_resources) - + if args.output: perm = await check_file_permission(args.output, args) if not perm: Exit(1) - + if args.output_directory: perm = await check_directory_permission(args.output_directory, args) if not perm: Exit(1) - + if args.domain: if not args.silent: - logger(f"Loading provider configuration file from {configpath}", "info" , args.no_color) - - logger(f"Enumerating subdomain for {args.domain}", "info" , args.no_color) + logger( + f"Loading provider configuration file from {configpath}", + "info", + args.no_color, + ) + + logger( + f"Enumerating subdomain for {args.domain}", "info", args.no_color + ) await _domain_handler_(args.domain) Exit() - + if args.domain_list: if not args.silent: - logger(f"Loading provider configuration file from {configpath}", "info" , args.no_color) + logger( + f"Loading provider configuration file from {configpath}", + "info", + args.no_color, + ) domains = await reader(args.domain_list, args) if domains: for domain in domains: if not args.silent: - logger(f"Enumerating subdomain for {domain}", "info" , args.no_color) - + logger( + f"Enumerating subdomain for {domain}", "info", args.no_color + ) + await _domain_handler_(domain) Exit() - + if sys.stdin.isatty(): - logger(f"subdominator exits due to no inputs provided, please use subdominator -h for more details", "warn", args.no_color) - Exit() + logger( + f"subdominator exits due to no inputs provided, please use subdominator -h for more details", + "warn", + args.no_color, + ) + Exit() else: if not args.silent: - logger(f"Loading provider configuration file from {configpath}", "info" , args.no_color) + logger( + f"Loading provider configuration file from {configpath}", + "info", + args.no_color, + ) for domain in sys.stdin: if domain: domain = domain.strip() if not args.silent: - logger(f"Enumerating subdomain for {domain}", "info" , args.no_color) + logger( + f"Enumerating subdomain for {domain}", "info", args.no_color + ) await _domain_handler_(domain) Exit() - except KeyboardInterrupt as e: - Exit() - + except (KeyboardInterrupt, CleanExit) as e: + await cleanup() + raise + + +async def cleanup(): + try: + tasks = [ + task for task in asyncio.all_tasks() if task is not asyncio.current_task() + ] + for task in tasks: + task.cancel() + if tasks: + await asyncio.wait_for( + asyncio.gather(*tasks, return_exceptions=True), timeout=2.0 + ) + from .models.models import async_engine + + await async_engine.dispose() + except Exception: + pass + def main_handler(): try: asyncio.run(handler()) - except KeyboardInterrupt as e: + except KeyboardInterrupt: sys.exit(1) + except CleanExit: + sys.exit(0) diff --git a/subdominator/modules/shell/shell.py b/subdominator/modules/shell/shell.py index 45ae80c..1d0b39f 100644 --- a/subdominator/modules/shell/shell.py +++ b/subdominator/modules/shell/shell.py @@ -9,7 +9,6 @@ from rich.console import Console from rich.table import Table from jinja2 import Environment, FileSystemLoader -from weasyprint import HTML console = Console() @@ -191,7 +190,6 @@ async def generate_report(self,domain,output_file,format_type): console.print(f"[bold green]HTML report saved as {output_file}[/bold green]") elif format_type.lower() == "pdf": self.generate_pdf_report(html_report, output_file) - console.print(f"[bold green]PDF report saved as {output_file}[/bold green]") else: console.print(f"[bold yellow]Report generation only supports pdf/html format, please use a valid report generation format.[/bold yellow]") return @@ -201,7 +199,14 @@ def generate_html_report(self, report_data, template_path): return template.render(domain=report_data["domain"], subdomains=report_data["subdomains"]) def generate_pdf_report(self,html_report, output_file): + try: + from weasyprint import HTML + except ModuleNotFoundError: + console.print("[bold red]PDF report requires weasyprint module[/bold red]") + console.print("[bold green]Hint:[/bold green] pip install 'subdominator[PDF]'") + return HTML(string=html_report).write_pdf(output_file) + console.print(f"[bold green]PDF report saved as {output_file}[/bold green]") def load_subdomains_from_file(self, filename): if not os.path.exists(filename): @@ -228,4 +233,4 @@ async def do_help(self): [bold][green]help[/green][/bold] [bold]- Show this help menu[/bold] [bold][green]System Commands[/green][/bold] [bold]- ls, clear, pwd, cd, cat, echo, mkdir, rm, cp, mv[/bold] """ - ) \ No newline at end of file + ) diff --git a/subdominator/modules/subscraper/abuseipdb/abuseipdb.py b/subdominator/modules/subscraper/abuseipdb/abuseipdb.py index e0888b3..c9239f1 100644 --- a/subdominator/modules/subscraper/abuseipdb/abuseipdb.py +++ b/subdominator/modules/subscraper/abuseipdb/abuseipdb.py @@ -1,40 +1,54 @@ import re -from subdominator.modules.utils.utils import check_subdomain,UserAgents +from subdominator.modules.utils.utils import check_subdomain, UserAgents import httpx from subdominator.modules.logger.logger import logger + abuseipdbs = [] + async def abuseipdb(domain: str, session: httpx.AsyncClient, args) -> list[str]: try: - - if args.include_resources and "abuseipdb" not in args.include_resources and not args.all: + if ( + args.include_resources + and "abuseipdb" not in args.include_resources + and not args.all + ): return abuseipdbs - + if args.exclude_resources and "abuseipdb" in args.exclude_resources: return abuseipdbs - + parsed_domain = check_subdomain(domain) if parsed_domain.subdomain: return abuseipdbs - url = f"https://www.abuseipdb.com/whois/{domain}" - headers = { - "User-Agent": UserAgents(), - "Cookie": "abuseipdb_session=" - } - response: httpx.Response = await session.request("GET", url, timeout=args.timeout, headers=headers) + url = f"https://www.abuseipdb.com/whois/{domain}" + headers = {"User-Agent": UserAgents(), "Cookie": "abuseipdb_session="} + response: httpx.Response = await session.request( + "GET", url, timeout=args.timeout, headers=headers + ) data = response.text if response.status_code != 200: return abuseipdbs - tags = re.findall(r'
  • \w.*
    • ', data) + tags = re.findall(r"
  • \w.*
    • ", data) subdomains = [re.sub("", "", tag) + f".{domain}" for tag in tags] abuseipdbs.extend(subdomains) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout exception occurred in the AbusiIpdb API due to: {e}", "warn", args.no_color) + logger( + f"Timeout exception occurred in the AbusiIpdb API due to: {e}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in AbuseIpdb module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in AbuseIpdb module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Abuseipdb API: {len(abuseipdbs)}", "info") - return abuseipdbs \ No newline at end of file + logger( + f"Total Subdomains found by Abuseipdb API: {len(abuseipdbs)}", "info" + ) + return abuseipdbs diff --git a/subdominator/modules/subscraper/alienvault/alientvault.py b/subdominator/modules/subscraper/alienvault/alientvault.py index 9a43a8a..dcf4690 100644 --- a/subdominator/modules/subscraper/alienvault/alientvault.py +++ b/subdominator/modules/subscraper/alienvault/alientvault.py @@ -1,33 +1,51 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents + alienvaults = [] -async def alienvault(domain: str, session: httpx.AsyncClient,args) -> list[str]: + +async def alienvault(domain: str, session: httpx.AsyncClient, args) -> list[str]: try: - if args.include_resources and "alienvault" not in args.include_resources and not args.all: + if ( + args.include_resources + and "alienvault" not in args.include_resources + and not args.all + ): return alienvaults - + if args.exclude_resources and "alienvault" in args.exclude_resources: return alienvaults - + headers = {"User-Agents": UserAgents()} url = f"https://otx.alienvault.com/api/v1/indicators/hostname/{domain}/passive_dns" - response: httpx.Response = await session.request("GET",url, timeout=args.timeout, headers=headers) + response: httpx.Response = await session.request( + "GET", url, timeout=args.timeout, headers=headers + ) if response.status_code != 200: return alienvaults data = response.json() - for entries in data['passive_dns']: - subdomain = entries['hostname'] + for entries in data["passive_dns"]: + subdomain = entries["hostname"] if subdomain.endswith(f".{domain}"): alienvaults.append(subdomain) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Alienvault API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for Alienvault API, due to: {e}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in Alienvalut module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in Alienvalut module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Alienvault API: {len(alienvaults)}", "info") - return alienvaults \ No newline at end of file + logger( + f"Total Subdomains found by Alienvault API: {len(alienvaults)}", "info" + ) + return alienvaults diff --git a/subdominator/modules/subscraper/anubis/anubis.py b/subdominator/modules/subscraper/anubis/anubis.py index f6a5986..305f606 100644 --- a/subdominator/modules/subscraper/anubis/anubis.py +++ b/subdominator/modules/subscraper/anubis/anubis.py @@ -1,33 +1,50 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents + anubiss = [] + async def anubis(domain: str, session: httpx.AsyncClient, args): try: - - if args.include_resources and "anubis" not in args.include_resources and not args.all: + if ( + args.include_resources + and "anubis" not in args.include_resources + and not args.all + ): return anubiss - + if args.exclude_resources and "anubis" in args.exclude_resources: return anubiss - + headers = {"User-Agent": UserAgents()} url = f"https://anubisdb.com/anubis/subdomains/{domain}" - response: httpx.Response = await session.get(url, timeout=args.timeout,headers=headers, follow_redirects=True) + response: httpx.Response = await session.get( + url, timeout=args.timeout, headers=headers, follow_redirects=True + ) if response.status_code != 200: return anubiss data = response.json() for subdomain in data: - if subdomain.endswith(f".{domain}") : + if subdomain.endswith(f".{domain}"): anubiss.append(subdomain) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Anubis API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for Anubis API, due to: {e}", "warn", args.no_color + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in Anubis API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in Anubis API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Anubis API: {len(anubiss)}", "info", args.no_color) - return anubiss \ No newline at end of file + logger( + f"Total Subdomains found by Anubis API: {len(anubiss)}", + "info", + args.no_color, + ) + return anubiss diff --git a/subdominator/modules/subscraper/arpsyndicate/arpsyndicate.py b/subdominator/modules/subscraper/arpsyndicate/arpsyndicate.py index bbad677..a36a247 100644 --- a/subdominator/modules/subscraper/arpsyndicate/arpsyndicate.py +++ b/subdominator/modules/subscraper/arpsyndicate/arpsyndicate.py @@ -2,22 +2,30 @@ from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import singlekeyloader from urllib.parse import unquote + arpsyndicates = [] -async def arpsyndicate(domain: str, session: httpx.AsyncClient, configs: str,args): + +async def arpsyndicate(domain: str, session: httpx.AsyncClient, configs: str, args): try: - if args.include_resources and "arpsyndicate" not in args.include_resources and not args.all: + if ( + args.include_resources + and "arpsyndicate" not in args.include_resources + and not args.all + ): return arpsyndicates - + if args.exclude_resources and "arpsyndicate" in args.exclude_resources: return arpsyndicates - - randomkey = await singlekeyloader(configs,"arpsyndicate") + + randomkey = await singlekeyloader(configs, "arpsyndicate") if randomkey is None: return arpsyndicates - + url = f"https://api.subdomain.center/beta/?domain={domain}&auth={randomkey}" - response: httpx.Response = await session.request("GET",url,timeout=args.timeout) + response: httpx.Response = await session.request( + "GET", url, timeout=args.timeout + ) if response.status_code != 200: return [] data = response.json() @@ -28,11 +36,21 @@ async def arpsyndicate(domain: str, session: httpx.AsyncClient, configs: str,arg arpsyndicates.append(subdomain) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Arpsyndicate due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for Arpsyndicate due to: {e}", "warn", args.no_color + ) except Exception as e: if args.verbose: - logger(f"Exception in Arpsyndicate API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in Arpsyndicate API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Arpsyndicate: {len(arpsyndicates)}", "info", args.no_color) - return arpsyndicates \ No newline at end of file + logger( + f"Total Subdomains found by Arpsyndicate: {len(arpsyndicates)}", + "info", + args.no_color, + ) + return arpsyndicates diff --git a/subdominator/modules/subscraper/bevigil/bevigil.py b/subdominator/modules/subscraper/bevigil/bevigil.py index 7b46b40..6cce363 100644 --- a/subdominator/modules/subscraper/bevigil/bevigil.py +++ b/subdominator/modules/subscraper/bevigil/bevigil.py @@ -1,30 +1,41 @@ import httpx from subdominator.modules.logger.logger import logger -from subdominator.modules.utils.utils import UserAgents,singlekeyloader +from subdominator.modules.utils.utils import UserAgents, singlekeyloader + bevigils = [] -async def bevigil(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def bevigil( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "bevigil" not in args.include_resources and not args.all: + if ( + args.include_resources + and "bevigil" not in args.include_resources + and not args.all + ): return bevigils - + if args.exclude_resources and "bevigil" in args.exclude_resources: return bevigils - - randomkey = await singlekeyloader(configs,"bevigil") - + + randomkey = await singlekeyloader(configs, "bevigil") + if randomkey is None: return bevigils - + url = f"https://osint.bevigil.com/api/{domain}/subdomains" - headers = { - 'User-Agent': UserAgents(), - 'X-Access-Token': randomkey - } - response: httpx.Response = await session.request("GET",url, headers=headers, timeout=args.timeout) + headers = {"User-Agent": UserAgents(), "X-Access-Token": randomkey} + response: httpx.Response = await session.request( + "GET", url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"Bevigil blocking our request, {username} please check your api usage for this key: {randomkey}", "warn", args.no_color) + logger( + f"Bevigil blocking our request, {username} please check your api usage for this key: {randomkey}", + "warn", + args.no_color, + ) return [] data = response.json() subdomains = data.get("subdomains", []) @@ -32,11 +43,21 @@ async def bevigil(domain: str, session: httpx.AsyncClient, configs: str, usernam bevigils.append(subdomain) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Bevigil API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for Bevigil API, due to: {e}", "warn", args.no_color + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in Bevigil API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in Bevigil API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomins found by Bevigil API: {len(bevigils)}", "info", args.no_color) - return bevigils \ No newline at end of file + logger( + f"Total Subdomins found by Bevigil API: {len(bevigils)}", + "info", + args.no_color, + ) + return bevigils diff --git a/subdominator/modules/subscraper/binaryedge/binaryedge.py b/subdominator/modules/subscraper/binaryedge/binaryedge.py index 4072705..29af24b 100644 --- a/subdominator/modules/subscraper/binaryedge/binaryedge.py +++ b/subdominator/modules/subscraper/binaryedge/binaryedge.py @@ -1,32 +1,53 @@ import httpx from subdominator.modules.logger.logger import logger -from subdominator.modules.utils.utils import UserAgents,singlekeyloader +from subdominator.modules.utils.utils import UserAgents, singlekeyloader + binaryedges = [] -async def binaryget(domain: str, session: httpx.AsyncClient, randkey: str, pagenum: int, pagesize: int, args): + +async def binaryget( + domain: str, + session: httpx.AsyncClient, + randkey: str, + pagenum: int, + pagesize: int, + args, +): try: url = f"https://api.binaryedge.io/v2/query/domains/subdomain/{domain}?page={pagenum}&pagesize={pagesize}" - auth = { - 'User-Agent': UserAgents(), - 'X-Key': f'{randkey}' - } - response: httpx.Response = await session.request("GET",url, timeout=args.timeout, headers=auth) + auth = {"User-Agent": UserAgents(), "X-Key": f"{randkey}"} + response: httpx.Response = await session.request( + "GET", url, timeout=args.timeout, headers=auth + ) if response.status_code != 200: - return + return data = response.json() subdomains = data.get("events", []) if subdomains: return subdomains except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Binaryegde API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for Binaryegde API, due to: {e}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception occurred in the Binaryedge request module due to: {e}, {type(e)}","warn", args.no_color) + logger( + f"Exception occurred in the Binaryedge request module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) + async def binaryedge(domain, session, configs, username, args) -> list[str]: try: - if args.include_resources and "binaryedge" not in args.include_resources and not args.all: + if ( + args.include_resources + and "binaryedge" not in args.include_resources + and not args.all + ): return binaryedges if args.exclude_resources and "binaryedge" in args.exclude_resources: return binaryedges @@ -36,7 +57,9 @@ async def binaryedge(domain, session, configs, username, args) -> list[str]: randomkey = await singlekeyloader(configs, "binaryedge") if randomkey is None: break - subdomains = await binaryget(domain, session, randomkey, pagenum, pagesize, args) + subdomains = await binaryget( + domain, session, randomkey, pagenum, pagesize, args + ) if subdomains: for subdomain in subdomains: binaryedges.append(subdomain) @@ -45,8 +68,16 @@ async def binaryedge(domain, session, configs, username, args) -> list[str]: pagenum += 1 except Exception as e: if args.verbose: - logger(f"Exception occurred in the Binaryedge API due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in the Binaryedge API due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Binaryedge API: {len(binaryedges)}", "info", args.no_color) - return binaryedges \ No newline at end of file + logger( + f"Total Subdomains found by Binaryedge API: {len(binaryedges)}", + "info", + args.no_color, + ) + return binaryedges diff --git a/subdominator/modules/subscraper/bufferover/bufferover.py b/subdominator/modules/subscraper/bufferover/bufferover.py index 807475e..1ba094b 100644 --- a/subdominator/modules/subscraper/bufferover/bufferover.py +++ b/subdominator/modules/subscraper/bufferover/bufferover.py @@ -1,44 +1,67 @@ import httpx from subdominator.modules.logger.logger import logger -from subdominator.modules.utils.utils import UserAgents,singlekeyloader +from subdominator.modules.utils.utils import UserAgents, singlekeyloader + bufferovers = [] -async def bufferover(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): - try: - if args.include_resources and "bufferover" not in args.include_resources and not args.all: + +async def bufferover( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): + try: + if ( + args.include_resources + and "bufferover" not in args.include_resources + and not args.all + ): return bufferovers - + if args.exclude_resources and "bufferover" in args.exclude_resources: return bufferovers - + randomkey = await singlekeyloader(configs, "bufferover") if randomkey is None: return bufferovers url = f"https://tls.bufferover.run/dns?q=.{domain}" - auth = { - 'User-Agent': UserAgents(), - 'x-api-key': randomkey - } - response: httpx.Response = await session.request("GET",url, headers=auth, timeout=args.timeout) + auth = {"User-Agent": UserAgents(), "x-api-key": randomkey} + response: httpx.Response = await session.request( + "GET", url, headers=auth, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"Bufferover blocking our request, {username} please check your api usage for this key: {randomkey}", "warn", args.no_color) + logger( + f"Bufferover blocking our request, {username} please check your api usage for this key: {randomkey}", + "warn", + args.no_color, + ) return bufferovers - + data = response.json() - if 'Results' in data: - results = data['Results'] + if "Results" in data: + results = data["Results"] for result in results: - elements = result.split(',') + elements = result.split(",") subdomain = elements[4].strip() bufferovers.append(subdomain) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Bufferover API due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Timeout reached for Bufferover API due to: {e}, {type(e)}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception occurred in Bufferover API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in Bufferover API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Bufferover API: {len(bufferovers)}", "info", args.no_color) - return bufferovers \ No newline at end of file + logger( + f"Total Subdomains found by Bufferover API: {len(bufferovers)}", + "info", + args.no_color, + ) + return bufferovers diff --git a/subdominator/modules/subscraper/builtwith/builtwith.py b/subdominator/modules/subscraper/builtwith/builtwith.py index ddd2e33..b0624a9 100644 --- a/subdominator/modules/subscraper/builtwith/builtwith.py +++ b/subdominator/modules/subscraper/builtwith/builtwith.py @@ -1,44 +1,64 @@ import httpx from subdominator.modules.logger.logger import logger -from subdominator.modules.utils.utils import UserAgents,singlekeyloader +from subdominator.modules.utils.utils import UserAgents, singlekeyloader + Builtwiths = [] -async def builtwith(domain: str,session: httpx.AsyncClient, configs: str, username: str, args): + +async def builtwith( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "builtwith" not in args.include_resources and not args.all: + if ( + args.include_resources + and "builtwith" not in args.include_resources + and not args.all + ): return Builtwiths - + if args.exclude_resources and "builtwith" in args.exclude_resources: return Builtwiths - + randomkey = await singlekeyloader(configs, "builtwith") if randomkey is None: return Builtwiths url = f"https://api.builtwith.com/v21/api.json?KEY={randomkey}&HIDETEXT=yes&HIDEDL=yes&NOLIVE=yes&NOMETA=yes&NOPII=yes&NOATTR=yes&LOOKUP={domain}" - headers = { - "User-Agent": UserAgents() - } - response: httpx.Response = await session.request("GET",url, headers=headers, timeout=args.timeout) + headers = {"User-Agent": UserAgents()} + response: httpx.Response = await session.request( + "GET", url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"Builtwith blocking our request, {username} please check your api usage for this key: {randomkey}", "warn", args.no_color) + logger( + f"Builtwith blocking our request, {username} please check your api usage for this key: {randomkey}", + "warn", + args.no_color, + ) return Builtwiths - + jdata = response.json() if isinstance(jdata, dict): results = jdata.get("Results", []) for result in results: for chunk in result.get("Result", {}).get("Paths", []): domain_name = chunk.get("Domain", "") - subdomain = chunk.get("SubDomain", "") + subdomain = chunk.get("SubDomain", "") if domain_name and subdomain: Builtwiths.append(f"{subdomain}.{domain_name}") except httpx.TimeoutException as e: - logger(f"Timeout reached for Builtwith API, due to: {e}", "warn", args.no_color) + logger(f"Timeout reached for Builtwith API, due to: {e}", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception occurred in Builtwith API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in Builtwith API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Builtwith API: {len(Builtwiths)}", "info", args.no_color) - return Builtwiths \ No newline at end of file + logger( + f"Total Subdomains found by Builtwith API: {len(Builtwiths)}", + "info", + args.no_color, + ) + return Builtwiths diff --git a/subdominator/modules/subscraper/c99/c99.py b/subdominator/modules/subscraper/c99/c99.py index 281e13d..bcdfc7e 100644 --- a/subdominator/modules/subscraper/c99/c99.py +++ b/subdominator/modules/subscraper/c99/c99.py @@ -1,36 +1,52 @@ import httpx from subdominator.modules.logger.logger import logger -from subdominator.modules.utils.utils import UserAgents,singlekeyloader +from subdominator.modules.utils.utils import UserAgents, singlekeyloader + C99s = [] -async def c99(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def c99( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "c99" not in args.include_resources and not args.all: + if ( + args.include_resources + and "c99" not in args.include_resources + and not args.all + ): return C99s - + if args.exclude_resources and "c99" in args.exclude_resources: return C99s - + randomkey = await singlekeyloader(configs, "c99") if randomkey is None: return C99s url = f"https://api.c99.nl/subdomainfinder?key={randomkey}&domain={domain}&json=true" headers = {"User-Agent": UserAgents()} - - response: httpx.Response = await session.request("GET",url, headers=headers,timeout=args.timeout) + + response: httpx.Response = await session.request( + "GET", url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: return C99s data = response.json() if "subdomain" in data: - subs= [entry["subdomain"] for entry in data] + subs = [entry["subdomain"] for entry in data] C99s.extend(subs) except httpx.TimeoutException as e: logger(f"Timeout Reached for C99 API due to: {e}", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception occurred in C99 API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in C99 API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by C99 API: {len(C99s)}", "info", args.no_color) - return C99s \ No newline at end of file + logger( + f"Total Subdomains found by C99 API: {len(C99s)}", "info", args.no_color + ) + return C99s diff --git a/subdominator/modules/subscraper/censys/censys.py b/subdominator/modules/subscraper/censys/censys.py index 3b0b8ae..e5792ae 100644 --- a/subdominator/modules/subscraper/censys/censys.py +++ b/subdominator/modules/subscraper/censys/censys.py @@ -1,51 +1,77 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import dualkeyloader + censyss = [] -async def censys(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): - + +async def censys( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "censys" not in args.include_resources and not args.all: + if ( + args.include_resources + and "censys" not in args.include_resources + and not args.all + ): return censyss - + if args.exclude_resources and "censys" in args.exclude_resources: return censyss - - randomtoken,randomsecret = await dualkeyloader(configs, "censys", False) - + + randomtoken, randomsecret = await dualkeyloader(configs, "censys", False) + if randomtoken is None or randomsecret is None: return censyss - + url = "https://search.censys.io/api/v2/certificates/search" maxpage = 10 maxdata = 100 cursor = None - params = {'q': domain, 'per_page': maxdata} - - for _ in range(maxpage+1): + params = {"q": domain, "per_page": maxdata} + + for _ in range(maxpage + 1): if cursor: - params['cursor'] = cursor - response: httpx.Response = await session.request("GET",url, auth=httpx.BasicAuth(randomtoken, randomsecret), params=params, timeout=args.timeout) + params["cursor"] = cursor + response: httpx.Response = await session.request( + "GET", + url, + auth=httpx.BasicAuth(randomtoken, randomsecret), + params=params, + timeout=args.timeout, + ) if response.status_code != 200: if args.show_key_info: - logger(f"Censys blocking our request, {username} please check your api usage for this keys: {randomsecret}, {randomtoken}", "warn", args.no_color) + logger( + f"Censys blocking our request, {username} please check your api usage for this keys: {randomsecret}, {randomtoken}", + "warn", + args.no_color, + ) return censyss - + data = response.json() - if 'result' in data and 'hits' in data['result']: - for hit in data['result']['hits']: - for name in hit.get('names', []): + if "result" in data and "hits" in data["result"]: + for hit in data["result"]["hits"]: + for name in hit.get("names", []): censyss.append(name) - cursor = data['result']['links'].get('next') + cursor = data["result"]["links"].get("next") if not cursor: - break + break except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Censys API due to: {e}", "warn", args.no_color) + logger(f"Timeout reached for Censys API due to: {e}", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception occurred in Censys API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in Censys API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains Found by Censys API: {len(censyss)}", "info", args.no_color) \ No newline at end of file + logger( + f"Total Subdomains Found by Censys API: {len(censyss)}", + "info", + args.no_color, + ) + return censyss diff --git a/subdominator/modules/subscraper/certspotter/certspotter.py b/subdominator/modules/subscraper/certspotter/certspotter.py index b24e85f..238f3f1 100644 --- a/subdominator/modules/subscraper/certspotter/certspotter.py +++ b/subdominator/modules/subscraper/certspotter/certspotter.py @@ -1,11 +1,16 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import singlekeyloader + certspotters = [] + async def certspotter(domain: str, session: httpx.AsyncClient, configs: str, args): try: - if not (args.all or (args.include_resources and "certspotter" in args.include_resources)): + if not ( + args.all + or (args.include_resources and "certspotter" in args.include_resources) + ): return certspotters randomkey = await singlekeyloader(configs, "certspotter") @@ -16,10 +21,16 @@ async def certspotter(domain: str, session: httpx.AsyncClient, configs: str, arg base_url = f"https://api.certspotter.com/v1/issuances?domain={domain}&include_subdomains=true&expand=dns_names" while base_url: - response : httpx.Response = await session.request("GET",base_url, headers=headers, timeout=args.timeout) + response: httpx.Response = await session.request( + "GET", base_url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: if args.verbose: - logger(f"CertSpotter API returned base response status : {response.status_code}", "warn", args.no_color) + logger( + f"CertSpotter API returned base response status : {response.status_code}", + "warn", + args.no_color, + ) return certspotters data = response.json() @@ -37,12 +48,24 @@ async def certspotter(domain: str, session: httpx.AsyncClient, configs: str, arg except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for CertSpotter API due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for CertSpotter API due to: {e}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in CertSpotter API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in CertSpotter API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by CertSpotter API: {len(certspotters)}", "info", args.no_color) - return certspotters + logger( + f"Total Subdomains found by CertSpotter API: {len(certspotters)}", + "info", + args.no_color, + ) + return certspotters diff --git a/subdominator/modules/subscraper/chaos/chaos.py b/subdominator/modules/subscraper/chaos/chaos.py index 3e35705..0069620 100644 --- a/subdominator/modules/subscraper/chaos/chaos.py +++ b/subdominator/modules/subscraper/chaos/chaos.py @@ -1,13 +1,19 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import singlekeyloader + chaoss = [] + async def chaos(domain: str, session: httpx.AsyncClient, configs: str, args): try: - if args.include_resources and "chaos" not in args.include_resources and not args.all: + if ( + args.include_resources + and "chaos" not in args.include_resources + and not args.all + ): return chaoss - + if args.exclude_resources and "chaos" in args.exclude_resources: return chaoss @@ -18,12 +24,18 @@ async def chaos(domain: str, session: httpx.AsyncClient, configs: str, args): url = f"https://dns.projectdiscovery.io/dns/{domain}/subdomains" headers = {"Authorization": randomkey} - response: httpx.Response = await session.request("GET", url, headers=headers, timeout=args.timeout) + response: httpx.Response = await session.request( + "GET", url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: if args.verbose: - logger(f"Chaos API returned response status: {response.status_code}", "warn", args.no_color) + logger( + f"Chaos API returned response status: {response.status_code}", + "warn", + args.no_color, + ) return chaoss - + data = response.json() if "subdomains" in data: for subdomain in data["subdomains"]: @@ -38,8 +50,16 @@ async def chaos(domain: str, session: httpx.AsyncClient, configs: str, args): logger(f"Timeout reached for Chaos API due to: {e}", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception error occurred in Chaos API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in Chaos API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Chaos API: {len(chaoss)}", "info", args.no_color) - return chaoss \ No newline at end of file + logger( + f"Total Subdomains found by Chaos API: {len(chaoss)}", + "info", + args.no_color, + ) + return chaoss diff --git a/subdominator/modules/subscraper/coderog/coderog.py b/subdominator/modules/subscraper/coderog/coderog.py index 7c97f7d..50e7a34 100644 --- a/subdominator/modules/subscraper/coderog/coderog.py +++ b/subdominator/modules/subscraper/coderog/coderog.py @@ -1,13 +1,19 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import singlekeyloader + coderogs = [] + async def coderog(domain: str, session: httpx.AsyncClient, configs: str, args): try: - if args.include_resources and "coderog" not in args.include_resources and not args.all: + if ( + args.include_resources + and "coderog" not in args.include_resources + and not args.all + ): return coderogs - + if args.exclude_resources and "coderog" in args.exclude_resources: return coderogs @@ -15,21 +21,33 @@ async def coderog(domain: str, session: httpx.AsyncClient, configs: str, args): if not randomkey: return coderogs - url = f"https://subdomain-finder5.p.rapidapi.com/subdomain-finder?domain={domain}" + url = ( + f"https://subdomain-finder5.p.rapidapi.com/subdomain-finder?domain={domain}" + ) headers = { "x-rapidapi-key": randomkey, - "x-rapidapi-host": "subdomain-finder5.p.rapidapi.com" + "x-rapidapi-host": "subdomain-finder5.p.rapidapi.com", } - response: httpx.Response = await session.request("GET", url, headers=headers, timeout=args.timeout) + response: httpx.Response = await session.request( + "GET", url, headers=headers, timeout=args.timeout + ) if response.status_code == 403: if args.verbose: - logger(f"CodeRog API blocked request. Ensure you are subscribed to the API service for key: {randomkey}", "warn", args.no_color) + logger( + f"CodeRog API blocked request. Ensure you are subscribed to the API service for key: {randomkey}", + "warn", + args.no_color, + ) return coderogs if response.status_code != 200: if args.verbose: - logger(f"CodeRog API returned response status: {response.status_code}. Check API usage for key: {randomkey}", "warn", args.no_color) + logger( + f"CodeRog API returned response status: {response.status_code}. Check API usage for key: {randomkey}", + "warn", + args.no_color, + ) return coderogs data = response.json() @@ -39,13 +57,25 @@ async def coderog(domain: str, session: httpx.AsyncClient, configs: str, args): coderogs.append(subdomain) except httpx.TimeoutException: if args.show_timeout_info: - logger("Timeout reached for CodeRog API due to server-side or client-side error.", "warn", args.no_color) + logger( + "Timeout reached for CodeRog API due to server-side or client-side error.", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception occurred in CodeRog API module: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in CodeRog API module: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by CodeRog API: {len(coderogs)}", "info", args.no_color) - return coderogs + logger( + f"Total Subdomains found by CodeRog API: {len(coderogs)}", + "info", + args.no_color, + ) + return coderogs diff --git a/subdominator/modules/subscraper/commoncrawl/commoncrawl.py b/subdominator/modules/subscraper/commoncrawl/commoncrawl.py index acb1ac6..74dfad2 100644 --- a/subdominator/modules/subscraper/commoncrawl/commoncrawl.py +++ b/subdominator/modules/subscraper/commoncrawl/commoncrawl.py @@ -7,73 +7,110 @@ Commoncrawls = set() -async def indexDB(args): + +async def indexDB(session: httpx.AsyncClient, args): try: - headers = { - "User-Agent": UserAgents() - } - async with httpx.AsyncClient(headers=headers, timeout=args.timeout, verify=False,proxy=args.proxy) as session: - response = await session.request("GET", "https://index.commoncrawl.org/collinfo.json", follow_redirects=True) - return response.json() + headers = {"User-Agent": UserAgents()} + response = await session.request( + "GET", "https://index.commoncrawl.org/collinfo.json", follow_redirects=True + ) + return response.json() except httpx.RemoteProtocolError: pass except httpx.ReadTimeout: pass except httpx.TimeoutException as e: - logger(f"Timeout Reached for Commoncrawl API IndexDB due to: {e}", "warn", args.no_color) + logger( + f"Timeout Reached for Commoncrawl API IndexDB due to: {e}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception occurred in the Commoncrawl API IndexDB module due to: {e}", "warn", args.no_color) - -async def CcClient(args,searchurl: str,domain): + logger( + f"Exception occurred in the Commoncrawl API IndexDB module due to: {e}", + "warn", + args.no_color, + ) + + +async def CcClient(session: httpx.AsyncClient, args, searchurl: str, domain): try: - headers = { - "User-Agent": UserAgents() - } - async with httpx.AsyncClient(timeout=httpx.Timeout(read=300.0, connect=args.timeout, write=None, pool=None), headers=headers, verify=False, proxy=args.proxy) as request: - async with request.stream("GET", f"{searchurl}?url=*.{domain}", follow_redirects=True) as response: - async for url in response.aiter_lines(): - subdomains = await extracts(url, domain) - if subdomains: - for subdomain in subdomains: - subdomain = subdomain.lstrip("25").lstrip("2F").lstrip("40").lstrip(".") - if subdomain not in Commoncrawls and not subdomain.startswith("%3D") and not subdomain.startswith("3D"): - Commoncrawls.add(subdomain) + headers = {"User-Agent": UserAgents()} + timeout = httpx.Timeout(read=300.0, connect=args.timeout, write=None, pool=None) + async with session.stream( + "GET", + f"{searchurl}?url=*.{domain}", + timeout=timeout, + headers=headers, + follow_redirects=True, + ) as response: + async for url in response.aiter_lines(): + subdomains = await extracts(url, domain) + if subdomains: + for subdomain in subdomains: + subdomain = ( + subdomain.lstrip("25").lstrip("2F").lstrip("40").lstrip(".") + ) + if ( + subdomain not in Commoncrawls + and not subdomain.startswith("%3D") + and not subdomain.startswith("3D") + ): + Commoncrawls.add(subdomain) except httpx.RemoteProtocolError: - pass + pass except httpx.ReadTimeout: - pass + pass except httpx.TimeoutException as e: - logger(f"Timeout Reached for Commoncrawl API Client due to: {e}", "warn", args.no_color) + logger( + f"Timeout Reached for Commoncrawl API Client due to: {e}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception occurred in the Commoncrawl API Client module due to: {e}", "warn", args.no_color) + logger( + f"Exception occurred in the Commoncrawl API Client module due to: {e}", + "warn", + args.no_color, + ) + async def commoncrawl(Domain, args): try: - - if not (args.all or (args.include_resources and "commoncrawl" in args.include_resources)): + if not ( + args.all + or (args.include_resources and "commoncrawl" in args.include_resources) + ): return Commoncrawls - - indexurls = [] - added = set() - responsed = await indexDB(args) - if responsed is None: - return Commoncrawls - ctyear = datetime.now().year - years = [str(ctyear - i) for i in range(6)] - for year in years: - for index in responsed: - if year not in added: - if year in index.get("name"): - indexurls.append(index.get('cdx-api')) - added.add(year) - for url in indexurls: - await CcClient(args, url, Domain) + + async with httpx.AsyncClient( + timeout=args.timeout, verify=False, proxy=args.proxy + ) as session: + indexurls = [] + added = set() + responsed = await indexDB(session, args) + if responsed is None: + return Commoncrawls + ctyear = datetime.now().year + years = [str(ctyear - i) for i in range(6)] + for year in years: + for index in responsed: + if year not in added: + if year in index.get("name"): + indexurls.append(index.get("cdx-api")) + added.add(year) + for url in indexurls: + await CcClient(session, args, url, Domain) except Exception as e: if args.verbose: - logger(f"Exception occurred at the Commoncrawl API core module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred at the Commoncrawl API core module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: logger(f"Total Subdomains found by Commoncrawl API: {len(Commoncrawls)}") - return Commoncrawls \ No newline at end of file + return Commoncrawls diff --git a/subdominator/modules/subscraper/crtsh/crtsh.py b/subdominator/modules/subscraper/crtsh/crtsh.py index a056d9f..cdc921a 100644 --- a/subdominator/modules/subscraper/crtsh/crtsh.py +++ b/subdominator/modules/subscraper/crtsh/crtsh.py @@ -1,27 +1,39 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents + crtshs = [] + async def crtsh(domain: str, session: httpx.AsyncClient, args): try: - if args.include_resources and "crtsh" not in args.include_resources and not args.all: + if ( + args.include_resources + and "crtsh" not in args.include_resources + and not args.all + ): return crtshs - + if args.exclude_resources and "crtsh" in args.exclude_resources: return crtshs url = f"https://crt.sh/?q=%25.{domain}&output=json" - headers = { - "User-Agent": UserAgents() - } + headers = {"User-Agent": UserAgents()} - timeout = httpx.Timeout(timeout=args.timeout, connect=args.timeout, read=300, write=args.timeout) - response: httpx.Response = await session.request("GET", url, headers=headers, timeout=timeout) + timeout = httpx.Timeout( + timeout=args.timeout, connect=args.timeout, read=300, write=args.timeout + ) + response: httpx.Response = await session.request( + "GET", url, headers=headers, timeout=timeout + ) if response.status_code != 200: if args.verbose: - logger(f"crt.sh API returned bad response status: {response.status_code}.", "warn", args.no_color) + logger( + f"crt.sh API returned bad response status: {response.status_code}.", + "warn", + args.no_color, + ) return crtshs data = response.json() @@ -30,11 +42,21 @@ async def crtsh(domain: str, session: httpx.AsyncClient, args): crtshs.append(subdomain) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for crt.sh API due to : {e}", "warn", args.no_color) + logger( + f"Timeout reached for crt.sh API due to : {e}", "warn", args.no_color + ) except Exception as e: if args.verbose: - logger(f"Exception occurred in crt.sh API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in crt.sh API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by crt.sh API: {len(crtshs)}", "info", args.no_color) - return crtshs \ No newline at end of file + logger( + f"Total Subdomains found by crt.sh API: {len(crtshs)}", + "info", + args.no_color, + ) + return crtshs diff --git a/subdominator/modules/subscraper/cyfare/cyfare.py b/subdominator/modules/subscraper/cyfare/cyfare.py index d904c8e..eff3798 100644 --- a/subdominator/modules/subscraper/cyfare/cyfare.py +++ b/subdominator/modules/subscraper/cyfare/cyfare.py @@ -4,26 +4,37 @@ cyfares = [] + async def cyfare(domain: str, session: httpx.AsyncClient, args): try: - if args.include_resources and "cyfare" not in args.include_resources and not args.all: + if ( + args.include_resources + and "cyfare" not in args.include_resources + and not args.all + ): return cyfares - + if args.exclude_resources and "cyfare" in args.exclude_resources: return cyfares url = "https://cyfare.net/apps/VulnerabilityStudio/subfind/query.php" headers = { - "User-Agent": UserAgents(), + "User-Agent": UserAgents(), "Origin": "https://cyfare.net", - "Content-Type": "application/json" + "Content-Type": "application/json", } json_body = {"domain": domain} - response: httpx.Response = await session.request("POST", url, headers=headers, json=json_body, timeout=args.timeout) + response: httpx.Response = await session.request( + "POST", url, headers=headers, json=json_body, timeout=args.timeout + ) if response.status_code != 200: if args.verbose: - logger(f"Cyfare API returned bad response status: {response.status_code}.", "warn", args.no_color) + logger( + f"Cyfare API returned bad response status: {response.status_code}.", + "warn", + args.no_color, + ) return cyfares data = response.json() @@ -35,9 +46,17 @@ async def cyfare(domain: str, session: httpx.AsyncClient, args): except Exception as e: if args.verbose: - logger(f"Exception occurred in Cyfare API module: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in Cyfare API module: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Cyfare API: {len(cyfares)}", "info", args.no_color) - return cyfares + logger( + f"Total Subdomains found by Cyfare API: {len(cyfares)}", + "info", + args.no_color, + ) + return cyfares diff --git a/subdominator/modules/subscraper/digitalyama/digitalyama.py b/subdominator/modules/subscraper/digitalyama/digitalyama.py index 9102ae3..69c2ac2 100644 --- a/subdominator/modules/subscraper/digitalyama/digitalyama.py +++ b/subdominator/modules/subscraper/digitalyama/digitalyama.py @@ -1,31 +1,41 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents, singlekeyloader + digitalyamas = [] -async def digitalyama(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): - try: - if args.include_resources and "digitalyama" not in args.include_resources and not args.all: +async def digitalyama( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): + try: + if ( + args.include_resources + and "digitalyama" not in args.include_resources + and not args.all + ): return digitalyamas - + if args.exclude_resources and "digitalyama" in args.exclude_resources: return digitalyamas - + randomkey = await singlekeyloader(configs, "digitalyama") if randomkey is None: return digitalyamas - + url = "https://api.digitalyama.com/subdomain_finder" params = {"domain": domain} - headers = { - "User-Agent": UserAgents(), - "x-api-key": randomkey - } - response: httpx.Response = await session.request("GET", url, headers=headers, params=params, timeout=args.timeout) + headers = {"User-Agent": UserAgents(), "x-api-key": randomkey} + response: httpx.Response = await session.request( + "GET", url, headers=headers, params=params, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"DigitalYama blocking our request, {username} please check your API usage for this key: {randomkey}", "warn", args.no_color) + logger( + f"DigitalYama blocking our request, {username} please check your API usage for this key: {randomkey}", + "warn", + args.no_color, + ) return [] data = response.json() @@ -34,11 +44,23 @@ async def digitalyama(domain: str, session: httpx.AsyncClient, configs: str, use digitalyamas.append(subdomain) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for DigitalYama API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for DigitalYama API, due to: {e}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in DigitalYama API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in DigitalYama API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by DigitalYama API: {len(digitalyamas)}", "info", args.no_color) - return digitalyamas \ No newline at end of file + logger( + f"Total Subdomains found by DigitalYama API: {len(digitalyamas)}", + "info", + args.no_color, + ) + return digitalyamas diff --git a/subdominator/modules/subscraper/digitorus/digitorus.py b/subdominator/modules/subscraper/digitorus/digitorus.py index 70750f6..f7f1f0b 100644 --- a/subdominator/modules/subscraper/digitorus/digitorus.py +++ b/subdominator/modules/subscraper/digitorus/digitorus.py @@ -2,26 +2,38 @@ import re from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents + digitorus_subs = [] + async def digitorus(domain: str, session: httpx.AsyncClient, args): try: - if args.include_resources and "digitorus" not in args.include_resources and not args.all: + if ( + args.include_resources + and "digitorus" not in args.include_resources + and not args.all + ): return digitorus_subs - + if args.exclude_resources and "digitorus" in args.exclude_resources: return digitorus_subs url = f"https://certificatedetails.com/{domain}" - headers = {"User-Agent": UserAgents()} - response: httpx.Response = await session.get(url, headers=headers, timeout=args.timeout) + headers = {"User-Agent": UserAgents()} + response: httpx.Response = await session.get( + url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: - logger(f"Digitorus API returned bad response status: {response.status_code}.", "warn", args.no_color) + logger( + f"Digitorus API returned bad response status: {response.status_code}.", + "warn", + args.no_color, + ) return digitorus_subs data = response.text filterdomain = re.escape(domain) - pattern = r'(?i)(?:https?://)?([a-zA-Z0-9*_.-]+\.' + filterdomain + r')' + pattern = r"(?i)(?:https?://)?([a-zA-Z0-9*_.-]+\." + filterdomain + r")" subdomains = re.findall(pattern, data) if subdomains: digitorus_subs.extend(subdomains) @@ -30,9 +42,17 @@ async def digitorus(domain: str, session: httpx.AsyncClient, args): logger(f"Timeout reached for Digitorus API due to: {e}", "warn", args.no_color) except Exception as e: - logger(f"Exception occurred in Digitorus API module: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in Digitorus API module: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Digitorus API: {len(digitorus_subs)}", "info", args.no_color) - return digitorus_subs \ No newline at end of file + logger( + f"Total Subdomains found by Digitorus API: {len(digitorus_subs)}", + "info", + args.no_color, + ) + return digitorus_subs diff --git a/subdominator/modules/subscraper/dnsdumpster/dnsdumpster.py b/subdominator/modules/subscraper/dnsdumpster/dnsdumpster.py index 86059c7..1024cdf 100644 --- a/subdominator/modules/subscraper/dnsdumpster/dnsdumpster.py +++ b/subdominator/modules/subscraper/dnsdumpster/dnsdumpster.py @@ -1,11 +1,17 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import singlekeyloader + dnsdumpsters = [] -async def dnsdumpster(domain: str, session: httpx.AsyncClient, configs: str, args): + +async def dnsdumpster(domain: str, session: httpx.AsyncClient, configs: str, args): try: - if args.include_resources and "dnsdumpster" not in args.include_resources and not args.all: + if ( + args.include_resources + and "dnsdumpster" not in args.include_resources + and not args.all + ): return dnsdumpsters if args.exclude_resources and "dnsdumpster" in args.exclude_resources: @@ -21,7 +27,9 @@ async def dnsdumpster(domain: str, session: httpx.AsyncClient, configs: str, arg break params = {"page": page} headers = {"X-API-Key": randomkey} - response: httpx.Response = await session.request("GET", url, headers=headers, params=params, timeout=args.timeout) + response: httpx.Response = await session.request( + "GET", url, headers=headers, params=params, timeout=args.timeout + ) if response.status_code != 200: return dnsdumpsters data = response.json() @@ -37,11 +45,23 @@ async def dnsdumpster(domain: str, session: httpx.AsyncClient, configs: str, arg page += 1 except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Dnsdumpster API due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for Dnsdumpster API due to: {e}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception occurred in Dnsdumpster API module: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in Dnsdumpster API module: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Dnsdumpster API: {len(dnsdumpsters)}", "info", args.no_color) - return dnsdumpsters + logger( + f"Total Subdomains found by Dnsdumpster API: {len(dnsdumpsters)}", + "info", + args.no_color, + ) + return dnsdumpsters diff --git a/subdominator/modules/subscraper/dnsrepo/dnsrepo.py b/subdominator/modules/subscraper/dnsrepo/dnsrepo.py index d1b294a..a994816 100644 --- a/subdominator/modules/subscraper/dnsrepo/dnsrepo.py +++ b/subdominator/modules/subscraper/dnsrepo/dnsrepo.py @@ -1,48 +1,70 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents, dualkeyloader + dnsrepo_results = [] -async def dnsrepo(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def dnsrepo( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "dnsrepo" not in args.include_resources and not args.all: + if ( + args.include_resources + and "dnsrepo" not in args.include_resources + and not args.all + ): return dnsrepo_results - + if args.exclude_resources and "dnsrepo" in args.exclude_resources: return dnsrepo_results - + token, randomkey = await dualkeyloader(configs, "dnsrepo", False) - + if token is None or randomkey is None: return dnsrepo_results url = f"https://dnsarchive.net/api/?apikey={randomkey}&search={domain}" - headers = { - 'User-Agent': UserAgents(), - 'X-API-Access': token - } - response: httpx.Response = await session.request("GET", url, headers=headers, timeout=args.timeout) - + headers = {"User-Agent": UserAgents(), "X-API-Access": token} + response: httpx.Response = await session.request( + "GET", url, headers=headers, timeout=args.timeout + ) + if response.status_code != 200: if args.show_key_info: - logger(f"DNSRepo API blocking our request, {username}, please check your API usage for this key: {randomkey}", "warn", args.no_color) + logger( + f"DNSRepo API blocking our request, {username}, please check your API usage for this key: {randomkey}", + "warn", + args.no_color, + ) return [] data = response.json() if not isinstance(data, list): return dnsrepo_results subdomains = [ - entry["domain"].rstrip(".") for entry in data + entry["domain"].rstrip(".") + for entry in data if "domain" in entry and entry["domain"].rstrip(".").endswith(f".{domain}") ] dnsrepo_results.extend(subdomains) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for DNSRepo API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for DNSRepo API, due to: {e}", "warn", args.no_color + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in DNSRepo API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in DNSRepo API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by DNSRepo API: {len(dnsrepo_results)}", "info", args.no_color) - return dnsrepo_results \ No newline at end of file + logger( + f"Total Subdomains found by DNSRepo API: {len(dnsrepo_results)}", + "info", + args.no_color, + ) + return dnsrepo_results diff --git a/subdominator/modules/subscraper/facebook/facebook.py b/subdominator/modules/subscraper/facebook/facebook.py index a6487e8..694c6e2 100644 --- a/subdominator/modules/subscraper/facebook/facebook.py +++ b/subdominator/modules/subscraper/facebook/facebook.py @@ -1,44 +1,60 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import dualkeyloader + fbcerts = [] -async def facebook(domain: str, session: httpx.AsyncClient, configs: str, username: str, args) -> list[str]: + +async def facebook( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +) -> list[str]: try: - if args.include_resources and "facebook" not in args.include_resources and not args.all: + if ( + args.include_resources + and "facebook" not in args.include_resources + and not args.all + ): return fbcerts - + if args.exclude_resources and "facebook" in args.exclude_resources: return fbcerts - - randomid , randomsecret = await dualkeyloader(configs,"facebook",False) + + randomid, randomsecret = await dualkeyloader(configs, "facebook", False) if randomid is None or randomsecret is None: return fbcerts randomtoken = f"{randomid}|{randomsecret}" url = f"https://graph.facebook.com/v18.0/certificates?fields=domains&access_token={randomtoken}&query={domain}&limit=1000" while True: - response: httpx.Response = await session.request("GET",url, timeout=args.timeout) + response: httpx.Response = await session.request( + "GET", url, timeout=args.timeout + ) if response.status_code != 200: return fbcerts data = response.json() - for item in data['data']: - subdomains = item['domains'] + for item in data["data"]: + subdomains = item["domains"] for subdomain in subdomains: if subdomain.endswith(f"{domain}"): fbcerts.append(subdomain) pages = data.get("paging", {}) - next_page = pages.get('next') + next_page = pages.get("next") if next_page: url = next_page if not next_page: break except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Facebook API due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for Facebook API due to: {e}", "warn", args.no_color + ) except Exception as e: if args.verbose: - logger(f"Exception occurred in the Facebook API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in the Facebook API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: logger(f"Total Subdomains found by Facebook API: {len(fbcerts)}") - return fbcerts \ No newline at end of file + return fbcerts diff --git a/subdominator/modules/subscraper/fofa/fofa.py b/subdominator/modules/subscraper/fofa/fofa.py index c1210aa..2b15eff 100644 --- a/subdominator/modules/subscraper/fofa/fofa.py +++ b/subdominator/modules/subscraper/fofa/fofa.py @@ -2,40 +2,49 @@ import base64 from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import singlekeyloader + FOFA = [] -async def fofa(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): +async def fofa( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "fofa" not in args.include_resources and not args.all: + if ( + args.include_resources + and "fofa" not in args.include_resources + and not args.all + ): return FOFA - + if args.exclude_resources and "fofa" in args.exclude_resources: return FOFA - + randomkey = await singlekeyloader(configs, "fofa") if randomkey is None: return FOFA pagenum = 1 - domain_encoded = f"""domain="{domain}" """.encode('utf-8') - subdomains = base64.b64encode(domain_encoded).decode('utf-8') + domain_encoded = f"""domain="{domain}" """.encode("utf-8") + subdomains = base64.b64encode(domain_encoded).decode("utf-8") while True: url = f"https://fofa.info/api/v1/search/all?key={randomkey}&qbase64={subdomains}&page={pagenum}&full=true&size=1000" - response: httpx.Response = await session.request("GET", url, timeout=args.timeout) + response: httpx.Response = await session.request( + "GET", url, timeout=args.timeout + ) if response.status_code != 200: return FOFA data = response.json() if "results" not in data: return FOFA - for result in data.get('results', []): + for result in data.get("results", []): url = result[0] if url.startswith("https://"): url = url.replace("https://", "") elif url.startswith("http://"): url = url.replace("http://", "") - subdomain = url.split(':')[0] if ':' in url else url + subdomain = url.split(":")[0] if ":" in url else url FOFA.append(subdomain) - size = data.get('size') + size = data.get("size") if size < 1000: return FOFA pagenum += 1 @@ -44,8 +53,16 @@ async def fofa(domain: str, session: httpx.AsyncClient, configs: str, username: logger(f"Timeout reached for FOFA API due to: {e}", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception occurred in the FOFA API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in the FOFA API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by FOFA API: {len(FOFA)}", "info", args.no_color) - return FOFA + logger( + f"Total Subdomains found by FOFA API: {len(FOFA)}", + "info", + args.no_color, + ) + return FOFA diff --git a/subdominator/modules/subscraper/fullhunt/fullhunt.py b/subdominator/modules/subscraper/fullhunt/fullhunt.py index 982320e..6b6beb8 100644 --- a/subdominator/modules/subscraper/fullhunt/fullhunt.py +++ b/subdominator/modules/subscraper/fullhunt/fullhunt.py @@ -1,11 +1,19 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents, singlekeyloader + fullhunts = [] -async def fullhunt(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def fullhunt( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "fullhunt" not in args.include_resources and not args.all: + if ( + args.include_resources + and "fullhunt" not in args.include_resources + and not args.all + ): return fullhunts if args.exclude_resources and "fullhunt" in args.exclude_resources: @@ -15,25 +23,38 @@ async def fullhunt(domain: str, session: httpx.AsyncClient, configs: str, userna if randomkey is None: return fullhunts url = f"https://fullhunt.io/api/v1/domain/{domain}/subdomains" - headers = { - "User-Agent": UserAgents(), - "X-API-KEY": randomkey - } - response: httpx.Response = await session.request("GET", url, headers=headers, timeout=args.timeout) + headers = {"User-Agent": UserAgents(), "X-API-KEY": randomkey} + response: httpx.Response = await session.request( + "GET", url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"FullHunt blocking our request, {username}, please check your API usage for this key: {randomkey}","warn",args.no_color) + logger( + f"FullHunt blocking our request, {username}, please check your API usage for this key: {randomkey}", + "warn", + args.no_color, + ) return fullhunts data = response.json() subdomains = data.get("hosts", []) fullhunts.extend(subdomains) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for FullHunt API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for FullHunt API, due to: {e}", "warn", args.no_color + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in FullHunt API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in FullHunt API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total subdomains found by FullHunt API: {len(fullhunts)}", "info", args.no_color) - return fullhunts + logger( + f"Total subdomains found by FullHunt API: {len(fullhunts)}", + "info", + args.no_color, + ) + return fullhunts diff --git a/subdominator/modules/subscraper/google/google.py b/subdominator/modules/subscraper/google/google.py index 52e66de..ba9bea2 100644 --- a/subdominator/modules/subscraper/google/google.py +++ b/subdominator/modules/subscraper/google/google.py @@ -1,11 +1,19 @@ import httpx from subdominator.modules.logger.logger import logger -from subdominator.modules.utils.utils import UserAgents,dualkeyloader +from subdominator.modules.utils.utils import UserAgents, dualkeyloader + googles = [] -async def google(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def google( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "google" not in args.include_resources and not args.all: + if ( + args.include_resources + and "google" not in args.include_resources + and not args.all + ): return googles if args.exclude_resources and "google" in args.exclude_resources: @@ -20,10 +28,10 @@ async def google(domain: str, session: httpx.AsyncClient, configs: str, username if randomcx is None or randomkey is None: return googles url = f"https://customsearch.googleapis.com/customsearch/v1?q={dork}&cx={randomcx}&num=10&start={page}&key={randomkey}&alt=json" - headers = { - "User-Agent": UserAgents() - } - response: httpx.Response = await session.request("GET", url, headers=headers, timeout=args.timeout) + headers = {"User-Agent": UserAgents()} + response: httpx.Response = await session.request( + "GET", url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: return googles data = response.json() @@ -37,11 +45,21 @@ async def google(domain: str, session: httpx.AsyncClient, configs: str, username page += 1 except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Google API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for Google API, due to: {e}", "warn", args.no_color + ) except Exception as e: if args.verbose: - logger(f"Exception occurred in Google API module: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in Google API module: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total subdomains found by Google API: {len(googles)}", "info", args.no_color) - return googles \ No newline at end of file + logger( + f"Total subdomains found by Google API: {len(googles)}", + "info", + args.no_color, + ) + return googles diff --git a/subdominator/modules/subscraper/hackertarget/hackertarget.py b/subdominator/modules/subscraper/hackertarget/hackertarget.py index afb5126..bccab69 100644 --- a/subdominator/modules/subscraper/hackertarget/hackertarget.py +++ b/subdominator/modules/subscraper/hackertarget/hackertarget.py @@ -1,18 +1,25 @@ import httpx from subdominator.modules.logger.logger import logger + hackertargets = [] async def hackertarget(domain: str, session: httpx.AsyncClient, args) -> list[str]: try: - if args.include_resources and "hackertarget" not in args.include_resources and not args.all: + if ( + args.include_resources + and "hackertarget" not in args.include_resources + and not args.all + ): return hackertargets if args.exclude_resources and "hackertarget" in args.exclude_resources: return hackertargets url = f"https://api.hackertarget.com/hostsearch/?q={domain}" - response: httpx.Response = await session.request("GET", url, timeout=args.timeout) + response: httpx.Response = await session.request( + "GET", url, timeout=args.timeout + ) if response.status_code != 200: return hackertargets data = response.text.splitlines() @@ -23,11 +30,23 @@ async def hackertarget(domain: str, session: httpx.AsyncClient, args) -> list[st hackertargets.append(subdomain) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Hackertarget API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for Hackertarget API, due to: {e}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception occurred in Hackertarget API module: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in Hackertarget API module: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total subdomains found by Hackertarget API: {len(hackertargets)}", "info", args.no_color) - return hackertargets \ No newline at end of file + logger( + f"Total subdomains found by Hackertarget API: {len(hackertargets)}", + "info", + args.no_color, + ) + return hackertargets diff --git a/subdominator/modules/subscraper/hudsonrock/hudsonrock.py b/subdominator/modules/subscraper/hudsonrock/hudsonrock.py index 903c71c..87a1588 100644 --- a/subdominator/modules/subscraper/hudsonrock/hudsonrock.py +++ b/subdominator/modules/subscraper/hudsonrock/hudsonrock.py @@ -2,11 +2,17 @@ from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents from urllib.parse import urlparse + hudsonrocks = [] + async def hudsonrock(domain: str, session: httpx.AsyncClient, args): try: - if args.include_resources and "hudsonrock" not in args.include_resources and not args.all: + if ( + args.include_resources + and "hudsonrock" not in args.include_resources + and not args.all + ): return hudsonrocks if args.exclude_resources and "hudsonrock" in args.exclude_resources: @@ -14,7 +20,9 @@ async def hudsonrock(domain: str, session: httpx.AsyncClient, args): headers = {"User-Agent": UserAgents()} url = f"https://cavalier.hudsonrock.com/api/json/v2/osint-tools/urls-by-domain?domain={domain}" - response: httpx.Response = await session.get(url, timeout=args.timeout, headers=headers) + response: httpx.Response = await session.get( + url, timeout=args.timeout, headers=headers + ) if response.status_code != 200: return hudsonrocks @@ -27,15 +35,31 @@ async def hudsonrock(domain: str, session: httpx.AsyncClient, args): parsed_url = urlparse(record.get("url", "")) subdomain = parsed_url.netloc - if subdomain.endswith(f".{domain}") and "•" not in subdomain and subdomain not in hudsonrocks: + if ( + subdomain.endswith(f".{domain}") + and "•" not in subdomain + and subdomain not in hudsonrocks + ): hudsonrocks.append(subdomain) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for HudsonRock API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for HudsonRock API, due to: {e}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in HudsonRock API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in HudsonRock API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by HudsonRock API: {len(hudsonrocks)}", "info", args.no_color) - return hudsonrocks \ No newline at end of file + logger( + f"Total Subdomains found by HudsonRock API: {len(hudsonrocks)}", + "info", + args.no_color, + ) + return hudsonrocks diff --git a/subdominator/modules/subscraper/huntermap/huntermap.py b/subdominator/modules/subscraper/huntermap/huntermap.py index c3c70f0..445c1bb 100644 --- a/subdominator/modules/subscraper/huntermap/huntermap.py +++ b/subdominator/modules/subscraper/huntermap/huntermap.py @@ -4,11 +4,19 @@ from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import singlekeyloader import time + hunterhows = [] -async def huntermap(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def huntermap( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "huntermap" not in args.include_resources and not args.all: + if ( + args.include_resources + and "huntermap" not in args.include_resources + and not args.all + ): return hunterhows if args.exclude_resources and "huntermap" in args.exclude_resources: @@ -20,17 +28,23 @@ async def huntermap(domain: str, session: httpx.AsyncClient, configs: str, usern query = base64.urlsafe_b64encode(domain.encode("utf-8")).decode("ascii") page_size = 100 page = 1 - + while True: randomapikey = await singlekeyloader(configs, "huntermap") if not randomapikey: return hunterhows time.sleep(2.5) url = f"https://api.hunter.how/search?api-key={randomapikey}&query={query}&start_time={inititatetime}&end_time={endtime}&page={page}&page_size={page_size}" - response: httpx.Response = await session.request("GET", url, timeout=args.timeout) + response: httpx.Response = await session.request( + "GET", url, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"Huntermap blocking our request, {username}, check your API usage for this key: {randomapikey}", "warn", args.no_color) + logger( + f"Huntermap blocking our request, {username}, check your API usage for this key: {randomapikey}", + "warn", + args.no_color, + ) return hunterhows data = response.json() subdomains = data.get("data", {}).get("list", []) @@ -48,8 +62,16 @@ async def huntermap(domain: str, session: httpx.AsyncClient, configs: str, usern logger(f"Timeout reached for Huntermap API: {e}", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception occurred in Huntermap API module: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in Huntermap API module: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total subdomains found by Huntermap API: {len(hunterhows)}", "info", args.no_color) - return hunterhows \ No newline at end of file + logger( + f"Total subdomains found by Huntermap API: {len(hunterhows)}", + "info", + args.no_color, + ) + return hunterhows diff --git a/subdominator/modules/subscraper/intelx/intelx.py b/subdominator/modules/subscraper/intelx/intelx.py index 1ff1d7f..4251200 100644 --- a/subdominator/modules/subscraper/intelx/intelx.py +++ b/subdominator/modules/subscraper/intelx/intelx.py @@ -1,8 +1,10 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import dualkeyloader, UserAgents + intelxs = [] + async def getID(domain: str, session: httpx.AsyncClient, host: str, key: str, args): try: baseurl = f"https://{host}/phonebook/search?k={key}" @@ -15,19 +17,32 @@ async def getID(domain: str, session: httpx.AsyncClient, host: str, key: str, ar "Terminate": None, "Timeout": 20, } - response: httpx.Response = await session.post(baseurl, headers=auth, timeout=10, json=reqbody) + response: httpx.Response = await session.post( + baseurl, headers=auth, timeout=10, json=reqbody + ) if response.status_code != 200: return None data = response.json() return data.get("id") except Exception as e: if args.sec_deb: - logger(f"Exception in IntelX getID block: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in IntelX getID block: {e}, {type(e)}", + "warn", + args.no_color, + ) return None -async def intelx(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def intelx( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "intelx" not in args.include_resources and not args.all: + if ( + args.include_resources + and "intelx" not in args.include_resources + and not args.all + ): return intelxs if args.exclude_resources and "intelx" in args.exclude_resources: @@ -45,7 +60,9 @@ async def intelx(domain: str, session: httpx.AsyncClient, configs: str, username while True: baseurl = f"https://{randhost}/phonebook/search/result?k={randkey}&id={id}&limit=10000" headers = {"User-Agent": UserAgents()} - response: httpx.Response = await session.get(baseurl, headers=headers, timeout=args.timeout) + response: httpx.Response = await session.get( + baseurl, headers=headers, timeout=args.timeout + ) if response.status_code != 200: return intelxs data = response.json() @@ -60,8 +77,14 @@ async def intelx(domain: str, session: httpx.AsyncClient, configs: str, username logger(f"Timeout reached for IntelX API: {e}", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception in IntelX API module: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in IntelX API module: {e}, {type(e)}", "warn", args.no_color + ) finally: if args.verbose: - logger(f"Total subdomains found by IntelX API: {len(intelxs)}", "info", args.no_color) - return intelxs \ No newline at end of file + logger( + f"Total subdomains found by IntelX API: {len(intelxs)}", + "info", + args.no_color, + ) + return intelxs diff --git a/subdominator/modules/subscraper/leakix/leakix.py b/subdominator/modules/subscraper/leakix/leakix.py index 8d3ecb9..1c85af3 100644 --- a/subdominator/modules/subscraper/leakix/leakix.py +++ b/subdominator/modules/subscraper/leakix/leakix.py @@ -1,11 +1,19 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import singlekeyloader + leakixs = [] -async def leakix(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def leakix( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "leakix" not in args.include_resources and not args.all: + if ( + args.include_resources + and "leakix" not in args.include_resources + and not args.all + ): return leakixs if args.exclude_resources and "leakix" in args.exclude_resources: return leakixs @@ -16,11 +24,17 @@ async def leakix(domain: str, session: httpx.AsyncClient, configs: str, username url = f"https://leakix.net/api/subdomains/{domain}" headers = {"accept": "application/json", "api-key": randomkey} - response: httpx.Response = await session.request("GET",url, headers=headers, timeout=args.timeout) + response: httpx.Response = await session.request( + "GET", url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"LeakIX blocking request, {username} check API usage for key: {randomkey}", "alert", args.no_color) + logger( + f"LeakIX blocking request, {username} check API usage for key: {randomkey}", + "alert", + args.no_color, + ) return leakixs data = response.json() for item in data: @@ -32,8 +46,14 @@ async def leakix(domain: str, session: httpx.AsyncClient, configs: str, username logger("Timeout reached for LeakIX API", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception in LeakIX API module: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in LeakIX API module: {e}, {type(e)}", "warn", args.no_color + ) finally: if args.verbose: - logger(f"Total subdomains found by LeakIX API: {len(leakixs)}", "info", args.no_color) - return leakixs \ No newline at end of file + logger( + f"Total subdomains found by LeakIX API: {len(leakixs)}", + "info", + args.no_color, + ) + return leakixs diff --git a/subdominator/modules/subscraper/merklemap/merklemap.py b/subdominator/modules/subscraper/merklemap/merklemap.py index 7c2eadf..7afdb6c 100644 --- a/subdominator/modules/subscraper/merklemap/merklemap.py +++ b/subdominator/modules/subscraper/merklemap/merklemap.py @@ -1,16 +1,24 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import singlekeyloader + merklemaps = [] -async def merklemap(domain: str, session: httpx.AsyncClient, configs: str, username, args): + +async def merklemap( + domain: str, session: httpx.AsyncClient, configs: str, username, args +): try: - if args.include_resources and "merklemap" not in args.include_resources and not args.all: + if ( + args.include_resources + and "merklemap" not in args.include_resources + and not args.all + ): return merklemaps if args.exclude_resources and "merklemap" in args.exclude_resources: return merklemaps - randomkey = await singlekeyloader(configs,"merklemap") + randomkey = await singlekeyloader(configs, "merklemap") if randomkey is None: return merklemaps @@ -18,15 +26,23 @@ async def merklemap(domain: str, session: httpx.AsyncClient, configs: str, usern headers = {"Authorization": f"Bearer {randomkey}"} params = {"query": f"*.{domain}", "page": 0, "type": "wildcard"} while True: - response: httpx.Response = await session.request("GET",url, headers=headers, params=params, timeout=httpx.Timeout(connect=args.timeout, read=1000.0, write=None, pool=None)) + response: httpx.Response = await session.request( + "GET", + url, + headers=headers, + params=params, + timeout=httpx.Timeout( + connect=args.timeout, read=1000.0, write=None, pool=None + ), + ) if response.status_code != 200: return merklemaps - + data = response.json() results = data.get("results", []) if not results: - break + break for result in results: hostname = result.get("hostname", "") common_name = result.get("subject_common_name", "") @@ -34,14 +50,22 @@ async def merklemap(domain: str, session: httpx.AsyncClient, configs: str, usern merklemaps.append(hostname) if common_name and common_name.endswith(f".{domain}"): merklemaps.append(common_name) - params["page"] += 1 + params["page"] += 1 except httpx.TimeoutException: if args.show_timeout_info: logger("Timeout reached for Merklemap API", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception in Merklemap API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in Merklemap API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total subdomains found by Merklemap API: {len(merklemaps)}", "info", args.no_color) - return merklemaps \ No newline at end of file + logger( + f"Total subdomains found by Merklemap API: {len(merklemaps)}", + "info", + args.no_color, + ) + return merklemaps diff --git a/subdominator/modules/subscraper/myssl/myssl.py b/subdominator/modules/subscraper/myssl/myssl.py index 76500e6..890efb6 100644 --- a/subdominator/modules/subscraper/myssl/myssl.py +++ b/subdominator/modules/subscraper/myssl/myssl.py @@ -1,18 +1,26 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents + myssls = [] + async def myssl(domain: str, session: httpx.AsyncClient, args): try: - if args.include_resources and "myssl" not in args.include_resources and not args.all: + if ( + args.include_resources + and "myssl" not in args.include_resources + and not args.all + ): return myssls if args.exclude_resources and "myssl" in args.exclude_resources: return myssls url = f"https://myssl.com/api/v1/discover_sub_domain?domain={domain}" headers = {"User-Agent": UserAgents()} - response: httpx.Response = await session.get(url,headers=headers,timeout=args.timeout) + response: httpx.Response = await session.get( + url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: return myssls @@ -27,8 +35,16 @@ async def myssl(domain: str, session: httpx.AsyncClient, args): logger("Timeout reached for MySSL API", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception occurred in MySSL API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in MySSL API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by MySSL API: {len(myssls)}", "info", args.no_color) - return myssls \ No newline at end of file + logger( + f"Total Subdomains found by MySSL API: {len(myssls)}", + "info", + args.no_color, + ) + return myssls diff --git a/subdominator/modules/subscraper/netlas/netlas.py b/subdominator/modules/subscraper/netlas/netlas.py index e17db72..9ca925a 100644 --- a/subdominator/modules/subscraper/netlas/netlas.py +++ b/subdominator/modules/subscraper/netlas/netlas.py @@ -1,13 +1,21 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import singlekeyloader + netlass = [] -async def netlas(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def netlas( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: start = 0 page_size = 20 - if args.include_resources and "netlas" not in args.include_resources and not args.all: + if ( + args.include_resources + and "netlas" not in args.include_resources + and not args.all + ): return netlass if args.exclude_resources and "netlas" in args.exclude_resources: return netlass @@ -23,26 +31,42 @@ async def netlas(domain: str, session: httpx.AsyncClient, configs: str, username "GET", req_url, headers=headers, - timeout=httpx.Timeout(timeout=args.timeout, connect=args.timeout, pool=None, write=None, read=120), + timeout=httpx.Timeout( + timeout=args.timeout, + connect=args.timeout, + pool=None, + write=None, + read=120, + ), ) if response.status_code != 200: if args.show_key_info: - logger(f"Netlas API request failed. {username}, check API key usage: {randomkey}", "warn", args.no_color) + logger( + f"Netlas API request failed. {username}, check API key usage: {randomkey}", + "warn", + args.no_color, + ) return netlass data = response.json() items = data.get("items", []) if not items: - break + break for item in items: netlass.append(item["data"]["domain"]) - start += page_size + start += page_size except httpx.TimeoutException as e: if args.show_timeout_info: logger(f"Timeout reached for Netlas API: {e}", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception in Netlas API module: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in Netlas API module: {e}, {type(e)}", "warn", args.no_color + ) finally: if args.verbose: - logger(f"Total Subdomains found by Netlas API: {len(netlass)}", "info", args.no_color) - return netlass \ No newline at end of file + logger( + f"Total Subdomains found by Netlas API: {len(netlass)}", + "info", + args.no_color, + ) + return netlass diff --git a/subdominator/modules/subscraper/odin/odin.py b/subdominator/modules/subscraper/odin/odin.py index f301c7e..12e7917 100644 --- a/subdominator/modules/subscraper/odin/odin.py +++ b/subdominator/modules/subscraper/odin/odin.py @@ -1,17 +1,24 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents, singlekeyloader + odin_results = [] -async def odin(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def odin( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - - if args.include_resources and "odin" not in args.include_resources and not args.all: + if ( + args.include_resources + and "odin" not in args.include_resources + and not args.all + ): return odin_results - + if args.exclude_resources and "odin" in args.exclude_resources: return odin_results - + randomkey = await singlekeyloader(configs, "odin") if randomkey is None: return odin_results @@ -21,40 +28,54 @@ async def odin(domain: str, session: httpx.AsyncClient, configs: str, username: "Content-Type": "application/json", "User-Agent": UserAgents(), } - limit = 1000 - start = None + limit = 1000 + start = None while True: payload = { "domain": domain, "limit": limit, - "start": start if start else [] + "start": start if start else [], } - response: httpx.Response = await session.request("POST", url, headers=headers, json=payload, timeout=args.timeout) + response: httpx.Response = await session.request( + "POST", url, headers=headers, json=payload, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"Odin API request failed with status {response.status_code}. Check API key or rate limits.", "warn", args.no_color) + logger( + f"Odin API request failed with status {response.status_code}. Check API key or rate limits.", + "warn", + args.no_color, + ) return odin_results data = response.json() - + if not data.get("success"): return odin_results - + subdomains = data.get("data", []) odin_results.extend(subdomains) pagination = data.get("pagination", {}) start = pagination.get("last") - if not start or len(subdomains) == 0: + if not start or len(subdomains) == 0: break except httpx.TimeoutException as e: if args.show_timeout_info: logger(f"Timeout reached for Odin API, due to: {e}", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception occurred in Odin API module: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in Odin API module: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Odin API: {len(odin_results)}", "info", args.no_color) - return odin_results \ No newline at end of file + logger( + f"Total Subdomains found by Odin API: {len(odin_results)}", + "info", + args.no_color, + ) + return odin_results diff --git a/subdominator/modules/subscraper/quake/quake.py b/subdominator/modules/subscraper/quake/quake.py index 5301e9c..d5e24f9 100644 --- a/subdominator/modules/subscraper/quake/quake.py +++ b/subdominator/modules/subscraper/quake/quake.py @@ -1,11 +1,19 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import singlekeyloader, UserAgents + quakes = [] -async def quake(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def quake( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "quake" not in args.include_resources and not args.all: + if ( + args.include_resources + and "quake" not in args.include_resources + and not args.all + ): return quakes if args.exclude_resources and "quake" in args.exclude_resources: return quakes @@ -21,7 +29,7 @@ async def quake(domain: str, session: httpx.AsyncClient, configs: str, username: "Accept-Language": "en", "Connection": "close", "Content-Type": "application/json", - "X-Quaketoken": randomkey + "X-Quaketoken": randomkey, } data = { "query": f"domain: {domain}", @@ -30,10 +38,16 @@ async def quake(domain: str, session: httpx.AsyncClient, configs: str, username: "start": 0, "size": 500, } - response: httpx.Response = await session.request("POST",url,headers=headers,json=data,timeout=args.timeout) + response: httpx.Response = await session.request( + "POST", url, headers=headers, json=data, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"Quake API blocking request. {username}, check API key usage: {randomkey}", "warn", args.no_color) + logger( + f"Quake API blocking request. {username}, check API key usage: {randomkey}", + "warn", + args.no_color, + ) return quakes result = response.json() for entry in result.get("data", []): @@ -45,8 +59,16 @@ async def quake(domain: str, session: httpx.AsyncClient, configs: str, username: logger("Timeout reached for Quake API", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception in Quake API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in Quake API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Quake API: {len(quakes)}", "info", args.no_color) - return quakes \ No newline at end of file + logger( + f"Total Subdomains found by Quake API: {len(quakes)}", + "info", + args.no_color, + ) + return quakes diff --git a/subdominator/modules/subscraper/racent/racent.py b/subdominator/modules/subscraper/racent/racent.py index c76ffff..f92f2d3 100644 --- a/subdominator/modules/subscraper/racent/racent.py +++ b/subdominator/modules/subscraper/racent/racent.py @@ -1,23 +1,31 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents + racents = [] + async def racent(domain: str, session: httpx.AsyncClient, args): try: - if args.include_resources and "racent" not in args.include_resources and not args.all: + if ( + args.include_resources + and "racent" not in args.include_resources + and not args.all + ): return racents if args.exclude_resources and "racent" in args.exclude_resources: return racents url = f"https://face.racent.com/tool/query_ctlog?keyword={domain}" headers = {"User-Agent": UserAgents()} - response: httpx.Response = await session.get(url,headers=headers,timeout=args.timeout) + response: httpx.Response = await session.get( + url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: return racents data = response.json() if "CTLog 查询超过限制" in data: return racents - for subdomains in data.get['data']['list']: + for subdomains in data.get["data"]["list"]: for subdomain in subdomains.get("dnsnames", []): racents.append(subdomain) except httpx.TimeoutException as e: @@ -25,8 +33,16 @@ async def racent(domain: str, session: httpx.AsyncClient, args): logger(f"Timeout reached for Racent API due to: {e}", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception at Racent API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception at Racent API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Racent API: {len(racents)}", "info", args.no_color) - return racents \ No newline at end of file + logger( + f"Total Subdomains found by Racent API: {len(racents)}", + "info", + args.no_color, + ) + return racents diff --git a/subdominator/modules/subscraper/rapidapi/rapidapi.py b/subdominator/modules/subscraper/rapidapi/rapidapi.py index 785d913..c67e7c7 100644 --- a/subdominator/modules/subscraper/rapidapi/rapidapi.py +++ b/subdominator/modules/subscraper/rapidapi/rapidapi.py @@ -1,17 +1,23 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import singlekeyloader + rapids = [] + async def rapidapi(domain: str, session: httpx.AsyncClient, configs: str, args): try: - if args.include_resources and "rapidapi" not in args.include_resources and not args.all: + if ( + args.include_resources + and "rapidapi" not in args.include_resources + and not args.all + ): return rapids if args.exclude_resources and "rapidapi" in args.exclude_resources: return rapids - rapidapi_key = await singlekeyloader(configs,"rapidapi") - whoisxml_key = await singlekeyloader(configs,"whoisxmlapi") + rapidapi_key = await singlekeyloader(configs, "rapidapi") + whoisxml_key = await singlekeyloader(configs, "whoisxmlapi") if not rapidapi_key or not whoisxml_key: return rapids @@ -20,14 +26,20 @@ async def rapidapi(domain: str, session: httpx.AsyncClient, configs: str, args): params = {"domainName": domain, "apiKey": whoisxml_key, "outputFormat": "JSON"} headers = { "X-RapidAPI-Key": rapidapi_key, - "X-RapidAPI-Host": "subdomains-lookup.p.rapidapi.com" + "X-RapidAPI-Host": "subdomains-lookup.p.rapidapi.com", } - response: httpx.Response = await session.request("GET",url,params=params,headers=headers,timeout=args.timeout) - + response: httpx.Response = await session.request( + "GET", url, params=params, headers=headers, timeout=args.timeout + ) + if response.status_code != 200: if args.show_key_info: - logger(f"RapidAPI blocking request, check API usage for these keys: {whoisxml_key}, {rapidapi_key}", "warn", args.no_color) + logger( + f"RapidAPI blocking request, check API usage for these keys: {whoisxml_key}, {rapidapi_key}", + "warn", + args.no_color, + ) return rapids data = response.json() @@ -42,8 +54,16 @@ async def rapidapi(domain: str, session: httpx.AsyncClient, configs: str, args): logger(f"Timeout reached for RapidAPI due to: {e}", "info", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception occurred in Rapid API module due to: {e}, type: {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in Rapid API module due to: {e}, type: {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Rapid API: {len(rapids)}", "info", args.no_color) - return rapids \ No newline at end of file + logger( + f"Total Subdomains found by Rapid API: {len(rapids)}", + "info", + args.no_color, + ) + return rapids diff --git a/subdominator/modules/subscraper/rapiddns/rapiddns.py b/subdominator/modules/subscraper/rapiddns/rapiddns.py index 5e9117f..1487cf9 100644 --- a/subdominator/modules/subscraper/rapiddns/rapiddns.py +++ b/subdominator/modules/subscraper/rapiddns/rapiddns.py @@ -2,21 +2,29 @@ import re from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents + rapiddnss = [] + async def rapiddns(domain: str, session: httpx.AsyncClient, args): try: - if args.include_resources and "rapiddns" not in args.include_resources and not args.all: + if ( + args.include_resources + and "rapiddns" not in args.include_resources + and not args.all + ): return rapiddnss if args.exclude_resources and "rapiddns" in args.exclude_resources: return rapiddnss for pagenum in range(1, 8): url = f"https://rapiddns.io/subdomain/{domain}?page={pagenum}" headers = {"User-Agent": UserAgents()} - response: httpx.Response = await session.request("GET",url,headers=headers,timeout=args.timeout) + response: httpx.Response = await session.request( + "GET", url, headers=headers, timeout=args.timeout + ) data = response.text filterdomain = re.escape(domain) - pattern = rf'(?i)(?:https?://)?([a-zA-Z0-9*_.-]+\.{filterdomain})' + pattern = rf"(?i)(?:https?://)?([a-zA-Z0-9*_.-]+\.{filterdomain})" subdomains = re.findall(pattern, data) rapiddnss.extend(subdomains) if "Next" not in data: @@ -26,8 +34,16 @@ async def rapiddns(domain: str, session: httpx.AsyncClient, args): logger(f"Timeout reached for RapidDNS due to: {e}", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception occurred in RapidDNS module due to: {e}, type: {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in RapidDNS module due to: {e}, type: {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by RapidDNS: {len(rapiddnss)}", "info", args.no_color) - return rapiddnss \ No newline at end of file + logger( + f"Total Subdomains found by RapidDNS: {len(rapiddnss)}", + "info", + args.no_color, + ) + return rapiddnss diff --git a/subdominator/modules/subscraper/rapidfinder/rapidfinder.py b/subdominator/modules/subscraper/rapidfinder/rapidfinder.py index 1ba46c1..c0f0893 100644 --- a/subdominator/modules/subscraper/rapidfinder/rapidfinder.py +++ b/subdominator/modules/subscraper/rapidfinder/rapidfinder.py @@ -4,9 +4,16 @@ rapidfinders = [] -async def rapidfinder(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def rapidfinder( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "rapidfinder" not in args.include_resources and not args.all: + if ( + args.include_resources + and "rapidfinder" not in args.include_resources + and not args.all + ): return rapidfinders if args.exclude_resources and "rapidfinder" in args.exclude_resources: @@ -21,18 +28,28 @@ async def rapidfinder(domain: str, session: httpx.AsyncClient, configs: str, use headers = { "User-Agent": UserAgents(), "X-RapidAPI-Key": randomkey, - "X-RapidAPI-Host": "subdomain-finder3.p.rapidapi.com" + "X-RapidAPI-Host": "subdomain-finder3.p.rapidapi.com", } - response: httpx.Response = await session.get(url, headers=headers, timeout=args.timeout, params=params) + response: httpx.Response = await session.get( + url, headers=headers, timeout=args.timeout, params=params + ) if response.status_code == 403: if args.show_key_info: - logger(f"Rapidfinder blocking our request, {username} please check that you subscribed to the Rapidfinder API service: {randomkey}", "warn", args.no_color) + logger( + f"Rapidfinder blocking our request, {username} please check that you subscribed to the Rapidfinder API service: {randomkey}", + "warn", + args.no_color, + ) return rapidfinders if response.status_code != 200: if args.show_key_info: - logger(f"Rapidfinder blocking our request, {username} please check your API usage for this key: {randomkey}", "warn", args.no_color) + logger( + f"Rapidfinder blocking our request, {username} please check your API usage for this key: {randomkey}", + "warn", + args.no_color, + ) return rapidfinders data = response.json() @@ -40,11 +57,23 @@ async def rapidfinder(domain: str, session: httpx.AsyncClient, configs: str, use rapidfinders.append(item["subdomain"]) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Rapidfinder API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for Rapidfinder API, due to: {e}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in Rapidfinder API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in Rapidfinder API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Rapidfinder API: {len(rapidfinders)}", "info", args.no_color) - return rapidfinders + logger( + f"Total Subdomains found by Rapidfinder API: {len(rapidfinders)}", + "info", + args.no_color, + ) + return rapidfinders diff --git a/subdominator/modules/subscraper/rapidscan/rapidscan.py b/subdominator/modules/subscraper/rapidscan/rapidscan.py index 2cd64ff..4853daa 100644 --- a/subdominator/modules/subscraper/rapidscan/rapidscan.py +++ b/subdominator/modules/subscraper/rapidscan/rapidscan.py @@ -1,11 +1,19 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents, singlekeyloader + rapidscans = [] -async def rapidscan(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def rapidscan( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "rapidscan" not in args.include_resources and not args.all: + if ( + args.include_resources + and "rapidscan" not in args.include_resources + and not args.all + ): return rapidscans if args.exclude_resources and "rapidscan" in args.exclude_resources: @@ -20,19 +28,29 @@ async def rapidscan(domain: str, session: httpx.AsyncClient, configs: str, usern headers = { "User-Agent": UserAgents(), "X-RapidAPI-Key": randomkey, - "X-RapidAPI-Host": "subdomain-scan1.p.rapidapi.com" + "X-RapidAPI-Host": "subdomain-scan1.p.rapidapi.com", } - response: httpx.Response = await session.get(url, headers=headers, timeout=args.timeout, params=params) - + response: httpx.Response = await session.get( + url, headers=headers, timeout=args.timeout, params=params + ) + if response.status_code == 403: if args.show_key_info: - logger(f"Rapidscan blocking our request, {username} please check that you subscribed to the Rapidscan API service: {randomkey}", "warn", args.no_color) + logger( + f"Rapidscan blocking our request, {username} please check that you subscribed to the Rapidscan API service: {randomkey}", + "warn", + args.no_color, + ) return rapidscans if response.status_code != 200: if args.show_key_info: - logger(f"Rapidscan blocking our request, {username} please check your API usage for this key: {randomkey}", "warn", args.no_color) + logger( + f"Rapidscan blocking our request, {username} please check your API usage for this key: {randomkey}", + "warn", + args.no_color, + ) return rapidscans data = response.json() @@ -40,11 +58,21 @@ async def rapidscan(domain: str, session: httpx.AsyncClient, configs: str, usern rapidscans.append(subdomain) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Rapidscan API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for Rapidscan API, due to: {e}", "warn", args.no_color + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in Rapidscan API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in Rapidscan API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Rapidscan API: {len(rapidscans)}", "info", args.no_color) - return rapidscans \ No newline at end of file + logger( + f"Total Subdomains found by Rapidscan API: {len(rapidscans)}", + "info", + args.no_color, + ) + return rapidscans diff --git a/subdominator/modules/subscraper/redhuntlabs/redhuntlabs.py b/subdominator/modules/subscraper/redhuntlabs/redhuntlabs.py index d4d7613..08d8ae5 100644 --- a/subdominator/modules/subscraper/redhuntlabs/redhuntlabs.py +++ b/subdominator/modules/subscraper/redhuntlabs/redhuntlabs.py @@ -1,13 +1,25 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents, dualkeyloader + redhuntlabs_subdomains = [] -async def redhuntapi(url: str, domain: str, session: httpx.AsyncClient, randkey: str, pagenum: int, pagesize: int, args): + +async def redhuntapi( + url: str, + domain: str, + session: httpx.AsyncClient, + randkey: str, + pagenum: int, + pagesize: int, + args, +): try: base_url = f"{url}?domain={domain}&page_size={pagesize}&page={pagenum}" headers = {"User-Agent": UserAgents(), "X-BLOBR-KEY": randkey} - response: httpx.Response = await session.get(base_url, headers=headers, timeout=args.timeout) + response: httpx.Response = await session.get( + base_url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: return [] data = response.json() @@ -17,33 +29,53 @@ async def redhuntapi(url: str, domain: str, session: httpx.AsyncClient, randkey: logger("Timeout reached for RedHuntLabs API", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception in RedHuntLabs API request: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in RedHuntLabs API request: {e}, {type(e)}", + "warn", + args.no_color, + ) return [] -async def redhuntlabs(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): +async def redhuntlabs( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "redhuntlabs" not in args.include_resources and not args.all: + if ( + args.include_resources + and "redhuntlabs" not in args.include_resources + and not args.all + ): return redhuntlabs_subdomains if args.exclude_resources and "redhuntlabs" in args.exclude_resources: return redhuntlabs_subdomains - url,randkeys = await dualkeyloader(configs, "redhuntlabs", True) + url, randkeys = await dualkeyloader(configs, "redhuntlabs", True) if url is None or randkeys is None: return redhuntlabs_subdomains pagenum = 1 pagesize = 1000 while True: - subdomains = await redhuntapi(url, domain, session, randkeys, pagenum, pagesize, args) + subdomains = await redhuntapi( + url, domain, session, randkeys, pagenum, pagesize, args + ) if not subdomains: break redhuntlabs_subdomains.extend(subdomains) pagenum += 1 except Exception as e: if args.verbose: - logger(f"Exception in RedHuntLabs API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in RedHuntLabs API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by RedHuntLabs API: {len(redhuntlabs_subdomains)}", "info", args.no_color) - return redhuntlabs_subdomains \ No newline at end of file + logger( + f"Total Subdomains found by RedHuntLabs API: {len(redhuntlabs_subdomains)}", + "info", + args.no_color, + ) + return redhuntlabs_subdomains diff --git a/subdominator/modules/subscraper/rsecloud/rsecloud.py b/subdominator/modules/subscraper/rsecloud/rsecloud.py index bf69c40..0560be7 100644 --- a/subdominator/modules/subscraper/rsecloud/rsecloud.py +++ b/subdominator/modules/subscraper/rsecloud/rsecloud.py @@ -1,33 +1,62 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents, singlekeyloader + rsecloud_subdomains = [] -async def rsecloudapi(url: str, domain: str, session: httpx.AsyncClient, randkey: str, page: int, args): + +async def rsecloudapi( + url: str, domain: str, session: httpx.AsyncClient, randkey: str, page: int, args +): try: json_payload = {"domain": domain} - headers = {"User-Agent": UserAgents(), "Content-Type": "application/json", "X-API-Key": randkey} + headers = { + "User-Agent": UserAgents(), + "Content-Type": "application/json", + "X-API-Key": randkey, + } url = f"{url}?page={page}" - response: httpx.Response = await session.post(url, headers=headers, json=json_payload, timeout=args.timeout) + response: httpx.Response = await session.post( + url, headers=headers, json=json_payload, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"RseCloud blocking request, check API usage for key: {randkey}", "warn", args.no_color) + logger( + f"RseCloud blocking request, check API usage for key: {randkey}", + "warn", + args.no_color, + ) return [] data = response.json() if "error" in data: - return [],1 + return [], 1 return data.get("data", []), data.get("total_pages", 1) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for RseCloud API request due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for RseCloud API request due to: {e}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception in RseCloud API request due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in RseCloud API request due to: {e}, {type(e)}", + "warn", + args.no_color, + ) return [], 1 -async def rsecloud(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def rsecloud( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "rsecloud" not in args.include_resources and not args.all: + if ( + args.include_resources + and "rsecloud" not in args.include_resources + and not args.all + ): return rsecloud_subdomains if args.exclude_resources and "rsecloud" in args.exclude_resources: @@ -40,7 +69,9 @@ async def rsecloud(domain: str, session: httpx.AsyncClient, configs: str, userna url = "https://api.rsecloud.com/api/v1/subdomains" page = 1 while True: - subdomains, total_pages = await rsecloudapi(url, domain, session, randomkey, page, args) + subdomains, total_pages = await rsecloudapi( + url, domain, session, randomkey, page, args + ) if not subdomains: break rsecloud_subdomains.extend(subdomains) @@ -49,8 +80,16 @@ async def rsecloud(domain: str, session: httpx.AsyncClient, configs: str, userna page += 1 except Exception as e: if args.verbose: - logger(f"Exception in RseCloud API module: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in RseCloud API module: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by RseCloud API: {len(rsecloud_subdomains)}", "info", args.no_color) - return rsecloud_subdomains \ No newline at end of file + logger( + f"Total Subdomains found by RseCloud API: {len(rsecloud_subdomains)}", + "info", + args.no_color, + ) + return rsecloud_subdomains diff --git a/subdominator/modules/subscraper/securitytrails/securitytrails.py b/subdominator/modules/subscraper/securitytrails/securitytrails.py index 9a4ab47..29b93df 100644 --- a/subdominator/modules/subscraper/securitytrails/securitytrails.py +++ b/subdominator/modules/subscraper/securitytrails/securitytrails.py @@ -1,29 +1,58 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents, singlekeyloader + Securitytrails = [] -async def securitytrailsapi(url: str, domain: str, session: httpx.AsyncClient, randkey: str, args): + +async def securitytrailsapi( + url: str, domain: str, session: httpx.AsyncClient, randkey: str, args +): try: - headers = {"User-Agent": UserAgents(), "accept": "application/json", "APIKEY": randkey} - response: httpx.Response = await session.get(url, headers=headers, timeout=args.timeout) + headers = { + "User-Agent": UserAgents(), + "accept": "application/json", + "APIKEY": randkey, + } + response: httpx.Response = await session.get( + url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"SecurityTrails blocking request, check API usage for key: {randkey}", "warn", args.no_color) + logger( + f"SecurityTrails blocking request, check API usage for key: {randkey}", + "warn", + args.no_color, + ) return [] data = response.json() return [f"{sub}.{domain}" for sub in data.get("subdomains", [])] except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for SecurityTrails API due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for SecurityTrails API due to: {e}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception in SecurityTrails API request module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in SecurityTrails API request module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) return [] -async def securitytrails(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def securitytrails( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "securitytrails" not in args.include_resources and not args.all: + if ( + args.include_resources + and "securitytrails" not in args.include_resources + and not args.all + ): return Securitytrails if args.exclude_resources and "securitytrails" in args.exclude_resources: @@ -32,14 +61,22 @@ async def securitytrails(domain: str, session: httpx.AsyncClient, configs: str, randomkey = await singlekeyloader(configs, "securitytrails") if randomkey is None: return Securitytrails - + url = f"https://api.securitytrails.com/v1/domain/{domain}/subdomains?children_only=false&include_inactive=true" subdomains = await securitytrailsapi(url, domain, session, randomkey, args) Securitytrails.extend(subdomains) except Exception as e: if args.verbose: - logger(f"Exception in SecurityTrails API module: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in SecurityTrails API module: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by SecurityTrails API: {len(Securitytrails)}", "info", args.no_color) - return Securitytrails \ No newline at end of file + logger( + f"Total Subdomains found by SecurityTrails API: {len(Securitytrails)}", + "info", + args.no_color, + ) + return Securitytrails diff --git a/subdominator/modules/subscraper/shodan/shodan.py b/subdominator/modules/subscraper/shodan/shodan.py index 94cccbd..6363ad9 100644 --- a/subdominator/modules/subscraper/shodan/shodan.py +++ b/subdominator/modules/subscraper/shodan/shodan.py @@ -1,15 +1,25 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents, singlekeyloader + Shodans = [] -async def shodanapi(url: str, domain: str, session: httpx.AsyncClient, randkey: str, args): + +async def shodanapi( + url: str, domain: str, session: httpx.AsyncClient, randkey: str, args +): try: headers = {"User-Agent": UserAgents()} - response: httpx.Response = await session.get(url, headers=headers, timeout=args.timeout) + response: httpx.Response = await session.get( + url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"Shodan blocking request, check API usage for key: {randkey}", "warn", args.no_color) + logger( + f"Shodan blocking request, check API usage for key: {randkey}", + "warn", + args.no_color, + ) return [] data = response.json() return [f"{sub}.{domain}" for sub in data.get("subdomains", [])] @@ -21,13 +31,23 @@ async def shodanapi(url: str, domain: str, session: httpx.AsyncClient, randkey: logger(f"Request error in Shodan API: {e}", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception in Shodan API request module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in Shodan API request module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) return [] -async def shodan(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): +async def shodan( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "shodan" not in args.include_resources and not args.all: + if ( + args.include_resources + and "shodan" not in args.include_resources + and not args.all + ): return Shodans if args.exclude_resources and "shodan" in args.exclude_resources: @@ -41,8 +61,14 @@ async def shodan(domain: str, session: httpx.AsyncClient, configs: str, username Shodans.extend(subdomains) except Exception as e: if args.verbose: - logger(f"Exception in Shodan API module: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in Shodan API module: {e}, {type(e)}", "warn", args.no_color + ) finally: if args.verbose: - logger(f"Total Subdomains found by Shodan API: {len(Shodans)}", "info", args.no_color) - return Shodans \ No newline at end of file + logger( + f"Total Subdomains found by Shodan API: {len(Shodans)}", + "info", + args.no_color, + ) + return Shodans diff --git a/subdominator/modules/subscraper/shodanx/shodanx.py b/subdominator/modules/subscraper/shodanx/shodanx.py index 214ef54..b98ed86 100644 --- a/subdominator/modules/subscraper/shodanx/shodanx.py +++ b/subdominator/modules/subscraper/shodanx/shodanx.py @@ -3,16 +3,22 @@ from bs4 import BeautifulSoup, XMLParsedAsHTMLWarning, MarkupResemblesLocatorWarning from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents, check_subdomain + Shodanxs = [] + async def shodanx(domain: str, session: httpx.AsyncClient, args): try: - if args.include_resources and "shodanx" not in args.include_resources and not args.all: + if ( + args.include_resources + and "shodanx" not in args.include_resources + and not args.all + ): return Shodanxs if args.exclude_resources and "shodanx" in args.exclude_resources: return Shodanxs - + parsed_domain = check_subdomain(domain) if parsed_domain.subdomain: return Shodanxs @@ -20,7 +26,7 @@ async def shodanx(domain: str, session: httpx.AsyncClient, args): url = f"https://www.shodan.io/domain/{domain}" headers = {"User-Agent": UserAgents()} - response = await session.get(url,headers=headers,timeout=args.timeout) + response = await session.get(url, headers=headers, timeout=args.timeout) if response.status_code != 200: return Shodanxs data = response.text @@ -45,8 +51,16 @@ async def shodanx(domain: str, session: httpx.AsyncClient, args): logger(f"Request error in ShodanX: {e}", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception in ShodanX request block: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in ShodanX request block: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by ShodanX: {len(Shodanxs)}", "info", args.no_color) - return Shodanxs + logger( + f"Total Subdomains found by ShodanX: {len(Shodanxs)}", + "info", + args.no_color, + ) + return Shodanxs diff --git a/subdominator/modules/subscraper/shrewdeye/shrewdeye.py b/subdominator/modules/subscraper/shrewdeye/shrewdeye.py index cac175d..2890532 100644 --- a/subdominator/modules/subscraper/shrewdeye/shrewdeye.py +++ b/subdominator/modules/subscraper/shrewdeye/shrewdeye.py @@ -1,19 +1,27 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents + Shrewdeyes = [] + async def shrewdeye(domain: str, session: httpx.AsyncClient, args): try: - if args.include_resources and "shrewdeye" not in args.include_resources and not args.all: + if ( + args.include_resources + and "shrewdeye" not in args.include_resources + and not args.all + ): return Shrewdeyes - + if args.exclude_resources and "shrewdeye" in args.exclude_resources: return Shrewdeyes - + url = f"https://shrewdeye.app/domains/{domain}.txt" - headers = {"User-Agent":UserAgents()} - response = await session.request("GET",url,timeout=args.timeout,headers=headers) + headers = {"User-Agent": UserAgents()} + response = await session.request( + "GET", url, timeout=args.timeout, headers=headers + ) if response.status_code != 200: return Shrewdeyes data = response.text.strip() @@ -23,11 +31,21 @@ async def shrewdeye(domain: str, session: httpx.AsyncClient, args): Shrewdeyes.extend(subdomains) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Shrewdeye API due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for Shrewdeye API due to: {e}", "warn", args.no_color + ) except Exception as e: if args.verbose: - logger(f"Exception occurred in Shrewdeye API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception occurred in Shrewdeye API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Shrewdeye: {len(Shrewdeyes)}", "info", args.no_color) - return Shrewdeyes \ No newline at end of file + logger( + f"Total Subdomains found by Shrewdeye: {len(Shrewdeyes)}", + "info", + args.no_color, + ) + return Shrewdeyes diff --git a/subdominator/modules/subscraper/shrewdeye/zoomeyeapi/zoomeyeapi.py b/subdominator/modules/subscraper/shrewdeye/zoomeyeapi/zoomeyeapi.py index e6437aa..b403ea8 100644 --- a/subdominator/modules/subscraper/shrewdeye/zoomeyeapi/zoomeyeapi.py +++ b/subdominator/modules/subscraper/shrewdeye/zoomeyeapi/zoomeyeapi.py @@ -1,17 +1,25 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents, dualkeyloader + zoomeyes = [] -async def zoomeyeapi(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def zoomeyeapi( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "zoomeye" not in args.include_resources and not args.all: + if ( + args.include_resources + and "zoomeye" not in args.include_resources + and not args.all + ): return zoomeyes - + if args.exclude_resources and "zoomeye" in args.exclude_resources: return zoomeyes - - host,randomkey = await dualkeyloader(configs, "zoomeyeapi") + + host, randomkey = await dualkeyloader(configs, "zoomeyeapi") if randomkey is None: return zoomeyes @@ -26,12 +34,18 @@ async def zoomeyeapi(domain: str, session: httpx.AsyncClient, configs: str, user "Accept": "application/json", "Content-Type": "application/json", } - response: httpx.Response = await session.request("GET", url, headers=headers, timeout=args.timeout) + response: httpx.Response = await session.request( + "GET", url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"Zoomeye API blocking our request, {username} please check your API usage for this key: {randomkey}", "warn", args.no_color) + logger( + f"Zoomeye API blocking our request, {username} please check your API usage for this key: {randomkey}", + "warn", + args.no_color, + ) return zoomeyes - + data = response.json() if "list" not in data: return zoomeyes @@ -40,11 +54,21 @@ async def zoomeyeapi(domain: str, session: httpx.AsyncClient, configs: str, user page += 1 except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Zoomeye API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for Zoomeye API, due to: {e}", "warn", args.no_color + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in Zoomeye API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in Zoomeye API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Zoomeye API: {len(zoomeyes)}", "info", args.no_color) - return zoomeyes \ No newline at end of file + logger( + f"Total Subdomains found by Zoomeye API: {len(zoomeyes)}", + "info", + args.no_color, + ) + return zoomeyes diff --git a/subdominator/modules/subscraper/sitedossier/sitedossier.py b/subdominator/modules/subscraper/sitedossier/sitedossier.py index dc75f41..0162ba7 100644 --- a/subdominator/modules/subscraper/sitedossier/sitedossier.py +++ b/subdominator/modules/subscraper/sitedossier/sitedossier.py @@ -2,38 +2,56 @@ import re from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents + Sitedossiers = [] + async def sitedossier(domain: str, session: httpx.AsyncClient, args): try: - if args.include_resources and "sitedossier" not in args.include_resources and not args.all: + if ( + args.include_resources + and "sitedossier" not in args.include_resources + and not args.all + ): return Sitedossiers - + if args.exclude_resources and "sitedossier" in args.exclude_resources: return Sitedossiers - + page = 1 while True: url = f"http://www.sitedossier.com/parentdomain/{domain}/{page}" headers = {"User-Agent": UserAgents()} - response = await session.request("GET",url,timeout=args.timeout, headers=headers) + response = await session.request( + "GET", url, timeout=args.timeout, headers=headers + ) if response.status_code != 200: return Sitedossiers data = response.text filterdomain = re.escape(domain) - pattern = rf'(?i)(?:https?://)?([a-zA-Z0-9*_.-]+\.{filterdomain})' + pattern = rf"(?i)(?:https?://)?([a-zA-Z0-9*_.-]+\.{filterdomain})" subdomains = re.findall(pattern, data) Sitedossiers.extend(subdomains) if "Show next 100 items" not in data: - return Sitedossiers + return Sitedossiers page += 100 except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Sitedossier due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for Sitedossier due to: {e}", "warn", args.no_color + ) except Exception as e: if args.verbose: - logger(f"Exception in Sitedossier module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in Sitedossier module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Sitedossier: {len(Sitedossiers)}", "info", args.no_color) - return Sitedossiers \ No newline at end of file + logger( + f"Total Subdomains found by Sitedossier: {len(Sitedossiers)}", + "info", + args.no_color, + ) + return Sitedossiers diff --git a/subdominator/modules/subscraper/threatcrowd/threatcrowd.py b/subdominator/modules/subscraper/threatcrowd/threatcrowd.py index b518da2..017eee6 100644 --- a/subdominator/modules/subscraper/threatcrowd/threatcrowd.py +++ b/subdominator/modules/subscraper/threatcrowd/threatcrowd.py @@ -1,19 +1,29 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents + threatcrowds = [] + async def threatcrowd(domain: str, session: httpx.AsyncClient, args): try: - if args.include_resources and "threatcrowd" not in args.include_resources and not args.all: + if ( + args.include_resources + and "threatcrowd" not in args.include_resources + and not args.all + ): return threatcrowds if args.exclude_resources and "threatcrowd" in args.exclude_resources: return threatcrowds headers = {"User-Agent": UserAgents()} - url = f"http://ci-www.threatcrowd.org/searchApi/v2/domain/report/?domain={domain}" - response: httpx.Response = await session.get(url, timeout=args.timeout, headers=headers) + url = ( + f"http://ci-www.threatcrowd.org/searchApi/v2/domain/report/?domain={domain}" + ) + response: httpx.Response = await session.get( + url, timeout=args.timeout, headers=headers + ) if response.status_code != 200: return threatcrowds @@ -24,11 +34,23 @@ async def threatcrowd(domain: str, session: httpx.AsyncClient, args): threatcrowds.append(subdomain) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for ThreatCrowd API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for ThreatCrowd API, due to: {e}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in ThreatCrowd API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in ThreatCrowd API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by ThreatCrowd API: {len(threatcrowds)}", "info", args.no_color) - return threatcrowds \ No newline at end of file + logger( + f"Total Subdomains found by ThreatCrowd API: {len(threatcrowds)}", + "info", + args.no_color, + ) + return threatcrowds diff --git a/subdominator/modules/subscraper/trickest/trickest.py b/subdominator/modules/subscraper/trickest/trickest.py index 7d48cba..8dd335d 100644 --- a/subdominator/modules/subscraper/trickest/trickest.py +++ b/subdominator/modules/subscraper/trickest/trickest.py @@ -2,6 +2,7 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import singlekeyloader, UserAgents + sem = asyncio.Semaphore(50) Trickest = set() @@ -15,16 +16,27 @@ async def get_count(session, offset, domain, args): return data.get("total_count", 0) except httpx.TimeoutException: if args.show_timeout_info: - logger("Timeout reached for Trickest API while fetching count.", "warn", args.no_color) + logger( + "Timeout reached for Trickest API while fetching count.", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception in Trickest get_count: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in Trickest get_count: {e}, {type(e)}", + "warn", + args.no_color, + ) + async def fetcher(session, offset, domain, args): try: async with sem: url = f"https://api.trickest.io/solutions/v1/public/solution/a7cba1f1-df07-4a5c-876a-953f178996be/view?q=hostname ~ '.{domain}'&dataset_id=a0a49ca9-03bb-45e0-aa9a-ad59082ebdfc&limit=50&offset={offset}&select=hostname&orderby=hostname" - response = await session.get(url, timeout=httpx.Timeout(read=300.0, connect=args.timeout)) + response = await session.get( + url, timeout=httpx.Timeout(read=300.0, connect=args.timeout) + ) if response.status_code == 200: data = response.json() for result in data.get("results", []): @@ -33,26 +45,35 @@ async def fetcher(session, offset, domain, args): Trickest.add(subdomain) except httpx.TimeoutException: if args.show_timeout_info: - logger("Timeout reached for Trickest API while fetching data.", "warn", args.no_color) + logger( + "Timeout reached for Trickest API while fetching data.", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception in Trickest fetcher: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in Trickest fetcher: {e}, {type(e)}", "warn", args.no_color + ) + -async def trickest(domain: str, configs: str,args): +async def trickest(domain: str, configs: str, args): try: - if not (args.all or (args.include_resources and "trickest" in args.include_resources)): + if not ( + args.all + or (args.include_resources and "trickest" in args.include_resources) + ): return Trickest randomapikey = await singlekeyloader(configs, "trickest") if not randomapikey: return Trickest - headers = { - "User-Agent": UserAgents(), - "Authorization": f"Token {randomapikey}" - } + headers = {"User-Agent": UserAgents(), "Authorization": f"Token {randomapikey}"} tasks = [] - async with httpx.AsyncClient(verify=False, proxy=args.proxy, headers=headers) as session: + async with httpx.AsyncClient( + verify=False, proxy=args.proxy, headers=headers, timeout=args.timeout + ) as session: total = await get_count(session, 10, domain, args) if not total or total == 0: return Trickest @@ -63,11 +84,23 @@ async def trickest(domain: str, configs: str,args): await asyncio.gather(*tasks, return_exceptions=False) except httpx.TimeoutException: if args.show_timeout_info: - logger("Timeout reached while connecting to Trickest API.", "warn", args.no_color) + logger( + "Timeout reached while connecting to Trickest API.", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception in Trickest main function: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in Trickest main function: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total subdomains found by Trickest: {len(Trickest)}", "info", args.no_color) - return Trickest \ No newline at end of file + logger( + f"Total subdomains found by Trickest: {len(Trickest)}", + "info", + args.no_color, + ) + return Trickest diff --git a/subdominator/modules/subscraper/urlscan/urlscan.py b/subdominator/modules/subscraper/urlscan/urlscan.py index c607a8c..f4ccfeb 100644 --- a/subdominator/modules/subscraper/urlscan/urlscan.py +++ b/subdominator/modules/subscraper/urlscan/urlscan.py @@ -1,18 +1,26 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents + urlscans = [] + async def urlscan(domain: str, session: httpx.AsyncClient, args): try: - if args.include_resources and "urlscan" not in args.include_resources and not args.all: + if ( + args.include_resources + and "urlscan" not in args.include_resources + and not args.all + ): return urlscans - + if args.exclude_resources and "urlscan" in args.exclude_resources: return urlscans headers = {"User-Agent": UserAgents()} url = f"https://urlscan.io/api/v1/search/?q=page.domain:{domain}&size=10000" - response: httpx.Response = await session.request("GET", url, timeout=args.timeout, headers=headers) + response: httpx.Response = await session.request( + "GET", url, timeout=args.timeout, headers=headers + ) if response.status_code != 200: return [] data = response.json() @@ -25,8 +33,16 @@ async def urlscan(domain: str, session: httpx.AsyncClient, args): logger(f"Timeout reached for Urlscan due to: {e}", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception in Urlscan module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in Urlscan module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Urlscan: {len(urlscans)}", "info", args.no_color) - return urlscans + logger( + f"Total Subdomains found by Urlscan: {len(urlscans)}", + "info", + args.no_color, + ) + return urlscans diff --git a/subdominator/modules/subscraper/virustotal/virustotal.py b/subdominator/modules/subscraper/virustotal/virustotal.py index 3b73175..4ca0973 100644 --- a/subdominator/modules/subscraper/virustotal/virustotal.py +++ b/subdominator/modules/subscraper/virustotal/virustotal.py @@ -1,37 +1,48 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents, singlekeyloader + virustotal_results = [] -async def virustotal(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def virustotal( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "virustotal" not in args.include_resources and not args.all: + if ( + args.include_resources + and "virustotal" not in args.include_resources + and not args.all + ): return virustotal_results - + if args.exclude_resources and "virustotal" in args.exclude_resources: return virustotal_results - + randomkey = await singlekeyloader(configs, "virustotal") - + if randomkey is None: return virustotal_results - + cursor = None while True: url = f"https://www.virustotal.com/api/v3/domains/{domain}/subdomains?limit=40" if cursor: url = f"{url}&cursor={cursor}" - - headers = { - 'User-Agent': UserAgents(), - 'x-apikey': randomkey - } - response: httpx.Response = await session.request("GET", url, headers=headers, timeout=args.timeout) + + headers = {"User-Agent": UserAgents(), "x-apikey": randomkey} + response: httpx.Response = await session.request( + "GET", url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"VirusTotal blocking our request, {username} please check your API usage for this key: {randomkey}", "warn", args.no_color) + logger( + f"VirusTotal blocking our request, {username} please check your API usage for this key: {randomkey}", + "warn", + args.no_color, + ) return virustotal_results - + data = response.json() subdomains = [item["id"] for item in data.get("data", [])] virustotal_results.extend(subdomains) @@ -40,11 +51,23 @@ async def virustotal(domain: str, session: httpx.AsyncClient, configs: str, user break except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for VirusTotal API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for VirusTotal API, due to: {e}", + "warn", + args.no_color, + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in VirusTotal API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in VirusTotal API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by VirusTotal API: {len(virustotal_results)}", "info", args.no_color) - return virustotal_results + logger( + f"Total Subdomains found by VirusTotal API: {len(virustotal_results)}", + "info", + args.no_color, + ) + return virustotal_results diff --git a/subdominator/modules/subscraper/waybackarchive/waybackarchive.py b/subdominator/modules/subscraper/waybackarchive/waybackarchive.py index 3331169..f620977 100644 --- a/subdominator/modules/subscraper/waybackarchive/waybackarchive.py +++ b/subdominator/modules/subscraper/waybackarchive/waybackarchive.py @@ -1,36 +1,63 @@ import httpx from subdominator.modules.utils.utils import extracts, UserAgents from subdominator.modules.logger.logger import logger + Waybackurls = set() + async def waybackarchive(domain, args): try: - if not (args.all or (args.include_resources and "waybackarchive" in args.include_resources)): + if not ( + args.all + or (args.include_resources and "waybackarchive" in args.include_resources) + ): return Waybackurls - headers = { - "User-Agent": UserAgents() - } - async with httpx.AsyncClient(verify=False, proxy=args.proxy) as request: - async with request.stream( - "GET", f"https://web.archive.org/cdx/search/cdx?url=*.{domain}&collapse=urlkey&fl=original", - headers=headers, - timeout=httpx.Timeout(read=300.0, connect=args.timeout, write=None, pool=None), - follow_redirects=True - ) as response: - async for url in response.aiter_lines(): - subdomains = await extracts(url, domain) - if subdomains: - for subdomain in subdomains: - subdomain = subdomain.lstrip("25").lstrip("2F").lstrip("40").lstrip(".").lstrip("B0") - if subdomain not in Waybackurls and not subdomain.startswith("%3D") and not subdomain.startswith("3D"): - Waybackurls.add(subdomain) + headers = {"User-Agent": UserAgents()} + async with httpx.AsyncClient( + verify=False, proxy=args.proxy, timeout=args.timeout + ) as request: + timeout = httpx.Timeout( + read=300.0, connect=args.timeout, write=None, pool=None + ) + async with request.stream( + "GET", + f"https://web.archive.org/cdx/search/cdx?url=*.{domain}&collapse=urlkey&fl=original", + headers=headers, + timeout=timeout, + follow_redirects=True, + ) as response: + async for url in response.aiter_lines(): + subdomains = await extracts(url, domain) + if subdomains: + for subdomain in subdomains: + subdomain = ( + subdomain.lstrip("25") + .lstrip("2F") + .lstrip("40") + .lstrip(".") + .lstrip("B0") + ) + if ( + subdomain not in Waybackurls + and not subdomain.startswith("%3D") + and not subdomain.startswith("3D") + ): + Waybackurls.add(subdomain) except httpx.TimeoutException as e: if args.show_timeout_info: logger(f"Timeout reached for Webarchive due to: {e}", "warn", args.no_color) except Exception as e: if args.verbose: - logger(f"Exception in Waybackarchive module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception in Waybackarchive module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Waybackarchive: {len(Waybackurls)}", "info", args.no_color) - return Waybackurls \ No newline at end of file + logger( + f"Total Subdomains found by Waybackarchive: {len(Waybackurls)}", + "info", + args.no_color, + ) + return Waybackurls diff --git a/subdominator/modules/subscraper/whoisxml/whoisxml.py b/subdominator/modules/subscraper/whoisxml/whoisxml.py index d8b98bc..6b70b13 100644 --- a/subdominator/modules/subscraper/whoisxml/whoisxml.py +++ b/subdominator/modules/subscraper/whoisxml/whoisxml.py @@ -4,35 +4,62 @@ whoisxml_results = [] -async def whoisxml(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def whoisxml( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "whoisxml" not in args.include_resources and not args.all: + if ( + args.include_resources + and "whoisxml" not in args.include_resources + and not args.all + ): return whoisxml_results - + if args.exclude_resources and "whoisxml" in args.exclude_resources: return whoisxml_results - + randomkey = await singlekeyloader(configs, "whoisxmlapi") - + if randomkey is None: return whoisxml_results url = f"https://subdomains.whoisxmlapi.com/api/v1?apiKey={randomkey}&domainName={domain}" - headers = {'User-Agent': UserAgents()} - response: httpx.Response = await session.request("GET", url, headers=headers, timeout=args.timeout) + headers = {"User-Agent": UserAgents()} + response: httpx.Response = await session.request( + "GET", url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"WhoisXML API blocking our request, {username} please check your API usage for this key: {randomkey}", "warn", args.no_color) + logger( + f"WhoisXML API blocking our request, {username} please check your API usage for this key: {randomkey}", + "warn", + args.no_color, + ) return whoisxml_results data = response.json() - subdomains = [record["domain"] for record in data.get("result", {}).get("records", []) if "domain" in record] + subdomains = [ + record["domain"] + for record in data.get("result", {}).get("records", []) + if "domain" in record + ] whoisxml_results.extend(subdomains) except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for WhoisXML API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for WhoisXML API, due to: {e}", "warn", args.no_color + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in WhoisXML API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in WhoisXML API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by WhoisXML API: {len(whoisxml_results)}", "info", args.no_color) - return whoisxml_results \ No newline at end of file + logger( + f"Total Subdomains found by WhoisXML API: {len(whoisxml_results)}", + "info", + args.no_color, + ) + return whoisxml_results diff --git a/subdominator/modules/subscraper/zoomeyeapi/zoomeyeapi.py b/subdominator/modules/subscraper/zoomeyeapi/zoomeyeapi.py index e6437aa..b403ea8 100644 --- a/subdominator/modules/subscraper/zoomeyeapi/zoomeyeapi.py +++ b/subdominator/modules/subscraper/zoomeyeapi/zoomeyeapi.py @@ -1,17 +1,25 @@ import httpx from subdominator.modules.logger.logger import logger from subdominator.modules.utils.utils import UserAgents, dualkeyloader + zoomeyes = [] -async def zoomeyeapi(domain: str, session: httpx.AsyncClient, configs: str, username: str, args): + +async def zoomeyeapi( + domain: str, session: httpx.AsyncClient, configs: str, username: str, args +): try: - if args.include_resources and "zoomeye" not in args.include_resources and not args.all: + if ( + args.include_resources + and "zoomeye" not in args.include_resources + and not args.all + ): return zoomeyes - + if args.exclude_resources and "zoomeye" in args.exclude_resources: return zoomeyes - - host,randomkey = await dualkeyloader(configs, "zoomeyeapi") + + host, randomkey = await dualkeyloader(configs, "zoomeyeapi") if randomkey is None: return zoomeyes @@ -26,12 +34,18 @@ async def zoomeyeapi(domain: str, session: httpx.AsyncClient, configs: str, user "Accept": "application/json", "Content-Type": "application/json", } - response: httpx.Response = await session.request("GET", url, headers=headers, timeout=args.timeout) + response: httpx.Response = await session.request( + "GET", url, headers=headers, timeout=args.timeout + ) if response.status_code != 200: if args.show_key_info: - logger(f"Zoomeye API blocking our request, {username} please check your API usage for this key: {randomkey}", "warn", args.no_color) + logger( + f"Zoomeye API blocking our request, {username} please check your API usage for this key: {randomkey}", + "warn", + args.no_color, + ) return zoomeyes - + data = response.json() if "list" not in data: return zoomeyes @@ -40,11 +54,21 @@ async def zoomeyeapi(domain: str, session: httpx.AsyncClient, configs: str, user page += 1 except httpx.TimeoutException as e: if args.show_timeout_info: - logger(f"Timeout reached for Zoomeye API, due to: {e}", "warn", args.no_color) + logger( + f"Timeout reached for Zoomeye API, due to: {e}", "warn", args.no_color + ) except Exception as e: if args.verbose: - logger(f"Exception error occurred in Zoomeye API module due to: {e}, {type(e)}", "warn", args.no_color) + logger( + f"Exception error occurred in Zoomeye API module due to: {e}, {type(e)}", + "warn", + args.no_color, + ) finally: if args.verbose: - logger(f"Total Subdomains found by Zoomeye API: {len(zoomeyes)}", "info", args.no_color) - return zoomeyes \ No newline at end of file + logger( + f"Total Subdomains found by Zoomeye API: {len(zoomeyes)}", + "info", + args.no_color, + ) + return zoomeyes diff --git a/subdominator/modules/utils/utils.py b/subdominator/modules/utils/utils.py index 2c839a2..fef1093 100644 --- a/subdominator/modules/utils/utils.py +++ b/subdominator/modules/utils/utils.py @@ -3,14 +3,15 @@ import aiofiles import yaml import random -from typing import Optional,Tuple +from typing import Optional, Tuple from fake_useragent import UserAgent import os +import asyncio from subdominator.modules.logger.logger import logger -from colorama import Fore,Style +from colorama import Fore, Style import sys -red = Fore.RED +red = Fore.RED green = Fore.GREEN magenta = Fore.MAGENTA cyan = Fore.CYAN @@ -21,25 +22,60 @@ reset = Style.RESET_ALL bold = Style.BRIGHT + +class CleanExit(Exception): + pass + + def Exit(code=0): + try: + if asyncio.get_running_loop(): + raise CleanExit() + except RuntimeError: + pass sys.exit(code) + +async def async_exit(code=0): + try: + tasks = [ + task for task in asyncio.all_tasks() if task is not asyncio.current_task() + ] + for task in tasks: + task.cancel() + + if tasks: + await asyncio.wait_for( + asyncio.gather(*tasks, return_exceptions=True), timeout=2.0 + ) + except asyncio.CancelledError: + pass + except asyncio.TimeoutError: + pass + finally: + sys.exit(code) + + async def compiler(domain): pattern = re.compile(r"(?i)[a-zA-Z0-9\*_.-]+\." + re.escape(domain)) return pattern + async def extracts(response, domain): regex = await compiler(domain) matches = regex.findall(response) return set(matches) - + + def check_subdomain(domain): parsed = tldextract.extract(domain) return parsed + def UserAgents() -> str: return UserAgent().random + async def singlekeyloader(filename: str, source: str) -> Optional[str]: try: async with aiofiles.open(filename, "r") as streamr: @@ -53,25 +89,29 @@ async def singlekeyloader(filename: str, source: str) -> Optional[str]: return None except Exception as e: return None - -async def dualkeyloader(filename: str, source: str, splits=False) -> Optional[Tuple[str, str]]: + + +async def dualkeyloader( + filename: str, source: str, splits=False +) -> Optional[Tuple[str, str]]: try: async with aiofiles.open(filename, "r") as streamr: data = yaml.safe_load(await streamr.read()) if source not in data: - return None,None + return None, None randomkeys = random.choice(data.get(source, [])) - if len(randomkeys) != 0 : + if len(randomkeys) != 0: if splits: key1, key2 = randomkeys.rsplit(":", 1) else: key1, key2 = randomkeys.split(":") - return key1,key2 + return key1, key2 else: - return None,None + return None, None except Exception as e: return None + def filters(results): if not results: return [] @@ -92,23 +132,37 @@ async def reader(filename: str, args) -> list: return data.splitlines() except FileNotFoundError: logger(f"File '{filename}' not found. Please check if it exists.") - Exit(1) + raise CleanExit() except Exception as e: - logger(f"Exception in file reader module due to: {e} ({type(e).__name__})", "warn", args.no_color) - Exit(1) + logger( + f"Exception in file reader module due to: {e} ({type(e).__name__})", + "warn", + args.no_color, + ) + raise CleanExit() + async def check_file_permission(filename: str, args): try: async with aiofiles.open(filename, mode="a"): - pass + pass return True except PermissionError: - logger(f"Permission denied: User doesn't have write permission in '{filename}' file", "warn", args.no_color) + logger( + f"Permission denied: User doesn't have write permission in '{filename}' file", + "warn", + args.no_color, + ) return False except Exception as e: - logger(f"Exception in permission check: {e} ({type(e).__name__})", "warn", args.no_color) + logger( + f"Exception in permission check: {e} ({type(e).__name__})", + "warn", + args.no_color, + ) return False - + + async def check_directory_permission(directory: str, args): try: if not os.path.exists(directory): @@ -116,14 +170,23 @@ async def check_directory_permission(directory: str, args): test_file = os.path.join(directory, ".permission_check") async with aiofiles.open(test_file, mode="w") as test: await test.write("test") - os.remove(test_file) + os.remove(test_file) return True except PermissionError: - logger(f"Permission denied: User doesn't have write permission in '{directory}' directory", "warn", args.no_color) + logger( + f"Permission denied: User doesn't have write permission in '{directory}' directory", + "warn", + args.no_color, + ) return False except Exception as e: - logger(f"Exception in directory permission check: {e} ({type(e).__name__})", "warn", args.no_color) + logger( + f"Exception in directory permission check: {e} ({type(e).__name__})", + "warn", + args.no_color, + ) return False + def split_to_list(values: str, delimiter: str = ",") -> list: - return [val.strip() for val in values.split(delimiter) if val.strip()] \ No newline at end of file + return [val.strip() for val in values.split(delimiter) if val.strip()]