scraper.py

import requests
from bs4 import BeautifulSoup
import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# Custom imports
from helpers import cvss_color_calc, cve_status_color_calc
from constants import search_urls

def scrape_nvd(cve_id):
    """
    AA
    Scrapes the NVD NIST page for a given CVE ID.

    Args:
        cve_id (str): The CVE ID to scrape.

    Returns:
        dict: A dictionary containing the scraped CVE data. If the CVE ID is not
        valid, the dictionary will contain an "error" key with a descriptive
        string value.
    """

    # AA. Check if the CVE ID is valid
    url = search_urls["NVDNIST"]

    # Get the HTML content of the page
    response = requests.get(url + cve_id, verify=False)

    # Check if the request was successful, otherwise return an error
    if response.status_code == 200:

        # Parse the HTML content
        soup = BeautifulSoup(response.text, "html.parser")

        # Extract the vulnerability description
        description = soup.find("p", {"data-testid": "vuln-description"})
        description = (
            description.text.strip() if description else "No description found"
        )

        # Extract the CVSS score and vector
        cvss_score = soup.find("a", {"data-testid": "vuln-cvss3-panel-score"})
        cvss_score = cvss_score.text.strip() if cvss_score else "N/A"

        cvss_vector = soup.find("a", {"data-testid": "vuln-cvss3-panel-vector"})
        cvss_color = cvss_color_calc(cvss_score)
        cvss_vector = soup.find("span", {"data-testid": "vuln-cvss3-nist-vector"})
        cvss_vector = cvss_vector.text.strip() if cvss_vector else "N/A"

        # Extract the published and last modified dates
        published_on = soup.find("span", {"data-testid": "vuln-published-on"})
        published_on = published_on.text.strip() if published_on else "N/A"

        last_modified = soup.find("span", {"data-testid": "vuln-last-modified-on"})
        last_modified = last_modified.text.strip() if last_modified else "N/A"

        # Extract the vulnerability source
        vuln_source = soup.find(
            "span", {"data-testid": "vuln-current-description-source"}
        )
        vuln_source = vuln_source.text.strip() if vuln_source else "N/A"

        # Combine the extracted data into a dictionary
        cve_data = {
            "id": cve_id.upper(),
            "description": description,
            "cvss_score": cvss_score,
            "cvss_color": cvss_color,
            "cvss_vector": cvss_vector,
            "published_on": published_on,
            "last_modified": last_modified,
            "vuln_source": vuln_source,
        }
    else:
        cve_data = {
            "error": "Failed to retrieve CVE data. Status code: {}".format(
                response.status_code
            )
        }

    return cve_data


def scrape_cveorg(cve_id):
    """
    AA
    Scrapes the CVE.org page for a given CVE ID.

    Args:
        cve_id (str): The CVE ID to scrape.

    Returns:
        dict: A dictionary containing the scraped CVE data. If the CVE ID is not
        valid, the dictionary will contain an "error" key with a descriptive
        string value.
    """

    # Set up Selenium options
    chrome_options = Options()
    chrome_options.add_argument("--headless")  # Run in headless mode (no GUI)
    chrome_options.add_argument("--disable-gpu")
    chrome_options.add_argument("--no-sandbox")

    # Initialize WebDriver
    driver = webdriver.Chrome(options=chrome_options)

    # Navigate to the CVE.org page
    try:
        # Navigate to the CVE.org page
        url = search_urls["CVEOrg"]
        driver.get(url + cve_id)

        # Wait for the element to load
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.CLASS_NAME, "cve-state-tag"))
        )

        # Scrape the dynamically loaded content
        status_element = driver.find_element(By.CLASS_NAME, "cve-state-tag")
        status = status_element.text

        # Check if the product element exists before scraping it
        product_elements = driver.find_elements(
            By.XPATH, "//p[text()='Product']/following-sibling::p"
        )

        # If the list is not empty, the element exists
        if product_elements:
            product_name = product_elements[0].text
        else:
            product_name = "Product field not found"

        # Combine the extracted data into a dictionary
        cve_data = {
            "status": status,
            "status_color": cve_status_color_calc(status),
            "product_name": product_name,
        }
    except Exception as e:
        cve_data = {"error": "CVEOrg Failed"}
        print("CVEOrg exception")
    finally:
        driver.quit()

    return cve_data


def scrape_cvedetails(cve_id):
    """
    AA
    Scrapes the CVE Details page for a given CVE ID.

    Args:
        cve_id (str): The CVE ID to scrape.

    Returns:
        dict: A dictionary containing the scraped CVE data. If the CVE ID is not
        valid, the dictionary will contain an "error" key with a descriptive
        string value.
    """

    # Send a GET request to the CVE Details page
    url = search_urls["CVEDetails"]

    # Add the CVE ID to the URL
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
    }

    # Send a GET request to the CVE Details page
    response = requests.get(url + cve_id, headers=headers, verify=False)

    # Check if the request was successful
    if response.status_code == 200:

        # Parse the HTML content
        soup = BeautifulSoup(response.text, "html.parser")

        # Select the ul element and then get the li elements within it
        ul_element = soup.select_one("#contentdiv > div > main > div:nth-child(5) > ul")
        msf_modules_items = (
            ul_element.select("li.list-group-item") if ul_element else []
        )

        # Initialize an empty list to store the extracted data
        msf_modules = []

        # Iterate over the li elements
        for item in msf_modules_items:
            # Extract title with icon and description
            title_element = item.select_one("h5")
            title = title_element.get_text(strip=True) if title_element else None

            # Check if the title is not None
            if title and title != "No title found":
                # Extract disclosure date and first seen date
                date_elements = item.select("div.ms-2")
                disclosure_date = "No disclosure date found"
                first_seen = "No first seen date found"

                # Extract disclosure date
                if len(date_elements) > 0:
                    disclosure_date = (
                        date_elements[0]
                        .get_text(strip=True)
                        .replace("Disclosure Date: ", "")
                    )

                # Extract first seen date
                if len(date_elements) > 1:
                    first_seen = (
                        date_elements[1]
                        .get_text(strip=True)
                        .replace("First seen: ", "")
                    )

                # Extract module name
                module_name_element = item.select_one("div.ssc-text-secondary")
                module_name = (
                    module_name_element.get_text(strip=True)
                    if module_name_element
                    else "No module name found"
                )

                # Extract "More information" link
                more_info_element = item.select_one("a.ssc-ext-link")
                more_info_url = (
                    more_info_element["href"] if more_info_element else "No link found"
                )

                # Append the extracted data to the list
                msf_modules.append(
                    {
                        "title": title,
                        "disclosure_date": disclosure_date,
                        "first_seen": first_seen,
                        "module_name": module_name,
                        "more_info_url": more_info_url,
                    }
                )
        # Combine the extracted data into a dictionary
        cve_data = {"msf_modules": msf_modules}
    else:

        # If the request was not successful, return an error message
        cve_data = {
            "error": "Failed to retrieve CVE data. Status code: {}".format(
                response.status_code
            )
        }
    return cve_data


def scrape_exploitdb(cve_id):
    """
    AA
    Scrapes the ExploitDB page for a given CVE ID.

    Args:
        cve_id (str): The CVE ID to scrape.

    Returns:
        list: A list of dictionaries containing the scraped exploit data. Each
        dictionary has the following keys:

            - date (str): The date the exploit was added to ExploitDB.
            - download_link (str): The URL of the exploit download page.
            - title (str): The title of the exploit.
            - title_link (str): The URL of the exploit details page.
            - exploit_type (str): The type of exploit (e.g. remote, local, etc.).
            - platform (str): The platform the exploit targets.
            - author (str): The author of the exploit.
            - verified (bool): Whether the exploit has been verified by the
                ExploitDB team.

    If the request fails or no exploits are found, an empty list is returned.
    """

    # Set up the Chrome webdriver
    chrome_options = Options()
    chrome_options.add_argument("--headless")
    chrome_options.add_argument("--disable-gpu")
    chrome_options.add_argument("--no-sandbox")

    # Scrape exploitdb for exploits
    driver = webdriver.Chrome(options=chrome_options)

    # Set up the search URLs
    try:
        # Scrape exploitdb for exploits
        url = search_urls["ExploitDB"]
        driver.get(url + cve_id)

        # Wait for the page to fully load
        time.sleep(10)

        # Check if "No data available in table" message is present
        if driver.find_elements(By.CSS_SELECTOR, "td.dataTables_empty"):
            return []

        # Find the exploits table by its ID
        table = driver.find_element(By.ID, "exploits-table")

        # Get all the rows inside the tbody of the table
        rows = table.find_elements(By.CSS_SELECTOR, "tbody tr")

        # Initialize an empty list to store the extracted data
        exploits_data = []

        # Iterate over the rows
        for row in rows:

            # Find all the columns in the row
            columns = row.find_elements(By.TAG_NAME, "td")

            # Extract the needed information from the columns, with error handling
            try:
                date = columns[0].text
            except IndexError:
                date = None

            try:
                download_link = (
                    columns[1].find_element(By.TAG_NAME, "a").get_attribute("href")
                    if columns[1].find_elements(By.TAG_NAME, "a")
                    else None
                )
            except IndexError:
                download_link = None

            try:
                title = columns[4].text
            except IndexError:
                title = None
                
            try:
                title_link = (
                    columns[4].find_element(By.TAG_NAME, "a").get_attribute("href")
                    if columns[4].find_elements(By.TAG_NAME, "a")
                    else None
                )
            except IndexError:
                title_link = None

            try:
                exploit_type = columns[5].text
            except IndexError:
                exploit_type = None

            try:
                platform = columns[6].text
            except IndexError:
                platform = None

            try:
                author = columns[7].text
            except IndexError:
                author = None

            # Check if the exploit has been verified
            try:
                verified = False
                if columns[3].find_elements(By.CLASS_NAME, "mdi-check"):
                    verified = True
                elif columns[3].find_elements(By.CLASS_NAME, "mdi-close"):
                    verified = False
            except IndexError:
                verified = False

            # Add to the results list
            exploits_data.append(
                {
                    "date": date,
                    "download_link": download_link,
                    "title": title,
                    "title_link": title_link,
                    "exploit_type": exploit_type,
                    "platform": platform,
                    "author": author,
                    "verified": verified,
                }
            )

        # Return the list of extracted data
        if not exploits_data:
            print("EMPTY EXPLOIT DATA")
            return "No data available"  # or another default value
        else:
            print("EXPLOIT DATA:", exploits_data)
            return exploits_data

    finally:
        # Close the driver
        driver.quit()