tinystartups.py

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from urllib.parse import urlparse
import time
from websiteparser import SocialMediaScraper
from concurrent.futures import ThreadPoolExecutor 
import asyncio
import json
import undetected_chromedriver as uc
import os
from dotenv import load_dotenv
load_dotenv()

def is_outgoing_link(href, base_domain="tinystartups.com"):
    """
    Check if a link is outgoing (external) by comparing domains
    """
    try:
        if not href:
            return False
            
        # Handle relative URLs
        if href.startswith('/') or href.startswith('#') or href.startswith('?'):
            return False
            
        # Handle mailto, tel, javascript links
        if href.startswith(('mailto:', 'tel:', 'javascript:')):
            return False
            
        # Parse the URL to get the domain
        parsed_url = urlparse(href)
        
        # If no scheme, it's likely a relative URL
        if not parsed_url.scheme:
            return False
            
        # Extract domain from the href
        link_domain = parsed_url.netloc.lower()
        
        # Remove www. prefix for comparison
        link_domain = link_domain.replace('www.', '')
        base_domain = base_domain.replace('www.', '')
        
        # Check if it's an outgoing link
        return link_domain != base_domain and link_domain != ''
        
    except Exception as e:
        print(f"Error parsing URL '{href}': {str(e)}")
        return False
def scrape_tinystartups_links():
    """
    Scrape outgoing href links from anchor tags within div elements with role='listitem' 
    from tinystartups.com
    """
    
    # Initialize the Chrome driver (make sure chromedriver is in your PATH)
    # driver = webdriver.Chrome()
    driver = uc.Chrome(headless=True,use_subprocess=True)

    
    try:
        # Navigate to the URL
        url = "https://www.tinystartups.com/"
        print(f"Navigating to: {url}")
        driver.get(url)
        
        # Wait for page to load
        wait = WebDriverWait(driver, 10)
        
        # Wait for the page to load and find the div elements
        print("Waiting for page to load...")
        time.sleep(3)  # Give the page time to fully load
        
        # Find all div elements with role='listitem'
        print("Finding div elements with role='listitem'...")
        elements = driver.find_elements(By.XPATH, "//div[contains(@role, 'listitem')]")
        
        print(f"Found {len(elements)} div elements with role='listitem'")
        
        # Store all outgoing href links
        outgoing_links = []
        internal_links = []
        
        # Iterate through each div element
        for i, div_element in enumerate(elements):
            try:
                # Find anchor tags within this div
                anchor_tags = div_element.find_elements(By.TAG_NAME, "a")
                
                print(f"Div {i+1}: Found {len(anchor_tags)} anchor tags")
                
                # Extract href from each anchor tag
                for anchor in anchor_tags:
                    href = anchor.get_attribute("href")
                    if href:  # Only process non-empty href values
                        if is_outgoing_link(href):
                            outgoing_links.append(href)
                            print(f"  - OUTGOING: {href}")
                        else:
                            internal_links.append(href)
                            print(f"  - internal: {href}")
                        
            except NoSuchElementException:
                print(f"Div {i+1}: No anchor tags found")
                continue
        
        print(f"\nTotal outgoing links found: {len(outgoing_links)}")
        print(f"Total internal links found: {len(internal_links)}")
        print(f"Total unique outgoing links: {len(set(outgoing_links))}")
        
        return outgoing_links, internal_links
        
    except TimeoutException:
        print("Timeout: Page took too long to load")
        return [], []
    
    except Exception as e:
        print(f"An error occurred: {str(e)}")
        return [], []
    
    finally:
        # Close the browser
        driver.quit()

def save_links_to_file(links, filename="tinystartups_outgoing_links.json"):
    """Save the scraped outgoing links to a text file"""
    try:
        with open(filename, 'w') as json_file:
            json.dump(links, json_file, indent=4)
    except Exception as e:
        print(f"Error saving links to file: {str(e)}")

async def tinyStartupScraperExecutor():
     # Scrape the links
        outgoing_links, internal_links = scrape_tinystartups_links()
        GEMINI_API_KEY = os.getenv("GEMINI_API_KEY") # Replace with your actual API key
        
        if outgoing_links:
            unique_outgoing_links = list(set(outgoing_links))
            print("\n" + "="*50)
            print("UNIQUE OUTGOING LINKS FOUND:")
            print("="*50)
            scraper = SocialMediaScraper(GEMINI_API_KEY)
            
            # Limit concurrency to 10
            semaphore = asyncio.Semaphore(10)

            async def limited_scrape(url):
                async with semaphore:
                    return await scraper.scrape_social_links(url)

            tasks = [limited_scrape(url) for url in unique_outgoing_links]
            results = await asyncio.gather(*tasks)

            scraper.save_links_to_file(results)
            
        else:
            print("No outgoing links were found or an error occurred.")

# Main execution
if __name__ == "__main__":
    asyncio.run(tinyStartupScraperExecutor())