web-parser.py

import re
import requests # type: ignore

def fetch_and_parse(url):
    # Standard headers to look like a real browser
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/121.0.0.0'
    }
    
    print(f"Fetching: {url}...")
    try:
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status() # Check for 404/500 errors
        html_content = response.text
    except Exception as e:
        print(f"Failed to fetch: {e}")
        return

    # Updated regex to be a bit more flexible with attributes
    # This looks for <a> tags specifically as an example
    link_pattern = re.compile(r'<a\s+(?:[^>]*?\s+)?href="([^"]*)"[^>]*>(.*?)</a>', re.IGNORECASE)
    
    links = link_pattern.findall(html_content)
    
    print(f"\nFound {len(links)} links:")
    for href, text in links[:15]:  # Show first 15 results
        clean_text = re.sub(r'<[^>]+>', '', text).strip() # Strip nested tags in text
        print(f"Link: {href} | Text: {clean_text}")

if __name__ == "__main__":
    # Change this to any site you want to test
    target_url = "https://www.wikipedia.org" 
    fetch_and_parse(target_url)