main.py

import requests
from bs4 import BeautifulSoup
import pandas as pd


def fetch_web_page(url):
    response = requests.get(url)
    if response.status_code == 200:
        return response.text
    else:
        print('Failed to retrieve the webpage')
        return None

def parse_html(html_content):
    soup = BeautifulSoup(html_content, 'html.parser')
    return soup

def extract_data(soup):
    data = []
    articles = soup.find_all('article')
    for article in articles:
        # title = article.find('h2').text
        link = article.find('a').get('href')
        data.append({'link': link})
    return data

def save_to_csv(data, filename):
    df = pd.DataFrame(data)
    df.to_csv(filename, index=False)

def main():
    url = 'https://www.wikipedia.org/'
    html_content = fetch_web_page(url)
    if html_content:
        soup = parse_html(html_content)
        data = extract_data(soup)
        save_to_csv(data, 'scraped_data.csv')
        print('Data saved to scraped_data.csv')

if __name__ == '__main__':
    main()