README.md

Scraper

code usage example

import asyncio

def main():
    URL = "http://localhost:8080/search" # searxng
    query = (
		"text here"
    )

    data = fetch_search_results(URL, query)
    output = extract_relevant_fields(data)

    urls = list(output["results"].values())
    extracted_texts = asyncio.run(extract_text_from_urls(urls))

    for url, text in extracted_texts.items():
        print(f"Extracted content from {url}:\n{text}\n")

if __name__ == "__main__":
    main()