main.py

import os
from dotenv import load_dotenv
from supabase import create_client, Client

load_dotenv()

url: str = os.environ.get("SUPABASE_URL")
key: str = os.environ.get("SUPABASE_KEY")
supabase: Client = create_client(url, key)

has_robots = (
        supabase.table('has_robots')
        .select("hostname")
        .execute()
    ).data

while len(has_robots) > 0:
    hostname = has_robots[0]['hostname']
    print(f"https://{hostname}/robots.txt")
    print(f"https://{hostname}")

    print("0: not a personal website\n1: personal website but not enthusiastic\n2: enthusiastic non-personal website\n3: enthusiastic personal website")

    result = int(input("\n:"))

    if result == 0:
        supabase.table("banned_hostnames").upsert({"url": hostname}, on_conflict="url").execute()
    elif result == 1:
        pass
    elif result == 2:
        supabase.table("crawl_only").upsert({"hostname": hostname}, on_conflict="hostname").execute()
    elif result == 3:
        supabase.table("approved_hostnames").upsert({"url": hostname}, on_conflict="url").execute()
        supabase.table("queue").insert({"url": "https://" + hostname}).execute()
 
    supabase.table('has_robots').delete().eq("hostname", hostname).execute()
    has_robots.pop(0)