diff --git a/oifey/alias.py b/oifey/alias.py index cb0c737d..11215cd8 100644 --- a/oifey/alias.py +++ b/oifey/alias.py @@ -5,7 +5,7 @@ embed_color = 0xe364d0 -search_text = util.text.search_text +search_text = util.text.standardize_text character_min = 3 character_max = 30 diff --git a/oifey/pool.py b/oifey/pool.py index 3872d012..3150ab7c 100644 --- a/oifey/pool.py +++ b/oifey/pool.py @@ -17,7 +17,7 @@ file = util.file.json_read(f"{root}/{file_path}") for key, value in file.items(): - key = util.text.search_text(key) + key = util.text.standardize_text(key) if not key in lexicon: lexicon[key] = [] @@ -41,7 +41,7 @@ def table_array(k): if value[k]: for i, p in value[k].items(): - p = util.text.search_text(p) + p = util.text.standardize_text(p) if not p in result: result.append(p) @@ -52,7 +52,7 @@ def table_array(k): self.hard = table_array("HARD_ALT_NAME") if use_lexicon: - lexicon_key = util.text.search_text(self.name) + lexicon_key = util.text.standardize_text(self.name) if lexicon_key in lexicon: for x in lexicon[lexicon_key]: @@ -188,7 +188,7 @@ def organize(self) -> None: # common function to add hard alt names def append_hard(k, v): - k = util.text.search_text(k) + k = util.text.standardize_text(k) if k not in self.hard: self.hard[k] = v @@ -198,7 +198,7 @@ def append_hard(k, v): return False def append_alt(k, v): - k = util.text.search_text(k) + k = util.text.standardize_text(k) if not value.id in self.alt: self.alt[value.id] = [] @@ -235,7 +235,7 @@ def append_alt(k, v): def search(self, og_text, ctx = None) -> SearchResult: # if context exists to get aliases from if ctx: - text = util.text.search_text(og_text, ignore_space = True) + text = util.text.standardize_text(og_text, ignore_space = True) # organize aliases aliases = sql.user.get(ctx.author.id).get("alias") or [] @@ -263,10 +263,10 @@ def search(self, og_text, ctx = None) -> SearchResult: text = text[:regex.start()] + new + text[regex.end():] - text = util.text.search_text(text) + text = util.text.standardize_text(text) else: - text = util.text.search_text(og_text) + text = util.text.standardize_text(og_text) finds = [] diff --git a/oifey/util/text.py b/oifey/util/text.py index 02da6dd4..88da9d2f 100644 --- a/oifey/util/text.py +++ b/oifey/util/text.py @@ -2,7 +2,7 @@ replace_char = {'%27': '', '%C3%A1': 'a', '%C3%AD': 'i', '%C3%BA': 'u', '%C3%A7': 'c', '%22': '', '%C3%A9': 'e', 'é': 'e', 'ð': 'd', 'á': 'a', 'ö': 'o', 'ý': 'y', 'þ': 'p', 'ú': 'u', 'ó': 'o', 'í': 'i', 'ø': 'o', 'æ': 'ae', 'Þ': 'p', 'ò': 'o', 'ù': 'u', 'ñ': 'n', 'ä': 'a'} -def search_text(text, ignore_plus = False, ignore_space = False, blank = '') -> str: +def standardize_text(text, ignore_plus = False, ignore_space = False, blank ='') -> str: text = text.lower() plus = "" diff --git a/requirements.txt b/requirements.txt index a9cc54d2..e3c40596 100644 Binary files a/requirements.txt and b/requirements.txt differ diff --git a/scraper/api_data/api_data.py b/scraper/api_data/api_data.py new file mode 100644 index 00000000..a5838c2a --- /dev/null +++ b/scraper/api_data/api_data.py @@ -0,0 +1,161 @@ +from time import sleep +from pprint import pprint +from mwbot import Bot +from dotenv import load_dotenv +import asyncio +import os +import json +import sys +from special_heroes import get_harmonized, get_emblem, get_legendary, get_duo +from utils import is_superboon_or_superbane, convert_game_title, image_asset_url +import queries +from local_db import LocalDB +# prevent "runtime error" errors +if sys.platform.startswith("win"): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + +load_dotenv() + +async def main(): + client = Bot(sitename="https://feheroes.fandom.com", api="https://feheroes.fandom.com/api.php", index="https://feheroes.fandom.com/wiki/Main_Page", username=os.environ["WIKI_BOT_USERNAME"], password=os.environ["WIKI_BOT_PASSWORD"]) + await client.login() + scraping_output = {} + + try: + with open("marker.json", "r") as markerFile: + marker_data = json.load(markerFile) + last_successful_run = marker_data.get("lastSuccessfulRun", '1990-01-01') + except: + last_successful_run = '1990-01-01' + + current_run = await queries.get_next_hero_date(client, last_successful_run) + version = await queries.get_unit_release_update(client, current_run) + unit_data = await queries.get_new_units(client, current_run) + new_units_for_queries = [] + + for entry in unit_data: + heroFullName = entry['title']['Page'] + unitProperties = {} + dataInside = entry["title"] + formattedWikipage = dataInside["WikiName"].replace(" ", "_") + + unitProperties["name"] = dataInside["Name"] + unitProperties["title"] = dataInside["Title"] + unitProperties["description"] = dataInside["Description"] + unitProperties["move"] = dataInside["MoveType"] + unitProperties["artist"] = dataInside["Artist"] + unitProperties["color"], unitProperties["weapon_type"] = dataInside["WeaponType"].split(" ") + unitProperties["id"] = dataInside["ID"] + en_voices = dataInside["ActorEN"].split(' • ') + jp_voices = dataInside["ActorJP"].split(' • ') + unitProperties["voice_en"] = en_voices[0] + if len(en_voices) > 1: + unitProperties["backpack_voice"] = en_voices[1] + else: unitProperties["backpack_voice"] = None + unitProperties["voice_jp"] = jp_voices[0] + if len(jp_voices) > 1: + unitProperties["backpack_voice_jp"] = jp_voices[1] + else: unitProperties["backpack_voice_jp"] = None + unitProperties["internal_id"] = dataInside["TagID"] + unitProperties["resplendent"] = None + unitProperties["resplendent_voice"] = None + unitProperties["resplendent_voice_jp"] = None + unitProperties["images"] = { + "portrait": image_asset_url(f"{formattedWikipage}_Face.webp"), + "attack": image_asset_url(f"{formattedWikipage}_BtlFace.webp"), + "special": image_asset_url(f"{formattedWikipage}_BtlFace_C.webp"), + "damage": image_asset_url(f"{formattedWikipage}_BtlFace_D.webp"), + } + unitProperties["resplendent_images"] = None + unitProperties["release"] = current_run + unitProperties["version"] = version + unitProperties["origin"] = " + ".join(convert_game_title(title) for title in dataInside["Origin"].split(",")) + if len(dataInside["Gender"]) != 1: + # database data is either Female, Male or N, but we only store the first letter + unitProperties["gender"] = dataInside["Gender"][0] + else: + unitProperties["gender"] = "" + unitProperties["rarity"] = await queries.get_unit_rarity(client, heroFullName) + + dbProperties = dataInside["Properties"] + specialUnitProperties = { + "emblem": None, + "harmonized": None, + "duo": None, + "duel": None, + "aided": None, + "type": "" + } + + page = dataInside["Page"] + + if "emblem" in dbProperties: + specialUnitProperties["emblem"] = await get_emblem(client, page) + elif "harmonized" in dbProperties: + specialUnitProperties["harmonized"] = await get_harmonized(client, page) + elif "legendary" in dbProperties: + legendaryProperties = await get_legendary(client, page) + specialUnitProperties = {**specialUnitProperties, **legendaryProperties} + elif "duo" in dbProperties: + specialUnitProperties["duo"] = await get_duo(client, page) + elif "aided" in dbProperties: + specialUnitProperties["type"] = "aided" + elif "chosen" in dbProperties: + specialUnitProperties["type"] = "chosen" + elif "entwined" in dbProperties: + specialUnitProperties["type"] = "entwined" + + unitProperties = {**unitProperties, **specialUnitProperties } + + scraping_output[heroFullName] = {} + new_units_for_queries.append("\"" + heroFullName + "\"") + + sleep(1) + skills = await queries.get_unit_skills(client, heroFullName) + scraping_output[dataInside["Page"]] = {**scraping_output[dataInside["Page"]], **skills, **unitProperties} + + if new_units_for_queries: + unitStatsPayload = { + "tables": "UnitStats", + "fields": "_pageName=Page, Lv1HP5, HPGR3, Lv1Atk5, AtkGR3, Lv1Spd5, SpdGR3, Lv1Def5, DefGR3, Lv1Res5, ResGR3", + "where": f"_pageName in ({', '.join(new_units_for_queries)})" + } + unitStatsQuery = await client.call_get_api("cargoquery", **unitStatsPayload) + + for element in unitStatsQuery["cargoquery"]: + innerData = element["title"] + page = innerData["Page"] + scraping_output[page]["base"] = { + "hp": int(innerData["Lv1HP5"]), + "atk": int(innerData["Lv1Atk5"]), + "spd": int(innerData["Lv1Spd5"]), + "def": int(innerData["Lv1Def5"]), + "res": int(innerData["Lv1Res5"]) + } + scraping_output[page]["growth"] = { + "hp": int(innerData["HPGR3"]), + "atk": int(innerData["AtkGR3"]), + "spd": int(innerData["SpdGR3"]), + "def": int(innerData["DefGR3"]), + "res": int(innerData["ResGR3"]) + } + supertraits = is_superboon_or_superbane(scraping_output[page]["growth"]) + scraping_output[innerData["Page"]] = {**scraping_output[innerData["Page"]], **supertraits} + else: + print(f"WARN: No units found for date {current_run}") + + # time to write to the local DB! + db = LocalDB(wiki_client=client) + + for unit_name, unit_dict in scraping_output.items(): + await db.add_unit(unit_dict) + + + with open("marker.json", "w") as markerFile: + json.dump({ "lastSuccessfulRun": current_run }, markerFile) + + pprint(scraping_output) + return scraping_output + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scraper/api_data/local_db.py b/scraper/api_data/local_db.py new file mode 100644 index 00000000..620eacec --- /dev/null +++ b/scraper/api_data/local_db.py @@ -0,0 +1,155 @@ +import os + +import psycopg +from psycopg.types import TypeInfo +from psycopg.types.hstore import register_hstore + +from scraper.api_data.queries import get_romanized_artist_name, get_romanized_va_name +from scraper.api_data.utils import is_japanese +from scraper.maps import entrymap, wepmap, restrictions +from scraper.utility import compress, skill_family_id, skill_family_name +import scraper.maps + + +class LocalDB: + def __init__(self, wiki_client, user='ninian', password=os.environ['DB_PASSWORD'], address='127.0.0.1', dbname='ninian'): + self.connection = psycopg.connect(f"user={user} password={password} hostaddr={address} dbname={dbname} gssencmode=disable") + type_info = TypeInfo.fetch(self.connection, "hstore") + register_hstore(type_info, self.connection) + self.wiki = wiki_client # we still gotta look some things up on-the-fly + + async def add_unit(self, unit): + # SKILLS + if 'weapon' in unit: + for skill in unit['weapon']: + self.add_skill(skill, 'Weapon') + if 'assist' in unit: + for skill in unit['assist']: + self.add_skill(skill, 'Assist') + if 'special' in unit: + for skill in unit['special']: + self.add_skill(skill, 'Special') + if 'passive_a' in unit: + for skill in unit['passive_a']: + self.add_skill(skill, 'Passive A') + if 'passive_b' in unit: + for skill in unit['passive_b']: + self.add_skill(skill, 'Passive B') + if 'passive_c' in unit: + for skill in unit['passive_c']: + self.add_skill(skill, 'Passive C') + if 'passive_x' in unit: + for skill in unit['passive_x']: + self.add_skill(skill, 'Passive X') + + # ARTISTS AND VAs + with self.connection.cursor() as db: + for artist in ['artist', 'resplendent']: + if unit[artist]: + if is_japanese(unit[artist]): + jp_name = unit[artist] + en_name = await get_romanized_artist_name(self.wiki, unit[artist]) + unit[artist] = en_name + else: + jp_name = None + en_name = unit[artist] + db.execute(""" + insert into artists (english_name, japanese_name) + values (%s::text, %s::text) + on conflict do nothing;""", + [en_name, jp_name] + ) + for va in ['voice_en', 'resplendent_voice', 'voice_jp', 'resplendent_voice_jp']: + if unit[va]: + if is_japanese(unit[va]): + jp_name = unit[va] + en_name = await get_romanized_va_name(self.wiki, unit[va]) + unit[va] = en_name + else: + jp_name = None + en_name = unit[va] + db.execute(""" + insert into voice_actors (english_name, japanese_name) + values (%s::text, %s::text) + on conflict do nothing;""", + [en_name, jp_name] + ) + self.connection.commit() + + #TODO 1. query for all skills missing a description 2. pull skill info for all these new skills + titles = unit['origin'].split(',') + continents = [''] + chibi = '' + for title in entrymap: + if title in titles[0] and title != '': + continents[0] = entrymap[title]['pool'].capitalize() + chibi = entrymap[title]['chibi'] + elif len(titles) > 1 and title in titles[1] and title != '': + continents.append(entrymap[title]['pool'].capitalize()) + if not continents: + print(f"WARN: no origin found for {unit['name']} ({unit['origin']})!") + + with self.connection.cursor() as db: + db.execute("select continent, name from characters where %s=any(array_append(aliases, name)) limit 1;", [unit['name']]) + database_row = db.fetchone() + # If we already have a character with the same name or alias... + if database_row is not None and continents[0] == database_row[0] and unit['name'] == database_row[1]: + new_character = False + elif database_row is not None and (continents[0] == database_row[0] or unit['name'] == database_row[1]): + if input(f"Is {continents[0]} {unit['name']} the same person as {database_row[0]} {database_row[1]}? (y/n) - ")[0].lower() == 'y': + new_character = False + else: + new_character = True + if new_character: + db.execute(""" + insert into characters (continent, name) + values (%s::text, %s::text);""", + [continents[0], unit['name']] + ) + character_origin = continents[0] + character_fkey = unit['name'] + else: + character_origin = database_row[0] + character_fkey = database_row[1] + + weapon_classification = f"{unit['color']} {unit['weapon_type']}" + emoji = f"<:{restrictions[weapon_classification]}:{wepmap[weapon_classification]}>" + db.execute("select count(*) from feh_units where simplified_name like %s;", [compress(unit['name'])]) + alt_count = db.fetchone()[0] + if alt_count > 0: + alt_count += 1 + else: + alt_count = None + # TODO MISSING: rarity, summon_pool, quotes, prf, resplendent voice, unit_type + db.execute(""" + insert into feh_units (id, simplified_name, alt_number, name, emoji, title, description, artist, move_type, weapon_type, color, version, bases, growths, superboons, superbanes, images, resplendent_images, gender, rarity, internal_id, resplendent_artist, resplendent_english_va, resplendent_japanese_va, continents, chibi, character_fkey, character_origin, english_voice_actor, japanese_voice_actor, origin_games, duel_bst, duo_skill, harmonized_skill, emblem_effect, weapon, assist, special, passive_a, passive_b, passive_c, passive_x, backpack_fkey, backpack_origin, backpack_english_va, backpack_japanese_va) + values (%s::integer, %s::text, %s::integer, %s::text, %s::text, %s::text, %s::text, %s::text, %s::text, %s::text, %s::text, %s::text, %s::hstore, %s::hstore, %s::text[], %s::text[], %s::hstore, %s::hstore, %s::char, %s::integer, %s::text, %s::text, %s::text, %s::text, %s::text[], %s::text, %s::text, %s::text, %s::text, %s::text, %s::text[], %s::integer, %s::text, %s::text, %s::text, %s::hstore, %s::hstore, %s::hstore, %s::hstore, %s::hstore, %s::hstore, %s::hstore, %s::text, %s::text, %s::text, %s::text) + on conflict (id) do nothing;""", + [unit['id'], compress(unit['name']), alt_count, unit['name'], emoji, unit['title'], unit['description'], unit['artist'], unit['move'], unit['weapon_type'], unit['color'], unit['version'], hstore(unit['base']), hstore(unit['growth']), unit['superboons'], unit['superbanes'], hstore(unit['images']), hstore(unit['resplendent_images']), unit['gender'], unit['rarity'], unit['internal_id'], unit['resplendent'], unit['resplendent_voice'], unit['resplendent_voice_jp'], continents, chibi, character_fkey, character_origin, unit['voice_en'], unit['voice_jp'], titles, unit['duel'], unit['duo'], unit['harmonized'], unit['emblem'], hstore(unit['weapon']), hstore(unit['assist']), hstore(unit['special']), hstore(unit['passive_a']), hstore(unit['passive_b']), hstore(unit['passive_c']), hstore(unit['passive_x']), None, None, unit['backpack_voice'], unit['backpack_voice_jp']] + ) + self.connection.commit() + + def add_skill(self, skill_name, slot): + with self.connection.cursor() as db: + db.execute(""" + insert into skill_families (id, name, slot) + values (%s::text, %s::text, %s::text) + on conflict do nothing;""", + [skill_family_id(skill_name), skill_family_name(skill_name), slot] + ) + db.execute(""" + insert into skills (name, skill_family) + values (%s::text, %s::text) + on conflict do nothing;""", + [skill_name, skill_family_id(skill_name)] + ) + self.connection.commit() + +def hstore(dict): + if not dict: + return None + string = '' + for key, value in dict.items(): + string += f'"{key}" => "{value}",\n' + return string[:-2] # remove final newline and comma + diff --git a/scraper/api_data/new_skills.py b/scraper/api_data/new_skills.py new file mode 100644 index 00000000..9084824c --- /dev/null +++ b/scraper/api_data/new_skills.py @@ -0,0 +1,6 @@ +async def get_new_skills(client, skills): + with open("../../database/feh/skill.json", "r") as existingSkills: + new_skills = [] + for skill in skills: + if skill not in existingSkills: + new_skills.append(skill) diff --git a/scraper/api_data/poggers.sql b/scraper/api_data/poggers.sql new file mode 100644 index 00000000..74cc88f3 --- /dev/null +++ b/scraper/api_data/poggers.sql @@ -0,0 +1,6 @@ +--HARD ALT NAMES +select * from skills inner join skill_families on skills.skill_family = skill_families.id where skills.name like 'Odd Temp%' order by skills.name desc limit 1; + +--CHARACTER SELECTION FROM A SKILL +select distinct feh_units.* from feh_units cross join skeys(passive_c) as alt_name_c where alt_name_c='Odd Tempest 2'; +--pull in skills.name and skill_families.slot to build the above query diff --git a/scraper/api_data/queries.py b/scraper/api_data/queries.py new file mode 100644 index 00000000..6a1c3a69 --- /dev/null +++ b/scraper/api_data/queries.py @@ -0,0 +1,133 @@ +from aiohttp import payload +from datetime import datetime +import re + +async def get_unit_skills(client, unit): + unitSkillsPayload = { + "tables": "UnitSkills, Skills", + "fields": "UnitSkills._pageName=UnitPage, Skills.Name=SkillName, skillPos, unlockRarity, Scategory, Description", + "where": f"UnitSkills._pageName = '{unit.replace("'", "''")}'", + "join_on": "Skills.WikiName = UnitSkills.skill", + "order_by": "Scategory DESC, defaultRarity ASC", + } + unitSkillsQuery = await client.call_get_api("cargoquery", **unitSkillsPayload) + + output = { + "weapon": {}, + "assist": {}, + "special": {}, + "passive_a": {}, + "passive_b": {}, + "passive_c": {}, + "passive_x": {} + } + + for result in unitSkillsQuery["cargoquery"]: + content = result["title"] + + match content["Scategory"]: + case "passivea": + output["passive_a"][content["SkillName"]] = content["unlockRarity"] + case "passiveb": + output["passive_b"][content["SkillName"]] = content["unlockRarity"] + case "passivec": + output["passive_c"][content["SkillName"]] = content["unlockRarity"] + case "passivex": + output["passive_x"][content["SkillName"]] = content["unlockRarity"] + case "weapon": + output["weapon"][content["SkillName"]] = content["unlockRarity"] + case "assist": + output["assist"][content["SkillName"]] = content["unlockRarity"] + case "special": + output["special"][content["SkillName"]] = content["unlockRarity"] + + return output + +async def get_unit_rarity(client, unit): + payload = { + "tables": "SummoningAvailability", + "fields": "Rarity, Property", + "where": f"_pageName like '{unit}' and EndTime > '{datetime.now()}'", + "order_by": "Rarity ASC", + "limit": "1" + } + query = await client.call_get_api("cargoquery", **payload) + if re.match('.*specialrate', query["cargoquery"][0]["title"]["Property"], re.IGNORECASE): + return int(query["cargoquery"][0]["title"]["Rarity"]) - 1 + else: + return int(query["cargoquery"][0]["title"]["Rarity"]) + +async def get_unit_release_update(client, date): + versionPayload = { + "tables": "VersionUpdates", + "fields": "Major, Minor", + "where": f"Date(ReleaseTime) >= '{date}'", + "limit": "1", + "order_by": "ReleaseTime ASC", + } + versionQuery = await client.call_get_api("cargoquery", **versionPayload) + return f'{versionQuery["cargoquery"][0]["title"]["Major"]}.{versionQuery["cargoquery"][0]["title"]["Minor"]}' + +async def get_unit_stats(client, units): + pass + +# returns the date when units were added nearest to, but after, the given date. if that makes any sense. +#TODO this leaps ahead years at a time +async def get_next_hero_date(client, date): + datePayload = { + "tables": "Units", + "fields": "ReleaseDate", + "where": f"ReleaseDate > '{date}' and WikiName not like \"%ENEMY\"", + "order_by": "WikiName ASC", + "limit": 1 + } + dateQuery = await client.call_get_api("cargoquery", **datePayload) + return dateQuery["cargoquery"][0]["title"]["ReleaseDate"] + +# returns all units released on the given day +async def get_new_units(client, date): + unitIdentityPayload = { + "tables": "Units", + "fields": "_pageName=Page, Name, WikiName, Title, WeaponType, Description, Gender, MoveType, Origin, Gender, Artist, ActorEN, ActorJP, TagID, Properties, _ID=ID", + "where": f"ReleaseDate = '{date}' and WikiName not like \"%ENEMY\"", + "order_by": "WikiName DESC", + "limit": 2 # 95 heroes existed on launch + } + unitIdentityQuery = await client.call_get_api("cargoquery", **unitIdentityPayload) + return unitIdentityQuery["cargoquery"] + +async def get_romanized_artist_name(client, name): + payload = { + "tables": "Artists", + "fields": "NameUSEN", + "where": f'Name like "{name}"', + "limit": 1 + } + query = await client.call_get_api("cargoquery", **payload) + try: + result = query["cargoquery"][0]['title']['NameUSEN'] + except: + print(f"WARN: error looking up artist {name}") + result = None + if result: + return result + else: + return None + +async def get_romanized_va_name(client, name): + payload = { + "tables": "VoiceActors", + "fields": "Name", + "where": f'NameJPJA like "{name}"', + "limit": 1 + } + query = await client.call_get_api("cargoquery", **payload) + try: + result = query["cargoquery"][0]['title']['Name'] + except: + print(f"WARN: error looking up VA {name}") + result = None + if result: + return result + else: + return None diff --git a/scraper/api_data/special_heroes.py b/scraper/api_data/special_heroes.py new file mode 100644 index 00000000..baec01fe --- /dev/null +++ b/scraper/api_data/special_heroes.py @@ -0,0 +1,55 @@ +async def get_duo(client, page): + duoPayload = { + "tables": "DuoHero", + "fields": "DuoSkill, Duel", + "where": "_pageName = \"" + page + "\"", + } + duoQuery = await client.call_get_api("cargoquery", **duoPayload) + return duoQuery["cargoquery"][0]["title"]["DuoSkill"] + +async def get_legendary(client, page): + legendaryPayload = { + "tables": "LegendaryHero", + "fields": "LegendaryEffect, AllyBoostHP, AllyBoostAtk, AllyBoostSpd, AllyBoostDef, AllyBoostRes, Duel", + "where": "_pageName = \"" + page + "\"", + } + legendaryQuery = await client.call_get_api("cargoquery", **legendaryPayload) + legendaryData = legendaryQuery["cargoquery"][0]["title"] + stats = {} + boost_map = { + "AllyBoostHP": "hp", + "AllyBoostAtk": "atk", + "AllyBoostSpd": "spd", + "AllyBoostDef": "def", + "AllyBoostRes": "res", + } + + for src, dst in boost_map.items(): + value = int(legendaryData.get(src, 0)) + if value != 0: + stats[dst] = value + returnedData = { + "type": "legendary", + "blessing": legendaryData["LegendaryEffect"], + "stats": stats, + "duel": legendaryData["Duel"], + } + return returnedData + +async def get_emblem(client, page): + emblemPayload = { + "tables": "EmblemHero", + "fields": "Effect", + "where": "_pageName = \"" + page + "\"", + } + emblemQuery = await client.call_get_api("cargoquery", **emblemPayload) + return emblemQuery["cargoquery"][0]["title"]["Effect"] + +async def get_harmonized(client, page): + harmonizedPayload = { + "tables": "HarmonizedHero", + "fields": "HarmonizedSkill", + "where": "_pageName = \"" + page + "\"", + } + harmonizedQuery = await client.call_get_api("cargoquery", **harmonizedPayload) + return harmonizedQuery["cargoquery"][0]["title"]["HarmonizedSkill"] \ No newline at end of file diff --git a/scraper/api_data/utils.py b/scraper/api_data/utils.py new file mode 100644 index 00000000..22676a63 --- /dev/null +++ b/scraper/api_data/utils.py @@ -0,0 +1,42 @@ +import hashlib +import re + +def is_superboon_or_superbane(growthRates): + superboons = [] + superbanes = [] + for stat in growthRates: + growthRate = growthRates[stat] + match growthRate: + case 30 | 50 | 75 | 95: + superbanes.append(stat) + case 25 | 45 | 70 | 90: + superboons.append(stat) + result = { + "superbanes": superbanes, + "superboons": superboons, + } + return result + + +# convert game titles as received from the db +def convert_game_title(title): + match title: + case "Fire Emblem Echoes: Shadows of Valentia": + return "Echoes" + case "Fire Emblem Warriors: Three Hopes": + return "Three Houses" + case _: + return title.replace("Fire Emblem", "").replace(":", "").strip() + + +def image_asset_url(image_name): + hash = hashlib.md5(image_name.encode()) + image_hash = hash.hexdigest() + first_folder = image_hash[0] + second_folder = image_hash[0:2] + + return f"https://static.wikia.nocookie.net/feheroes_gamepedia_en/images/{first_folder}/{second_folder}/{image_name}" + +def is_japanese(text): + japanese_pattern = re.compile(r'[\u3040-\u30FF\u4E00-\u9FFF]') + return bool(japanese_pattern.search(text)) diff --git a/scraper/maps.py b/scraper/maps.py index b2eabc73..dc3e43c1 100644 --- a/scraper/maps.py +++ b/scraper/maps.py @@ -1,49 +1,49 @@ lords = [ -'Alfonse', -'Veronica', -'Marth', -'Caeda', -'Kris', -'Alm', -'Celica', -'Sigurd', -'Deirdre', -'Seliph', -'Julia', -'Leif', -'Nanna', -'Roy', -'Lilina', -'Eliwood', -'Lyn', -'Hector', -'Ninian', -'Eirika', -'Ephraim', -'Ike', -'Elincia', -'Micaiah', -'Sothe', -'Chrom', -'Lucina', -'Robin', -'Corrin', -'Azura', -'Ryoma', -'Xander', -'Byleth', -'Edelgard', -'Dimitri', -'Claude', -'Yuri', -'Shez', -'Alear', -'Alfred', -'Ivy', -'Diamant', -'Timerra', -'Itsuki', -'Tsubasa' + 'Alfonse', + 'Veronica', + 'Marth', + 'Caeda', + 'Kris', + 'Alm', + 'Celica', + 'Sigurd', + 'Deirdre', + 'Seliph', + 'Julia', + 'Leif', + 'Nanna', + 'Roy', + 'Lilina', + 'Eliwood', + 'Lyn', + 'Hector', + 'Ninian', + 'Eirika', + 'Ephraim', + 'Ike', + 'Elincia', + 'Micaiah', + 'Sothe', + 'Chrom', + 'Lucina', + 'Robin', + 'Corrin', + 'Azura', + 'Ryoma', + 'Xander', + 'Byleth', + 'Edelgard', + 'Dimitri', + 'Claude', + 'Yuri', + 'Shez', + 'Alear', + 'Alfred', + 'Ivy', + 'Diamant', + 'Timerra', + 'Itsuki', + 'Tsubasa' ] entrymap = { @@ -59,17 +59,10 @@ "pool": "archanea", "chibi": "archanea" }, - "Shadow Dragon / (New) Mystery": { "pool": "archanea", "chibi": "archanea" }, - - "Shadow Dragon": { - "pool": "archanea", - "chibi": "archanea" - }, - "Shadows of Valentia": { "pool": "valentia", "chibi": "fe15" @@ -102,7 +95,6 @@ "pool": "tellius", "chibi": "fe9" }, - "Radiant Dawn": { "pool": "tellius", "chibi": "fe10" @@ -119,22 +111,18 @@ "pool": "fodlan", "chibi": "fodlan" }, - "Three Hopes": { "pool": "fodlan", "chibi": "fodlan" }, - "Engage": { "pool": "elyos", "chibi": "fe17" }, - "Tokyo Mirage Sessions": { "pool": "tokyo", "chibi": "tms" }, - '': { "pool": "", "chibi": "" @@ -171,28 +159,27 @@ restrictions = { "Red Sword": "Red_Sword", "Red Dagger": "Red_Dagger", - "Red bow": "Red_Bow", + "Red Bow": "Red_Bow", "Red Tome": "Red_Tome", "Red Breath": "Red_Breath", "Red Beast": "Red_Beast", "Blue Lance": "Blue_Lance", "Blue Dagger": "Blue_Dagger", - "Blue bow": "Blue_Bow", + "Blue Bow": "Blue_Bow", "Blue Tome": "Blue_Tome", "Blue Breath": "Blue_Breath", "Blue Beast": "Blue_Beast", "Green Axe": "Green_Axe", "Green Dagger": "Green_Dagger", - "Green bow": "Green_Bow", + "Green Bow": "Green_Bow", "Green Tome": "Green_Tome", "Green Breath": "Green_Breath", "Green Beast": "Green_Beast", "Colorless Staff": "Colorless_Staff", "Staff": "Colorless_Staff", "Colorless Dagger": "Colorless_Dagger", - "Colorless bow": "Colorless_Bow", + "Colorless Bow": "Colorless_Bow", "Colorless Tome": "Colorless_Tome", "Colorless Breath": "Colorless_Breath", "Colorless Beast": "Colorless_Beast", - } diff --git a/scraper/requirements.txt b/scraper/requirements.txt index db09495c..59512103 100644 --- a/scraper/requirements.txt +++ b/scraper/requirements.txt @@ -1,5 +1,6 @@ # these are needed to run the webscraper on your machine, but not the bot itself. beautifulsoup4 +mwbot Pillow requests unidecode \ No newline at end of file diff --git a/scraper/utility.py b/scraper/utility.py index c297a997..57069cd9 100644 --- a/scraper/utility.py +++ b/scraper/utility.py @@ -26,6 +26,12 @@ def compress(string): new_string = new_string.lower() return(new_string) +def skill_family_id(skill): + return compress(skill).rstrip(string.digits) + +def skill_family_name(skill): + return skill.replace('+', '').replace('/', ' ').rstrip(string.digits) + def is_brave(unit): res = [i for i in unit['ALT_NAME'] if 'Brave' in i] if (len(res) > 0): @@ -37,4 +43,4 @@ def remove_num(arg): return arg.rstrip(string.digits) -sortById() \ No newline at end of file +#sortById() \ No newline at end of file