From 5a14a9b8fdfdaf538d95367451ec0276b45cf315 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sat, 21 Feb 2026 10:16:39 +0100 Subject: [PATCH 01/26] --wip-- [skip ci] --- .env.dist | 1 - app/adapters/cache/valkey_cache.py | 21 - app/adapters/tasks/stub_task_queue.py | 26 + app/api/dependencies.py | 98 ++- app/api/routers/gamemodes.py | 20 +- app/api/routers/heroes.py | 53 +- app/api/routers/maps.py | 17 +- app/api/routers/players.py | 102 ++- app/api/routers/roles.py | 15 +- app/config.py | 23 +- app/controllers.py | 146 ---- app/domain/ports/cache.py | 18 - app/domain/services/__init__.py | 17 + app/domain/services/base_service.py | 171 +++++ app/domain/services/gamemode_service.py | 32 + app/domain/services/hero_service.py | 248 +++++++ app/domain/services/map_service.py | 42 ++ app/domain/services/player_service.py | 684 ++++++++++++++++++ app/domain/services/role_service.py | 41 ++ app/gamemodes/controllers/__init__.py | 0 .../controllers/list_gamemodes_controller.py | 28 - app/helpers.py | 27 + app/heroes/controllers/__init__.py | 0 app/heroes/controllers/get_hero_controller.py | 140 ---- .../get_hero_stats_summary_controller.py | 59 -- .../controllers/list_heroes_controller.py | 49 -- app/maps/controllers/__init__.py | 0 app/maps/controllers/list_maps_controller.py | 29 - app/monitoring/metrics.py | 7 + app/players/controllers/__init__.py | 0 .../controllers/base_player_controller.py | 415 ----------- .../get_player_career_controller.py | 201 ----- .../get_player_career_stats_controller.py | 170 ----- .../get_player_stats_summary_controller.py | 166 ----- .../controllers/search_players_controller.py | 43 -- app/roles/controllers/__init__.py | 0 .../controllers/list_roles_controller.py | 40 - pyproject.toml | 3 + tests/conftest.py | 9 +- tests/heroes/controllers/conftest.py | 13 - .../controllers/test_heroes_controllers.py | 11 +- tests/heroes/test_hero_routes.py | 6 +- tests/heroes/test_hero_stats_route.py | 4 +- tests/heroes/test_heroes_route.py | 4 +- tests/players/test_player_career_route.py | 4 +- tests/players/test_player_stats_route.py | 14 +- .../test_player_stats_summary_route.py | 12 +- tests/players/test_player_summary_route.py | 4 +- tests/players/test_search_players_route.py | 2 +- tests/roles/test_roles_route.py | 4 +- uv.lock | 2 +- 51 files changed, 1584 insertions(+), 1657 deletions(-) create mode 100644 app/adapters/tasks/stub_task_queue.py delete mode 100644 app/controllers.py create mode 100644 app/domain/services/base_service.py create mode 100644 app/domain/services/gamemode_service.py create mode 100644 app/domain/services/hero_service.py create mode 100644 app/domain/services/map_service.py create mode 100644 app/domain/services/player_service.py create mode 100644 app/domain/services/role_service.py delete mode 100644 app/gamemodes/controllers/__init__.py delete mode 100644 app/gamemodes/controllers/list_gamemodes_controller.py delete mode 100644 app/heroes/controllers/__init__.py delete mode 100644 app/heroes/controllers/get_hero_controller.py delete mode 100644 app/heroes/controllers/get_hero_stats_summary_controller.py delete mode 100644 app/heroes/controllers/list_heroes_controller.py delete mode 100644 app/maps/controllers/__init__.py delete mode 100644 app/maps/controllers/list_maps_controller.py delete mode 100644 app/players/controllers/__init__.py delete mode 100644 app/players/controllers/base_player_controller.py delete mode 100644 app/players/controllers/get_player_career_controller.py delete mode 100644 app/players/controllers/get_player_career_stats_controller.py delete mode 100644 app/players/controllers/get_player_stats_summary_controller.py delete mode 100644 app/players/controllers/search_players_controller.py delete mode 100644 app/roles/controllers/__init__.py delete mode 100644 app/roles/controllers/list_roles_controller.py delete mode 100644 tests/heroes/controllers/conftest.py diff --git a/.env.dist b/.env.dist index 1abc2eaf..6a1caa33 100644 --- a/.env.dist +++ b/.env.dist @@ -45,7 +45,6 @@ UNKNOWN_PLAYER_MAX_RETRY_AFTER=21600 # Cache configuration CACHE_TTL_HEADER=X-Cache-TTL -PLAYER_CACHE_TIMEOUT=86400 HEROES_PATH_CACHE_TIMEOUT=86400 HERO_PATH_CACHE_TIMEOUT=86400 CSV_CACHE_TIMEOUT=86400 diff --git a/app/adapters/cache/valkey_cache.py b/app/adapters/cache/valkey_cache.py index 17b70e93..f59ab393 100644 --- a/app/adapters/cache/valkey_cache.py +++ b/app/adapters/cache/valkey_cache.py @@ -147,27 +147,6 @@ async def update_api_cache( ex=expire, ) - @handle_valkey_error(default_return=None) - async def get_player_cache(self, player_id: str) -> dict | list | None: - """Get the Player Cache value associated with a given cache key""" - player_key = f"{settings.player_cache_key_prefix}:{player_id}" - player_cache = await self.valkey_server.get(player_key) - if not player_cache or not isinstance(player_cache, bytes): - return None - # Reset the TTL before returning the value - await self.valkey_server.expire(player_key, settings.player_cache_timeout) - return self._decompress_json_value(player_cache) - - @handle_valkey_error(default_return=None) - async def update_player_cache(self, player_id: str, value: dict) -> None: - """Update or set a Player Cache value""" - compressed_value = self._compress_json_value(value) - await self.valkey_server.set( - f"{settings.player_cache_key_prefix}:{player_id}", - value=compressed_value, - ex=settings.player_cache_timeout, - ) - @handle_valkey_error(default_return=False) async def is_being_rate_limited(self) -> bool: """Check if Blizzard rate limit is currently active""" diff --git a/app/adapters/tasks/stub_task_queue.py b/app/adapters/tasks/stub_task_queue.py new file mode 100644 index 00000000..fb60d212 --- /dev/null +++ b/app/adapters/tasks/stub_task_queue.py @@ -0,0 +1,26 @@ +"""Stub task queue adapter — Phase 4 placeholder until arq is wired in Phase 5""" + +from app.overfast_logger import logger + + +class StubTaskQueue: + """No-op implementation of TaskQueuePort. + + Logs enqueue calls for observability but does not actually execute tasks. + Will be replaced by ArqTaskQueue in Phase 5. + """ + + async def enqueue( + self, + task_name: str, + *_args, + job_id: str | None = None, + **_kwargs, + ) -> str: + """Log and discard the task — no background processing yet.""" + logger.debug(f"StubTaskQueue: skipping enqueue({task_name}, job_id={job_id})") + return job_id or "" + + async def is_job_pending_or_running(self, _job_id: str) -> bool: + """Always returns False — stub never has pending jobs.""" + return False diff --git a/app/api/dependencies.py b/app/api/dependencies.py index 2b952379..7c675c4a 100644 --- a/app/api/dependencies.py +++ b/app/api/dependencies.py @@ -5,19 +5,99 @@ from fastapi import Depends from app.adapters.blizzard.client import BlizzardClient -from app.cache_manager import CacheManager +from app.adapters.cache.valkey_cache import ValkeyCache +from app.adapters.storage.sqlite_storage import SQLiteStorage +from app.adapters.tasks.stub_task_queue import StubTaskQueue +from app.domain.ports import BlizzardClientPort, CachePort, StoragePort, TaskQueuePort +from app.domain.services import ( + GamemodeService, + HeroService, + MapService, + PlayerService, + RoleService, +) +# --------------------------------------------------------------------------- +# Low-level adapter dependencies +# --------------------------------------------------------------------------- -def get_blizzard_client() -> BlizzardClient: - """Dependency for Blizzard HTTP client""" + +def get_blizzard_client() -> BlizzardClientPort: + """Dependency for Blizzard HTTP client (Singleton).""" return BlizzardClient() -def get_cache_manager() -> CacheManager: - """Dependency for cache manager (Valkey)""" - return CacheManager() +def get_cache() -> CachePort: + """Dependency for Valkey cache (Singleton).""" + return ValkeyCache() + + +def get_storage() -> StoragePort: + """Dependency for SQLite persistent storage (Singleton).""" + return SQLiteStorage() + + +def get_task_queue() -> TaskQueuePort: + """Dependency for background task queue (Stub until Phase 5).""" + return StubTaskQueue() + + +# --------------------------------------------------------------------------- +# Service dependencies +# --------------------------------------------------------------------------- + + +def get_hero_service( + cache: CachePort = Depends(get_cache), + storage: StoragePort = Depends(get_storage), + blizzard_client: BlizzardClientPort = Depends(get_blizzard_client), + task_queue: TaskQueuePort = Depends(get_task_queue), +) -> HeroService: + return HeroService(cache, storage, blizzard_client, task_queue) + + +def get_map_service( + cache: CachePort = Depends(get_cache), + storage: StoragePort = Depends(get_storage), + blizzard_client: BlizzardClientPort = Depends(get_blizzard_client), + task_queue: TaskQueuePort = Depends(get_task_queue), +) -> MapService: + return MapService(cache, storage, blizzard_client, task_queue) + + +def get_gamemode_service( + cache: CachePort = Depends(get_cache), + storage: StoragePort = Depends(get_storage), + blizzard_client: BlizzardClientPort = Depends(get_blizzard_client), + task_queue: TaskQueuePort = Depends(get_task_queue), +) -> GamemodeService: + return GamemodeService(cache, storage, blizzard_client, task_queue) + + +def get_role_service( + cache: CachePort = Depends(get_cache), + storage: StoragePort = Depends(get_storage), + blizzard_client: BlizzardClientPort = Depends(get_blizzard_client), + task_queue: TaskQueuePort = Depends(get_task_queue), +) -> RoleService: + return RoleService(cache, storage, blizzard_client, task_queue) + + +def get_player_service( + cache: CachePort = Depends(get_cache), + storage: StoragePort = Depends(get_storage), + blizzard_client: BlizzardClientPort = Depends(get_blizzard_client), + task_queue: TaskQueuePort = Depends(get_task_queue), +) -> PlayerService: + return PlayerService(cache, storage, blizzard_client, task_queue) + +# --------------------------------------------------------------------------- +# Type aliases for cleaner router injection +# --------------------------------------------------------------------------- -# Type aliases for cleaner dependency injection -BlizzardClientDep = Annotated[BlizzardClient, Depends(get_blizzard_client)] -CacheManagerDep = Annotated[CacheManager, Depends(get_cache_manager)] +HeroServiceDep = Annotated[HeroService, Depends(get_hero_service)] +MapServiceDep = Annotated[MapService, Depends(get_map_service)] +GamemodeServiceDep = Annotated[GamemodeService, Depends(get_gamemode_service)] +RoleServiceDep = Annotated[RoleService, Depends(get_role_service)] +PlayerServiceDep = Annotated[PlayerService, Depends(get_player_service)] diff --git a/app/api/routers/gamemodes.py b/app/api/routers/gamemodes.py index 0528a859..5f3ff539 100644 --- a/app/api/routers/gamemodes.py +++ b/app/api/routers/gamemodes.py @@ -4,10 +4,11 @@ from fastapi import APIRouter, Request, Response +from app.api.dependencies import GamemodeServiceDep +from app.config import settings from app.enums import RouteTag -from app.gamemodes.controllers.list_gamemodes_controller import ListGamemodesController from app.gamemodes.models import GamemodeDetails -from app.helpers import success_responses +from app.helpers import apply_swr_headers, success_responses router = APIRouter() @@ -19,10 +20,19 @@ summary="Get a list of gamemodes", description=( "Get a list of Overwatch gamemodes : Assault, Escort, Flashpoint, Hybrid, etc." - f"
**Cache TTL : {ListGamemodesController.get_human_readable_timeout()}.**" + f"
**Cache TTL : {settings.csv_cache_timeout} seconds.**" ), operation_id="list_map_gamemodes", response_model=list[GamemodeDetails], ) -async def list_map_gamemodes(request: Request, response: Response) -> Any: - return await ListGamemodesController(request, response).process_request() +async def list_map_gamemodes( + request: Request, + response: Response, + service: GamemodeServiceDep, +) -> Any: + cache_key = request.url.path + ( + f"?{request.query_params}" if request.query_params else "" + ) + data, is_stale, age = await service.list_gamemodes(cache_key=cache_key) + apply_swr_headers(response, settings.csv_cache_timeout, is_stale, age) + return data diff --git a/app/api/routers/heroes.py b/app/api/routers/heroes.py index 68c4f9a1..c2c1fda9 100644 --- a/app/api/routers/heroes.py +++ b/app/api/routers/heroes.py @@ -4,13 +4,10 @@ from fastapi import APIRouter, Path, Query, Request, Response, status +from app.api.dependencies import HeroServiceDep +from app.config import settings from app.enums import Locale, RouteTag -from app.helpers import routes_responses -from app.heroes.controllers.get_hero_controller import GetHeroController -from app.heroes.controllers.get_hero_stats_summary_controller import ( - GetHeroStatsSummaryController, -) -from app.heroes.controllers.list_heroes_controller import ListHeroesController +from app.helpers import apply_swr_headers, routes_responses from app.heroes.enums import HeroGamemode, HeroKey from app.heroes.models import ( BadRequestErrorMessage, @@ -38,7 +35,7 @@ summary="Get a list of heroes", description=( "Get a list of Overwatch heroes, which can be filtered using roles or gamemodes. " - f"
**Cache TTL : {ListHeroesController.get_human_readable_timeout()}.**" + f"
**Cache TTL : {settings.heroes_path_cache_timeout} seconds.**" ), operation_id="list_heroes", response_model=list[HeroShort], @@ -46,17 +43,21 @@ async def list_heroes( request: Request, response: Response, + service: HeroServiceDep, role: Annotated[Role | None, Query(title="Role filter")] = None, locale: Annotated[ Locale, Query(title="Locale to be displayed") ] = Locale.ENGLISH_US, gamemode: Annotated[HeroGamemode | None, Query(title="Gamemode filter")] = None, ) -> Any: - return await ListHeroesController(request, response).process_request( - role=role, - locale=locale, - gamemode=gamemode, + cache_key = request.url.path + ( + f"?{request.query_params}" if request.query_params else "" ) + data, is_stale, age = await service.list_heroes( + locale=locale, role=role, gamemode=gamemode, cache_key=cache_key + ) + apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale, age) + return data @router.get( @@ -73,7 +74,7 @@ async def list_heroes( description=( "Get hero statistics usage, filtered by platform, region, role, etc." "Only Role Queue gamemodes are concerned." - f"
**Cache TTL : {GetHeroStatsSummaryController.get_human_readable_timeout()}.**" + f"
**Cache TTL : {settings.hero_stats_cache_timeout} seconds.**" ), operation_id="get_hero_stats", response_model=list[HeroStatsSummary], @@ -81,6 +82,7 @@ async def list_heroes( async def get_hero_stats( request: Request, response: Response, + service: HeroServiceDep, platform: Annotated[ PlayerPlatform, Query(title="Player platform filter", examples=["pc"]) ], @@ -88,7 +90,7 @@ async def get_hero_stats( PlayerGamemode, Query( title="Gamemode", - description=("Filter on a specific gamemode."), + description="Filter on a specific gamemode.", examples=["competitive"], ), ], @@ -96,7 +98,7 @@ async def get_hero_stats( PlayerRegion, Query( title="Region", - description=("Filter on a specific player region."), + description="Filter on a specific player region.", examples=["europe"], ), ], @@ -121,15 +123,21 @@ async def get_hero_stats( ), ] = "hero:asc", ) -> Any: - return await GetHeroStatsSummaryController(request, response).process_request( + cache_key = request.url.path + ( + f"?{request.query_params}" if request.query_params else "" + ) + data, is_stale, age = await service.get_hero_stats( platform=platform, gamemode=gamemode, region=region, role=role, - map=map_, + map_filter=map_, competitive_division=competitive_division, order_by=order_by, + cache_key=cache_key, ) + apply_swr_headers(response, settings.hero_stats_cache_timeout, is_stale, age) + return data @router.get( @@ -145,7 +153,7 @@ async def get_hero_stats( summary="Get hero data", description=( "Get data about an Overwatch hero : description, abilities, stadium powers, story, etc. " - f"
**Cache TTL : {GetHeroController.get_human_readable_timeout()}.**" + f"
**Cache TTL : {settings.hero_path_cache_timeout} seconds.**" ), operation_id="get_hero", response_model=Hero, @@ -153,12 +161,17 @@ async def get_hero_stats( async def get_hero( request: Request, response: Response, + service: HeroServiceDep, hero_key: Annotated[HeroKey, Path(title="Key name of the hero")], # ty: ignore[invalid-type-form] locale: Annotated[ Locale, Query(title="Locale to be displayed") ] = Locale.ENGLISH_US, ) -> Any: - return await GetHeroController(request, response).process_request( - hero_key=hero_key, - locale=locale, + cache_key = request.url.path + ( + f"?{request.query_params}" if request.query_params else "" + ) + data, is_stale, age = await service.get_hero( + hero_key=str(hero_key), locale=locale, cache_key=cache_key ) + apply_swr_headers(response, settings.hero_path_cache_timeout, is_stale, age) + return data diff --git a/app/api/routers/maps.py b/app/api/routers/maps.py index a65c3842..8cbf4bae 100644 --- a/app/api/routers/maps.py +++ b/app/api/routers/maps.py @@ -4,10 +4,11 @@ from fastapi import APIRouter, Query, Request, Response +from app.api.dependencies import MapServiceDep +from app.config import settings from app.enums import RouteTag from app.gamemodes.enums import MapGamemode -from app.helpers import success_responses -from app.maps.controllers.list_maps_controller import ListMapsController +from app.helpers import apply_swr_headers, success_responses from app.maps.models import Map router = APIRouter() @@ -20,7 +21,7 @@ summary="Get a list of maps", description=( "Get a list of Overwatch maps : Hanamura, King's Row, Dorado, etc." - f"
**Cache TTL : {ListMapsController.get_human_readable_timeout()}.**" + f"
**Cache TTL : {settings.csv_cache_timeout} seconds.**" ), operation_id="list_maps", response_model=list[Map], @@ -28,6 +29,7 @@ async def list_maps( request: Request, response: Response, + service: MapServiceDep, gamemode: Annotated[ MapGamemode | None, # ty: ignore[invalid-type-form] Query( @@ -36,6 +38,11 @@ async def list_maps( ), ] = None, ) -> Any: - return await ListMapsController(request, response).process_request( - gamemode=gamemode + cache_key = request.url.path + ( + f"?{request.query_params}" if request.query_params else "" ) + data, is_stale, age = await service.list_maps( + gamemode=gamemode, cache_key=cache_key + ) + apply_swr_headers(response, settings.csv_cache_timeout, is_stale, age) + return data diff --git a/app/api/routers/players.py b/app/api/routers/players.py index c6895811..f3e38671 100644 --- a/app/api/routers/players.py +++ b/app/api/routers/players.py @@ -4,18 +4,11 @@ from fastapi import APIRouter, Depends, Path, Query, Request, Response, status +from app.api.dependencies import PlayerServiceDep +from app.config import settings from app.enums import RouteTag +from app.helpers import apply_swr_headers from app.helpers import routes_responses as common_routes_responses -from app.players.controllers.get_player_career_controller import ( - GetPlayerCareerController, -) -from app.players.controllers.get_player_career_stats_controller import ( - GetPlayerCareerStatsController, -) -from app.players.controllers.get_player_stats_summary_controller import ( - GetPlayerStatsSummaryController, -) -from app.players.controllers.search_players_controller import SearchPlayersController from app.players.enums import ( HeroKeyCareerFilter, PlayerGamemode, @@ -106,7 +99,7 @@ async def get_player_career_common_parameters( description=( "Search for a given player by using its username or BattleTag (with # replaced by -). " "
You should be able to find the associated player_id to use in order to request career data." - f"
**Cache TTL : {SearchPlayersController.get_human_readable_timeout()}.**" + f"
**Cache TTL : {settings.search_account_path_cache_timeout} seconds.**" ), operation_id="search_players", response_model=PlayerSearchResult, @@ -114,6 +107,7 @@ async def get_player_career_common_parameters( async def search_players( request: Request, response: Response, + service: PlayerServiceDep, name: Annotated[ str, Query( @@ -131,12 +125,20 @@ async def search_players( offset: Annotated[int, Query(title="Offset of the results", ge=0)] = 0, limit: Annotated[int, Query(title="Limit of results per page", gt=0)] = 20, ) -> Any: - return await SearchPlayersController(request, response).process_request( + cache_key = request.url.path + ( + f"?{request.query_params}" if request.query_params else "" + ) + data = await service.search_players( name=name, order_by=order_by, offset=offset, limit=limit, + cache_key=cache_key, + ) + response.headers[settings.cache_ttl_header] = str( + settings.search_account_path_cache_timeout ) + return data @router.get( @@ -146,7 +148,7 @@ async def search_players( summary="Get player summary", description=( "Get player summary : name, avatar, competitive ranks, etc. " - f"
**Cache TTL : {GetPlayerCareerController.get_human_readable_timeout()}.**" + f"
**Cache TTL : {settings.career_path_cache_timeout} seconds.**" ), operation_id="get_player_summary", response_model=PlayerSummary, @@ -154,12 +156,18 @@ async def search_players( async def get_player_summary( request: Request, response: Response, + service: PlayerServiceDep, commons: CommonsPlayerDep, ) -> Any: - return await GetPlayerCareerController(request, response).process_request( - summary=True, - player_id=commons.get("player_id"), + cache_key = request.url.path + ( + f"?{request.query_params}" if request.query_params else "" + ) + data, is_stale, age = await service.get_player_summary( + player_id=commons["player_id"], + cache_key=cache_key, ) + apply_swr_headers(response, settings.career_path_cache_timeout, is_stale, age) + return data @router.get( @@ -176,7 +184,7 @@ async def get_player_summary( "
Depending on filters, data from both competitive and quickplay, " "and/or pc and console will be merged." "
Default behaviour : all gamemodes and platforms are taken in account." - f"
**Cache TTL : {GetPlayerStatsSummaryController.get_human_readable_timeout()}.**" + f"
**Cache TTL : {settings.career_path_cache_timeout} seconds.**" ), operation_id="get_player_stats_summary", response_model=PlayerStatsSummary, @@ -184,6 +192,7 @@ async def get_player_summary( async def get_player_stats_summary( request: Request, response: Response, + service: PlayerServiceDep, commons: CommonsPlayerDep, gamemode: Annotated[ PlayerGamemode | None, @@ -208,11 +217,17 @@ async def get_player_stats_summary( ), ] = None, ) -> Any: - return await GetPlayerStatsSummaryController(request, response).process_request( - player_id=commons.get("player_id"), - platform=platform, + cache_key = request.url.path + ( + f"?{request.query_params}" if request.query_params else "" + ) + data, is_stale, age = await service.get_player_stats_summary( + player_id=commons["player_id"], gamemode=gamemode, + platform=platform, + cache_key=cache_key, ) + apply_swr_headers(response, settings.career_path_cache_timeout, is_stale, age) + return data @router.get( @@ -226,7 +241,7 @@ async def get_player_stats_summary( "(combat, game, best, hero specific, average, etc.). Filter them on " "specific platform and gamemode (mandatory). You can even retrieve " "data about a specific hero of your choice." - f"
**Cache TTL : {GetPlayerCareerStatsController.get_human_readable_timeout()}.**" + f"
**Cache TTL : {settings.career_path_cache_timeout} seconds.**" ), operation_id="get_player_career_stats", response_model=PlayerCareerStats, @@ -234,12 +249,21 @@ async def get_player_stats_summary( async def get_player_career_stats( request: Request, response: Response, + service: PlayerServiceDep, commons: CommonsPlayerCareerDep, ) -> Any: - return await GetPlayerCareerStatsController(request, response).process_request( - stats=True, - **commons, + cache_key = request.url.path + ( + f"?{request.query_params}" if request.query_params else "" ) + data, is_stale, age = await service.get_player_career_stats( + player_id=commons["player_id"], + gamemode=commons.get("gamemode"), + platform=commons.get("platform"), + hero=commons.get("hero"), + cache_key=cache_key, + ) + apply_swr_headers(response, settings.career_path_cache_timeout, is_stale, age) + return data @router.get( @@ -251,7 +275,7 @@ async def get_player_career_stats( description=( "This endpoint exposes the same data as the previous one, except it also " "exposes labels of the categories and statistics." - f"
**Cache TTL : {GetPlayerCareerController.get_human_readable_timeout()}.**" + f"
**Cache TTL : {settings.career_path_cache_timeout} seconds.**" ), operation_id="get_player_stats", response_model=CareerStats, @@ -259,12 +283,21 @@ async def get_player_career_stats( async def get_player_stats( request: Request, response: Response, + service: PlayerServiceDep, commons: CommonsPlayerCareerDep, ) -> Any: - return await GetPlayerCareerController(request, response).process_request( - stats=True, - **commons, + cache_key = request.url.path + ( + f"?{request.query_params}" if request.query_params else "" + ) + data, is_stale, age = await service.get_player_stats( + player_id=commons["player_id"], + gamemode=commons.get("gamemode"), + platform=commons.get("platform"), + hero=commons.get("hero"), + cache_key=cache_key, ) + apply_swr_headers(response, settings.career_path_cache_timeout, is_stale, age) + return data @router.get( @@ -274,7 +307,7 @@ async def get_player_stats( summary="Get all player data", description=( "Get all player data : summary and statistics with labels." - f"
**Cache TTL : {GetPlayerCareerController.get_human_readable_timeout()}.**" + f"
**Cache TTL : {settings.career_path_cache_timeout} seconds.**" ), operation_id="get_player_career", response_model=Player, @@ -282,6 +315,7 @@ async def get_player_stats( async def get_player_career( request: Request, response: Response, + service: PlayerServiceDep, commons: CommonsPlayerDep, gamemode: Annotated[ PlayerGamemode | None, @@ -300,8 +334,14 @@ async def get_player_career( ), ] = None, ) -> Any: - return await GetPlayerCareerController(request, response).process_request( - player_id=commons.get("player_id"), + cache_key = request.url.path + ( + f"?{request.query_params}" if request.query_params else "" + ) + data, is_stale, age = await service.get_player_career( + player_id=commons["player_id"], gamemode=gamemode, platform=platform, + cache_key=cache_key, ) + apply_swr_headers(response, settings.career_path_cache_timeout, is_stale, age) + return data diff --git a/app/api/routers/roles.py b/app/api/routers/roles.py index 332e5c0b..6cb3d729 100644 --- a/app/api/routers/roles.py +++ b/app/api/routers/roles.py @@ -4,9 +4,10 @@ from fastapi import APIRouter, Query, Request, Response +from app.api.dependencies import RoleServiceDep +from app.config import settings from app.enums import Locale, RouteTag -from app.helpers import routes_responses -from app.roles.controllers.list_roles_controller import ListRolesController +from app.helpers import apply_swr_headers, routes_responses from app.roles.models import RoleDetail router = APIRouter() @@ -19,7 +20,7 @@ summary="Get a list of roles", description=( "Get a list of available Overwatch roles." - f"
**Cache TTL : {ListRolesController.get_human_readable_timeout()}.**" + f"
**Cache TTL : {settings.heroes_path_cache_timeout} seconds.**" ), operation_id="list_roles", response_model=list[RoleDetail], @@ -27,8 +28,14 @@ async def list_roles( request: Request, response: Response, + service: RoleServiceDep, locale: Annotated[ Locale, Query(title="Locale to be displayed") ] = Locale.ENGLISH_US, ) -> Any: - return await ListRolesController(request, response).process_request(locale=locale) + cache_key = request.url.path + ( + f"?{request.query_params}" if request.query_params else "" + ) + data, is_stale, age = await service.list_roles(locale=locale, cache_key=cache_key) + apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale, age) + return data diff --git a/app/config.py b/app/config.py index ec45d4b0..388f6f56 100644 --- a/app/config.py +++ b/app/config.py @@ -133,14 +133,6 @@ class Settings(BaseSettings): # Used by nginx as main API cache. api_cache_key_prefix: str = "api-cache" - # Prefix for keys in Player Cache (Valkey). Used by player classes - # in order to avoid parsing data which has already been parsed. - player_cache_key_prefix: str = "player-cache" - - # Cache TTL for Player Cache. Whenever a key is accessed, its TTL is reset. - # It will only expires if not accessed during TTL time. - player_cache_timeout: int = 259200 - # Cache TTL for heroes list data (seconds) heroes_path_cache_timeout: int = 86400 @@ -159,6 +151,21 @@ class Settings(BaseSettings): # Cache TTL for hero stats data (seconds) hero_stats_cache_timeout: int = 3600 + ############ + # SWR STALENESS THRESHOLDS + ############ + + # Age (seconds) after which static data is considered stale and triggers + # a background refresh while still serving the cached response. + heroes_staleness_threshold: int = 86400 # 24 hours + maps_staleness_threshold: int = 86400 + gamemodes_staleness_threshold: int = 86400 + roles_staleness_threshold: int = 86400 + hero_stats_staleness_threshold: int = 3600 # 1 hour (same as cache TTL) + + # Age (seconds) after which a player profile is considered stale. + player_staleness_threshold: int = 1800 # 30 min + ############ # UNKNOWN PLAYERS SYSTEM ############ diff --git a/app/controllers.py b/app/controllers.py deleted file mode 100644 index fe5ce181..00000000 --- a/app/controllers.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Abstract API Controller module""" - -import json -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING - -from fastapi import HTTPException, Request, Response - -from .adapters.storage import SQLiteStorage -from .cache_manager import CacheManager -from .config import settings -from .exceptions import ParserBlizzardError, ParserParsingError -from .helpers import get_human_readable_duration, overfast_internal_error -from .monitoring.metrics import storage_write_errors_total -from .overfast_logger import logger - -if TYPE_CHECKING: - from .domain.ports import CachePort, StoragePort - - -class AbstractController(ABC): - """Generic Abstract API Controller, containing attributes structure and methods - in order to quickly be able to create concrete controllers. A controller can - be associated with several parsers (one parser = one Blizzard page parsing). - The API Cache system is handled here. - """ - - # Cache manager for Valkey operations (Protocol type for dependency inversion) - cache_manager: CachePort = CacheManager() - - # Storage adapter for persistent data (Protocol type for dependency inversion) - storage: StoragePort = SQLiteStorage() - - def __init__(self, request: Request, response: Response): - self.cache_key = CacheManager.get_cache_key_from_request(request) - self.response = response - - @property - @classmethod - @abstractmethod - def parser_classes(cls) -> list[type]: - """Parser classes used for parsing the Blizzard page retrieved with this controller""" - - @property - @classmethod - @abstractmethod - def timeout(cls) -> int: - """Timeout used for API Cache storage for this specific controller""" - - @classmethod - def get_human_readable_timeout(cls) -> str: - return get_human_readable_duration(cls.timeout) - - async def update_static_cache( - self, data: dict | list, storage_key: str, data_type: str = "json" - ) -> None: - """ - Dual-write static data to both Valkey cache and SQLite storage. - - Args: - data: Data to cache (will be JSON-serialized) - storage_key: Key for SQLite storage (e.g., "heroes:en-us") - data_type: Type of data ("json" or "html") - """ - # Update API Cache (Valkey) - async - await self.cache_manager.update_api_cache(self.cache_key, data, self.timeout) - - # Update persistent storage (SQLite) - Phase 3 dual-write - try: - json_data = json.dumps(data, separators=(",", ":")) - await self.storage.set_static_data( - key=storage_key, data=json_data, data_type=data_type - ) - except OSError as error: - # Disk/file I/O errors - logger.warning( - f"Storage write failed (disk error) for {storage_key}: {error}" - ) - self._track_storage_error("disk_error") - except RuntimeError as error: - # Compression/serialization errors - logger.warning( - f"Storage write failed (compression error) for {storage_key}: {error}" - ) - self._track_storage_error("compression_error") - except Exception as error: # noqa: BLE001 - # Unexpected errors - logger.error(f"Storage write failed (unknown) for {storage_key}: {error}") - self._track_storage_error("unknown") - - @staticmethod - def _track_storage_error(error_type: str) -> None: - """Track storage write errors in Prometheus if enabled""" - if settings.prometheus_enabled: - storage_write_errors_total.labels(error_type=error_type).inc() - - async def process_request(self, **kwargs) -> dict | list: - """Main method used to process the request from user and return final data. Raises - an HTTPException in case of error when retrieving or parsing data. - - The main steps are : - - Instanciate the dedicated parser classes in order to retrieve Blizzard data - - Depending on the parser, an intermediary cache can be used in the process - - Filter the data using kwargs parameters, then merge the data from parsers - - Update related API Cache and return the final data - """ - - # Instance parsers and request data - parsers_data = [] - for parser_class in self.parser_classes: - parser = parser_class(**kwargs) - - try: - await parser.parse() - except ParserBlizzardError as error: - raise HTTPException( - status_code=error.status_code, - detail=error.message, - ) from error - except ParserParsingError as error: - raise overfast_internal_error(parser.blizzard_url, error) from error - - # Filter the data to obtain final parser data - logger.info("Filtering the data using query...") - parsers_data.append(parser.filter_request_using_query(**kwargs)) - - # Merge parsers data together - computed_data = self.merge_parsers_data(parsers_data, **kwargs) - - # Update API Cache - async - await self.cache_manager.update_api_cache( - self.cache_key, computed_data, self.timeout - ) - - # Ensure response headers contains Cache TTL - self.response.headers[settings.cache_ttl_header] = str(self.timeout) - - logger.info("Done ! Returning filtered data...") - return computed_data - - def merge_parsers_data(self, parsers_data: list[dict | list], **_) -> dict | list: - """Merge parsers data together. It depends on the given route and datas, - and needs to be overriden in case a given Controller is associated - with several Parsers. - """ - return parsers_data[0] diff --git a/app/domain/ports/cache.py b/app/domain/ports/cache.py index fb866ede..4ab43065 100644 --- a/app/domain/ports/cache.py +++ b/app/domain/ports/cache.py @@ -56,24 +56,6 @@ async def update_api_cache( """ ... - async def get_player_cache(self, player_id: str) -> dict | list | None: - """ - Get the Player Cache value associated with a given player ID. - - Returns decompressed JSON data or None if not found. - Resets the TTL on successful retrieval. - """ - ... - - async def update_player_cache(self, player_id: str, value: dict) -> None: - """ - Update or set a Player Cache value. - - Value is JSON-serialized and compressed before storage. - Uses configured player cache TTL. - """ - ... - # Rate limiting methods async def is_being_rate_limited(self) -> bool: """Check if Blizzard rate limit is currently active""" diff --git a/app/domain/services/__init__.py b/app/domain/services/__init__.py index e69de29b..cc79ccac 100644 --- a/app/domain/services/__init__.py +++ b/app/domain/services/__init__.py @@ -0,0 +1,17 @@ +"""Domain services — SWR orchestration layer""" + +from .base_service import BaseService +from .gamemode_service import GamemodeService +from .hero_service import HeroService +from .map_service import MapService +from .player_service import PlayerService +from .role_service import RoleService + +__all__ = [ + "BaseService", + "GamemodeService", + "HeroService", + "MapService", + "PlayerService", + "RoleService", +] diff --git a/app/domain/services/base_service.py b/app/domain/services/base_service.py new file mode 100644 index 00000000..4e895933 --- /dev/null +++ b/app/domain/services/base_service.py @@ -0,0 +1,171 @@ +"""Base service providing Stale-While-Revalidate orchestration for all domain services""" + +import json +import time +from typing import TYPE_CHECKING, Any + +from app.config import settings +from app.overfast_logger import logger + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + + from app.domain.ports import ( + BlizzardClientPort, + CachePort, + StoragePort, + TaskQueuePort, + ) + + +class BaseService: + """Base service providing Stale-While-Revalidate (SWR) orchestration. + + All domain services inherit from this class to get SWR behaviour: + 1. Check SQLite persistent storage (cache key). + 2. If found and fresh (age < staleness_threshold) → return data, update Valkey. + 3. If found but stale → return data, update Valkey, trigger background refresh. + 4. If not found (cold start) → fetch from Blizzard, store in both, return data. + + Note: Valkey API-cache check happens at the Nginx/Lua level before FastAPI is reached, + so this class only handles the SQLite → Blizzard fallback path. + """ + + def __init__( + self, + cache: CachePort, + storage: StoragePort, + blizzard_client: BlizzardClientPort, + task_queue: TaskQueuePort, + ) -> None: + self.cache = cache + self.storage = storage + self.blizzard_client = blizzard_client + self.task_queue = task_queue + + # ------------------------------------------------------------------ + # SWR core — used by static-data services (heroes, maps, …) + # ------------------------------------------------------------------ + + async def _get_or_fetch_static( + self, + *, + storage_key: str, + fetcher: Callable[[], Awaitable[Any]], + cache_key: str, + cache_ttl: int, + staleness_threshold: int, + entity_type: str, + ) -> tuple[Any, bool, int]: + """SWR orchestration for static data backed by SQLite. + + Args: + storage_key: Key in the ``static_data`` SQLite table. + fetcher: Async callable that fetches and parses fresh data from Blizzard. + Must return the final *list* or *dict* to store as JSON. + cache_key: Valkey API-cache key to update after serving data. + cache_ttl: TTL in seconds for the Valkey API-cache entry. + staleness_threshold: Seconds after which stored data is considered stale. + entity_type: Human-readable label used in metrics / logs. + + Returns: + ``(data, is_stale, age_seconds)`` tuple. + ``age_seconds`` is 0 on a cold-start fetch. + """ + stored = await self.storage.get_static_data(storage_key) + + if stored: + data = json.loads(stored["data"]) + age = int(time.time()) - stored["updated_at"] + is_stale = age >= staleness_threshold + + if is_stale: + logger.info( + f"[SWR] {entity_type} data is stale (age={age}s, " + f"threshold={staleness_threshold}s) — serving stale + triggering refresh" + ) + await self._enqueue_refresh(entity_type, storage_key) + self._track_stale_response(entity_type) + else: + logger.info( + f"[SWR] {entity_type} data is fresh (age={age}s) — serving from SQLite" + ) + + self._track_storage_hit("hit") + await self._update_api_cache(cache_key, data, cache_ttl) + return data, is_stale, age + + # Cold start — fetch synchronously from Blizzard + logger.info(f"[SWR] {entity_type} not in SQLite — fetching from Blizzard") + self._track_storage_hit("miss") + data = await fetcher() + await self._persist_static(storage_key, data, cache_key, cache_ttl) + return data, False, 0 + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + async def _persist_static( + self, storage_key: str, data: Any, cache_key: str, cache_ttl: int + ) -> None: + """Write static data to both SQLite and Valkey API cache.""" + try: + await self.storage.set_static_data( + key=storage_key, + data=json.dumps(data, separators=(",", ":")), + data_type="json", + ) + except Exception as exc: # noqa: BLE001 + logger.warning(f"[SWR] SQLite write failed for {storage_key}: {exc}") + + await self._update_api_cache(cache_key, data, cache_ttl) + + async def _update_api_cache( + self, cache_key: str, data: Any, cache_ttl: int + ) -> None: + """Write data to Valkey API cache, swallowing errors.""" + try: + await self.cache.update_api_cache(cache_key, data, cache_ttl) + except Exception as exc: # noqa: BLE001 + logger.warning(f"[SWR] Valkey write failed for {cache_key}: {exc}") + + async def _enqueue_refresh(self, entity_type: str, entity_id: str) -> None: + """Enqueue a background refresh task (deduplication via job_id).""" + job_id = f"refresh:{entity_type}:{entity_id}" + try: + already_running = await self.task_queue.is_job_pending_or_running(job_id) + if not already_running: + await self.task_queue.enqueue( + f"refresh_{entity_type}", + entity_id, + job_id=job_id, + ) + if settings.prometheus_enabled: + from app.monitoring.metrics import ( + background_refresh_triggered_total, + ) + + background_refresh_triggered_total.labels( + entity_type=entity_type + ).inc() + except Exception as exc: # noqa: BLE001 + logger.warning( + f"[SWR] Failed to enqueue refresh for {entity_type}/{entity_id}: {exc}" + ) + + @staticmethod + def _track_storage_hit(result: str) -> None: + if settings.prometheus_enabled: + from app.monitoring.metrics import storage_hits_total + + storage_hits_total.labels(result=result).inc() + + @staticmethod + def _track_stale_response(entity_type: str) -> None: + if settings.prometheus_enabled: + from app.monitoring.metrics import stale_responses_total + + stale_responses_total.inc() + # entity_type tracked via background_refresh_triggered_total + _ = entity_type diff --git a/app/domain/services/gamemode_service.py b/app/domain/services/gamemode_service.py new file mode 100644 index 00000000..0d9d60b2 --- /dev/null +++ b/app/domain/services/gamemode_service.py @@ -0,0 +1,32 @@ +"""Gamemode domain service — gamemodes list""" + +from app.adapters.blizzard.parsers.gamemodes import parse_gamemodes_csv +from app.config import settings +from app.domain.services.base_service import BaseService + + +class GamemodeService(BaseService): + """Domain service for gamemode data.""" + + async def list_gamemodes( + self, + cache_key: str, + ) -> tuple[list[dict], bool, int]: + """Return the gamemodes list. + + Returns: + (data, is_stale, age_seconds) + """ + storage_key = "gamemodes:all" + + async def _fetch() -> list[dict]: + return parse_gamemodes_csv() + + return await self._get_or_fetch_static( + storage_key=storage_key, + fetcher=_fetch, + cache_key=cache_key, + cache_ttl=settings.csv_cache_timeout, + staleness_threshold=settings.gamemodes_staleness_threshold, + entity_type="gamemodes", + ) diff --git a/app/domain/services/hero_service.py b/app/domain/services/hero_service.py new file mode 100644 index 00000000..fa9d5792 --- /dev/null +++ b/app/domain/services/hero_service.py @@ -0,0 +1,248 @@ +"""Hero domain service — heroes list, hero detail, hero stats""" + +from typing import TYPE_CHECKING, Any + +from fastapi import HTTPException + +from app.adapters.blizzard.parsers.hero import parse_hero +from app.adapters.blizzard.parsers.hero_stats_summary import parse_hero_stats_summary +from app.adapters.blizzard.parsers.heroes import ( + fetch_heroes_html, + filter_heroes, + parse_heroes_html, +) +from app.adapters.blizzard.parsers.heroes_stats import parse_heroes_stats +from app.config import settings +from app.domain.services.base_service import BaseService +from app.enums import Locale +from app.exceptions import ParserBlizzardError, ParserParsingError +from app.helpers import overfast_internal_error +from app.overfast_logger import logger + +if TYPE_CHECKING: + from app.heroes.enums import HeroGamemode + from app.maps.enums import MapKey + from app.players.enums import ( + CompetitiveDivisionFilter, + PlayerGamemode, + PlayerPlatform, + PlayerRegion, + ) + from app.roles.enums import Role + + +class HeroService(BaseService): + """Domain service for hero data: list, detail, and usage statistics.""" + + # ------------------------------------------------------------------ + # Heroes list (GET /heroes) + # ------------------------------------------------------------------ + + async def list_heroes( + self, + locale: Locale, + role: Role | None, + gamemode: HeroGamemode | None, + cache_key: str, + ) -> tuple[list[dict], bool, int]: + """Return the heroes list (with optional role/gamemode filters). + + SWR: stores the *full* (unfiltered) heroes list per locale in SQLite; + filters are applied after retrieval so all filter combinations benefit + from the same cache entry. + + Returns: + (data, is_stale, age_seconds) + """ + storage_key = f"heroes:{locale}" + + async def _fetch() -> list[dict]: + html = await fetch_heroes_html(self.blizzard_client, locale) # ty: ignore[invalid-argument-type] + return parse_heroes_html(html) + + data, is_stale, age = await self._get_or_fetch_static( + storage_key=storage_key, + fetcher=_fetch, + cache_key=cache_key, + cache_ttl=settings.heroes_path_cache_timeout, + staleness_threshold=settings.heroes_staleness_threshold, + entity_type="heroes", + ) + + return filter_heroes(data, role, gamemode), is_stale, age + + # ------------------------------------------------------------------ + # Single hero (GET /heroes/{hero_key}) + # ------------------------------------------------------------------ + + async def get_hero( + self, + hero_key: str, + locale: Locale, + cache_key: str, + ) -> tuple[dict, bool, int]: + """Return full hero details merged with portrait and hitpoints. + + Single-hero data is not stored persistently (it is derived from three + sources that each have their own caching); the Valkey API cache is still + updated after each fetch. + + Returns: + (data, is_stale, age_seconds) + """ + try: + hero_data = await parse_hero(self.blizzard_client, hero_key, locale) # ty: ignore[invalid-argument-type] + heroes_html = await fetch_heroes_html(self.blizzard_client, locale) # ty: ignore[invalid-argument-type] + heroes_list = parse_heroes_html(heroes_html) + heroes_stats = parse_heroes_stats() + data = _merge_hero_data(hero_data, heroes_list, heroes_stats, hero_key) + except ParserBlizzardError as exc: + raise HTTPException( + status_code=exc.status_code, detail=exc.message + ) from exc + except ParserParsingError as exc: + blizzard_url = ( + f"{settings.blizzard_host}/{locale}{settings.heroes_path}{hero_key}/" + ) + raise overfast_internal_error(blizzard_url, exc) from exc + + await self._update_api_cache(cache_key, data, settings.hero_path_cache_timeout) + return data, False, 0 + + # ------------------------------------------------------------------ + # Hero stats summary (GET /heroes/stats) + # ------------------------------------------------------------------ + + async def get_hero_stats( + self, + platform: PlayerPlatform, + gamemode: PlayerGamemode, + region: PlayerRegion, + role: Role | None, + map_filter: MapKey | None, # ty: ignore[invalid-type-form] + competitive_division: CompetitiveDivisionFilter | None, # ty: ignore[invalid-type-form] + order_by: str, + cache_key: str, + ) -> tuple[list[dict], bool, int]: + """Return hero usage statistics with SWR. + + Returns: + (data, is_stale, age_seconds) + """ + storage_key = _build_hero_stats_storage_key( + platform, gamemode, region, map_filter, competitive_division + ) + + async def _fetch() -> list[dict]: + try: + return await parse_hero_stats_summary( + self.blizzard_client, # ty: ignore[invalid-argument-type] + platform=platform, + gamemode=gamemode, + region=region, + role=role, + map_filter=map_filter, + competitive_division=competitive_division, + order_by=order_by, + ) + except ParserBlizzardError as exc: + raise HTTPException( + status_code=exc.status_code, detail=exc.message + ) from exc + + data, is_stale, age = await self._get_or_fetch_static( + storage_key=storage_key, + fetcher=_fetch, + cache_key=cache_key, + cache_ttl=settings.hero_stats_cache_timeout, + staleness_threshold=settings.hero_stats_staleness_threshold, + entity_type="hero_stats", + ) + + # Apply post-fetch filters (role, order_by) on stale data from SQLite + if is_stale and age > 0: + data = _filter_hero_stats(data, role, order_by) + + return data, is_stale, age + + +# --------------------------------------------------------------------------- +# Module-level helpers (kept accessible for tests) +# --------------------------------------------------------------------------- + + +def _merge_hero_data( + hero_data: dict, + heroes_list: list[dict], + heroes_stats: dict, + hero_key: str, +) -> dict: + """Merge data from hero details, heroes list, and heroes stats.""" + # Portrait from heroes list + try: + portrait_value = next( + hero["portrait"] for hero in heroes_list if hero["key"] == hero_key + ) + except StopIteration: + portrait_value = None + else: + hero_data = dict_insert_value_before_key( + hero_data, "role", "portrait", portrait_value + ) + + # Hitpoints from stats CSV + try: + hitpoints = heroes_stats[hero_key]["hitpoints"] + except KeyError: + hitpoints = None + else: + hero_data = dict_insert_value_before_key( + hero_data, "abilities", "hitpoints", hitpoints + ) + + return hero_data + + +def dict_insert_value_before_key( + data: dict, + key: str, + new_key: str, + new_value: Any, +) -> dict: + """Insert ``new_key: new_value`` before ``key`` in ``data``.""" + if key not in data: + raise KeyError + pos = list(data.keys()).index(key) + items = list(data.items()) + items.insert(pos, (new_key, new_value)) + return dict(items) + + +def _build_hero_stats_storage_key( + platform: Any, + gamemode: Any, + region: Any, + map_filter: Any, + competitive_division: Any, +) -> str: + map_val = map_filter.value if map_filter else "all-maps" + tier_val = competitive_division.value if competitive_division else "null" + return ( + f"hero_stats:{platform.value}:{gamemode.value}:{region.value}" + f":{map_val}:{tier_val}" + ) + + +def _filter_hero_stats( + data: list[dict], + role: Any, + order_by: str, +) -> list[dict]: + """Re-apply role filter and ordering when serving stale data from SQLite.""" + logger.debug("[SWR] Re-applying hero_stats filters on stale data") + if role: + data = [h for h in data if h.get("role") == role.value] + + # Re-sort according to order_by + field, direction = order_by.split(":") + return sorted(data, key=lambda h: h.get(field, ""), reverse=(direction == "desc")) diff --git a/app/domain/services/map_service.py b/app/domain/services/map_service.py new file mode 100644 index 00000000..2e3c205d --- /dev/null +++ b/app/domain/services/map_service.py @@ -0,0 +1,42 @@ +"""Map domain service — maps list""" + +from app.adapters.blizzard.parsers.maps import parse_maps_csv +from app.config import settings +from app.domain.services.base_service import BaseService + + +class MapService(BaseService): + """Domain service for maps data.""" + + async def list_maps( + self, + gamemode: str | None, + cache_key: str, + ) -> tuple[list[dict], bool, int]: + """Return the maps list (with optional gamemode filter). + + Stores the full (unfiltered) maps list in SQLite; gamemode filter is + applied after retrieval. + + Returns: + (data, is_stale, age_seconds) + """ + storage_key = "maps:all" + + async def _fetch() -> list[dict]: + return parse_maps_csv() + + data, is_stale, age = await self._get_or_fetch_static( + storage_key=storage_key, + fetcher=_fetch, + cache_key=cache_key, + cache_ttl=settings.csv_cache_timeout, + staleness_threshold=settings.maps_staleness_threshold, + entity_type="maps", + ) + + if gamemode: + gamemode_val = gamemode.value if hasattr(gamemode, "value") else gamemode + data = [m for m in data if gamemode_val in m.get("gamemodes", [])] + + return data, is_stale, age diff --git a/app/domain/services/player_service.py b/app/domain/services/player_service.py new file mode 100644 index 00000000..11f55f10 --- /dev/null +++ b/app/domain/services/player_service.py @@ -0,0 +1,684 @@ +"""Player domain service — career, stats, summary, and search""" + +import time +from typing import TYPE_CHECKING, cast + +from fastapi import HTTPException, status + +from app.adapters.blizzard.parsers.player_career_stats import ( + parse_player_career_stats_from_html, +) +from app.adapters.blizzard.parsers.player_profile import ( + extract_name_from_profile_html, + fetch_player_html, + filter_all_stats_data, + filter_stats_by_query, + parse_player_profile_html, +) +from app.adapters.blizzard.parsers.player_search import parse_player_search +from app.adapters.blizzard.parsers.player_stats import ( + parse_player_stats_summary_from_html, +) +from app.adapters.blizzard.parsers.player_summary import ( + fetch_player_summary_json, + parse_player_summary_json, +) +from app.adapters.blizzard.parsers.utils import is_blizzard_id +from app.config import settings +from app.domain.services.base_service import BaseService +from app.exceptions import ParserBlizzardError, ParserParsingError +from app.helpers import overfast_internal_error +from app.monitoring.metrics import ( + sqlite_battletag_lookup_total, + sqlite_cache_hit_total, + storage_hits_total, +) +from app.overfast_logger import logger + +if TYPE_CHECKING: + from app.players.enums import ( + HeroKeyCareerFilter, + PlayerGamemode, + PlayerPlatform, + ) + + +class PlayerService(BaseService): + """Domain service for all player-related endpoints. + + Wraps identity resolution, SQLite profile caching, and SWR staleness logic + that was previously scattered across multiple controllers. + """ + + # ------------------------------------------------------------------ + # Search (Valkey-only, no SQLite, no SWR) + # ------------------------------------------------------------------ + + async def search_players( + self, + name: str, + order_by: str, + offset: int, + limit: int, + cache_key: str, + ) -> dict: + """Search for players by name — Valkey-only cache, no persistent storage.""" + try: + data = await parse_player_search( + self.blizzard_client, + name=name, + order_by=order_by, + offset=offset, + limit=limit, + ) + except ParserParsingError as exc: + search_name = name.split("-", 1)[0] + blizzard_url = ( + f"{settings.blizzard_host}{settings.search_account_path}/{search_name}/" + ) + raise overfast_internal_error(blizzard_url, exc) from exc + + await self._update_api_cache( + cache_key, data, settings.search_account_path_cache_timeout + ) + return data + + # ------------------------------------------------------------------ + # Player summary (GET /players/{player_id}/summary) + # ------------------------------------------------------------------ + + async def get_player_summary( + self, + player_id: str, + cache_key: str, + ) -> tuple[dict, bool, int]: + """Return player summary (name, avatar, competitive ranks, …). + + Returns: + (data, is_stale, age_seconds) + """ + return await self._execute_player_request( + player_id=player_id, + cache_key=cache_key, + summary=True, + stats=False, + ) + + # ------------------------------------------------------------------ + # Player career (GET /players/{player_id}) + # ------------------------------------------------------------------ + + async def get_player_career( + self, + player_id: str, + gamemode: PlayerGamemode | None, + platform: PlayerPlatform | None, + cache_key: str, + ) -> tuple[dict, bool, int]: + """Return full player data: summary + stats. + + Returns: + (data, is_stale, age_seconds) + """ + return await self._execute_player_request( + player_id=player_id, + cache_key=cache_key, + summary=False, + stats=False, + gamemode=gamemode, + platform=platform, + ) + + # ------------------------------------------------------------------ + # Player stats (GET /players/{player_id}/stats) + # ------------------------------------------------------------------ + + async def get_player_stats( + self, + player_id: str, + gamemode: PlayerGamemode | None, + platform: PlayerPlatform | None, + hero: HeroKeyCareerFilter | None, # ty: ignore[invalid-type-form] + cache_key: str, + ) -> tuple[dict, bool, int]: + """Return player stats with category labels. + + Returns: + (data, is_stale, age_seconds) + """ + return await self._execute_player_request( + player_id=player_id, + cache_key=cache_key, + summary=False, + stats=True, + gamemode=gamemode, + platform=platform, + hero=hero, + ) + + # ------------------------------------------------------------------ + # Player stats summary (GET /players/{player_id}/stats/summary) + # ------------------------------------------------------------------ + + async def get_player_stats_summary( + self, + player_id: str, + gamemode: PlayerGamemode | None, + platform: PlayerPlatform | None, + cache_key: str, + ) -> tuple[dict, bool, int]: + """Return player statistics summary (winrate, kda, …). + + Returns: + (data, is_stale, age_seconds) + """ + cache_key_player = player_id + battletag_input: str | None = None + player_summary: dict = {} + + try: + ( + blizzard_id, + player_summary, + cached_html, + battletag_input, + ) = await self._resolve_player_identity(player_id) + cache_key_player = blizzard_id or player_id + + data = await self._fetch_stats_summary_with_cache( + blizzard_id=cache_key_player, + player_summary=player_summary, + gamemode=gamemode, + platform=platform, + battletag_input=battletag_input, + cached_html=cached_html, + ) + except Exception as exc: # noqa: BLE001 + await self._handle_player_exceptions( + exc, cache_key_player, battletag_input, player_summary + ) + + is_stale, age = self._check_player_staleness(cache_key_player) + await self._update_api_cache( + cache_key, data, settings.career_path_cache_timeout + ) + return data, is_stale, age + + # ------------------------------------------------------------------ + # Player career stats (GET /players/{player_id}/stats/career) + # ------------------------------------------------------------------ + + async def get_player_career_stats( + self, + player_id: str, + gamemode: PlayerGamemode | None, + platform: PlayerPlatform | None, + hero: HeroKeyCareerFilter | None, # ty: ignore[invalid-type-form] + cache_key: str, + ) -> tuple[dict, bool, int]: + """Return player career stats (no labels). + + Returns: + (data, is_stale, age_seconds) + """ + cache_key_player = player_id + battletag_input: str | None = None + player_summary: dict = {} + + try: + ( + blizzard_id, + player_summary, + cached_html, + battletag_input, + ) = await self._resolve_player_identity(player_id) + cache_key_player = blizzard_id or player_id + + data = await self._fetch_career_stats_with_cache( + blizzard_id=cache_key_player, + player_summary=player_summary, + platform=platform, + gamemode=gamemode, + hero=hero, + battletag_input=battletag_input, + cached_html=cached_html, + ) + except Exception as exc: # noqa: BLE001 + await self._handle_player_exceptions( + exc, cache_key_player, battletag_input, player_summary + ) + + is_stale, age = self._check_player_staleness(cache_key_player) + await self._update_api_cache( + cache_key, data, settings.career_path_cache_timeout + ) + return data, is_stale, age + + # ------------------------------------------------------------------ + # Core request execution (career + summary) + # ------------------------------------------------------------------ + + async def _execute_player_request( + self, + player_id: str, + cache_key: str, + summary: bool, + stats: bool, + gamemode: PlayerGamemode | None = None, + platform: PlayerPlatform | None = None, + hero: HeroKeyCareerFilter | None = None, # ty: ignore[invalid-type-form] + ) -> tuple[dict, bool, int]: + """Shared execution path for summary, career, and stats endpoints.""" + cache_key_player = player_id + battletag_input: str | None = None + player_summary: dict = {} + + try: + ( + blizzard_id, + player_summary, + cached_html, + battletag_input, + ) = await self._resolve_player_identity(player_id) + cache_key_player = blizzard_id or player_id + + _html, profile_data = await self._fetch_profile_with_cache( + blizzard_id=cache_key_player, + player_summary=player_summary, + battletag_input=battletag_input, + cached_html=cached_html, + ) + + data = self._filter_profile_data( + profile_data, + summary_filter=summary, + stats_filter=stats, + platform_filter=platform, + gamemode_filter=gamemode, + hero_filter=hero, + ) + except Exception as exc: # noqa: BLE001 + await self._handle_player_exceptions( + exc, cache_key_player, battletag_input, player_summary + ) + + is_stale, age = self._check_player_staleness(cache_key_player) + await self._update_api_cache( + cache_key, data, settings.career_path_cache_timeout + ) + return data, is_stale, age + + # ------------------------------------------------------------------ + # Profile caching helpers + # ------------------------------------------------------------------ + + async def get_player_profile_cache(self, player_id: str) -> dict | None: + """Get player profile from SQLite storage.""" + profile = await self.storage.get_player_profile(player_id) + if not profile: + if settings.prometheus_enabled: + sqlite_cache_hit_total.labels( + table="player_profiles", result="miss" + ).inc() + storage_hits_total.labels(result="miss").inc() + return None + + if settings.prometheus_enabled: + sqlite_cache_hit_total.labels(table="player_profiles", result="hit").inc() + storage_hits_total.labels(result="hit").inc() + + return { + "profile": profile["html"], + "summary": profile["summary"], + "battletag": profile.get("battletag"), + "name": profile.get("name"), + "updated_at": profile.get("updated_at", 0), + } + + async def update_player_profile_cache( + self, + player_id: str, + player_summary: dict, + html: str, + battletag: str | None = None, + name: str | None = None, + ) -> None: + """Store player profile in SQLite.""" + await self.storage.set_player_profile( + player_id=player_id, + html=html, + summary=player_summary if player_summary else None, + battletag=battletag, + name=name, + ) + + def _check_player_staleness(self, _player_id: str) -> tuple[bool, int]: + """Return (is_stale, age_seconds) based purely on time — best effort.""" + # This is a lightweight check; the actual updated_at would require + # another async storage lookup. We return (False, 0) here and let the + # background refresh (Phase 5) handle true staleness. + # Phase 5 will update this to return proper staleness info. + return False, 0 + + async def _fetch_profile_with_cache( + self, + blizzard_id: str, + player_summary: dict, + battletag_input: str | None, + cached_html: str | None = None, + ) -> tuple[str, dict]: + """Fetch player profile with SQLite cache.""" + if cached_html: + logger.info("Using cached HTML from identity resolution") + profile_data = parse_player_profile_html(cached_html, player_summary) + name = extract_name_from_profile_html(cached_html) or player_summary.get( + "name" + ) + await self.update_player_profile_cache( + blizzard_id, player_summary, cached_html, battletag_input, name + ) + return cached_html, profile_data + + player_cache = await self.get_player_profile_cache(blizzard_id) + + if ( + player_cache is not None + and player_summary + and player_cache["summary"]["lastUpdated"] == player_summary["lastUpdated"] + ): + logger.info("Player profile cache found and up-to-date") + html = cast("str", player_cache["profile"]) + profile_data = parse_player_profile_html(html, player_summary) + + if battletag_input and not player_cache.get("battletag"): + cached_name = player_cache.get("name") + await self.update_player_profile_cache( + blizzard_id, + player_summary, + html, + battletag_input, + cached_name if isinstance(cached_name, str) else None, + ) + return html, profile_data + + logger.info("Player profile cache miss — fetching from Blizzard") + html, _ = await fetch_player_html(self.blizzard_client, blizzard_id) + profile_data = parse_player_profile_html(html, player_summary) + name = extract_name_from_profile_html(html) or player_summary.get("name") + await self.update_player_profile_cache( + blizzard_id, player_summary, html, battletag_input, name + ) + return html, profile_data + + async def _fetch_stats_summary_with_cache( + self, + blizzard_id: str, + player_summary: dict, + gamemode: PlayerGamemode | None, + platform: PlayerPlatform | None, + battletag_input: str | None = None, + cached_html: str | None = None, + ) -> dict: + """Fetch player stats summary with SQLite cache.""" + if cached_html: + return parse_player_stats_summary_from_html( + cached_html, player_summary, gamemode, platform + ) + + if not blizzard_id: + msg = "Unable to resolve player identity" + raise ParserParsingError(msg) + + player_cache = await self.get_player_profile_cache(blizzard_id) + + if ( + player_cache is not None + and player_summary + and player_cache["summary"]["lastUpdated"] == player_summary["lastUpdated"] + ): + html = cast("str", player_cache["profile"]) + return parse_player_stats_summary_from_html( + html, player_summary, gamemode, platform + ) + + html, _ = await fetch_player_html(self.blizzard_client, blizzard_id) + name = extract_name_from_profile_html(html) + data = parse_player_stats_summary_from_html( + html, player_summary, gamemode, platform + ) + await self.update_player_profile_cache( + blizzard_id, player_summary, html, battletag_input, name + ) + return data + + async def _fetch_career_stats_with_cache( + self, + blizzard_id: str, + player_summary: dict, + platform: PlayerPlatform | None, + gamemode: PlayerGamemode | None, + hero: HeroKeyCareerFilter | None, # ty: ignore[invalid-type-form] + battletag_input: str | None = None, + cached_html: str | None = None, + ) -> dict: + """Fetch player career stats with SQLite cache.""" + if cached_html: + return parse_player_career_stats_from_html( + cached_html, player_summary, platform, gamemode, hero + ) + + if not blizzard_id: + msg = "Unable to resolve player identity" + raise ParserParsingError(msg) + + player_cache = await self.get_player_profile_cache(blizzard_id) + + if ( + player_cache is not None + and player_summary + and player_cache["summary"]["lastUpdated"] == player_summary["lastUpdated"] + ): + html = cast("str", player_cache["profile"]) + return parse_player_career_stats_from_html( + html, player_summary, platform, gamemode, hero + ) + + html, _ = await fetch_player_html(self.blizzard_client, blizzard_id) + name = extract_name_from_profile_html(html) + data = parse_player_career_stats_from_html( + html, player_summary, platform, gamemode, hero + ) + await self.update_player_profile_cache( + blizzard_id, player_summary, html, battletag_input, name + ) + return data + + @staticmethod + def _filter_profile_data( + profile_data: dict, + summary_filter: bool, + stats_filter: bool, + platform_filter: PlayerPlatform | None, + gamemode_filter: PlayerGamemode | None, + hero_filter: HeroKeyCareerFilter | None, # ty: ignore[invalid-type-form] + ) -> dict: + if summary_filter: + return profile_data.get("summary") or {} + if stats_filter: + return filter_stats_by_query( + profile_data.get("stats") or {}, + platform_filter, + gamemode_filter, + hero_filter, + ) + return { + "summary": profile_data.get("summary") or {}, + "stats": filter_all_stats_data( + profile_data.get("stats") or {}, + platform_filter, + gamemode_filter, + ), + } + + # ------------------------------------------------------------------ + # Identity resolution + # ------------------------------------------------------------------ + + async def _resolve_player_identity( + self, player_id: str + ) -> tuple[str | None, dict, str | None, str | None]: + """Resolve BattleTag or Blizzard ID to a canonical (blizzard_id, summary, html, battletag).""" + logger.info("Retrieving Player Summary...") + + if is_blizzard_id(player_id): + logger.info("Player ID is a Blizzard ID — attempting reverse enrichment") + player_summary, html = await self._enrich_from_blizzard_id(player_id) + return player_id, player_summary, html, None + + battletag_input = player_id + search_json = await fetch_player_summary_json(self.blizzard_client, player_id) + player_summary = parse_player_summary_json(search_json, player_id) + + if player_summary: + logger.info("Player Summary retrieved!") + blizzard_id = player_summary.get("url") + return blizzard_id, player_summary, None, battletag_input + + logger.info( + "Player not found in search — checking SQLite for cached Blizzard ID" + ) + cached_blizzard_id = await self.storage.get_player_id_by_battletag( + battletag_input + ) + + if cached_blizzard_id: + if settings.prometheus_enabled: + sqlite_battletag_lookup_total.labels(result="hit").inc() + player_summary = parse_player_summary_json( + search_json, player_id, cached_blizzard_id + ) + if player_summary: + return cached_blizzard_id, player_summary, None, battletag_input + elif settings.prometheus_enabled: + sqlite_battletag_lookup_total.labels(result="miss").inc() + + logger.info("No cached mapping — resolving via Blizzard redirect") + html, blizzard_id = await fetch_player_html(self.blizzard_client, player_id) + + if blizzard_id and search_json: + player_summary = parse_player_summary_json( + search_json, player_id, blizzard_id + ) + if player_summary: + return blizzard_id, player_summary, html, battletag_input + + return blizzard_id, {}, html, battletag_input + + async def _enrich_from_blizzard_id( + self, blizzard_id: str + ) -> tuple[dict, str | None]: + """Reverse-enrich: fetch HTML → extract name → search for summary.""" + html, _ = await fetch_player_html(self.blizzard_client, blizzard_id) + if not html: + return {}, None + + try: + player_name = extract_name_from_profile_html(html) + if player_name: + search_json = await fetch_player_summary_json( + self.blizzard_client, player_name + ) + player_summary = parse_player_summary_json( + search_json, player_name, blizzard_id + ) + if player_summary: + return player_summary, html + except Exception as exc: # noqa: BLE001 + logger.warning(f"Reverse enrichment failed: {exc}") + + return {}, html + + # ------------------------------------------------------------------ + # Unknown player tracking + # ------------------------------------------------------------------ + + def _calculate_retry_after(self, check_count: int) -> int: + base = settings.unknown_player_initial_retry + multiplier = settings.unknown_player_retry_multiplier + max_retry = settings.unknown_player_max_retry + retry_after = base * (multiplier ** (check_count - 1)) + return min(int(retry_after), max_retry) + + async def _mark_player_unknown( + self, + blizzard_id: str, + exception: HTTPException, + battletag: str | None = None, + ) -> None: + if not settings.unknown_players_cache_enabled: + return + if exception.status_code != status.HTTP_404_NOT_FOUND: + return + + player_status = await self.cache.get_player_status(blizzard_id) + check_count = player_status["check_count"] + 1 if player_status else 1 + retry_after = self._calculate_retry_after(check_count) + next_check_at = int(time.time()) + retry_after + + await self.cache.set_player_status( + blizzard_id, check_count, retry_after, battletag=battletag + ) + + exception.detail = { + "error": "Player not found", + "retry_after": retry_after, + "next_check_at": next_check_at, + "check_count": check_count, + } + + logger.info( + f"Marked player {blizzard_id} as unknown (check #{check_count}, " + f"retry in {retry_after}s)" + ) + + async def _handle_player_exceptions( + self, + error: Exception, + cache_key: str, + battletag_input: str | None, + player_summary: dict, + ) -> None: + """Translate all player exceptions to HTTPException and always raise.""" + if isinstance(error, ParserBlizzardError): + exc = HTTPException(status_code=error.status_code, detail=error.message) + if error.status_code == status.HTTP_404_NOT_FOUND: + await self._mark_player_unknown( + cache_key, exc, battletag=battletag_input + ) + raise exc from error + + if isinstance(error, ParserParsingError): + if "Could not find main content in HTML" in str(error): + exc = HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Player not found", + ) + await self._mark_player_unknown( + cache_key, exc, battletag=battletag_input + ) + raise exc from error + + blizzard_url = ( + f"{settings.blizzard_host}{settings.career_path}/" + f"{player_summary.get('url', cache_key) if player_summary else cache_key}/" + ) + raise overfast_internal_error(blizzard_url, error) from error + + if isinstance(error, HTTPException): + if error.status_code == status.HTTP_404_NOT_FOUND: + await self._mark_player_unknown( + cache_key, error, battletag=battletag_input + ) + raise error + + raise error diff --git a/app/domain/services/role_service.py b/app/domain/services/role_service.py new file mode 100644 index 00000000..dbb06c2d --- /dev/null +++ b/app/domain/services/role_service.py @@ -0,0 +1,41 @@ +"""Role domain service — roles list""" + +from app.adapters.blizzard.parsers.roles import fetch_roles_html, parse_roles_html +from app.config import settings +from app.domain.services.base_service import BaseService +from app.enums import Locale +from app.exceptions import ParserParsingError +from app.helpers import overfast_internal_error + + +class RoleService(BaseService): + """Domain service for role data.""" + + async def list_roles( + self, + locale: Locale, + cache_key: str, + ) -> tuple[list[dict], bool, int]: + """Return the roles list. + + Returns: + (data, is_stale, age_seconds) + """ + storage_key = f"roles:{locale}" + + async def _fetch() -> list[dict]: + try: + html = await fetch_roles_html(self.blizzard_client, locale) # ty: ignore[invalid-argument-type] + return parse_roles_html(html) + except ParserParsingError as exc: + blizzard_url = f"{settings.blizzard_host}/{locale}{settings.home_path}" + raise overfast_internal_error(blizzard_url, exc) from exc + + return await self._get_or_fetch_static( + storage_key=storage_key, + fetcher=_fetch, + cache_key=cache_key, + cache_ttl=settings.heroes_path_cache_timeout, + staleness_threshold=settings.roles_staleness_threshold, + entity_type="roles", + ) diff --git a/app/gamemodes/controllers/__init__.py b/app/gamemodes/controllers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/gamemodes/controllers/list_gamemodes_controller.py b/app/gamemodes/controllers/list_gamemodes_controller.py deleted file mode 100644 index 84c07d8a..00000000 --- a/app/gamemodes/controllers/list_gamemodes_controller.py +++ /dev/null @@ -1,28 +0,0 @@ -"""List Gamemodes Controller module""" - -from typing import ClassVar - -from app.adapters.blizzard.parsers.gamemodes import parse_gamemodes -from app.config import settings -from app.controllers import AbstractController - - -class ListGamemodesController(AbstractController): - """List Gamemodes Controller used in order to retrieve a list of - available Overwatch gamemodes. - """ - - parser_classes: ClassVar[list[type]] = [] - timeout = settings.csv_cache_timeout - - async def process_request(self, **kwargs) -> list[dict]: # noqa: ARG002 - """Process request using stateless parser function""" - data = parse_gamemodes() - - # Dual-write to API Cache (Valkey) and Storage (SQLite) - storage_key = "gamemodes:en-us" # Gamemodes are not localized - await self.update_static_cache(data, storage_key, data_type="json") - - self.response.headers[settings.cache_ttl_header] = str(self.timeout) - - return data diff --git a/app/helpers.py b/app/helpers.py index 4cb3903b..27468970 100644 --- a/app/helpers.py +++ b/app/helpers.py @@ -5,6 +5,7 @@ from datetime import UTC, datetime from functools import cache from pathlib import Path +from typing import TYPE_CHECKING import httpx from fastapi import HTTPException, status @@ -18,6 +19,9 @@ ) from .overfast_logger import logger +if TYPE_CHECKING: + from fastapi import Response + # Typical routes responses to return success_responses = { status.HTTP_200_OK: { @@ -225,3 +229,26 @@ def get_human_readable_duration(duration: int) -> str: duration_parts.append(f"{minutes} minute{'s' if minutes > 1 else ''}") return ", ".join(duration_parts) + + +def apply_swr_headers( + response: Response, + cache_ttl: int, + is_stale: bool, + age_seconds: int, +) -> None: + """Add standard SWR and cache metadata headers to the response. + + Always sets ``X-Cache-TTL``. + When ``is_stale`` is True, additionally sets RFC-5861 ``Cache-Control``, + ``Age``, and ``X-Cache-Status`` so downstream proxies and clients can + handle stale content correctly. + """ + response.headers[settings.cache_ttl_header] = str(cache_ttl) + + if is_stale: + response.headers["Cache-Control"] = ( + f"max-age={cache_ttl}, stale-while-revalidate={cache_ttl * 2}" + ) + response.headers["Age"] = str(age_seconds) + response.headers["X-Cache-Status"] = "stale" diff --git a/app/heroes/controllers/__init__.py b/app/heroes/controllers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/heroes/controllers/get_hero_controller.py b/app/heroes/controllers/get_hero_controller.py deleted file mode 100644 index b23aa8d5..00000000 --- a/app/heroes/controllers/get_hero_controller.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Hero Controller module""" - -from typing import Any, ClassVar - -from fastapi import HTTPException - -from app.adapters.blizzard import BlizzardClient -from app.adapters.blizzard.parsers.hero import parse_hero -from app.adapters.blizzard.parsers.heroes import parse_heroes -from app.adapters.blizzard.parsers.heroes_stats import parse_heroes_stats -from app.config import settings -from app.controllers import AbstractController -from app.enums import Locale -from app.exceptions import ParserBlizzardError, ParserParsingError -from app.helpers import overfast_internal_error - - -class GetHeroController(AbstractController): - """Hero Controller used in order to retrieve data about a single - Overwatch hero. - """ - - parser_classes: ClassVar[list[type]] = [] - timeout = settings.hero_path_cache_timeout - - async def process_request(self, **kwargs) -> dict: - """Process request using stateless parser functions""" - hero_key = kwargs["hero_key"] - locale = kwargs.get("locale") or Locale.ENGLISH_US - - client = BlizzardClient() - - try: - # Fetch data from all three sources - hero_data = await parse_hero(client, str(hero_key), locale) - heroes_list = await parse_heroes(client, locale) - heroes_stats = parse_heroes_stats() - - # Merge data - data = self._merge_hero_data(hero_data, heroes_list, heroes_stats, hero_key) - except ParserBlizzardError as error: - raise HTTPException( - status_code=error.status_code, - detail=error.message, - ) from error - except ParserParsingError as error: - # Get Blizzard URL for error reporting - blizzard_url = ( - f"{settings.blizzard_host}/{locale}{settings.heroes_path}{hero_key}/" - ) - raise overfast_internal_error(blizzard_url, error) from error - - # Update API Cache - await self.cache_manager.update_api_cache(self.cache_key, data, self.timeout) - - # Note: Single hero data is not stored in persistent storage (Phase 3) - # It's derived from heroes list + hero details, both of which are cached separately - - self.response.headers[settings.cache_ttl_header] = str(self.timeout) - - return data - - @staticmethod - def _merge_hero_data( - hero_data: dict, - heroes_list: list[dict], - heroes_stats: dict, - hero_key: str, - ) -> dict: - """ - Merge data from hero details, heroes list, and heroes stats - - Args: - hero_data: Detailed hero data from hero parser - heroes_list: List of all heroes (for portrait) - heroes_stats: Stats dict (for hitpoints) - hero_key: Hero key for lookups - """ - # Add portrait from heroes list - try: - portrait_value = next( - hero["portrait"] for hero in heroes_list if hero["key"] == hero_key - ) - except StopIteration: - # The hero key may not be here in some specific edge cases, - # for example if the hero has been released but is not in the - # heroes list yet, or the list cache is outdated - portrait_value = None - else: - hero_data = dict_insert_value_before_key( - hero_data, - "role", - "portrait", - portrait_value, - ) - - # Add hitpoints from stats - try: - hitpoints = heroes_stats[hero_key]["hitpoints"] - except KeyError: - # Hero hitpoints may not be here if the CSV file - # containing the data hasn't been updated - hitpoints = None - else: - hero_data = dict_insert_value_before_key( - hero_data, - "abilities", - "hitpoints", - hitpoints, - ) - - return hero_data - - # Keep legacy method name for backward compatibility with tests - @staticmethod - def _dict_insert_value_before_key( - data: dict, - key: str, - new_key: str, - new_value: Any, - ) -> dict: - """Backward compatibility wrapper for tests""" - return dict_insert_value_before_key(data, key, new_key, new_value) - - -def dict_insert_value_before_key( - data: dict, - key: str, - new_key: str, - new_value: Any, -) -> dict: - """Insert a given key/value pair before another key in a given dict""" - if key not in data: - raise KeyError - - key_pos = list(data.keys()).index(key) - data_items = list(data.items()) - data_items.insert(key_pos, (new_key, new_value)) - - return dict(data_items) diff --git a/app/heroes/controllers/get_hero_stats_summary_controller.py b/app/heroes/controllers/get_hero_stats_summary_controller.py deleted file mode 100644 index d6e39b4d..00000000 --- a/app/heroes/controllers/get_hero_stats_summary_controller.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Hero Stats Summary Controller module""" - -from typing import ClassVar - -from fastapi import HTTPException - -from app.adapters.blizzard import BlizzardClient -from app.adapters.blizzard.parsers.hero_stats_summary import parse_hero_stats_summary -from app.config import settings -from app.controllers import AbstractController -from app.exceptions import ParserBlizzardError - - -class GetHeroStatsSummaryController(AbstractController): - """Get Hero Stats Summary Controller used in order to - retrieve usage statistics for Overwatch heroes. - """ - - parser_classes: ClassVar[list[type]] = [] - timeout = settings.hero_stats_cache_timeout - - async def process_request(self, **kwargs) -> list[dict]: - """Process request using stateless parser function""" - client = BlizzardClient() - - try: - data = await parse_hero_stats_summary( - client, - platform=kwargs["platform"], - gamemode=kwargs["gamemode"], - region=kwargs["region"], - role=kwargs.get("role"), - map_filter=kwargs.get("map"), - competitive_division=kwargs.get("competitive_division"), - order_by=kwargs.get("order_by", "hero:asc"), - ) - except ParserBlizzardError as error: - raise HTTPException( - status_code=error.status_code, - detail=error.message, - ) from error - - # Dual-write to API Cache (Valkey) and Storage (SQLite) - storage_key = self._build_storage_key(**kwargs) - await self.update_static_cache(data, storage_key, data_type="json") - - self.response.headers[settings.cache_ttl_header] = str(self.timeout) - - return data - - @staticmethod - def _build_storage_key(**kwargs) -> str: - """Build parameterized storage key for hero stats""" - platform = kwargs["platform"].value - gamemode = kwargs["gamemode"].value - region = kwargs["region"].value - map_filter = kwargs.get("map", "all-maps") - tier = kwargs.get("competitive_division", "null") - return f"hero_stats:{platform}:{gamemode}:{region}:{map_filter}:{tier}" diff --git a/app/heroes/controllers/list_heroes_controller.py b/app/heroes/controllers/list_heroes_controller.py deleted file mode 100644 index 0bd65f93..00000000 --- a/app/heroes/controllers/list_heroes_controller.py +++ /dev/null @@ -1,49 +0,0 @@ -"""List Heroes Controller module""" - -from typing import ClassVar - -from app.adapters.blizzard import BlizzardClient -from app.adapters.blizzard.parsers.heroes import parse_heroes -from app.config import settings -from app.controllers import AbstractController -from app.enums import Locale -from app.exceptions import ParserParsingError -from app.helpers import overfast_internal_error - - -class ListHeroesController(AbstractController): - """List Heroes Controller used in order to - retrieve a list of available Overwatch heroes. - """ - - # Keep parser_classes for backward compatibility, but it's no longer used - parser_classes: ClassVar[list[type]] = [] - timeout = settings.heroes_path_cache_timeout - - async def process_request(self, **kwargs) -> list[dict]: - """Process request using stateless parser function""" - # Extract params - role = kwargs.get("role") - locale = kwargs.get("locale") or Locale.ENGLISH_US - gamemode = kwargs.get("gamemode") - - # Use stateless parser function - client = BlizzardClient() - - try: - data = await parse_heroes( - client, locale=locale, role=role, gamemode=gamemode - ) - except ParserParsingError as error: - # Get Blizzard URL for error reporting - blizzard_url = f"{settings.blizzard_host}/{locale}{settings.heroes_path}" - raise overfast_internal_error(blizzard_url, error) from error - - # Dual-write to API Cache (Valkey) and Storage (SQLite) - storage_key = f"heroes:{locale}" - await self.update_static_cache(data, storage_key, data_type="json") - - # Ensure response headers contains Cache TTL - self.response.headers[settings.cache_ttl_header] = str(self.timeout) - - return data diff --git a/app/maps/controllers/__init__.py b/app/maps/controllers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/maps/controllers/list_maps_controller.py b/app/maps/controllers/list_maps_controller.py deleted file mode 100644 index fbccabf3..00000000 --- a/app/maps/controllers/list_maps_controller.py +++ /dev/null @@ -1,29 +0,0 @@ -"""List Maps Controller module""" - -from typing import ClassVar - -from app.adapters.blizzard.parsers.maps import parse_maps -from app.config import settings -from app.controllers import AbstractController - - -class ListMapsController(AbstractController): - """List Maps Controller used in order to retrieve a list of - available Overwatch maps. - """ - - parser_classes: ClassVar[list[type]] = [] - timeout = settings.csv_cache_timeout - - async def process_request(self, **kwargs) -> list[dict]: - """Process request using stateless parser function""" - gamemode = kwargs.get("gamemode") - data = parse_maps(gamemode=gamemode) - - # Dual-write to API Cache (Valkey) and Storage (SQLite) - storage_key = "maps:en-us" # Maps are not localized - await self.update_static_cache(data, storage_key, data_type="json") - - self.response.headers[settings.cache_ttl_header] = str(self.timeout) - - return data diff --git a/app/monitoring/metrics.py b/app/monitoring/metrics.py index 164fb6b5..2678740f 100644 --- a/app/monitoring/metrics.py +++ b/app/monitoring/metrics.py @@ -66,6 +66,13 @@ "Stale responses served from persistent storage (SWR)", ) +# Background refresh tasks triggered (Phase 4 onwards) +background_refresh_triggered_total = Counter( + "background_refresh_triggered_total", + "Background refresh tasks triggered by SWR staleness detection", + ["entity_type"], # "heroes", "maps", "gamemodes", "roles", "player", "hero_stats" +) + ######################## # Background Task Metrics (Phase 5) ######################## diff --git a/app/players/controllers/__init__.py b/app/players/controllers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/players/controllers/base_player_controller.py b/app/players/controllers/base_player_controller.py deleted file mode 100644 index 2316279d..00000000 --- a/app/players/controllers/base_player_controller.py +++ /dev/null @@ -1,415 +0,0 @@ -"""Base Player Controller module""" - -import time -from typing import TYPE_CHECKING - -from fastapi import HTTPException, status - -from app.adapters.blizzard.parsers.player_profile import ( - extract_name_from_profile_html, - fetch_player_html, -) -from app.adapters.blizzard.parsers.player_summary import ( - fetch_player_summary_json, - parse_player_summary_json, -) -from app.adapters.blizzard.parsers.utils import is_blizzard_id -from app.config import settings -from app.controllers import AbstractController -from app.exceptions import ParserBlizzardError, ParserParsingError -from app.helpers import overfast_internal_error -from app.monitoring.metrics import ( - sqlite_battletag_lookup_total, - sqlite_cache_hit_total, - storage_hits_total, -) -from app.overfast_logger import logger - -if TYPE_CHECKING: - from app.domain.ports import BlizzardClientPort - - -class BasePlayerController(AbstractController): - """Base Player Controller used in order to ensure specific exceptions - are properly handled. For instance, not found players should be cached - to prevent spamming Blizzard with calls. - """ - - # Player Profile Cache Methods (SQLite-based, replacing Valkey Player Cache) - - async def get_player_profile_cache( - self, player_id: str - ) -> dict[str, str | dict | None] | None: - """ - Get player profile from persistent storage. - Returns dict compatible with old Valkey cache format: {"summary": {...}, "profile": "..."} - - Phase 3.5B: player_id is now always Blizzard ID. - - Args: - player_id: Blizzard ID (canonical key) - - Returns: - Dict with 'summary', 'profile', 'battletag', 'name' keys, or None if not found - """ - profile = await self.storage.get_player_profile(player_id) - if not profile: - # Track cache miss - if settings.prometheus_enabled: - sqlite_cache_hit_total.labels( - table="player_profiles", result="miss" - ).inc() - storage_hits_total.labels(result="miss").inc() - return None - - # Track cache hit - if settings.prometheus_enabled: - sqlite_cache_hit_total.labels(table="player_profiles", result="hit").inc() - storage_hits_total.labels(result="hit").inc() - - # Storage now returns full summary from summary_json column - return { - "profile": profile["html"], - "summary": profile["summary"], # Full summary with all fields - "battletag": profile.get("battletag"), # Optional metadata - "name": profile.get("name"), # Display name - } - - async def update_player_profile_cache( - self, - player_id: str, - player_summary: dict, - html: str, - battletag: str | None = None, - name: str | None = None, - ) -> None: - """ - Update player profile in persistent storage. - - Phase 3.5B: Uses Blizzard ID as key, stores battletag/name as optional metadata. - - Args: - player_id: Blizzard ID (canonical key) - player_summary: Full summary from search endpoint (all fields), may be empty dict - html: Raw career page HTML - battletag: Full BattleTag from user input (e.g., "TeKrop-2217"), optional - name: Display name from HTML or summary (e.g., "TeKrop"), optional - """ - await self.storage.set_player_profile( - player_id=player_id, - html=html, - summary=player_summary if player_summary else None, # None if empty dict - battletag=battletag, - name=name, - ) - - # Unknown Player Tracking Methods (Valkey-based with exponential backoff) - # Early rejection is handled at the nginx/Lua layer via the cooldown key. - # Python only handles the write path: marking players as unknown on 404 - # and deleting their status when they become known. - - def _calculate_retry_after(self, check_count: int) -> int: - """ - Calculate retry_after time using exponential backoff. - - Formula: min(base * multiplier^(check_count - 1), max) - Example: 600 * 3^0 = 600s (10min), 600 * 3^1 = 1800s (30min), etc. - - Args: - check_count: Number of failed checks - - Returns: - Seconds to wait before next check (capped at max) - """ - base = settings.unknown_player_initial_retry - multiplier = settings.unknown_player_retry_multiplier - max_retry = settings.unknown_player_max_retry - - retry_after = base * (multiplier ** (check_count - 1)) - return min(int(retry_after), max_retry) - - async def mark_player_unknown_on_404( - self, - blizzard_id: str, - exception: HTTPException, - battletag: str | None = None, - ) -> None: - """ - Mark player as unknown if 404 exception is raised. - Implements exponential backoff for retries. - - Stores by Blizzard ID (primary) with optional BattleTag cooldown key - for early rejection before identity resolution. - - Updates the exception detail to match PlayerNotFoundError format - with retry_after, next_check_at, and check_count fields. - - Args: - blizzard_id: Resolved Blizzard ID (always available after redirect) - exception: HTTPException to modify and check for 404 status - battletag: Optional BattleTag from user request (enables early check) - """ - if not settings.unknown_players_cache_enabled: - return - - if exception.status_code != status.HTTP_404_NOT_FOUND: - return - - # Get existing status to increment check count (works even if cooldown expired) - player_status = await self.cache_manager.get_player_status(blizzard_id) - check_count = player_status["check_count"] + 1 if player_status else 1 - - # Calculate exponential backoff and next check time - retry_after = self._calculate_retry_after(check_count) - next_check_at = int(time.time()) + retry_after - - await self.cache_manager.set_player_status( - blizzard_id, check_count, retry_after, battletag=battletag - ) - - # Update exception detail to match PlayerNotFoundError model - exception.detail = { - "error": "Player not found", - "retry_after": retry_after, - "next_check_at": next_check_at, - "check_count": check_count, - } - - logger.info( - f"Marked player {blizzard_id} as unknown" - f"{f' (battletag: {battletag})' if battletag else ''} " - f"(check #{check_count}, retry in {retry_after}s, next check at {next_check_at})" - ) - - async def _enrich_summary_from_blizzard_id( - self, client: BlizzardClientPort, blizzard_id: str - ) -> tuple[dict, str | None]: - """Reverse enrichment: fetch HTML, extract name, search for matching summary. - - When user provides a Blizzard ID directly, we can still enrich the response - with player summary by extracting the name from HTML and searching. - - Args: - client: Blizzard API client - blizzard_id: Blizzard ID to enrich - - Returns: - Tuple of (player_summary, html): - - player_summary: Enriched summary dict if found, empty dict otherwise - - html: Profile HTML (always returned if fetch succeeds) - """ - # Fetch HTML to extract player name - html, _ = await fetch_player_html(client, blizzard_id) - - if not html: - logger.warning("Could not fetch HTML for Blizzard ID, no enrichment") - return {}, None - - # Extract name from HTML (e.g., "TeKrop" from profile) - try: - player_name = extract_name_from_profile_html(html) - - if player_name: - logger.info( - f"Extracted name '{player_name}', searching for matching summary" - ) - # Search with the name to find summary - search_json = await fetch_player_summary_json(client, player_name) - # Find the entry that matches our Blizzard ID - player_summary = parse_player_summary_json( - search_json, player_name, blizzard_id - ) - - if player_summary: - logger.info("Successfully enriched summary from name-based search") - return player_summary, html - - logger.warning( - f"Name '{player_name}' found in HTML but no matching summary in search" - ) - else: - logger.warning("Could not extract name from HTML for enrichment") - except Exception as e: # noqa: BLE001 - logger.warning(f"Error during reverse enrichment: {e}") - - # Enrichment failed but we have HTML - return what we have - return {}, html - - async def _resolve_player_identity( - self, client: BlizzardClientPort, player_id: str - ) -> tuple[str | None, dict, str | None, str | None]: - """Resolve player identity via search and Blizzard ID redirect if needed. - - This method implements the identity resolution flow: - 1. If Blizzard ID: Attempt reverse enrichment (fetch HTML → extract name → search) - 2. If BattleTag: Fetch and parse search results - 3. If not found in search: Check SQLite for cached BattleTag→Blizzard ID mapping - 4. If still not found: Attempt Blizzard ID resolution via redirect (returns HTML) - 5. Re-parse search results with Blizzard ID if obtained - - Phase 3.5B optimizations: - - SQLite lookup before redirect to avoid redundant Blizzard calls - - Reverse enrichment: When Blizzard ID provided, extract name and search for summary - This builds summary cache even when users provide direct Blizzard IDs - - Args: - client: Blizzard API client - player_id: Player identifier (BattleTag or Blizzard ID) - - Returns: - Tuple of (blizzard_id, player_summary, profile_html, battletag_input): - - blizzard_id: Canonical Blizzard ID (cache key), None if not resolved yet - - player_summary: dict from search (may be empty if direct Blizzard ID or not found) - - profile_html: HTML from profile page if fetched during resolution, None otherwise - - battletag_input: Original BattleTag from user input if applicable, None for Blizzard ID input - """ - logger.info("Retrieving Player Summary...") - - # Blizzard ID provided - attempt reverse enrichment - if is_blizzard_id(player_id): - logger.info( - "Player ID is a Blizzard ID, attempting reverse enrichment from name" - ) - player_summary, html = await self._enrich_summary_from_blizzard_id( - client, player_id - ) - return player_id, player_summary, html, None - - # User provided BattleTag - track it for storage - battletag_input = player_id - - # Fetch and parse search results - search_json = await fetch_player_summary_json(client, player_id) - player_summary = parse_player_summary_json(search_json, player_id) - - if player_summary: - logger.info("Player Summary retrieved!") - blizzard_id = player_summary.get("url") - return blizzard_id, player_summary, None, battletag_input - - # Player not found in search - check SQLite cache before attempting redirect - logger.info( - "Player not found in search, checking SQLite for cached Blizzard ID mapping" - ) - cached_blizzard_id = await self.storage.get_player_id_by_battletag( - battletag_input - ) - - if cached_blizzard_id: - # Track BattleTag lookup hit - if settings.prometheus_enabled: - sqlite_battletag_lookup_total.labels(result="hit").inc() - - logger.info( - f"Found cached Blizzard ID {cached_blizzard_id} for {battletag_input}, " - "skipping redirect" - ) - # Re-parse search with cached Blizzard ID (handles multiple matches) - player_summary = parse_player_summary_json( - search_json, player_id, cached_blizzard_id - ) - if player_summary: - logger.info("Successfully resolved player using cached Blizzard ID") - # No HTML since we skipped the redirect - return cached_blizzard_id, player_summary, None, battletag_input - - logger.warning( - "Cached Blizzard ID found but couldn't resolve player in search results" - ) - # Track BattleTag lookup miss - elif settings.prometheus_enabled: - sqlite_battletag_lookup_total.labels(result="miss").inc() - - # Last resort: attempt Blizzard ID resolution via redirect - logger.info("No cached mapping found, attempting Blizzard ID resolution") - # Step 1: Capture HTML from this call to avoid double-fetch - html, blizzard_id = await fetch_player_html(client, player_id) - - if blizzard_id and search_json: - logger.info( - f"Got Blizzard ID from redirect: {blizzard_id}, re-parsing search results" - ) - player_summary = parse_player_summary_json( - search_json, player_id, blizzard_id - ) - - if player_summary: - logger.info("Successfully resolved player via Blizzard ID") - # Return the HTML we already fetched - return blizzard_id, player_summary, html, battletag_input - - logger.warning( - "Could not resolve player even with Blizzard ID from redirect" - ) - - # Always return HTML if we fetched it, even if blizzard_id or parsing failed - # This avoids a second fetch in the controller - return blizzard_id, {}, html, battletag_input - - async def handle_player_request_exceptions( - self, - error: Exception, - cache_key: str, - battletag_input: str | None, - player_summary: dict, - ): - """ - Shared exception handler for player controllers to eliminate code duplication. - - Handles ParserBlizzardError, ParserParsingError, and HTTPException uniformly - across all player endpoints. Marks players as unknown on 404 errors. - - This method always raises - it processes the error and then re-raises it. - - Args: - error: The exception that was caught - cache_key: The cache key (usually Blizzard ID or player_id) - battletag_input: BattleTag from user input (optional) - player_summary: Player summary dict (may be empty) - - Raises: - HTTPException: Always raises after processing the error - """ - if isinstance(error, ParserBlizzardError): - exception = HTTPException( - status_code=error.status_code, - detail=error.message, - ) - # Mark unknown on 404 from Blizzard - if error.status_code == status.HTTP_404_NOT_FOUND: - await self.mark_player_unknown_on_404( - cache_key, exception, battletag=battletag_input - ) - raise exception from error - - if isinstance(error, ParserParsingError): - # Check if error message indicates player not found - # This can happen when HTML structure is malformed or missing expected elements - if "Could not find main content in HTML" in str(error): - exception = HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Player not found", - ) - # Mark unknown with Blizzard ID (primary) and BattleTag (if available) - await self.mark_player_unknown_on_404( - cache_key, exception, battletag=battletag_input - ) - raise exception from error - - # Get Blizzard URL for error reporting - blizzard_url = ( - f"{settings.blizzard_host}{settings.career_path}/" - f"{player_summary.get('url', cache_key) if player_summary else cache_key}/" - ) - raise overfast_internal_error(blizzard_url, error) from error - - if isinstance(error, HTTPException): - # Mark unknown on any 404 (explicit HTTPException) - if error.status_code == status.HTTP_404_NOT_FOUND: - await self.mark_player_unknown_on_404( - cache_key, error, battletag=battletag_input - ) - raise error - - # Unknown exception type - re-raise as-is - raise error diff --git a/app/players/controllers/get_player_career_controller.py b/app/players/controllers/get_player_career_controller.py deleted file mode 100644 index d918f0cf..00000000 --- a/app/players/controllers/get_player_career_controller.py +++ /dev/null @@ -1,201 +0,0 @@ -"""Player Career Controller module""" - -from typing import TYPE_CHECKING, ClassVar, cast - -from app.adapters.blizzard import BlizzardClient -from app.adapters.blizzard.parsers.player_profile import ( - extract_name_from_profile_html, - fetch_player_html, - filter_all_stats_data, - filter_stats_by_query, - parse_player_profile_html, -) -from app.config import settings -from app.overfast_logger import logger - -if TYPE_CHECKING: - from app.domain.ports import BlizzardClientPort - from app.players.enums import PlayerGamemode, PlayerPlatform - -from .base_player_controller import BasePlayerController - - -class GetPlayerCareerController(BasePlayerController): - """Player Career Controller used in order to retrieve data about a player - Overwatch career : summary, statistics about heroes, etc. - """ - - parser_classes: ClassVar[list] = [] - timeout = settings.career_path_cache_timeout - - async def process_request(self, **kwargs) -> dict: - """Process request with Player Cache support and unknown player guard""" - player_id = kwargs["player_id"] - client = BlizzardClient() - - # Initialize variables for exception handling (must be in scope for except block) - cache_key = player_id - battletag_input = None - player_summary = {} - - try: - # Step 1: Resolve player identity (search + Blizzard ID resolution) - # Phase 3.5B: Returns (blizzard_id, summary, cached_html, battletag_input) - ( - blizzard_id, - player_summary, - cached_html, - battletag_input, - ) = await self._resolve_player_identity(client, player_id) - - # Use Blizzard ID as canonical key (fallback to player_id if not resolved yet) - cache_key = blizzard_id or player_id - - # Step 2: Fetch profile with cache optimization - # Pass cached_html to avoid redundant Blizzard call - _html, profile_data = await self._fetch_profile_with_cache( - client, - cache_key, - player_summary, - battletag_input, - cached_html=cached_html, - ) - - # Step 3: Apply filters - data = self._filter_profile_data( - profile_data, - bool(kwargs.get("summary")), - bool(kwargs.get("stats")), - kwargs.get("platform"), - kwargs.get("gamemode"), - kwargs.get("hero"), - ) - - except Exception as error: # noqa: BLE001 - # Use shared exception handler (always raises) - await self.handle_player_request_exceptions( - error, cache_key, battletag_input, player_summary - ) - - # Update API Cache - await self.cache_manager.update_api_cache(self.cache_key, data, self.timeout) - self.response.headers[settings.cache_ttl_header] = str(self.timeout) - - return data - - async def _fetch_profile_with_cache( - self, - client: BlizzardClientPort, - blizzard_id: str, - player_summary: dict, - battletag_input: str | None, - cached_html: str | None = None, - ) -> tuple[str, dict]: - """Fetch player profile HTML with cache optimization. - - Phase 3.5B: Uses Blizzard ID as cache key, extracts name from HTML, - stores battletag as optional metadata. - - Args: - client: Blizzard API client - blizzard_id: Blizzard ID (canonical cache key) - player_summary: Player summary from search (may be empty for direct Blizzard ID requests) - battletag_input: BattleTag from user input (None if user provided Blizzard ID) - cached_html: HTML already fetched during identity resolution (Step 1 optimization) - - Returns: - Tuple of (html, profile_data) - """ - # Step 1 optimization: Use cached HTML from _resolve_player_identity - if cached_html: - logger.info( - "Using cached HTML from identity resolution (avoiding double-fetch)" - ) - profile_data = parse_player_profile_html(cached_html, player_summary) - - # Extract name and store profile - name = extract_name_from_profile_html(cached_html) or player_summary.get( - "name" - ) - await self.update_player_profile_cache( - blizzard_id, player_summary, cached_html, battletag_input, name - ) - - return cached_html, profile_data - - # Check Player Cache (SQLite storage) using Blizzard ID as key - logger.info("Checking Player Cache...") - player_cache = await self.get_player_profile_cache(blizzard_id) - - if ( - player_cache is not None - and player_summary # Have summary to compare - and player_cache["summary"]["lastUpdated"] # ty: ignore[invalid-argument-type, not-subscriptable] - == player_summary["lastUpdated"] - ): - logger.info("Player Cache found and up-to-date, using it") - html = cast("str", player_cache["profile"]) - profile_data = parse_player_profile_html(html, player_summary) - - # Update battletag if provided (progressive enhancement) - if battletag_input and not player_cache.get("battletag"): - logger.info(f"Enriching cache with BattleTag: {battletag_input}") - cached_name = player_cache.get("name") - name_str = cached_name if isinstance(cached_name, str) else None - await self.update_player_profile_cache( - blizzard_id, player_summary, html, battletag_input, name_str - ) - - return html, profile_data - - # Fetch from Blizzard with Blizzard ID - logger.info("Player Cache not found or not up-to-date, calling Blizzard") - html, _ = await fetch_player_html(client, blizzard_id) - profile_data = parse_player_profile_html(html, player_summary) - - # Extract name and Update Player Cache (SQLite storage) - name = extract_name_from_profile_html(html) or player_summary.get("name") - await self.update_player_profile_cache( - blizzard_id, player_summary, html, battletag_input, name - ) - - return html, profile_data - - async def _fetch_player_html( - self, client: BlizzardClientPort, player_id: str - ) -> tuple[str, str | None]: - """Fetch player HTML from Blizzard and extract Blizzard ID from redirect""" - return await fetch_player_html(client, player_id) - - def _filter_profile_data( - self, - profile_data: dict, - summary_filter: bool, - stats_filter: bool, - platform_filter: PlayerPlatform | None, - gamemode_filter: PlayerGamemode | None, - hero_filter: str | None, - ) -> dict: - """Apply query filters to profile data""" - # If only summary requested - if summary_filter: - return profile_data.get("summary") or {} - - # If only stats requested - if stats_filter: - return filter_stats_by_query( - profile_data.get("stats") or {}, - platform_filter, - gamemode_filter, - hero_filter, - ) - - # Both summary and stats (with optional platform/gamemode filters) - return { - "summary": profile_data.get("summary") or {}, - "stats": filter_all_stats_data( - profile_data.get("stats") or {}, - platform_filter, - gamemode_filter, - ), - } diff --git a/app/players/controllers/get_player_career_stats_controller.py b/app/players/controllers/get_player_career_stats_controller.py deleted file mode 100644 index b7ffb8ed..00000000 --- a/app/players/controllers/get_player_career_stats_controller.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Player Career Stats Controller module""" - -from typing import TYPE_CHECKING, ClassVar, cast - -from app.adapters.blizzard import BlizzardClient -from app.adapters.blizzard.parsers.player_career_stats import ( - parse_player_career_stats_from_html, -) -from app.adapters.blizzard.parsers.player_profile import ( - extract_name_from_profile_html, - fetch_player_html, -) -from app.config import settings -from app.exceptions import ParserParsingError -from app.overfast_logger import logger - -if TYPE_CHECKING: - from app.domain.ports import BlizzardClientPort - from app.players.enums import PlayerGamemode, PlayerPlatform - -from .base_player_controller import BasePlayerController - - -class GetPlayerCareerStatsController(BasePlayerController): - """Player Career Stats Controller used in order to retrieve career - statistics of a player without labels, easily explorable - """ - - parser_classes: ClassVar[list] = [] - timeout = settings.career_path_cache_timeout - - async def process_request(self, **kwargs) -> dict: - """Process request with Player Cache support and unknown player guard""" - player_id = kwargs["player_id"] - platform = kwargs.get("platform") - gamemode = kwargs.get("gamemode") - hero = kwargs.get("hero") - client = BlizzardClient() - - # Initialize variables for exception handling (must be in scope for except block) - cache_key = player_id - battletag_input = None - player_summary = {} - - try: - # Step 1: Resolve player identity (search + Blizzard ID resolution) - # Returns 4-tuple: (blizzard_id, summary, html, battletag_input) - ( - blizzard_id, - player_summary, - cached_html, - battletag_input, - ) = await self._resolve_player_identity(client, player_id) - - # Use Blizzard ID as canonical key (fallback to player_id if not resolved yet) - cache_key = blizzard_id or player_id - - # Step 2: Fetch career stats with cache optimization - # Pass cached_html to avoid redundant Blizzard call - data = await self._fetch_career_stats_with_cache( - client, - cache_key, - player_summary, - platform, - gamemode, - hero, - battletag_input=battletag_input, - cached_html=cached_html, - ) - - except Exception as error: # noqa: BLE001 - # Use shared exception handler (always raises) - await self.handle_player_request_exceptions( - error, cache_key, battletag_input, player_summary - ) - - # Update API Cache - await self.cache_manager.update_api_cache(self.cache_key, data, self.timeout) - self.response.headers[settings.cache_ttl_header] = str(self.timeout) - - return data - - async def _fetch_career_stats_with_cache( - self, - client: BlizzardClientPort, - blizzard_id: str | None, - player_summary: dict, - platform: PlayerPlatform | None, - gamemode: PlayerGamemode | None, - hero: str | None, - battletag_input: str | None = None, - cached_html: str | None = None, - ) -> dict: - """Fetch player career stats with cache optimization. - - Uses Blizzard ID as the canonical identifier for all caching operations. - BattleTag is stored as optional metadata but never used as a cache key. - - Args: - client: Blizzard API client - blizzard_id: Player's Blizzard ID (canonical identifier) - player_summary: Player summary from search - platform: Optional platform filter - gamemode: Optional gamemode filter - hero: Optional hero filter - battletag_input: BattleTag from user's request URL (optional metadata) - cached_html: HTML already fetched during identity resolution - - Returns: - Player career stats data - """ - # Step 1 optimization: Use cached HTML from _resolve_player_identity - if cached_html: - logger.info( - "Using cached HTML from identity resolution (avoiding double-fetch)" - ) - return parse_player_career_stats_from_html( - cached_html, player_summary, platform, gamemode, hero - ) - - # No Blizzard ID - should not happen but handle defensively - if not blizzard_id: - msg = "Unable to resolve player identity" - logger.warning("No Blizzard ID available, cannot fetch career stats") - raise ParserParsingError(msg) - - # Check Player Cache (SQLite storage) - keyed by Blizzard ID - logger.info(f"Checking Player Cache for Blizzard ID: {blizzard_id}") - player_cache = await self.get_player_profile_cache(blizzard_id) - - if ( - player_cache is not None - and player_summary - and player_cache["summary"]["lastUpdated"] # ty: ignore[invalid-argument-type, not-subscriptable] - == player_summary["lastUpdated"] - ): - logger.info("Player Cache found and up-to-date, using it") - html = cast("str", player_cache["profile"]) - return parse_player_career_stats_from_html( - html, player_summary, platform, gamemode, hero - ) - - # Fetch from Blizzard using Blizzard ID - logger.info( - f"Player Cache not found or not up-to-date, fetching from Blizzard: {blizzard_id}" - ) - html, _ = await fetch_player_html(client, blizzard_id) - - # Extract name from HTML for metadata - name = extract_name_from_profile_html(html) - - # Parse career stats - data = parse_player_career_stats_from_html( - html, player_summary, platform, gamemode, hero - ) - - # Update Player Cache with Blizzard ID as key - # Progressive enhancement: if cache exists but is missing battletag, update it - if player_cache and not player_cache.get("battletag") and battletag_input: - logger.info(f"Updating cache with battletag metadata: {battletag_input}") - - await self.update_player_profile_cache( - blizzard_id, - player_summary, - html, - battletag=battletag_input, # Optional metadata from user's request - name=name, # Display name extracted from HTML - ) - - return data diff --git a/app/players/controllers/get_player_stats_summary_controller.py b/app/players/controllers/get_player_stats_summary_controller.py deleted file mode 100644 index 13c3d69e..00000000 --- a/app/players/controllers/get_player_stats_summary_controller.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Player Stats Summary Controller module""" - -from typing import TYPE_CHECKING, ClassVar, cast - -from app.adapters.blizzard import BlizzardClient -from app.adapters.blizzard.parsers.player_profile import ( - extract_name_from_profile_html, - fetch_player_html, -) -from app.adapters.blizzard.parsers.player_stats import ( - parse_player_stats_summary_from_html, -) -from app.config import settings -from app.exceptions import ParserParsingError -from app.overfast_logger import logger - -if TYPE_CHECKING: - from app.domain.ports import BlizzardClientPort - from app.players.enums import PlayerGamemode, PlayerPlatform - -from .base_player_controller import BasePlayerController - - -class GetPlayerStatsSummaryController(BasePlayerController): - """Player Stats Summary Controller used in order to retrieve essential - stats of a player, often used for tracking progress: winrate, kda, damage, etc. - """ - - parser_classes: ClassVar[list] = [] - timeout = settings.career_path_cache_timeout - - async def process_request(self, **kwargs) -> dict: - """Process request with Player Cache support and unknown player guard""" - player_id = kwargs["player_id"] - gamemode = kwargs.get("gamemode") - platform = kwargs.get("platform") - client = BlizzardClient() - - # Initialize variables for exception handling (must be in scope for except block) - cache_key = player_id - battletag_input = None - player_summary = {} - - try: - # Step 1: Resolve player identity (search + Blizzard ID resolution) - # Returns 4-tuple: (blizzard_id, summary, html, battletag_input) - ( - blizzard_id, - player_summary, - cached_html, - battletag_input, - ) = await self._resolve_player_identity(client, player_id) - - # Use Blizzard ID as cache key (canonical identifier) - cache_key = blizzard_id or player_id - - # Step 2: Fetch stats with cache optimization - # Pass cached_html to avoid redundant Blizzard call - data = await self._fetch_stats_with_cache( - client, - cache_key, - player_summary, - gamemode, - platform, - battletag_input=battletag_input, - cached_html=cached_html, - ) - - except Exception as error: # noqa: BLE001 - # Use shared exception handler (always raises) - await self.handle_player_request_exceptions( - error, cache_key, battletag_input, player_summary - ) - - # Update API Cache - await self.cache_manager.update_api_cache(self.cache_key, data, self.timeout) - self.response.headers[settings.cache_ttl_header] = str(self.timeout) - - return data - - async def _fetch_stats_with_cache( - self, - client: BlizzardClientPort, - blizzard_id: str | None, - player_summary: dict, - gamemode: PlayerGamemode | None, - platform: PlayerPlatform | None, - battletag_input: str | None = None, - cached_html: str | None = None, - ) -> dict: - """Fetch player stats with cache optimization. - - Uses Blizzard ID as the canonical identifier for all caching operations. - BattleTag is stored as optional metadata but never used as a cache key. - - Args: - client: Blizzard API client - blizzard_id: Player's Blizzard ID (canonical identifier) - player_summary: Player summary from search - gamemode: Optional gamemode filter - platform: Optional platform filter - battletag_input: BattleTag from user's request URL (optional metadata) - cached_html: HTML already fetched during identity resolution - - Returns: - Player stats summary data - """ - # Step 1 optimization: Use cached HTML from _resolve_player_identity - if cached_html: - logger.info( - "Using cached HTML from identity resolution (avoiding double-fetch)" - ) - return parse_player_stats_summary_from_html( - cached_html, player_summary, gamemode, platform - ) - - # No Blizzard ID - should not happen but handle defensively - if not blizzard_id: - msg = "Unable to resolve player identity" - logger.warning("No Blizzard ID available, cannot fetch stats") - raise ParserParsingError(msg) - - # Check Player Cache (SQLite storage) - keyed by Blizzard ID - logger.info(f"Checking Player Cache for Blizzard ID: {blizzard_id}") - player_cache = await self.get_player_profile_cache(blizzard_id) - - if ( - player_cache is not None - and player_summary - and player_cache["summary"]["lastUpdated"] # ty: ignore[invalid-argument-type, not-subscriptable] - == player_summary["lastUpdated"] - ): - logger.info("Player Cache found and up-to-date, using it") - html = cast("str", player_cache["profile"]) - return parse_player_stats_summary_from_html( - html, player_summary, gamemode, platform - ) - - # Fetch from Blizzard using Blizzard ID - logger.info( - f"Player Cache not found or not up-to-date, fetching from Blizzard: {blizzard_id}" - ) - html, _ = await fetch_player_html(client, blizzard_id) - - # Extract name from HTML for metadata - name = extract_name_from_profile_html(html) - - # Parse stats - data = parse_player_stats_summary_from_html( - html, player_summary, gamemode, platform - ) - - # Update Player Cache with Blizzard ID as key - # Progressive enhancement: if cache exists but is missing battletag, update it - if player_cache and not player_cache.get("battletag") and battletag_input: - logger.info(f"Updating cache with battletag metadata: {battletag_input}") - - await self.update_player_profile_cache( - blizzard_id, - player_summary, - html, - battletag=battletag_input, # Optional metadata from user's request - name=name, # Display name extracted from HTML - ) - - return data diff --git a/app/players/controllers/search_players_controller.py b/app/players/controllers/search_players_controller.py deleted file mode 100644 index 63015375..00000000 --- a/app/players/controllers/search_players_controller.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Search Players Controller module""" - -from typing import ClassVar - -from app.adapters.blizzard import BlizzardClient -from app.adapters.blizzard.parsers.player_search import parse_player_search -from app.config import settings -from app.controllers import AbstractController -from app.exceptions import ParserParsingError -from app.helpers import overfast_internal_error - - -class SearchPlayersController(AbstractController): - """Search Players Controller used in order to find an Overwatch player""" - - parser_classes: ClassVar[list] = [] - timeout = settings.search_account_path_cache_timeout - - async def process_request(self, **kwargs) -> dict: - """Process request using stateless parser function""" - client = BlizzardClient() - - try: - data = await parse_player_search( - client, - name=kwargs["name"], - order_by=kwargs.get("order_by", "name:asc"), - offset=kwargs.get("offset", 0), - limit=kwargs.get("limit", 10), - ) - except ParserParsingError as error: - # Get Blizzard URL for error reporting - search_name = kwargs["name"].split("-", 1)[0] - blizzard_url = ( - f"{settings.blizzard_host}{settings.search_account_path}/{search_name}/" - ) - raise overfast_internal_error(blizzard_url, error) from error - - # Update API Cache - await self.cache_manager.update_api_cache(self.cache_key, data, self.timeout) - self.response.headers[settings.cache_ttl_header] = str(self.timeout) - - return data diff --git a/app/roles/controllers/__init__.py b/app/roles/controllers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/roles/controllers/list_roles_controller.py b/app/roles/controllers/list_roles_controller.py deleted file mode 100644 index be8a2ca6..00000000 --- a/app/roles/controllers/list_roles_controller.py +++ /dev/null @@ -1,40 +0,0 @@ -"""List Roles Controller module""" - -from typing import ClassVar - -from app.adapters.blizzard import BlizzardClient -from app.adapters.blizzard.parsers.roles import parse_roles -from app.config import settings -from app.controllers import AbstractController -from app.enums import Locale -from app.exceptions import ParserParsingError -from app.helpers import overfast_internal_error - - -class ListRolesController(AbstractController): - """List Roles Controller used in order to - retrieve a list of available Overwatch roles. - """ - - parser_classes: ClassVar[list[type]] = [] - timeout = settings.heroes_path_cache_timeout - - async def process_request(self, **kwargs) -> list[dict]: - """Process request using stateless parser function""" - locale = kwargs.get("locale") or Locale.ENGLISH_US - client = BlizzardClient() - - try: - data = await parse_roles(client, locale=locale) - except ParserParsingError as error: - # Get Blizzard URL for error reporting - blizzard_url = f"{settings.blizzard_host}/{locale}{settings.home_path}" - raise overfast_internal_error(blizzard_url, error) from error - - # Dual-write to API Cache (Valkey) and Storage (SQLite) - storage_key = f"roles:{locale}" - await self.update_static_cache(data, storage_key, data_type="json") - - self.response.headers[settings.cache_ttl_header] = str(self.timeout) - - return data diff --git a/pyproject.toml b/pyproject.toml index 94daad2b..daf44c50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -136,6 +136,9 @@ allowed-confusables = ["（", "）", "："] "tests/**" = ["SLF001"] # Ignore private member access on tests "models.py" = ["TC001"] # Ignore type-checking block (Pydantic models definition) "app/api/routers/**" = ["TC001"] # Enums used in FastAPI path/query parameters need runtime import +"app/api/dependencies.py" = ["TC001"] # Port types needed at runtime for FastAPI Depends +"app/domain/services/**" = ["TC001"] # Locale/enums used in runtime method annotations +"app/domain/services/base_service.py" = ["TC001", "PLC0415"] # Lazy metrics imports are intentional [tool.ruff.lint.isort] # Consider app as first-party for imports in tests diff --git a/tests/conftest.py b/tests/conftest.py index 8fd25f18..942905cb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,7 @@ from fastapi.testclient import TestClient from app.adapters.storage import SQLiteStorage +from app.api.dependencies import get_storage from app.main import app @@ -45,6 +46,8 @@ async def _patch_before_every_test( await valkey_server.flushdb() await storage_db.clear_all_data() + app.dependency_overrides[get_storage] = lambda: storage_db + with ( patch("app.helpers.settings.discord_webhook_enabled", False), patch("app.helpers.settings.profiler", None), @@ -52,12 +55,10 @@ async def _patch_before_every_test( "app.cache_manager.CacheManager.valkey_server", valkey_server, ), - patch( - "app.controllers.AbstractController.storage", - storage_db, - ), ): yield + app.dependency_overrides.pop(get_storage, None) + await valkey_server.flushdb() await storage_db.clear_all_data() diff --git a/tests/heroes/controllers/conftest.py b/tests/heroes/controllers/conftest.py deleted file mode 100644 index 9f73f668..00000000 --- a/tests/heroes/controllers/conftest.py +++ /dev/null @@ -1,13 +0,0 @@ -from unittest.mock import MagicMock, Mock, patch - -import pytest - -from app.heroes.controllers.get_hero_controller import GetHeroController - - -@pytest.fixture(scope="package") -def get_hero_controller() -> GetHeroController: - with patch( - "app.controllers.AbstractController.__init__", MagicMock(return_value=None) - ): - return GetHeroController(request=Mock(), response=Mock()) diff --git a/tests/heroes/controllers/test_heroes_controllers.py b/tests/heroes/controllers/test_heroes_controllers.py index 6e244e96..a480af1f 100644 --- a/tests/heroes/controllers/test_heroes_controllers.py +++ b/tests/heroes/controllers/test_heroes_controllers.py @@ -2,7 +2,7 @@ import pytest -from app.heroes.controllers.get_hero_controller import GetHeroController +from app.domain.services.hero_service import dict_insert_value_before_key @pytest.mark.parametrize( @@ -21,9 +21,7 @@ def test_dict_insert_value_before_key_with_key_error( new_value: Any, ): with pytest.raises(KeyError): - GetHeroController._dict_insert_value_before_key( - input_dict, key, new_key, new_value - ) + dict_insert_value_before_key(input_dict, key, new_key, new_value) @pytest.mark.parametrize( @@ -63,8 +61,5 @@ def test_dict_insert_value_before_key_valid( result_dict: dict, ): assert ( - GetHeroController._dict_insert_value_before_key( - input_dict, key, new_key, new_value - ) - == result_dict + dict_insert_value_before_key(input_dict, key, new_key, new_value) == result_dict ) diff --git a/tests/heroes/test_hero_routes.py b/tests/heroes/test_hero_routes.py index 1687f31e..ce96fa81 100644 --- a/tests/heroes/test_hero_routes.py +++ b/tests/heroes/test_hero_routes.py @@ -72,8 +72,8 @@ def test_get_hero_blizzard_error(client: TestClient): def test_get_hero_internal_error(client: TestClient): with patch( - "app.heroes.controllers.get_hero_controller.GetHeroController.process_request", - return_value={"invalid_key": "invalid_value"}, + "app.domain.services.hero_service.HeroService.get_hero", + return_value=({"invalid_key": "invalid_value"}, False, 0), ): response = client.get(f"/heroes/{HeroKey.ANA}") # ty: ignore[unresolved-attribute] assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR @@ -119,7 +119,7 @@ def test_get_hero_no_portrait( ], ), patch( - "app.adapters.blizzard.parsers.heroes.filter_heroes", + "app.domain.services.hero_service.parse_heroes_html", return_value=[], ), ): diff --git a/tests/heroes/test_hero_stats_route.py b/tests/heroes/test_hero_stats_route.py index 9cbb0a37..408e8b16 100644 --- a/tests/heroes/test_hero_stats_route.py +++ b/tests/heroes/test_hero_stats_route.py @@ -96,8 +96,8 @@ def test_get_hero_stats_blizzard_error(client: TestClient): def test_get_heroes_internal_error(client: TestClient): with patch( - "app.heroes.controllers.get_hero_stats_summary_controller.GetHeroStatsSummaryController.process_request", - return_value=[{"invalid_key": "invalid_value"}], + "app.domain.services.hero_service.HeroService.get_hero_stats", + return_value=([{"invalid_key": "invalid_value"}], False, 0), ): response = client.get( "/heroes/stats", diff --git a/tests/heroes/test_heroes_route.py b/tests/heroes/test_heroes_route.py index e12d4147..5fd2cfdc 100644 --- a/tests/heroes/test_heroes_route.py +++ b/tests/heroes/test_heroes_route.py @@ -69,8 +69,8 @@ def test_get_heroes_blizzard_error(client: TestClient): def test_get_heroes_internal_error(client: TestClient): with patch( - "app.heroes.controllers.list_heroes_controller.ListHeroesController.process_request", - return_value=[{"invalid_key": "invalid_value"}], + "app.domain.services.hero_service.HeroService.list_heroes", + return_value=([{"invalid_key": "invalid_value"}], False, 0), ): response = client.get("/heroes") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/tests/players/test_player_career_route.py b/tests/players/test_player_career_route.py index c3ea3604..70845fa3 100644 --- a/tests/players/test_player_career_route.py +++ b/tests/players/test_player_career_route.py @@ -97,8 +97,8 @@ def test_get_player_career_blizzard_remote_protocol_error(client: TestClient): def test_get_player_career_internal_error(client: TestClient): with patch( - "app.players.controllers.get_player_career_controller.GetPlayerCareerController.process_request", - return_value={"invalid_key": "invalid_value"}, + "app.domain.services.player_service.PlayerService.get_player_career", + return_value=({"invalid_key": "invalid_value"}, False, 0), ): response = client.get("/players/TeKrop-2217") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/tests/players/test_player_stats_route.py b/tests/players/test_player_stats_route.py index 507c299e..4ee57608 100644 --- a/tests/players/test_player_stats_route.py +++ b/tests/players/test_player_stats_route.py @@ -200,11 +200,11 @@ def test_get_player_stats_blizzard_forbidden_error(client: TestClient, uri: str) [ ( "/stats", - "app.players.controllers.get_player_career_controller.GetPlayerCareerController.process_request", + "app.domain.services.player_service.PlayerService.get_player_stats", ), ( "/stats/career", - "app.players.controllers.get_player_career_stats_controller.GetPlayerCareerStatsController.process_request", + "app.domain.services.player_service.PlayerService.get_player_career_stats", ), ], ) @@ -213,9 +213,13 @@ def test_get_player_stats_internal_error( ): with patch( patch_target, - return_value={ - "ana": [{"category": "invalid_value", "stats": [{"key": "test"}]}], - }, + return_value=( + { + "ana": [{"category": "invalid_value", "stats": [{"key": "test"}]}], + }, + False, + 0, + ), ): response = client.get( f"/players/TeKrop-2217{uri}", diff --git a/tests/players/test_player_stats_summary_route.py b/tests/players/test_player_stats_summary_route.py index ee6da96a..cb00df54 100644 --- a/tests/players/test_player_stats_summary_route.py +++ b/tests/players/test_player_stats_summary_route.py @@ -130,10 +130,14 @@ def test_get_player_stats_summary_blizzard_timeout(client: TestClient): @pytest.mark.parametrize("player_html_data", ["TeKrop-2217"], indirect=True) def test_get_player_stats_summary_internal_error(client: TestClient): with patch( - "app.players.controllers.get_player_stats_summary_controller.GetPlayerStatsSummaryController.process_request", - return_value={ - "general": [{"category": "invalid_value", "stats": [{"key": "test"}]}], - }, + "app.domain.services.player_service.PlayerService.get_player_stats_summary", + return_value=( + { + "general": [{"category": "invalid_value", "stats": [{"key": "test"}]}], + }, + False, + 0, + ), ): response = client.get( "/players/TeKrop-2217/stats/summary", diff --git a/tests/players/test_player_summary_route.py b/tests/players/test_player_summary_route.py index 7cc4f365..03795b95 100644 --- a/tests/players/test_player_summary_route.py +++ b/tests/players/test_player_summary_route.py @@ -74,8 +74,8 @@ def test_get_player_summary_blizzard_timeout(client: TestClient): def test_get_player_summary_internal_error(client: TestClient): with patch( - "app.players.controllers.get_player_career_controller.GetPlayerCareerController.process_request", - return_value={"invalid_key": "invalid_value"}, + "app.domain.services.player_service.PlayerService.get_player_summary", + return_value=({"invalid_key": "invalid_value"}, False, 0), ): response = client.get("/players/TeKrop-2217/summary") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/tests/players/test_search_players_route.py b/tests/players/test_search_players_route.py index d0c06244..38049ae3 100644 --- a/tests/players/test_search_players_route.py +++ b/tests/players/test_search_players_route.py @@ -161,7 +161,7 @@ def test_search_players_ordering( def test_search_players_internal_error(client: TestClient): with patch( - "app.players.controllers.search_players_controller.SearchPlayersController.process_request", + "app.domain.services.player_service.PlayerService.search_players", return_value={"invalid_key": "invalid_value"}, ): response = client.get("/players", params={"name": "Test"}) diff --git a/tests/roles/test_roles_route.py b/tests/roles/test_roles_route.py index 5f533d0c..4a225740 100644 --- a/tests/roles/test_roles_route.py +++ b/tests/roles/test_roles_route.py @@ -36,8 +36,8 @@ def test_get_roles_blizzard_error(client: TestClient): def test_get_roles_internal_error(client: TestClient): with patch( - "app.roles.controllers.list_roles_controller.ListRolesController.process_request", - return_value=[{"invalid_key": "invalid_value"}], + "app.domain.services.role_service.RoleService.list_roles", + return_value=([{"invalid_key": "invalid_value"}], False, 0), ): response = client.get("/roles") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/uv.lock b/uv.lock index 1458e4bb..5a6c718e 100644 --- a/uv.lock +++ b/uv.lock @@ -604,7 +604,7 @@ wheels = [ [[package]] name = "overfast-api" -version = "3.41.8" +version = "3.42.1" source = { virtual = "." } dependencies = [ { name = "aiosqlite" }, From e7b940abd6244fd89beef5788c39463f5a3bb323 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sat, 21 Feb 2026 13:50:47 +0100 Subject: [PATCH 02/26] --wip-- [skip ci] --- app/adapters/blizzard/parsers/hero.py | 6 +- app/adapters/blizzard/parsers/heroes.py | 7 +- app/adapters/tasks/asyncio_task_queue.py | 53 ++++++ app/adapters/tasks/stub_task_queue.py | 26 --- app/api/dependencies.py | 4 +- app/api/routers/gamemodes.py | 9 +- app/api/routers/heroes.py | 29 +-- app/api/routers/maps.py | 11 +- app/api/routers/players.py | 46 ++--- app/api/routers/roles.py | 9 +- app/domain/services/base_service.py | 172 ++++++++++-------- app/domain/services/gamemode_service.py | 13 +- app/domain/services/hero_service.py | 67 +++---- app/domain/services/map_service.py | 27 ++- app/domain/services/player_service.py | 38 ++-- app/domain/services/role_service.py | 13 +- app/helpers.py | 17 +- pyproject.toml | 1 - .../{controllers => services}/__init__.py | 0 .../test_hero_service.py} | 0 tests/heroes/test_hero_routes.py | 2 +- tests/heroes/test_hero_stats_route.py | 2 +- tests/heroes/test_heroes_route.py | 2 +- tests/players/test_player_career_route.py | 2 +- tests/players/test_player_stats_route.py | 1 - .../test_player_stats_summary_route.py | 1 - tests/players/test_player_summary_route.py | 2 +- tests/roles/test_roles_route.py | 2 +- 28 files changed, 280 insertions(+), 282 deletions(-) create mode 100644 app/adapters/tasks/asyncio_task_queue.py delete mode 100644 app/adapters/tasks/stub_task_queue.py rename tests/heroes/{controllers => services}/__init__.py (100%) rename tests/heroes/{controllers/test_heroes_controllers.py => services/test_hero_service.py} (100%) diff --git a/app/adapters/blizzard/parsers/hero.py b/app/adapters/blizzard/parsers/hero.py index 76fb89f3..35da774b 100644 --- a/app/adapters/blizzard/parsers/hero.py +++ b/app/adapters/blizzard/parsers/hero.py @@ -16,7 +16,7 @@ if TYPE_CHECKING: from selectolax.lexbor import LexborNode - from app.adapters.blizzard.client import BlizzardClient + from app.domain.ports import BlizzardClientPort from app.enums import Locale from app.exceptions import ParserBlizzardError, ParserParsingError @@ -26,7 +26,7 @@ async def fetch_hero_html( - client: BlizzardClient, + client: BlizzardClientPort, hero_key: str, locale: Locale = Locale.ENGLISH_US, ) -> str: @@ -278,7 +278,7 @@ def _parse_hero_stadium_powers(stadium_wrapper: LexborNode) -> list[dict]: async def parse_hero( - client: BlizzardClient, + client: BlizzardClientPort, hero_key: str, locale: Locale = Locale.ENGLISH_US, ) -> dict: diff --git a/app/adapters/blizzard/parsers/heroes.py b/app/adapters/blizzard/parsers/heroes.py index 23c122dc..f5fbe875 100644 --- a/app/adapters/blizzard/parsers/heroes.py +++ b/app/adapters/blizzard/parsers/heroes.py @@ -9,16 +9,17 @@ validate_response_status, ) from app.config import settings +from app.domain.ports import BlizzardClientPort from app.enums import Locale from app.exceptions import ParserParsingError from app.heroes.enums import HeroGamemode if TYPE_CHECKING: - from app.adapters.blizzard.client import BlizzardClient + from app.domain.ports import BlizzardClientPort async def fetch_heroes_html( - client: BlizzardClient, + client: BlizzardClientPort, locale: Locale = Locale.ENGLISH_US, ) -> str: """ @@ -95,7 +96,7 @@ def filter_heroes( async def parse_heroes( - client: BlizzardClient, + client: BlizzardClientPort, locale: Locale = Locale.ENGLISH_US, role: str | None = None, gamemode: HeroGamemode | None = None, diff --git a/app/adapters/tasks/asyncio_task_queue.py b/app/adapters/tasks/asyncio_task_queue.py new file mode 100644 index 00000000..c79c824d --- /dev/null +++ b/app/adapters/tasks/asyncio_task_queue.py @@ -0,0 +1,53 @@ +"""AsyncIO task queue adapter — Phase 4 in-process background tasks with deduplication""" + +import asyncio +from typing import ClassVar + +from app.overfast_logger import logger + + +class AsyncioTaskQueue: + """In-process task queue backed by asyncio.create_task(). + + Uses a class-level set for deduplication so that concurrent requests + don't trigger duplicate refreshes for the same entity. + + Note: Tasks are no-ops (log only) until Phase 5 wires real refresh + coroutines via the arq task registry. The deduplication infrastructure + is already functional. + """ + + _pending_jobs: ClassVar[set[str]] = set() + + async def enqueue( + self, + task_name: str, + *_args, + job_id: str | None = None, + **_kwargs, + ) -> str: + """Schedule a background task if not already pending.""" + effective_id = job_id or f"{task_name}" + if effective_id in self._pending_jobs: + logger.debug(f"[TaskQueue] Skipping duplicate job: {effective_id}") + return effective_id + + self._pending_jobs.add(effective_id) + + async def _run() -> None: + try: + logger.info( + f"[TaskQueue] Running task '{task_name}' (job_id={effective_id})" + ) + # Phase 5: dispatch to real refresh coroutine via registry + finally: + self._pending_jobs.discard(effective_id) + + task = asyncio.create_task(_run(), name=effective_id) + # Keep a strong reference to prevent GC before the task completes + task.add_done_callback(lambda _: None) + return effective_id + + async def is_job_pending_or_running(self, job_id: str) -> bool: + """Return True if a job with this ID is already in-flight.""" + return job_id in self._pending_jobs diff --git a/app/adapters/tasks/stub_task_queue.py b/app/adapters/tasks/stub_task_queue.py deleted file mode 100644 index fb60d212..00000000 --- a/app/adapters/tasks/stub_task_queue.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Stub task queue adapter — Phase 4 placeholder until arq is wired in Phase 5""" - -from app.overfast_logger import logger - - -class StubTaskQueue: - """No-op implementation of TaskQueuePort. - - Logs enqueue calls for observability but does not actually execute tasks. - Will be replaced by ArqTaskQueue in Phase 5. - """ - - async def enqueue( - self, - task_name: str, - *_args, - job_id: str | None = None, - **_kwargs, - ) -> str: - """Log and discard the task — no background processing yet.""" - logger.debug(f"StubTaskQueue: skipping enqueue({task_name}, job_id={job_id})") - return job_id or "" - - async def is_job_pending_or_running(self, _job_id: str) -> bool: - """Always returns False — stub never has pending jobs.""" - return False diff --git a/app/api/dependencies.py b/app/api/dependencies.py index 7c675c4a..bf6a5b0e 100644 --- a/app/api/dependencies.py +++ b/app/api/dependencies.py @@ -7,7 +7,7 @@ from app.adapters.blizzard.client import BlizzardClient from app.adapters.cache.valkey_cache import ValkeyCache from app.adapters.storage.sqlite_storage import SQLiteStorage -from app.adapters.tasks.stub_task_queue import StubTaskQueue +from app.adapters.tasks.asyncio_task_queue import AsyncioTaskQueue from app.domain.ports import BlizzardClientPort, CachePort, StoragePort, TaskQueuePort from app.domain.services import ( GamemodeService, @@ -39,7 +39,7 @@ def get_storage() -> StoragePort: def get_task_queue() -> TaskQueuePort: """Dependency for background task queue (Stub until Phase 5).""" - return StubTaskQueue() + return AsyncioTaskQueue() # --------------------------------------------------------------------------- diff --git a/app/api/routers/gamemodes.py b/app/api/routers/gamemodes.py index 5f3ff539..74ba0f0a 100644 --- a/app/api/routers/gamemodes.py +++ b/app/api/routers/gamemodes.py @@ -8,7 +8,7 @@ from app.config import settings from app.enums import RouteTag from app.gamemodes.models import GamemodeDetails -from app.helpers import apply_swr_headers, success_responses +from app.helpers import apply_swr_headers, build_cache_key, success_responses router = APIRouter() @@ -30,9 +30,6 @@ async def list_map_gamemodes( response: Response, service: GamemodeServiceDep, ) -> Any: - cache_key = request.url.path + ( - f"?{request.query_params}" if request.query_params else "" - ) - data, is_stale, age = await service.list_gamemodes(cache_key=cache_key) - apply_swr_headers(response, settings.csv_cache_timeout, is_stale, age) + data, is_stale = await service.list_gamemodes(cache_key=build_cache_key(request)) + apply_swr_headers(response, settings.csv_cache_timeout, is_stale) return data diff --git a/app/api/routers/heroes.py b/app/api/routers/heroes.py index c2c1fda9..23193027 100644 --- a/app/api/routers/heroes.py +++ b/app/api/routers/heroes.py @@ -7,7 +7,7 @@ from app.api.dependencies import HeroServiceDep from app.config import settings from app.enums import Locale, RouteTag -from app.helpers import apply_swr_headers, routes_responses +from app.helpers import apply_swr_headers, build_cache_key, routes_responses from app.heroes.enums import HeroGamemode, HeroKey from app.heroes.models import ( BadRequestErrorMessage, @@ -50,13 +50,10 @@ async def list_heroes( ] = Locale.ENGLISH_US, gamemode: Annotated[HeroGamemode | None, Query(title="Gamemode filter")] = None, ) -> Any: - cache_key = request.url.path + ( - f"?{request.query_params}" if request.query_params else "" + data, is_stale = await service.list_heroes( + locale=locale, role=role, gamemode=gamemode, cache_key=build_cache_key(request) ) - data, is_stale, age = await service.list_heroes( - locale=locale, role=role, gamemode=gamemode, cache_key=cache_key - ) - apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale, age) + apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale) return data @@ -123,10 +120,7 @@ async def get_hero_stats( ), ] = "hero:asc", ) -> Any: - cache_key = request.url.path + ( - f"?{request.query_params}" if request.query_params else "" - ) - data, is_stale, age = await service.get_hero_stats( + data, is_stale = await service.get_hero_stats( platform=platform, gamemode=gamemode, region=region, @@ -134,9 +128,9 @@ async def get_hero_stats( map_filter=map_, competitive_division=competitive_division, order_by=order_by, - cache_key=cache_key, + cache_key=build_cache_key(request), ) - apply_swr_headers(response, settings.hero_stats_cache_timeout, is_stale, age) + apply_swr_headers(response, settings.hero_stats_cache_timeout, is_stale) return data @@ -167,11 +161,8 @@ async def get_hero( Locale, Query(title="Locale to be displayed") ] = Locale.ENGLISH_US, ) -> Any: - cache_key = request.url.path + ( - f"?{request.query_params}" if request.query_params else "" - ) - data, is_stale, age = await service.get_hero( - hero_key=str(hero_key), locale=locale, cache_key=cache_key + data, is_stale = await service.get_hero( + hero_key=str(hero_key), locale=locale, cache_key=build_cache_key(request) ) - apply_swr_headers(response, settings.hero_path_cache_timeout, is_stale, age) + apply_swr_headers(response, settings.hero_path_cache_timeout, is_stale) return data diff --git a/app/api/routers/maps.py b/app/api/routers/maps.py index 8cbf4bae..da292874 100644 --- a/app/api/routers/maps.py +++ b/app/api/routers/maps.py @@ -8,7 +8,7 @@ from app.config import settings from app.enums import RouteTag from app.gamemodes.enums import MapGamemode -from app.helpers import apply_swr_headers, success_responses +from app.helpers import apply_swr_headers, build_cache_key, success_responses from app.maps.models import Map router = APIRouter() @@ -38,11 +38,8 @@ async def list_maps( ), ] = None, ) -> Any: - cache_key = request.url.path + ( - f"?{request.query_params}" if request.query_params else "" + data, is_stale = await service.list_maps( + gamemode=gamemode, cache_key=build_cache_key(request) ) - data, is_stale, age = await service.list_maps( - gamemode=gamemode, cache_key=cache_key - ) - apply_swr_headers(response, settings.csv_cache_timeout, is_stale, age) + apply_swr_headers(response, settings.csv_cache_timeout, is_stale) return data diff --git a/app/api/routers/players.py b/app/api/routers/players.py index f3e38671..fbf51d32 100644 --- a/app/api/routers/players.py +++ b/app/api/routers/players.py @@ -7,7 +7,7 @@ from app.api.dependencies import PlayerServiceDep from app.config import settings from app.enums import RouteTag -from app.helpers import apply_swr_headers +from app.helpers import apply_swr_headers, build_cache_key from app.helpers import routes_responses as common_routes_responses from app.players.enums import ( HeroKeyCareerFilter, @@ -125,9 +125,7 @@ async def search_players( offset: Annotated[int, Query(title="Offset of the results", ge=0)] = 0, limit: Annotated[int, Query(title="Limit of results per page", gt=0)] = 20, ) -> Any: - cache_key = request.url.path + ( - f"?{request.query_params}" if request.query_params else "" - ) + cache_key = build_cache_key(request) data = await service.search_players( name=name, order_by=order_by, @@ -159,14 +157,12 @@ async def get_player_summary( service: PlayerServiceDep, commons: CommonsPlayerDep, ) -> Any: - cache_key = request.url.path + ( - f"?{request.query_params}" if request.query_params else "" - ) - data, is_stale, age = await service.get_player_summary( + cache_key = build_cache_key(request) + data, is_stale = await service.get_player_summary( player_id=commons["player_id"], cache_key=cache_key, ) - apply_swr_headers(response, settings.career_path_cache_timeout, is_stale, age) + apply_swr_headers(response, settings.career_path_cache_timeout, is_stale) return data @@ -217,16 +213,14 @@ async def get_player_stats_summary( ), ] = None, ) -> Any: - cache_key = request.url.path + ( - f"?{request.query_params}" if request.query_params else "" - ) - data, is_stale, age = await service.get_player_stats_summary( + cache_key = build_cache_key(request) + data, is_stale = await service.get_player_stats_summary( player_id=commons["player_id"], gamemode=gamemode, platform=platform, cache_key=cache_key, ) - apply_swr_headers(response, settings.career_path_cache_timeout, is_stale, age) + apply_swr_headers(response, settings.career_path_cache_timeout, is_stale) return data @@ -252,17 +246,15 @@ async def get_player_career_stats( service: PlayerServiceDep, commons: CommonsPlayerCareerDep, ) -> Any: - cache_key = request.url.path + ( - f"?{request.query_params}" if request.query_params else "" - ) - data, is_stale, age = await service.get_player_career_stats( + cache_key = build_cache_key(request) + data, is_stale = await service.get_player_career_stats( player_id=commons["player_id"], gamemode=commons.get("gamemode"), platform=commons.get("platform"), hero=commons.get("hero"), cache_key=cache_key, ) - apply_swr_headers(response, settings.career_path_cache_timeout, is_stale, age) + apply_swr_headers(response, settings.career_path_cache_timeout, is_stale) return data @@ -286,17 +278,15 @@ async def get_player_stats( service: PlayerServiceDep, commons: CommonsPlayerCareerDep, ) -> Any: - cache_key = request.url.path + ( - f"?{request.query_params}" if request.query_params else "" - ) - data, is_stale, age = await service.get_player_stats( + cache_key = build_cache_key(request) + data, is_stale = await service.get_player_stats( player_id=commons["player_id"], gamemode=commons.get("gamemode"), platform=commons.get("platform"), hero=commons.get("hero"), cache_key=cache_key, ) - apply_swr_headers(response, settings.career_path_cache_timeout, is_stale, age) + apply_swr_headers(response, settings.career_path_cache_timeout, is_stale) return data @@ -334,14 +324,12 @@ async def get_player_career( ), ] = None, ) -> Any: - cache_key = request.url.path + ( - f"?{request.query_params}" if request.query_params else "" - ) - data, is_stale, age = await service.get_player_career( + cache_key = build_cache_key(request) + data, is_stale = await service.get_player_career( player_id=commons["player_id"], gamemode=gamemode, platform=platform, cache_key=cache_key, ) - apply_swr_headers(response, settings.career_path_cache_timeout, is_stale, age) + apply_swr_headers(response, settings.career_path_cache_timeout, is_stale) return data diff --git a/app/api/routers/roles.py b/app/api/routers/roles.py index 6cb3d729..5e890600 100644 --- a/app/api/routers/roles.py +++ b/app/api/routers/roles.py @@ -7,7 +7,7 @@ from app.api.dependencies import RoleServiceDep from app.config import settings from app.enums import Locale, RouteTag -from app.helpers import apply_swr_headers, routes_responses +from app.helpers import apply_swr_headers, build_cache_key, routes_responses from app.roles.models import RoleDetail router = APIRouter() @@ -33,9 +33,8 @@ async def list_roles( Locale, Query(title="Locale to be displayed") ] = Locale.ENGLISH_US, ) -> Any: - cache_key = request.url.path + ( - f"?{request.query_params}" if request.query_params else "" + data, is_stale = await service.list_roles( + locale=locale, cache_key=build_cache_key(request) ) - data, is_stale, age = await service.list_roles(locale=locale, cache_key=cache_key) - apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale, age) + apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale) return data diff --git a/app/domain/services/base_service.py b/app/domain/services/base_service.py index 4e895933..1837bb88 100644 --- a/app/domain/services/base_service.py +++ b/app/domain/services/base_service.py @@ -2,13 +2,18 @@ import json import time +from enum import StrEnum from typing import TYPE_CHECKING, Any -from app.config import settings +from app.monitoring.metrics import ( + background_refresh_triggered_total, + stale_responses_total, + storage_hits_total, +) from app.overfast_logger import logger if TYPE_CHECKING: - from collections.abc import Awaitable, Callable + from collections.abc import Callable from app.domain.ports import ( BlizzardClientPort, @@ -18,17 +23,31 @@ ) +class StorageTable(StrEnum): + """Persistent storage table identifiers used across services.""" + + STATIC_DATA = "static_data" + PLAYER_PROFILES = "player_profiles" + + class BaseService: """Base service providing Stale-While-Revalidate (SWR) orchestration. - All domain services inherit from this class to get SWR behaviour: - 1. Check SQLite persistent storage (cache key). - 2. If found and fresh (age < staleness_threshold) → return data, update Valkey. - 3. If found but stale → return data, update Valkey, trigger background refresh. - 4. If not found (cold start) → fetch from Blizzard, store in both, return data. + The generic ``get_or_fetch`` method implements the SWR flow for data backed + by SQLite persistent storage: + + 1. Hit SQLite. If found and *fresh* → return + update Valkey. + 2. If found but *stale* → return + update Valkey + trigger background refresh. + 3. On cold start (miss) → fetch synchronously from Blizzard, store, return. + + Concrete services call ``get_or_fetch`` with domain-specific ``fetcher``, + ``parser``, and optional ``filter`` callables. - Note: Valkey API-cache check happens at the Nginx/Lua level before FastAPI is reached, - so this class only handles the SQLite → Blizzard fallback path. + Note: Valkey API-cache reads happen at the Nginx/Lua layer *before* FastAPI + is reached; services only ever *write* to the API cache. + + Player data uses a different staleness strategy (``lastUpdated`` field from + Blizzard) so ``PlayerService`` manages its own request flow independently. """ def __init__( @@ -44,82 +63,115 @@ def __init__( self.task_queue = task_queue # ------------------------------------------------------------------ - # SWR core — used by static-data services (heroes, maps, …) + # Generic SWR orchestration # ------------------------------------------------------------------ - async def _get_or_fetch_static( + async def get_or_fetch( self, *, storage_key: str, - fetcher: Callable[[], Awaitable[Any]], + fetcher: Callable[[], Any], cache_key: str, cache_ttl: int, staleness_threshold: int, entity_type: str, - ) -> tuple[Any, bool, int]: - """SWR orchestration for static data backed by SQLite. + table: StorageTable = StorageTable.STATIC_DATA, + parser: Callable[[Any], Any] | None = None, + filter: Callable[[Any], Any] | None = None, # noqa: A002 + ) -> tuple[Any, bool]: + """SWR orchestration for data backed by SQLite. Args: - storage_key: Key in the ``static_data`` SQLite table. - fetcher: Async callable that fetches and parses fresh data from Blizzard. - Must return the final *list* or *dict* to store as JSON. + storage_key: Key in the SQLite table. + fetcher: Async callable that retrieves raw data (HTML, JSON, or + any form) from the upstream source (Blizzard or local CSV). cache_key: Valkey API-cache key to update after serving data. cache_ttl: TTL in seconds for the Valkey API-cache entry. staleness_threshold: Seconds after which stored data is considered stale. entity_type: Human-readable label used in metrics / logs. + table: SQLite table to read/write (default: static_data). + parser: Optional callable that converts raw fetcher output into the + stored/returned format. Defaults to identity (raw data as-is). + filter: Optional callable applied to parsed data before returning. + Not stored — re-applied on each request when serving stale data. Returns: - ``(data, is_stale, age_seconds)`` tuple. - ``age_seconds`` is 0 on a cold-start fetch. + ``(data, is_stale)`` tuple. """ - stored = await self.storage.get_static_data(storage_key) + stored = await self._load_from_storage(storage_key, table) - if stored: - data = json.loads(stored["data"]) + if stored is not None: + data = stored["data"] age = int(time.time()) - stored["updated_at"] is_stale = age >= staleness_threshold if is_stale: logger.info( - f"[SWR] {entity_type} data is stale (age={age}s, " - f"threshold={staleness_threshold}s) — serving stale + triggering refresh" + f"[SWR] {entity_type} stale (age={age}s, " + f"threshold={staleness_threshold}s) — serving + triggering refresh" ) await self._enqueue_refresh(entity_type, storage_key) - self._track_stale_response(entity_type) + stale_responses_total.inc() + background_refresh_triggered_total.labels(entity_type=entity_type).inc() else: logger.info( - f"[SWR] {entity_type} data is fresh (age={age}s) — serving from SQLite" + f"[SWR] {entity_type} fresh (age={age}s) — serving from SQLite" ) - self._track_storage_hit("hit") + storage_hits_total.labels(result="hit").inc() + + if filter is not None: + data = filter(data) + await self._update_api_cache(cache_key, data, cache_ttl) - return data, is_stale, age + return data, is_stale + + # Cold start — fetch synchronously + logger.info(f"[SWR] {entity_type} not in SQLite — fetching from source") + storage_hits_total.labels(result="miss").inc() + + raw = await fetcher() + data = parser(raw) if parser is not None else raw + await self._store_in_storage(storage_key, data, table) - # Cold start — fetch synchronously from Blizzard - logger.info(f"[SWR] {entity_type} not in SQLite — fetching from Blizzard") - self._track_storage_hit("miss") - data = await fetcher() - await self._persist_static(storage_key, data, cache_key, cache_ttl) - return data, False, 0 + filtered = filter(data) if filter is not None else data + await self._update_api_cache(cache_key, filtered, cache_ttl) + return filtered, False # ------------------------------------------------------------------ - # Helpers + # Storage helpers — overridable by concrete services # ------------------------------------------------------------------ - async def _persist_static( - self, storage_key: str, data: Any, cache_key: str, cache_ttl: int + async def _load_from_storage( + self, storage_key: str, table: StorageTable + ) -> dict[str, Any] | None: + """Load data from the given SQLite table. Returns ``None`` on miss.""" + if table == StorageTable.STATIC_DATA: + result = await self.storage.get_static_data(storage_key) + if result: + return { + "data": json.loads(result["data"]), + "updated_at": result["updated_at"], + } + return None + + async def _store_in_storage( + self, storage_key: str, data: Any, table: StorageTable ) -> None: - """Write static data to both SQLite and Valkey API cache.""" - try: - await self.storage.set_static_data( - key=storage_key, - data=json.dumps(data, separators=(",", ":")), - data_type="json", - ) - except Exception as exc: # noqa: BLE001 - logger.warning(f"[SWR] SQLite write failed for {storage_key}: {exc}") + """Persist data to the given SQLite table (default: static_data JSON).""" + if table == StorageTable.STATIC_DATA: + try: + await self.storage.set_static_data( + key=storage_key, + data=json.dumps(data, separators=(",", ":")), + data_type="json", + ) + except Exception as exc: # noqa: BLE001 + logger.warning(f"[SWR] SQLite write failed for {storage_key}: {exc}") - await self._update_api_cache(cache_key, data, cache_ttl) + # ------------------------------------------------------------------ + # Shared low-level helpers + # ------------------------------------------------------------------ async def _update_api_cache( self, cache_key: str, data: Any, cache_ttl: int @@ -131,7 +183,7 @@ async def _update_api_cache( logger.warning(f"[SWR] Valkey write failed for {cache_key}: {exc}") async def _enqueue_refresh(self, entity_type: str, entity_id: str) -> None: - """Enqueue a background refresh task (deduplication via job_id).""" + """Enqueue a background refresh, deduplicating via job_id.""" job_id = f"refresh:{entity_type}:{entity_id}" try: already_running = await self.task_queue.is_job_pending_or_running(job_id) @@ -141,31 +193,7 @@ async def _enqueue_refresh(self, entity_type: str, entity_id: str) -> None: entity_id, job_id=job_id, ) - if settings.prometheus_enabled: - from app.monitoring.metrics import ( - background_refresh_triggered_total, - ) - - background_refresh_triggered_total.labels( - entity_type=entity_type - ).inc() except Exception as exc: # noqa: BLE001 logger.warning( f"[SWR] Failed to enqueue refresh for {entity_type}/{entity_id}: {exc}" ) - - @staticmethod - def _track_storage_hit(result: str) -> None: - if settings.prometheus_enabled: - from app.monitoring.metrics import storage_hits_total - - storage_hits_total.labels(result=result).inc() - - @staticmethod - def _track_stale_response(entity_type: str) -> None: - if settings.prometheus_enabled: - from app.monitoring.metrics import stale_responses_total - - stale_responses_total.inc() - # entity_type tracked via background_refresh_triggered_total - _ = entity_type diff --git a/app/domain/services/gamemode_service.py b/app/domain/services/gamemode_service.py index 0d9d60b2..1650f376 100644 --- a/app/domain/services/gamemode_service.py +++ b/app/domain/services/gamemode_service.py @@ -11,19 +11,14 @@ class GamemodeService(BaseService): async def list_gamemodes( self, cache_key: str, - ) -> tuple[list[dict], bool, int]: - """Return the gamemodes list. - - Returns: - (data, is_stale, age_seconds) - """ - storage_key = "gamemodes:all" + ) -> tuple[list[dict], bool]: + """Return the gamemodes list.""" async def _fetch() -> list[dict]: return parse_gamemodes_csv() - return await self._get_or_fetch_static( - storage_key=storage_key, + return await self.get_or_fetch( + storage_key="gamemodes:all", fetcher=_fetch, cache_key=cache_key, cache_ttl=settings.csv_cache_timeout, diff --git a/app/domain/services/hero_service.py b/app/domain/services/hero_service.py index fa9d5792..6fb50cee 100644 --- a/app/domain/services/hero_service.py +++ b/app/domain/services/hero_service.py @@ -44,33 +44,30 @@ async def list_heroes( role: Role | None, gamemode: HeroGamemode | None, cache_key: str, - ) -> tuple[list[dict], bool, int]: + ) -> tuple[list[dict], bool]: """Return the heroes list (with optional role/gamemode filters). - SWR: stores the *full* (unfiltered) heroes list per locale in SQLite; - filters are applied after retrieval so all filter combinations benefit - from the same cache entry. - - Returns: - (data, is_stale, age_seconds) + Stores the *full* (unfiltered) heroes list per locale in SQLite so + that all filter combinations benefit from the same cache entry. """ - storage_key = f"heroes:{locale}" async def _fetch() -> list[dict]: - html = await fetch_heroes_html(self.blizzard_client, locale) # ty: ignore[invalid-argument-type] + html = await fetch_heroes_html(self.blizzard_client, locale) return parse_heroes_html(html) - data, is_stale, age = await self._get_or_fetch_static( - storage_key=storage_key, + def _filter(data: list[dict]) -> list[dict]: + return filter_heroes(data, role, gamemode) + + return await self.get_or_fetch( + storage_key=f"heroes:{locale}", fetcher=_fetch, + filter=_filter, cache_key=cache_key, cache_ttl=settings.heroes_path_cache_timeout, staleness_threshold=settings.heroes_staleness_threshold, entity_type="heroes", ) - return filter_heroes(data, role, gamemode), is_stale, age - # ------------------------------------------------------------------ # Single hero (GET /heroes/{hero_key}) # ------------------------------------------------------------------ @@ -80,19 +77,15 @@ async def get_hero( hero_key: str, locale: Locale, cache_key: str, - ) -> tuple[dict, bool, int]: + ) -> tuple[dict, bool]: """Return full hero details merged with portrait and hitpoints. - Single-hero data is not stored persistently (it is derived from three - sources that each have their own caching); the Valkey API cache is still - updated after each fetch. - - Returns: - (data, is_stale, age_seconds) + Single-hero data is not stored persistently; the Valkey API cache is + still updated on every fetch. """ try: - hero_data = await parse_hero(self.blizzard_client, hero_key, locale) # ty: ignore[invalid-argument-type] - heroes_html = await fetch_heroes_html(self.blizzard_client, locale) # ty: ignore[invalid-argument-type] + hero_data = await parse_hero(self.blizzard_client, hero_key, locale) + heroes_html = await fetch_heroes_html(self.blizzard_client, locale) heroes_list = parse_heroes_html(heroes_html) heroes_stats = parse_heroes_stats() data = _merge_hero_data(hero_data, heroes_list, heroes_stats, hero_key) @@ -107,7 +100,7 @@ async def get_hero( raise overfast_internal_error(blizzard_url, exc) from exc await self._update_api_cache(cache_key, data, settings.hero_path_cache_timeout) - return data, False, 0 + return data, False # ------------------------------------------------------------------ # Hero stats summary (GET /heroes/stats) @@ -123,12 +116,8 @@ async def get_hero_stats( competitive_division: CompetitiveDivisionFilter | None, # ty: ignore[invalid-type-form] order_by: str, cache_key: str, - ) -> tuple[list[dict], bool, int]: - """Return hero usage statistics with SWR. - - Returns: - (data, is_stale, age_seconds) - """ + ) -> tuple[list[dict], bool]: + """Return hero usage statistics with SWR.""" storage_key = _build_hero_stats_storage_key( platform, gamemode, region, map_filter, competitive_division ) @@ -150,21 +139,19 @@ async def _fetch() -> list[dict]: status_code=exc.status_code, detail=exc.message ) from exc - data, is_stale, age = await self._get_or_fetch_static( + def _filter(data: list[dict]) -> list[dict]: + return _filter_hero_stats(data, role, order_by) + + return await self.get_or_fetch( storage_key=storage_key, fetcher=_fetch, + filter=_filter, cache_key=cache_key, cache_ttl=settings.hero_stats_cache_timeout, staleness_threshold=settings.hero_stats_staleness_threshold, entity_type="hero_stats", ) - # Apply post-fetch filters (role, order_by) on stale data from SQLite - if is_stale and age > 0: - data = _filter_hero_stats(data, role, order_by) - - return data, is_stale, age - # --------------------------------------------------------------------------- # Module-level helpers (kept accessible for tests) @@ -178,7 +165,6 @@ def _merge_hero_data( hero_key: str, ) -> dict: """Merge data from hero details, heroes list, and heroes stats.""" - # Portrait from heroes list try: portrait_value = next( hero["portrait"] for hero in heroes_list if hero["key"] == hero_key @@ -190,7 +176,6 @@ def _merge_hero_data( hero_data, "role", "portrait", portrait_value ) - # Hitpoints from stats CSV try: hitpoints = heroes_stats[hero_key]["hitpoints"] except KeyError: @@ -238,11 +223,9 @@ def _filter_hero_stats( role: Any, order_by: str, ) -> list[dict]: - """Re-apply role filter and ordering when serving stale data from SQLite.""" - logger.debug("[SWR] Re-applying hero_stats filters on stale data") + """Re-apply role filter and ordering (used both on stale data and cold-start).""" + logger.debug("[SWR] Applying hero_stats filters") if role: data = [h for h in data if h.get("role") == role.value] - - # Re-sort according to order_by field, direction = order_by.split(":") return sorted(data, key=lambda h: h.get(field, ""), reverse=(direction == "desc")) diff --git a/app/domain/services/map_service.py b/app/domain/services/map_service.py index 2e3c205d..5ac269cc 100644 --- a/app/domain/services/map_service.py +++ b/app/domain/services/map_service.py @@ -12,31 +12,26 @@ async def list_maps( self, gamemode: str | None, cache_key: str, - ) -> tuple[list[dict], bool, int]: + ) -> tuple[list[dict], bool]: """Return the maps list (with optional gamemode filter). - Stores the full (unfiltered) maps list in SQLite; gamemode filter is - applied after retrieval. - - Returns: - (data, is_stale, age_seconds) + Stores the full (unfiltered) maps list in SQLite. """ - storage_key = "maps:all" - async def _fetch() -> list[dict]: return parse_maps_csv() - data, is_stale, age = await self._get_or_fetch_static( - storage_key=storage_key, + def _filter(data: list[dict]) -> list[dict]: + if not gamemode: + return data + gamemode_val = gamemode.value if hasattr(gamemode, "value") else gamemode + return [m for m in data if gamemode_val in m.get("gamemodes", [])] + + return await self.get_or_fetch( + storage_key="maps:all", fetcher=_fetch, + filter=_filter, cache_key=cache_key, cache_ttl=settings.csv_cache_timeout, staleness_threshold=settings.maps_staleness_threshold, entity_type="maps", ) - - if gamemode: - gamemode_val = gamemode.value if hasattr(gamemode, "value") else gamemode - data = [m for m in data if gamemode_val in m.get("gamemodes", [])] - - return data, is_stale, age diff --git a/app/domain/services/player_service.py b/app/domain/services/player_service.py index 11f55f10..fa66be1a 100644 --- a/app/domain/services/player_service.py +++ b/app/domain/services/player_service.py @@ -91,7 +91,7 @@ async def get_player_summary( self, player_id: str, cache_key: str, - ) -> tuple[dict, bool, int]: + ) -> tuple[dict, bool]: """Return player summary (name, avatar, competitive ranks, …). Returns: @@ -114,7 +114,7 @@ async def get_player_career( gamemode: PlayerGamemode | None, platform: PlayerPlatform | None, cache_key: str, - ) -> tuple[dict, bool, int]: + ) -> tuple[dict, bool]: """Return full player data: summary + stats. Returns: @@ -140,7 +140,7 @@ async def get_player_stats( platform: PlayerPlatform | None, hero: HeroKeyCareerFilter | None, # ty: ignore[invalid-type-form] cache_key: str, - ) -> tuple[dict, bool, int]: + ) -> tuple[dict, bool]: """Return player stats with category labels. Returns: @@ -166,7 +166,7 @@ async def get_player_stats_summary( gamemode: PlayerGamemode | None, platform: PlayerPlatform | None, cache_key: str, - ) -> tuple[dict, bool, int]: + ) -> tuple[dict, bool]: """Return player statistics summary (winrate, kda, …). Returns: @@ -198,11 +198,11 @@ async def get_player_stats_summary( exc, cache_key_player, battletag_input, player_summary ) - is_stale, age = self._check_player_staleness(cache_key_player) + is_stale = self._check_player_staleness(cache_key_player) await self._update_api_cache( cache_key, data, settings.career_path_cache_timeout ) - return data, is_stale, age + return data, is_stale # ------------------------------------------------------------------ # Player career stats (GET /players/{player_id}/stats/career) @@ -215,7 +215,7 @@ async def get_player_career_stats( platform: PlayerPlatform | None, hero: HeroKeyCareerFilter | None, # ty: ignore[invalid-type-form] cache_key: str, - ) -> tuple[dict, bool, int]: + ) -> tuple[dict, bool]: """Return player career stats (no labels). Returns: @@ -248,11 +248,11 @@ async def get_player_career_stats( exc, cache_key_player, battletag_input, player_summary ) - is_stale, age = self._check_player_staleness(cache_key_player) + is_stale = self._check_player_staleness(cache_key_player) await self._update_api_cache( cache_key, data, settings.career_path_cache_timeout ) - return data, is_stale, age + return data, is_stale # ------------------------------------------------------------------ # Core request execution (career + summary) @@ -267,7 +267,7 @@ async def _execute_player_request( gamemode: PlayerGamemode | None = None, platform: PlayerPlatform | None = None, hero: HeroKeyCareerFilter | None = None, # ty: ignore[invalid-type-form] - ) -> tuple[dict, bool, int]: + ) -> tuple[dict, bool]: """Shared execution path for summary, career, and stats endpoints.""" cache_key_player = player_id battletag_input: str | None = None @@ -302,11 +302,11 @@ async def _execute_player_request( exc, cache_key_player, battletag_input, player_summary ) - is_stale, age = self._check_player_staleness(cache_key_player) + is_stale = self._check_player_staleness(cache_key_player) await self._update_api_cache( cache_key, data, settings.career_path_cache_timeout ) - return data, is_stale, age + return data, is_stale # ------------------------------------------------------------------ # Profile caching helpers @@ -352,13 +352,13 @@ async def update_player_profile_cache( name=name, ) - def _check_player_staleness(self, _player_id: str) -> tuple[bool, int]: - """Return (is_stale, age_seconds) based purely on time — best effort.""" - # This is a lightweight check; the actual updated_at would require - # another async storage lookup. We return (False, 0) here and let the - # background refresh (Phase 5) handle true staleness. - # Phase 5 will update this to return proper staleness info. - return False, 0 + def _check_player_staleness(self, _player_id: str) -> bool: + """Return is_stale based purely on time — best effort. + + Phase 5 will perform an actual async storage lookup; for now always + returns False (fresh) and lets the background refresh handle real staleness. + """ + return False async def _fetch_profile_with_cache( self, diff --git a/app/domain/services/role_service.py b/app/domain/services/role_service.py index dbb06c2d..cb1b2feb 100644 --- a/app/domain/services/role_service.py +++ b/app/domain/services/role_service.py @@ -15,13 +15,8 @@ async def list_roles( self, locale: Locale, cache_key: str, - ) -> tuple[list[dict], bool, int]: - """Return the roles list. - - Returns: - (data, is_stale, age_seconds) - """ - storage_key = f"roles:{locale}" + ) -> tuple[list[dict], bool]: + """Return the roles list.""" async def _fetch() -> list[dict]: try: @@ -31,8 +26,8 @@ async def _fetch() -> list[dict]: blizzard_url = f"{settings.blizzard_host}/{locale}{settings.home_path}" raise overfast_internal_error(blizzard_url, exc) from exc - return await self._get_or_fetch_static( - storage_key=storage_key, + return await self.get_or_fetch( + storage_key=f"roles:{locale}", fetcher=_fetch, cache_key=cache_key, cache_ttl=settings.heroes_path_cache_timeout, diff --git a/app/helpers.py b/app/helpers.py index 27468970..a3ae7eb9 100644 --- a/app/helpers.py +++ b/app/helpers.py @@ -20,7 +20,7 @@ from .overfast_logger import logger if TYPE_CHECKING: - from fastapi import Response + from fastapi import Request, Response # Typical routes responses to return success_responses = { @@ -231,18 +231,24 @@ def get_human_readable_duration(duration: int) -> str: return ", ".join(duration_parts) +def build_cache_key(request: Request) -> str: + """Build a canonical cache key from the request URL path + query string.""" + qs = str(request.query_params) + return f"{request.url.path}?{qs}" if qs else request.url.path + + def apply_swr_headers( response: Response, cache_ttl: int, is_stale: bool, - age_seconds: int, ) -> None: """Add standard SWR and cache metadata headers to the response. Always sets ``X-Cache-TTL``. - When ``is_stale`` is True, additionally sets RFC-5861 ``Cache-Control``, - ``Age``, and ``X-Cache-Status`` so downstream proxies and clients can - handle stale content correctly. + When ``is_stale`` is True, additionally sets RFC-5861 ``Cache-Control`` + and ``X-Cache-Status`` so downstream proxies (nginx/Lua) can handle stale + content correctly. The ``Age`` header is intentionally left to nginx, which + knows the actual time the response has been in its cache. """ response.headers[settings.cache_ttl_header] = str(cache_ttl) @@ -250,5 +256,4 @@ def apply_swr_headers( response.headers["Cache-Control"] = ( f"max-age={cache_ttl}, stale-while-revalidate={cache_ttl * 2}" ) - response.headers["Age"] = str(age_seconds) response.headers["X-Cache-Status"] = "stale" diff --git a/pyproject.toml b/pyproject.toml index daf44c50..9fcb0dd9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -138,7 +138,6 @@ allowed-confusables = ["（", "）", "："] "app/api/routers/**" = ["TC001"] # Enums used in FastAPI path/query parameters need runtime import "app/api/dependencies.py" = ["TC001"] # Port types needed at runtime for FastAPI Depends "app/domain/services/**" = ["TC001"] # Locale/enums used in runtime method annotations -"app/domain/services/base_service.py" = ["TC001", "PLC0415"] # Lazy metrics imports are intentional [tool.ruff.lint.isort] # Consider app as first-party for imports in tests diff --git a/tests/heroes/controllers/__init__.py b/tests/heroes/services/__init__.py similarity index 100% rename from tests/heroes/controllers/__init__.py rename to tests/heroes/services/__init__.py diff --git a/tests/heroes/controllers/test_heroes_controllers.py b/tests/heroes/services/test_hero_service.py similarity index 100% rename from tests/heroes/controllers/test_heroes_controllers.py rename to tests/heroes/services/test_hero_service.py diff --git a/tests/heroes/test_hero_routes.py b/tests/heroes/test_hero_routes.py index ce96fa81..3ad1c61d 100644 --- a/tests/heroes/test_hero_routes.py +++ b/tests/heroes/test_hero_routes.py @@ -73,7 +73,7 @@ def test_get_hero_blizzard_error(client: TestClient): def test_get_hero_internal_error(client: TestClient): with patch( "app.domain.services.hero_service.HeroService.get_hero", - return_value=({"invalid_key": "invalid_value"}, False, 0), + return_value=({"invalid_key": "invalid_value"}, False), ): response = client.get(f"/heroes/{HeroKey.ANA}") # ty: ignore[unresolved-attribute] assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/tests/heroes/test_hero_stats_route.py b/tests/heroes/test_hero_stats_route.py index 408e8b16..dd0af1b7 100644 --- a/tests/heroes/test_hero_stats_route.py +++ b/tests/heroes/test_hero_stats_route.py @@ -97,7 +97,7 @@ def test_get_hero_stats_blizzard_error(client: TestClient): def test_get_heroes_internal_error(client: TestClient): with patch( "app.domain.services.hero_service.HeroService.get_hero_stats", - return_value=([{"invalid_key": "invalid_value"}], False, 0), + return_value=([{"invalid_key": "invalid_value"}], False), ): response = client.get( "/heroes/stats", diff --git a/tests/heroes/test_heroes_route.py b/tests/heroes/test_heroes_route.py index 5fd2cfdc..2d68a8ba 100644 --- a/tests/heroes/test_heroes_route.py +++ b/tests/heroes/test_heroes_route.py @@ -70,7 +70,7 @@ def test_get_heroes_blizzard_error(client: TestClient): def test_get_heroes_internal_error(client: TestClient): with patch( "app.domain.services.hero_service.HeroService.list_heroes", - return_value=([{"invalid_key": "invalid_value"}], False, 0), + return_value=([{"invalid_key": "invalid_value"}], False), ): response = client.get("/heroes") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/tests/players/test_player_career_route.py b/tests/players/test_player_career_route.py index 70845fa3..8f8a72ec 100644 --- a/tests/players/test_player_career_route.py +++ b/tests/players/test_player_career_route.py @@ -98,7 +98,7 @@ def test_get_player_career_blizzard_remote_protocol_error(client: TestClient): def test_get_player_career_internal_error(client: TestClient): with patch( "app.domain.services.player_service.PlayerService.get_player_career", - return_value=({"invalid_key": "invalid_value"}, False, 0), + return_value=({"invalid_key": "invalid_value"}, False), ): response = client.get("/players/TeKrop-2217") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/tests/players/test_player_stats_route.py b/tests/players/test_player_stats_route.py index 4ee57608..64bfee7f 100644 --- a/tests/players/test_player_stats_route.py +++ b/tests/players/test_player_stats_route.py @@ -218,7 +218,6 @@ def test_get_player_stats_internal_error( "ana": [{"category": "invalid_value", "stats": [{"key": "test"}]}], }, False, - 0, ), ): response = client.get( diff --git a/tests/players/test_player_stats_summary_route.py b/tests/players/test_player_stats_summary_route.py index cb00df54..3c05fc66 100644 --- a/tests/players/test_player_stats_summary_route.py +++ b/tests/players/test_player_stats_summary_route.py @@ -136,7 +136,6 @@ def test_get_player_stats_summary_internal_error(client: TestClient): "general": [{"category": "invalid_value", "stats": [{"key": "test"}]}], }, False, - 0, ), ): response = client.get( diff --git a/tests/players/test_player_summary_route.py b/tests/players/test_player_summary_route.py index 03795b95..d877d33d 100644 --- a/tests/players/test_player_summary_route.py +++ b/tests/players/test_player_summary_route.py @@ -75,7 +75,7 @@ def test_get_player_summary_blizzard_timeout(client: TestClient): def test_get_player_summary_internal_error(client: TestClient): with patch( "app.domain.services.player_service.PlayerService.get_player_summary", - return_value=({"invalid_key": "invalid_value"}, False, 0), + return_value=({"invalid_key": "invalid_value"}, False), ): response = client.get("/players/TeKrop-2217/summary") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/tests/roles/test_roles_route.py b/tests/roles/test_roles_route.py index 4a225740..e924348d 100644 --- a/tests/roles/test_roles_route.py +++ b/tests/roles/test_roles_route.py @@ -37,7 +37,7 @@ def test_get_roles_blizzard_error(client: TestClient): def test_get_roles_internal_error(client: TestClient): with patch( "app.domain.services.role_service.RoleService.list_roles", - return_value=([{"invalid_key": "invalid_value"}], False, 0), + return_value=([{"invalid_key": "invalid_value"}], False), ): response = client.get("/roles") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR From 5eaca90b47afae79f6e853f4787fc9c932c70490 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sat, 21 Feb 2026 13:57:36 +0100 Subject: [PATCH 03/26] --wip-- [skip ci] --- .../blizzard/parsers/hero_stats_summary.py | 6 +-- .../blizzard/parsers/heroes_hitpoints.py | 28 +++++++++++++ app/adapters/blizzard/parsers/heroes_stats.py | 39 ------------------- app/adapters/blizzard/parsers/roles.py | 6 +-- app/domain/models/__init__.py | 11 ++++++ app/domain/services/base_service.py | 10 ++--- app/domain/services/hero_service.py | 18 ++++----- app/domain/services/map_service.py | 2 +- app/domain/services/role_service.py | 2 +- 9 files changed, 61 insertions(+), 61 deletions(-) create mode 100644 app/adapters/blizzard/parsers/heroes_hitpoints.py delete mode 100644 app/adapters/blizzard/parsers/heroes_stats.py diff --git a/app/adapters/blizzard/parsers/hero_stats_summary.py b/app/adapters/blizzard/parsers/hero_stats_summary.py index fe4fd5c3..7011ffad 100644 --- a/app/adapters/blizzard/parsers/hero_stats_summary.py +++ b/app/adapters/blizzard/parsers/hero_stats_summary.py @@ -10,7 +10,7 @@ from app.players.enums import PlayerGamemode, PlayerPlatform, PlayerRegion if TYPE_CHECKING: - from app.adapters.blizzard.client import BlizzardClient + from app.domain.ports import BlizzardClientPort # Mappings for query parameters PLATFORM_MAPPING: dict[PlayerPlatform, str] = { @@ -25,7 +25,7 @@ async def fetch_hero_stats_json( - client: BlizzardClient, + client: BlizzardClientPort, platform: PlayerPlatform, gamemode: PlayerGamemode, region: PlayerRegion, @@ -125,7 +125,7 @@ def _normalize_rate(rate: float) -> float: async def parse_hero_stats_summary( - client: BlizzardClient, + client: BlizzardClientPort, platform: PlayerPlatform, gamemode: PlayerGamemode, region: PlayerRegion, diff --git a/app/adapters/blizzard/parsers/heroes_hitpoints.py b/app/adapters/blizzard/parsers/heroes_hitpoints.py new file mode 100644 index 00000000..9985078a --- /dev/null +++ b/app/adapters/blizzard/parsers/heroes_hitpoints.py @@ -0,0 +1,28 @@ +"""Stateless parser functions for heroes hitpoints data (HP, armor, shields) from CSV""" + +from app.adapters.csv import CSVReader + +HITPOINTS_KEYS = {"health", "armor", "shields"} + + +def parse_heroes_hitpoints() -> dict[str, dict]: + """Parse heroes hitpoints (health/armor/shields) from the heroes CSV file. + + Returns: + Dict mapping hero key to hitpoints data. + Example: {"ana": {"hitpoints": {"health": 200, "armor": 0, "shields": 0, "total": 200}}} + """ + csv_reader = CSVReader() + csv_data = csv_reader.read_csv_file("heroes") + + return { + row["key"]: {"hitpoints": _get_hitpoints(row)} + for row in csv_data + } + + +def _get_hitpoints(row: dict) -> dict: + """Extract hitpoints data from a hero CSV row.""" + hitpoints = {hp_key: int(row[hp_key]) for hp_key in HITPOINTS_KEYS} + hitpoints["total"] = sum(hitpoints.values()) + return hitpoints diff --git a/app/adapters/blizzard/parsers/heroes_stats.py b/app/adapters/blizzard/parsers/heroes_stats.py deleted file mode 100644 index 6a950468..00000000 --- a/app/adapters/blizzard/parsers/heroes_stats.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Stateless parser functions for heroes stats data (HP, armor, shields)""" - -from app.adapters.csv import CSVReader - -HITPOINTS_KEYS = {"health", "armor", "shields"} - - -def parse_heroes_stats_csv() -> dict[str, dict]: - """ - Parse heroes stats (hitpoints) from CSV file - - Returns: - Dict mapping hero key to stats dict with hitpoints data - Example: {"ana": {"hitpoints": {"health": 200, "armor": 0, "shields": 0, "total": 200}}} - """ - csv_reader = CSVReader() - csv_data = csv_reader.read_csv_file("heroes") - - return { - hero_stats["key"]: {"hitpoints": _get_hitpoints(hero_stats)} - for hero_stats in csv_data - } - - -def _get_hitpoints(hero_stats: dict) -> dict: - """Extract hitpoints data from hero CSV row""" - hitpoints = {hp_key: int(hero_stats[hp_key]) for hp_key in HITPOINTS_KEYS} - hitpoints["total"] = sum(hitpoints.values()) - return hitpoints - - -def parse_heroes_stats() -> dict[str, dict]: - """ - High-level function to parse heroes stats - - Returns: - Dict mapping hero key to stats - """ - return parse_heroes_stats_csv() diff --git a/app/adapters/blizzard/parsers/roles.py b/app/adapters/blizzard/parsers/roles.py index c379caa1..2e624296 100644 --- a/app/adapters/blizzard/parsers/roles.py +++ b/app/adapters/blizzard/parsers/roles.py @@ -14,11 +14,11 @@ from app.roles.helpers import get_role_from_icon_url if TYPE_CHECKING: - from app.adapters.blizzard.client import BlizzardClient + from app.domain.ports import BlizzardClientPort async def fetch_roles_html( - client: BlizzardClient, + client: BlizzardClientPort, locale: Locale = Locale.ENGLISH_US, ) -> str: """Fetch roles HTML from Blizzard homepage""" @@ -86,7 +86,7 @@ def parse_roles_html(html: str) -> list[dict]: async def parse_roles( - client: BlizzardClient, + client: BlizzardClientPort, locale: Locale = Locale.ENGLISH_US, ) -> list[dict]: """ diff --git a/app/domain/models/__init__.py b/app/domain/models/__init__.py index e69de29b..2ba48204 100644 --- a/app/domain/models/__init__.py +++ b/app/domain/models/__init__.py @@ -0,0 +1,11 @@ +"""Domain models package. + +Domain model classes (dataclasses or Pydantic models representing core business +entities independently of FastAPI response schemas) will live here from Phase 5 +onward, when background refresh workers need structured types to pass between +the task queue and the service layer. + +The folder is intentionally empty during Phase 4 — all response shapes are +currently expressed directly via the existing Pydantic models in each feature +module (e.g. app/heroes/models.py). +""" diff --git a/app/domain/services/base_service.py b/app/domain/services/base_service.py index 1837bb88..3f0c96e4 100644 --- a/app/domain/services/base_service.py +++ b/app/domain/services/base_service.py @@ -77,7 +77,7 @@ async def get_or_fetch( entity_type: str, table: StorageTable = StorageTable.STATIC_DATA, parser: Callable[[Any], Any] | None = None, - filter: Callable[[Any], Any] | None = None, # noqa: A002 + result_filter: Callable[[Any], Any] | None = None, ) -> tuple[Any, bool]: """SWR orchestration for data backed by SQLite. @@ -92,7 +92,7 @@ async def get_or_fetch( table: SQLite table to read/write (default: static_data). parser: Optional callable that converts raw fetcher output into the stored/returned format. Defaults to identity (raw data as-is). - filter: Optional callable applied to parsed data before returning. + result_filter: Optional callable applied to parsed data before returning. Not stored — re-applied on each request when serving stale data. Returns: @@ -120,8 +120,8 @@ async def get_or_fetch( storage_hits_total.labels(result="hit").inc() - if filter is not None: - data = filter(data) + if result_filter is not None: + data = result_filter(data) await self._update_api_cache(cache_key, data, cache_ttl) return data, is_stale @@ -134,7 +134,7 @@ async def get_or_fetch( data = parser(raw) if parser is not None else raw await self._store_in_storage(storage_key, data, table) - filtered = filter(data) if filter is not None else data + filtered = result_filter(data) if result_filter is not None else data await self._update_api_cache(cache_key, filtered, cache_ttl) return filtered, False diff --git a/app/domain/services/hero_service.py b/app/domain/services/hero_service.py index 6fb50cee..1560e103 100644 --- a/app/domain/services/hero_service.py +++ b/app/domain/services/hero_service.py @@ -11,7 +11,7 @@ filter_heroes, parse_heroes_html, ) -from app.adapters.blizzard.parsers.heroes_stats import parse_heroes_stats +from app.adapters.blizzard.parsers.heroes_hitpoints import parse_heroes_hitpoints from app.config import settings from app.domain.services.base_service import BaseService from app.enums import Locale @@ -61,7 +61,7 @@ def _filter(data: list[dict]) -> list[dict]: return await self.get_or_fetch( storage_key=f"heroes:{locale}", fetcher=_fetch, - filter=_filter, + result_filter=_filter, cache_key=cache_key, cache_ttl=settings.heroes_path_cache_timeout, staleness_threshold=settings.heroes_staleness_threshold, @@ -87,8 +87,8 @@ async def get_hero( hero_data = await parse_hero(self.blizzard_client, hero_key, locale) heroes_html = await fetch_heroes_html(self.blizzard_client, locale) heroes_list = parse_heroes_html(heroes_html) - heroes_stats = parse_heroes_stats() - data = _merge_hero_data(hero_data, heroes_list, heroes_stats, hero_key) + heroes_hitpoints = parse_heroes_hitpoints() + data = _merge_hero_data(hero_data, heroes_list, heroes_hitpoints, hero_key) except ParserBlizzardError as exc: raise HTTPException( status_code=exc.status_code, detail=exc.message @@ -125,7 +125,7 @@ async def get_hero_stats( async def _fetch() -> list[dict]: try: return await parse_hero_stats_summary( - self.blizzard_client, # ty: ignore[invalid-argument-type] + self.blizzard_client, platform=platform, gamemode=gamemode, region=region, @@ -145,7 +145,7 @@ def _filter(data: list[dict]) -> list[dict]: return await self.get_or_fetch( storage_key=storage_key, fetcher=_fetch, - filter=_filter, + result_filter=_filter, cache_key=cache_key, cache_ttl=settings.hero_stats_cache_timeout, staleness_threshold=settings.hero_stats_staleness_threshold, @@ -161,10 +161,10 @@ def _filter(data: list[dict]) -> list[dict]: def _merge_hero_data( hero_data: dict, heroes_list: list[dict], - heroes_stats: dict, + heroes_hitpoints: dict, hero_key: str, ) -> dict: - """Merge data from hero details, heroes list, and heroes stats.""" + """Merge data from hero details, heroes list, and heroes hitpoints.""" try: portrait_value = next( hero["portrait"] for hero in heroes_list if hero["key"] == hero_key @@ -177,7 +177,7 @@ def _merge_hero_data( ) try: - hitpoints = heroes_stats[hero_key]["hitpoints"] + hitpoints = heroes_hitpoints[hero_key]["hitpoints"] except KeyError: hitpoints = None else: diff --git a/app/domain/services/map_service.py b/app/domain/services/map_service.py index 5ac269cc..76f1d5fc 100644 --- a/app/domain/services/map_service.py +++ b/app/domain/services/map_service.py @@ -29,7 +29,7 @@ def _filter(data: list[dict]) -> list[dict]: return await self.get_or_fetch( storage_key="maps:all", fetcher=_fetch, - filter=_filter, + result_filter=_filter, cache_key=cache_key, cache_ttl=settings.csv_cache_timeout, staleness_threshold=settings.maps_staleness_threshold, diff --git a/app/domain/services/role_service.py b/app/domain/services/role_service.py index cb1b2feb..fbb2ddd5 100644 --- a/app/domain/services/role_service.py +++ b/app/domain/services/role_service.py @@ -20,7 +20,7 @@ async def list_roles( async def _fetch() -> list[dict]: try: - html = await fetch_roles_html(self.blizzard_client, locale) # ty: ignore[invalid-argument-type] + html = await fetch_roles_html(self.blizzard_client, locale) return parse_roles_html(html) except ParserParsingError as exc: blizzard_url = f"{settings.blizzard_host}/{locale}{settings.home_path}" From b315ac26cf2be95b2417b34254531b8b6a793ef7 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sat, 21 Feb 2026 14:17:07 +0100 Subject: [PATCH 04/26] --wip-- [skip ci] --- app/adapters/tasks/asyncio_task_queue.py | 33 ++-- app/domain/ports/task_queue.py | 18 ++- app/domain/services/base_service.py | 175 ++++----------------- app/domain/services/gamemode_service.py | 4 +- app/domain/services/hero_service.py | 4 +- app/domain/services/map_service.py | 4 +- app/domain/services/role_service.py | 4 +- app/domain/services/static_data_service.py | 145 +++++++++++++++++ 8 files changed, 223 insertions(+), 164 deletions(-) create mode 100644 app/domain/services/static_data_service.py diff --git a/app/adapters/tasks/asyncio_task_queue.py b/app/adapters/tasks/asyncio_task_queue.py index c79c824d..1dc8e99f 100644 --- a/app/adapters/tasks/asyncio_task_queue.py +++ b/app/adapters/tasks/asyncio_task_queue.py @@ -1,20 +1,24 @@ """AsyncIO task queue adapter — Phase 4 in-process background tasks with deduplication""" import asyncio -from typing import ClassVar +from typing import TYPE_CHECKING, Any, ClassVar from app.overfast_logger import logger +if TYPE_CHECKING: + from collections.abc import Coroutine + class AsyncioTaskQueue: """In-process task queue backed by asyncio.create_task(). - Uses a class-level set for deduplication so that concurrent requests - don't trigger duplicate refreshes for the same entity. + Uses a class-level set for deduplication so concurrent requests don't + trigger duplicate refreshes for the same entity. - Note: Tasks are no-ops (log only) until Phase 5 wires real refresh - coroutines via the arq task registry. The deduplication infrastructure - is already functional. + When a ``coro`` is provided to ``enqueue``, it is executed as a real + background task. Phase 5 will replace this adapter with an arq-backed + one that dispatches to a worker process instead, but the interface stays + the same. """ _pending_jobs: ClassVar[set[str]] = set() @@ -22,14 +26,17 @@ class AsyncioTaskQueue: async def enqueue( self, task_name: str, - *_args, + *_args: Any, job_id: str | None = None, - **_kwargs, + coro: Coroutine[Any, Any, Any] | None = None, + **_kwargs: Any, ) -> str: """Schedule a background task if not already pending.""" - effective_id = job_id or f"{task_name}" + effective_id = job_id or task_name if effective_id in self._pending_jobs: logger.debug(f"[TaskQueue] Skipping duplicate job: {effective_id}") + if coro is not None: + coro.close() # avoid "coroutine was never awaited" warning return effective_id self._pending_jobs.add(effective_id) @@ -39,12 +46,16 @@ async def _run() -> None: logger.info( f"[TaskQueue] Running task '{task_name}' (job_id={effective_id})" ) - # Phase 5: dispatch to real refresh coroutine via registry + if coro is not None: + await coro + except Exception as exc: # noqa: BLE001 + logger.warning( + f"[TaskQueue] Task '{task_name}' (job_id={effective_id}) failed: {exc}" + ) finally: self._pending_jobs.discard(effective_id) task = asyncio.create_task(_run(), name=effective_id) - # Keep a strong reference to prevent GC before the task completes task.add_done_callback(lambda _: None) return effective_id diff --git a/app/domain/ports/task_queue.py b/app/domain/ports/task_queue.py index 6942d16d..26e70379 100644 --- a/app/domain/ports/task_queue.py +++ b/app/domain/ports/task_queue.py @@ -1,21 +1,31 @@ """Task queue port protocol for background job processing""" -from typing import Any, Protocol +from typing import TYPE_CHECKING, Any, Protocol + +if TYPE_CHECKING: + from collections.abc import Coroutine class TaskQueuePort(Protocol): - """Protocol for background task queue operations (arq in v4)""" + """Protocol for background task queue operations (arq in Phase 5)""" async def enqueue( self, task_name: str, *args: Any, job_id: str | None = None, + coro: Coroutine[Any, Any, Any] | None = None, **kwargs: Any, ) -> str: - """Enqueue a background task, returns job ID""" + """Enqueue a background task. + + ``coro``, when provided, is executed immediately (Phase 4 asyncio) or + dispatched to a worker process (Phase 5 arq). ``task_name`` is kept for + arq compatibility and logging. + Returns the effective job ID. + """ ... async def is_job_pending_or_running(self, job_id: str) -> bool: - """Check if job is already pending or running (for deduplication)""" + """Return True if a job with this ID is already pending or running.""" ... diff --git a/app/domain/services/base_service.py b/app/domain/services/base_service.py index 3f0c96e4..8c52bc29 100644 --- a/app/domain/services/base_service.py +++ b/app/domain/services/base_service.py @@ -1,19 +1,25 @@ -"""Base service providing Stale-While-Revalidate orchestration for all domain services""" +"""Domain service base classes. + +``BaseService`` holds infrastructure adapters and low-level helpers shared by +*all* services (static + player). + +``StaticDataService`` extends it with the generic Stale-While-Revalidate flow +for data that is stored as JSON in the ``static_data`` SQLite table and where +staleness is determined by a configurable time threshold. All static-content +services (heroes, maps, gamemodes, roles) inherit from this class. + +``PlayerService`` inherits directly from ``BaseService`` and implements its own +staleness strategy (Blizzard ``lastUpdated`` comparison) and storage logic +(``player_profiles`` table). +""" -import json -import time from enum import StrEnum from typing import TYPE_CHECKING, Any -from app.monitoring.metrics import ( - background_refresh_triggered_total, - stale_responses_total, - storage_hits_total, -) from app.overfast_logger import logger if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Coroutine from app.domain.ports import ( BlizzardClientPort, @@ -24,30 +30,19 @@ class StorageTable(StrEnum): - """Persistent storage table identifiers used across services.""" + """Persistent storage table identifiers.""" STATIC_DATA = "static_data" PLAYER_PROFILES = "player_profiles" class BaseService: - """Base service providing Stale-While-Revalidate (SWR) orchestration. - - The generic ``get_or_fetch`` method implements the SWR flow for data backed - by SQLite persistent storage: - - 1. Hit SQLite. If found and *fresh* → return + update Valkey. - 2. If found but *stale* → return + update Valkey + trigger background refresh. - 3. On cold start (miss) → fetch synchronously from Blizzard, store, return. - - Concrete services call ``get_or_fetch`` with domain-specific ``fetcher``, - ``parser``, and optional ``filter`` callables. - - Note: Valkey API-cache reads happen at the Nginx/Lua layer *before* FastAPI - is reached; services only ever *write* to the API cache. + """Infrastructure holder shared by all domain services. - Player data uses a different staleness strategy (``lastUpdated`` field from - Blizzard) so ``PlayerService`` manages its own request flow independently. + Provides: + - Adapter references (cache, storage, blizzard_client, task_queue) + - ``_update_api_cache``: write to Valkey after serving data + - ``_enqueue_refresh``: deduplicated background refresh scheduling """ def __init__( @@ -62,117 +57,6 @@ def __init__( self.blizzard_client = blizzard_client self.task_queue = task_queue - # ------------------------------------------------------------------ - # Generic SWR orchestration - # ------------------------------------------------------------------ - - async def get_or_fetch( - self, - *, - storage_key: str, - fetcher: Callable[[], Any], - cache_key: str, - cache_ttl: int, - staleness_threshold: int, - entity_type: str, - table: StorageTable = StorageTable.STATIC_DATA, - parser: Callable[[Any], Any] | None = None, - result_filter: Callable[[Any], Any] | None = None, - ) -> tuple[Any, bool]: - """SWR orchestration for data backed by SQLite. - - Args: - storage_key: Key in the SQLite table. - fetcher: Async callable that retrieves raw data (HTML, JSON, or - any form) from the upstream source (Blizzard or local CSV). - cache_key: Valkey API-cache key to update after serving data. - cache_ttl: TTL in seconds for the Valkey API-cache entry. - staleness_threshold: Seconds after which stored data is considered stale. - entity_type: Human-readable label used in metrics / logs. - table: SQLite table to read/write (default: static_data). - parser: Optional callable that converts raw fetcher output into the - stored/returned format. Defaults to identity (raw data as-is). - result_filter: Optional callable applied to parsed data before returning. - Not stored — re-applied on each request when serving stale data. - - Returns: - ``(data, is_stale)`` tuple. - """ - stored = await self._load_from_storage(storage_key, table) - - if stored is not None: - data = stored["data"] - age = int(time.time()) - stored["updated_at"] - is_stale = age >= staleness_threshold - - if is_stale: - logger.info( - f"[SWR] {entity_type} stale (age={age}s, " - f"threshold={staleness_threshold}s) — serving + triggering refresh" - ) - await self._enqueue_refresh(entity_type, storage_key) - stale_responses_total.inc() - background_refresh_triggered_total.labels(entity_type=entity_type).inc() - else: - logger.info( - f"[SWR] {entity_type} fresh (age={age}s) — serving from SQLite" - ) - - storage_hits_total.labels(result="hit").inc() - - if result_filter is not None: - data = result_filter(data) - - await self._update_api_cache(cache_key, data, cache_ttl) - return data, is_stale - - # Cold start — fetch synchronously - logger.info(f"[SWR] {entity_type} not in SQLite — fetching from source") - storage_hits_total.labels(result="miss").inc() - - raw = await fetcher() - data = parser(raw) if parser is not None else raw - await self._store_in_storage(storage_key, data, table) - - filtered = result_filter(data) if result_filter is not None else data - await self._update_api_cache(cache_key, filtered, cache_ttl) - return filtered, False - - # ------------------------------------------------------------------ - # Storage helpers — overridable by concrete services - # ------------------------------------------------------------------ - - async def _load_from_storage( - self, storage_key: str, table: StorageTable - ) -> dict[str, Any] | None: - """Load data from the given SQLite table. Returns ``None`` on miss.""" - if table == StorageTable.STATIC_DATA: - result = await self.storage.get_static_data(storage_key) - if result: - return { - "data": json.loads(result["data"]), - "updated_at": result["updated_at"], - } - return None - - async def _store_in_storage( - self, storage_key: str, data: Any, table: StorageTable - ) -> None: - """Persist data to the given SQLite table (default: static_data JSON).""" - if table == StorageTable.STATIC_DATA: - try: - await self.storage.set_static_data( - key=storage_key, - data=json.dumps(data, separators=(",", ":")), - data_type="json", - ) - except Exception as exc: # noqa: BLE001 - logger.warning(f"[SWR] SQLite write failed for {storage_key}: {exc}") - - # ------------------------------------------------------------------ - # Shared low-level helpers - # ------------------------------------------------------------------ - async def _update_api_cache( self, cache_key: str, data: Any, cache_ttl: int ) -> None: @@ -182,16 +66,25 @@ async def _update_api_cache( except Exception as exc: # noqa: BLE001 logger.warning(f"[SWR] Valkey write failed for {cache_key}: {exc}") - async def _enqueue_refresh(self, entity_type: str, entity_id: str) -> None: - """Enqueue a background refresh, deduplicating via job_id.""" + async def _enqueue_refresh( + self, + entity_type: str, + entity_id: str, + refresh_coro: Coroutine[Any, Any, Any] | None = None, + ) -> None: + """Enqueue a background refresh, deduplicating via job_id. + + ``refresh_coro``, when provided, is passed to the task queue and + executed as the actual refresh work (Phase 4: asyncio, Phase 5: arq). + """ job_id = f"refresh:{entity_type}:{entity_id}" try: - already_running = await self.task_queue.is_job_pending_or_running(job_id) - if not already_running: + if not await self.task_queue.is_job_pending_or_running(job_id): await self.task_queue.enqueue( f"refresh_{entity_type}", entity_id, job_id=job_id, + coro=refresh_coro, ) except Exception as exc: # noqa: BLE001 logger.warning( diff --git a/app/domain/services/gamemode_service.py b/app/domain/services/gamemode_service.py index 1650f376..a7ecc331 100644 --- a/app/domain/services/gamemode_service.py +++ b/app/domain/services/gamemode_service.py @@ -2,10 +2,10 @@ from app.adapters.blizzard.parsers.gamemodes import parse_gamemodes_csv from app.config import settings -from app.domain.services.base_service import BaseService +from app.domain.services.static_data_service import StaticDataService -class GamemodeService(BaseService): +class GamemodeService(StaticDataService): """Domain service for gamemode data.""" async def list_gamemodes( diff --git a/app/domain/services/hero_service.py b/app/domain/services/hero_service.py index 1560e103..130f4b66 100644 --- a/app/domain/services/hero_service.py +++ b/app/domain/services/hero_service.py @@ -13,7 +13,7 @@ ) from app.adapters.blizzard.parsers.heroes_hitpoints import parse_heroes_hitpoints from app.config import settings -from app.domain.services.base_service import BaseService +from app.domain.services.static_data_service import StaticDataService from app.enums import Locale from app.exceptions import ParserBlizzardError, ParserParsingError from app.helpers import overfast_internal_error @@ -31,7 +31,7 @@ from app.roles.enums import Role -class HeroService(BaseService): +class HeroService(StaticDataService): """Domain service for hero data: list, detail, and usage statistics.""" # ------------------------------------------------------------------ diff --git a/app/domain/services/map_service.py b/app/domain/services/map_service.py index 76f1d5fc..0f9bb2dc 100644 --- a/app/domain/services/map_service.py +++ b/app/domain/services/map_service.py @@ -2,10 +2,10 @@ from app.adapters.blizzard.parsers.maps import parse_maps_csv from app.config import settings -from app.domain.services.base_service import BaseService +from app.domain.services.base_service import StaticDataService -class MapService(BaseService): +class MapService(StaticDataService): """Domain service for maps data.""" async def list_maps( diff --git a/app/domain/services/role_service.py b/app/domain/services/role_service.py index fbb2ddd5..2f66e6b3 100644 --- a/app/domain/services/role_service.py +++ b/app/domain/services/role_service.py @@ -2,13 +2,13 @@ from app.adapters.blizzard.parsers.roles import fetch_roles_html, parse_roles_html from app.config import settings -from app.domain.services.base_service import BaseService +from app.domain.services.static_data_service import StaticDataService from app.enums import Locale from app.exceptions import ParserParsingError from app.helpers import overfast_internal_error -class RoleService(BaseService): +class RoleService(StaticDataService): """Domain service for role data.""" async def list_roles( diff --git a/app/domain/services/static_data_service.py b/app/domain/services/static_data_service.py new file mode 100644 index 00000000..1bd1ebf0 --- /dev/null +++ b/app/domain/services/static_data_service.py @@ -0,0 +1,145 @@ +import json +import time +from typing import TYPE_CHECKING, Any + +from app.domain.services import BaseService +from app.monitoring.metrics import ( + background_refresh_triggered_total, + stale_responses_total, + storage_hits_total, +) +from app.overfast_logger import logger + +if TYPE_CHECKING: + from collections.abc import Callable + + +class StaticDataService(BaseService): + """SWR orchestration for static content backed by the ``static_data`` SQLite table. + + Staleness is determined by a configurable time threshold. Concrete static + services (heroes, maps, gamemodes, roles) call ``get_or_fetch`` with + domain-specific ``fetcher``, ``parser``, and optional ``result_filter`` + callables — no subclass-level overrides are needed for the storage layer. + + Note: Valkey API-cache *reads* happen at the Nginx/Lua layer before FastAPI + is reached; this service only ever *writes* to the API cache. + """ + + async def get_or_fetch( + self, + *, + storage_key: str, + fetcher: Callable[[], Any], + cache_key: str, + cache_ttl: int, + staleness_threshold: int, + entity_type: str, + parser: Callable[[Any], Any] | None = None, + result_filter: Callable[[Any], Any] | None = None, + ) -> tuple[Any, bool]: + """SWR orchestration for static data. + + Args: + storage_key: Key in the ``static_data`` SQLite table. + fetcher: Async callable that retrieves raw data from the upstream + source (Blizzard HTML/JSON or a local CSV file). + cache_key: Valkey API-cache key to update after serving data. + cache_ttl: TTL in seconds for the Valkey API-cache entry. + staleness_threshold: Seconds after which stored data is considered stale. + entity_type: Human-readable label used in metrics and logs. + parser: Optional callable that converts raw fetcher output into the + stored/returned format. Defaults to identity (raw data as-is). + result_filter: Optional callable applied to the stored data before + returning. Not persisted — re-applied on every request. + + Returns: + ``(data, is_stale)`` tuple. + """ + stored = await self._load_from_storage(storage_key) + + if stored is not None: + data = stored["data"] + age = int(time.time()) - stored["updated_at"] + is_stale = age >= staleness_threshold + + if is_stale: + logger.info( + f"[SWR] {entity_type} stale (age={age}s, " + f"threshold={staleness_threshold}s) — serving + triggering refresh" + ) + await self._enqueue_refresh( + entity_type, + storage_key, + refresh_coro=self._refresh_static( + storage_key, fetcher, parser, entity_type + ), + ) + stale_responses_total.inc() + background_refresh_triggered_total.labels(entity_type=entity_type).inc() + else: + logger.info( + f"[SWR] {entity_type} fresh (age={age}s) — serving from SQLite" + ) + + storage_hits_total.labels(result="hit").inc() + + if result_filter is not None: + data = result_filter(data) + + await self._update_api_cache(cache_key, data, cache_ttl) + return data, is_stale + + # Cold start — fetch synchronously + logger.info(f"[SWR] {entity_type} not in SQLite — fetching from source") + storage_hits_total.labels(result="miss").inc() + + raw = await fetcher() + data = parser(raw) if parser is not None else raw + await self._store_in_storage(storage_key, data) + + filtered = result_filter(data) if result_filter is not None else data + await self._update_api_cache(cache_key, filtered, cache_ttl) + return filtered, False + + async def _refresh_static( + self, + storage_key: str, + fetcher: Callable[[], Any], + parser: Callable[[Any], Any] | None, + entity_type: str, + ) -> None: + """Fetch fresh data and persist it — executed as a background task.""" + logger.info(f"[SWR] Background refresh started for {entity_type}/{storage_key}") + try: + raw = await fetcher() + data = parser(raw) if parser is not None else raw + await self._store_in_storage(storage_key, data) + logger.info( + f"[SWR] Background refresh complete for {entity_type}/{storage_key}" + ) + except Exception as exc: # noqa: BLE001 + logger.warning( + f"[SWR] Background refresh failed for {entity_type}/{storage_key}: {exc}" + ) + + async def _load_from_storage(self, storage_key: str) -> dict[str, Any] | None: + """Load data from the ``static_data`` SQLite table. Returns ``None`` on miss.""" + result = await self.storage.get_static_data(storage_key) + if result: + return { + "data": json.loads(result["data"]), + "updated_at": result["updated_at"], + } + return None + + async def _store_in_storage(self, storage_key: str, data: Any) -> None: + """Persist data to the ``static_data`` SQLite table as JSON.""" + try: + await self.storage.set_static_data( + key=storage_key, + data=json.dumps(data, separators=(",", ":")), + data_type="json", + ) + except Exception as exc: # noqa: BLE001 + logger.warning(f"[SWR] SQLite write failed for {storage_key}: {exc}") From 3641690698b757b6c5279706b02ec72bdffa14d5 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sat, 21 Feb 2026 14:19:40 +0100 Subject: [PATCH 05/26] --wip-- [skip ci] --- app/adapters/blizzard/parsers/heroes_hitpoints.py | 5 +---- app/domain/services/map_service.py | 3 ++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/app/adapters/blizzard/parsers/heroes_hitpoints.py b/app/adapters/blizzard/parsers/heroes_hitpoints.py index 9985078a..ad721046 100644 --- a/app/adapters/blizzard/parsers/heroes_hitpoints.py +++ b/app/adapters/blizzard/parsers/heroes_hitpoints.py @@ -15,10 +15,7 @@ def parse_heroes_hitpoints() -> dict[str, dict]: csv_reader = CSVReader() csv_data = csv_reader.read_csv_file("heroes") - return { - row["key"]: {"hitpoints": _get_hitpoints(row)} - for row in csv_data - } + return {row["key"]: {"hitpoints": _get_hitpoints(row)} for row in csv_data} def _get_hitpoints(row: dict) -> dict: diff --git a/app/domain/services/map_service.py b/app/domain/services/map_service.py index 0f9bb2dc..23af340f 100644 --- a/app/domain/services/map_service.py +++ b/app/domain/services/map_service.py @@ -2,7 +2,7 @@ from app.adapters.blizzard.parsers.maps import parse_maps_csv from app.config import settings -from app.domain.services.base_service import StaticDataService +from app.domain.services.static_data_service import StaticDataService class MapService(StaticDataService): @@ -17,6 +17,7 @@ async def list_maps( Stores the full (unfiltered) maps list in SQLite. """ + async def _fetch() -> list[dict]: return parse_maps_csv() From 879acc798d3c390462816852b4c2728917c41031 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sat, 21 Feb 2026 14:39:03 +0100 Subject: [PATCH 06/26] --wip-- [skip ci] --- app/adapters/csv/csv_reader.py | 28 +-- app/gamemodes/data/gamemodes.csv | 15 -- app/gamemodes/enums.py | 4 +- app/gamemodes/parsers/__init__.py | 0 app/gamemodes/parsers/gamemodes_parser.py | 27 --- app/helpers.py | 11 - app/heroes/commands/check_new_hero.py | 16 +- app/heroes/data/heroes.csv | 51 ----- app/heroes/enums.py | 4 +- app/heroes/parsers/__init__.py | 0 app/heroes/parsers/hero_parser.py | 216 ------------------ .../parsers/hero_stats_summary_parser.py | 169 -------------- app/heroes/parsers/heroes_parser.py | 40 ---- app/heroes/parsers/heroes_stats_parser.py | 23 -- app/maps/data/maps.csv | 58 ----- app/maps/enums.py | 4 +- app/maps/parsers/__init__.py | 0 app/maps/parsers/maps_parser.py | 34 --- app/parsers.py | 168 -------------- app/players/helpers.py | 6 +- app/roles/parsers/__init__.py | 0 app/roles/parsers/roles_parser.py | 36 --- tests/gamemodes/parsers/conftest.py | 8 - .../parsers/test_gamemodes_parser.py | 34 +-- tests/heroes/parsers/conftest.py | 26 --- tests/heroes/parsers/test_hero_parser.py | 101 -------- .../heroes/parsers/test_hero_stats_summary.py | 70 ++---- tests/heroes/parsers/test_heroes_parser.py | 53 ++++- tests/maps/parsers/conftest.py | 8 - tests/maps/parsers/test_maps_parser.py | 41 ++-- tests/roles/parsers/conftest.py | 8 - tests/roles/parsers/test_roles_parser.py | 28 ++- 32 files changed, 123 insertions(+), 1164 deletions(-) delete mode 100644 app/gamemodes/data/gamemodes.csv delete mode 100644 app/gamemodes/parsers/__init__.py delete mode 100644 app/gamemodes/parsers/gamemodes_parser.py delete mode 100644 app/heroes/data/heroes.csv delete mode 100644 app/heroes/parsers/__init__.py delete mode 100644 app/heroes/parsers/hero_parser.py delete mode 100644 app/heroes/parsers/hero_stats_summary_parser.py delete mode 100644 app/heroes/parsers/heroes_parser.py delete mode 100644 app/heroes/parsers/heroes_stats_parser.py delete mode 100644 app/maps/data/maps.csv delete mode 100644 app/maps/parsers/__init__.py delete mode 100644 app/maps/parsers/maps_parser.py delete mode 100644 app/parsers.py delete mode 100644 app/roles/parsers/__init__.py delete mode 100644 app/roles/parsers/roles_parser.py delete mode 100644 tests/gamemodes/parsers/conftest.py delete mode 100644 tests/heroes/parsers/conftest.py delete mode 100644 tests/heroes/parsers/test_hero_parser.py delete mode 100644 tests/maps/parsers/conftest.py delete mode 100644 tests/roles/parsers/conftest.py diff --git a/app/adapters/csv/csv_reader.py b/app/adapters/csv/csv_reader.py index d6f0a14d..8ee9e55d 100644 --- a/app/adapters/csv/csv_reader.py +++ b/app/adapters/csv/csv_reader.py @@ -5,35 +5,11 @@ class CSVReader: - """Adapter for reading CSV data files""" + """Adapter for reading CSV data files from app/adapters/csv/data/""" @staticmethod def read_csv_file(filename: str) -> list[dict[str, str]]: - """ - Read a CSV file from app/adapters/csv/data/ directory - - Args: - filename: Name of the CSV file without extension (e.g., "heroes", "maps", "gamemodes") - - Returns: - List of dictionaries with CSV rows - """ + """Read a CSV file by name (without extension) from the data/ directory.""" csv_path = Path(__file__).parent / "data" / f"{filename}.csv" with csv_path.open(encoding="utf-8") as csv_file: return list(csv.DictReader(csv_file, delimiter=",")) - - @staticmethod - def read_csv_file_legacy(filename: str) -> list[dict[str, str]]: - """ - Legacy method for reading CSV files from old location app/{module}/data/{module}.csv - This is kept for backward compatibility during migration. - - Args: - filename: Name of the module/CSV file (e.g., "heroes", "maps", "gamemodes") - - Returns: - List of dictionaries with CSV rows - """ - csv_path = Path.cwd() / "app" / filename / "data" / f"{filename}.csv" - with csv_path.open(encoding="utf-8") as csv_file: - return list(csv.DictReader(csv_file, delimiter=",")) diff --git a/app/gamemodes/data/gamemodes.csv b/app/gamemodes/data/gamemodes.csv deleted file mode 100644 index e74a8943..00000000 --- a/app/gamemodes/data/gamemodes.csv +++ /dev/null @@ -1,15 +0,0 @@ -key,name,description -assault,Assault,"Teams fight to capture or defend two successive points against the enemy team. It's an inactive Overwatch 1 gamemode, also called 2CP." -capture-the-flag,Capture the Flag,Teams compete to capture the enemy team’s flag while defending their own. -clash,Clash,"Vie for dominance across a series of capture points with dynamic spawns and linear map routes, so you spend less time running back to the battle and more time in the heart of it." -control,Control,Teams fight to hold a single objective. The first team to win two rounds wins the map. -deathmatch,Deathmatch,Race to reach 20 points first by racking up kills in a free-for-all format. -elimination,Elimination,"Dispatch all enemies to win the round. Win three rounds to claim victory. Available with teams of one, three, or six." -escort,Escort,"One team escorts a payload to its delivery point, while the other races to stop them." -flashpoint,Flashpoint,"Teams fight across our biggest PVP maps to date, New Junk City and Suravasa, to seize control of five different objectives in a fast-paced, best-of-five battle!" -hybrid,Hybrid,"Attackers capture a payload, then escort it to its destination; defenders try to hold them back." -payload-race,Payload Race,"Both teams get a payload to escort to the ending location while preventing the enemies from doing the same." -practice-range,Practice Range,"Learn the basics, practice and test your settings." -push,Push,Teams battle to take control of a robot and push it toward the enemy base. -team-deathmatch,Team Deathmatch,Team up and triumph over your enemies by scoring the most kills. -workshop,Workshop,"Experience custom and experimental gameplay, only in Custom Games." \ No newline at end of file diff --git a/app/gamemodes/enums.py b/app/gamemodes/enums.py index 5614f6b8..3cf8bda9 100644 --- a/app/gamemodes/enums.py +++ b/app/gamemodes/enums.py @@ -1,9 +1,9 @@ from enum import StrEnum -from app.helpers import read_csv_data_file +from app.adapters.csv import CSVReader # Dynamically create the MapGamemode enum by using the CSV File -gamemodes_data = read_csv_data_file("gamemodes") +gamemodes_data = CSVReader.read_csv_file("gamemodes") MapGamemode = StrEnum( "MapGamemode", { diff --git a/app/gamemodes/parsers/__init__.py b/app/gamemodes/parsers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/gamemodes/parsers/gamemodes_parser.py b/app/gamemodes/parsers/gamemodes_parser.py deleted file mode 100644 index 90c39e42..00000000 --- a/app/gamemodes/parsers/gamemodes_parser.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Gamemodes Parser module""" - -from app.parsers import CSVParser - - -class GamemodesParser(CSVParser): - """Overwatch map gamemodes list page Parser class""" - - filename = "gamemodes" - - def parse_data(self) -> list[dict]: - return [ - { - "key": gamemode["key"], - "name": gamemode["name"], - "icon": self.get_static_url( - f"{gamemode['key']}-icon", - extension="svg", - ), - "description": gamemode["description"], - "screenshot": self.get_static_url( - gamemode["key"], - extension="avif", - ), - } - for gamemode in self.csv_data - ] diff --git a/app/helpers.py b/app/helpers.py index a3ae7eb9..8b6ad492 100644 --- a/app/helpers.py +++ b/app/helpers.py @@ -1,10 +1,8 @@ """Parser Helpers module""" -import csv import traceback from datetime import UTC, datetime from functools import cache -from pathlib import Path from typing import TYPE_CHECKING import httpx @@ -203,15 +201,6 @@ def send_discord_webhook_message( ) -@cache -def read_csv_data_file(filename: str) -> list[dict[str, str]]: - """Helper method for obtaining CSV DictReader from a path""" - with Path(f"{Path.cwd()}/app/{filename}/data/{filename}.csv").open( - encoding="utf-8" - ) as csv_file: - return list(csv.DictReader(csv_file, delimiter=",")) - - @cache def get_human_readable_duration(duration: int) -> str: # Define the time units diff --git a/app/heroes/commands/check_new_hero.py b/app/heroes/commands/check_new_hero.py index 6941ee50..6104fc83 100644 --- a/app/heroes/commands/check_new_hero.py +++ b/app/heroes/commands/check_new_hero.py @@ -7,6 +7,7 @@ from fastapi import HTTPException +from app.adapters.blizzard.parsers.heroes import fetch_heroes_html, parse_heroes_html from app.config import settings from app.exceptions import ParserParsingError from app.helpers import send_discord_webhook_message @@ -14,22 +15,17 @@ from app.overfast_logger import logger from ..enums import HeroKey -from ..parsers.heroes_parser import HeroesParser async def get_distant_hero_keys(client: OverFastClient) -> set[str]: """Get a set of Overwatch hero keys from the Blizzard heroes page""" - heroes_parser = HeroesParser(client=client) - try: - await heroes_parser.parse() + html = await fetch_heroes_html(client) + heroes = parse_heroes_html(html) except (HTTPException, ParserParsingError) as error: raise SystemExit from error - if not isinstance(heroes_parser.data, list): - raise SystemExit - - return {hero["key"] for hero in heroes_parser.data} + return {hero["key"] for hero in heroes} def get_local_hero_keys() -> set[str]: @@ -46,7 +42,6 @@ async def main(): logger.info("OK ! Starting to check if a new hero is here...") - # Instanciate one HTTPX Client to use for all the updates client = OverFastClient() distant_hero_keys = await get_distant_hero_keys(client) @@ -54,7 +49,6 @@ async def main(): await client.aclose() - # Compare both sets. If we have a difference, notify the developer new_hero_keys = distant_hero_keys - local_hero_keys if len(new_hero_keys) > 0: logger.info("New hero keys were found : {}", new_hero_keys) @@ -73,7 +67,7 @@ async def main(): "inline": False, }, ], - color=0x2ECC71, # Green + color=0x2ECC71, ) else: logger.info("No new hero found. Exiting.") diff --git a/app/heroes/data/heroes.csv b/app/heroes/data/heroes.csv deleted file mode 100644 index f56c2394..00000000 --- a/app/heroes/data/heroes.csv +++ /dev/null @@ -1,51 +0,0 @@ -key,name,role,health,armor,shields -ana,Ana,support,250,0,0 -anran,Anran,damage,250,0,0 -ashe,Ashe,damage,250,0,0 -baptiste,Baptiste,support,250,0,0 -bastion,Bastion,damage,250,100,0 -brigitte,Brigitte,support,200,50,0 -cassidy,Cassidy,damage,250,0,0 -dva,D.Va,tank,325,325,0 -domina,Domina,tank,250,0,400 -doomfist,Doomfist,tank,525,0,0 -echo,Echo,damage,150,0,75 -emre,Emre,damage,250,0,0 -freja,Freja,damage,225,0,0 -genji,Genji,damage,250,0,0 -hazard,Hazard,tank,425,225,0 -hanzo,Hanzo,damage,250,0,0 -illari,Illari,support,250,0,0 -jetpack-cat,Jetpack Cat,support,225,0,0 -junker-queen,Junker Queen,tank,525,0,0 -junkrat,Junkrat,damage,250,0,0 -juno,Juno,support,75,0,150 -kiriko,Kiriko,support,225,0,0 -lifeweaver,Lifeweaver,support,200,0,50 -lucio,Lúcio,support,225,0,0 -mauga,Mauga,tank,575,150,0 -mei,Mei,damage,300,0,0 -mercy,Mercy,support,225,0,0 -mizuki,Mizuki,support,250,0,0 -moira,Moira,support,225,0,0 -orisa,Orisa,tank,300,300,0 -pharah,Pharah,damage,225,0,0 -ramattra,Ramattra,tank,425,100,0 -reaper,Reaper,damage,300,0,0 -reinhardt,Reinhardt,tank,400,300,0 -roadhog,Roadhog,tank,750,0,0 -sigma,Sigma,tank,350,0,275 -sojourn,Sojourn,damage,225,0,0 -soldier-76,Soldier: 76,damage,250,0,0 -sombra,Sombra,damage,225,0,0 -symmetra,Symmetra,damage,125,0,150 -torbjorn,Torbjörn,damage,225,75,0 -tracer,Tracer,damage,175,0,0 -vendetta,Vendetta,damage,175,100,0 -venture,Venture,damage,250,0,0 -widowmaker,Widowmaker,damage,225,0,0 -winston,Winston,tank,425,200,0 -wrecking-ball,Wrecking Ball,tank,450,125,150 -wuyang,Wuyang,support,225,0,0 -zarya,Zarya,tank,325,0,225 -zenyatta,Zenyatta,support,75,0,175 diff --git a/app/heroes/enums.py b/app/heroes/enums.py index 0661f3a7..9f6149d7 100644 --- a/app/heroes/enums.py +++ b/app/heroes/enums.py @@ -1,6 +1,6 @@ from enum import StrEnum -from app.helpers import read_csv_data_file +from app.adapters.csv import CSVReader class MediaType(StrEnum): @@ -12,7 +12,7 @@ class MediaType(StrEnum): # Dynamically create the HeroKey enum by using the CSV File -heroes_data = read_csv_data_file("heroes") +heroes_data = CSVReader.read_csv_file("heroes") HeroKey = StrEnum( "HeroKey", { diff --git a/app/heroes/parsers/__init__.py b/app/heroes/parsers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/heroes/parsers/hero_parser.py b/app/heroes/parsers/hero_parser.py deleted file mode 100644 index 034bd135..00000000 --- a/app/heroes/parsers/hero_parser.py +++ /dev/null @@ -1,216 +0,0 @@ -"""Hero page Parser module""" - -import re -from typing import TYPE_CHECKING, ClassVar - -from fastapi import status - -from app.config import settings -from app.enums import Locale -from app.exceptions import ParserBlizzardError -from app.overfast_logger import logger -from app.parsers import HTMLParser -from app.roles.helpers import get_role_from_icon_url - -from ..enums import MediaType - -if TYPE_CHECKING: - from selectolax.lexbor import LexborNode - - -class HeroParser(HTMLParser): - """Overwatch single hero page Parser class""" - - root_path = settings.heroes_path - valid_http_codes: ClassVar[list] = [ - 200, # Classic response - 404, # Hero Not Found response, we want to handle it here - ] - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.locale = kwargs.get("locale") or Locale.ENGLISH_US - - def get_blizzard_url(self, **kwargs) -> str: - return f"{super().get_blizzard_url(**kwargs)}{kwargs.get('hero_key')}/" - - async def parse_data(self) -> dict: - # We must check if we have the expected section for hero. If not, - # it means the hero hasn't been found and/or released yet. - if not ( - abilities_section := self.root_tag.css_first("div.abilities-container") - ): - raise ParserBlizzardError( - status_code=status.HTTP_404_NOT_FOUND, - message="Hero not found or not released yet", - ) - - overview_section = self.root_tag.css_first("blz-page-header") - lore_section = self.root_tag.css_first("blz-section.lore") - - return { - **self.__get_summary(overview_section), - "abilities": self.__get_abilities(abilities_section), - "story": self.__get_story(lore_section), - } - - def __get_summary(self, overview_section: LexborNode) -> dict: - header_section = overview_section.css_first("blz-header") - extra_list_items = overview_section.css_first("blz-list").css("blz-list-item") - birthday, age = self._get_birthday_and_age( - text=extra_list_items[2].css_first("p").text(), locale=self.locale - ) - icon_url = extra_list_items[0].css_first("blz-icon").attributes["src"] or "" - - return { - "name": header_section.css_first("h2").text(), - "description": header_section.css_first("p").text().strip(), - "role": get_role_from_icon_url(icon_url), - "location": extra_list_items[1].text().strip(), - "birthday": birthday, - "age": age, - } - - @staticmethod - def _get_birthday_and_age( - text: str, locale: Locale - ) -> tuple[str | None, int | None]: - """Get birthday and age from text for a given hero""" - - # Regex matching the birthday for every known locale - birthday_regex = r"^(.*) [\(（].*[:：] ?(\d+).*[\)）]$" - - result = re.match(birthday_regex, text) - if not result: - return None, None - - # Text corresponding to "Unknown" in the locale of the page - unknown_texts = { - Locale.GERMAN: "Unbekannt", - Locale.ENGLISH_EU: "Unknown", - Locale.ENGLISH_US: "Unknown", - Locale.SPANISH_EU: "Desconocido", - Locale.SPANISH_LATIN: "Desconocido", - Locale.FRENCH: "Inconnu", - Locale.ITALIANO: "Sconosciuto", - Locale.JAPANESE: "不明", - Locale.KOREAN: "알 수 없음", - Locale.POLISH: "Nieznane", - Locale.PORTUGUESE_BRAZIL: "Desconhecido", - Locale.RUSSIAN: "Неизвестно", - Locale.CHINESE_TAIWAN: "未知", - } - unknown_text = unknown_texts.get(locale, "Unknown") - - birthday = result[1] if result[1] != unknown_text else None - age = int(result[2]) if result[2] else None - - return birthday, age - - @staticmethod - def __get_abilities(abilities_section: LexborNode) -> list[dict]: - carousel_section_div = abilities_section.css_first("blz-carousel-section") - abilities_list_div = carousel_section_div.css_first("blz-carousel") - - abilities_desc = [ - ( - desc_div.css_first("p") - .text() - .strip() - .replace("\r", "") - .replace("\n", " ") - ) - for desc_div in abilities_list_div.css("blz-feature") - ] - - abilities_videos = [ - { - "thumbnail": video_div.attributes["poster"], - "link": { - "mp4": video_div.attributes["mp4"], - "webm": video_div.attributes["webm"], - }, - } - for video_div in carousel_section_div.css("blz-web-video") - ] - - return [ - { - "name": ability_div.attributes["label"], - "description": abilities_desc[ability_index].strip(), - "icon": ability_div.css_first("blz-image").attributes["src"], - "video": abilities_videos[ability_index], - } - for ability_index, ability_div in enumerate( - abilities_list_div.css_first("blz-tab-controls").css("blz-tab-control"), - ) - ] - - def __get_story(self, lore_section: LexborNode) -> dict: - showcase_section = lore_section.css_first("blz-showcase") - - return { - "summary": ( - showcase_section.css_first("blz-header p") - .text() - .strip() - .replace("\n", "") - ), - "media": self.__get_media(showcase_section), - "chapters": self.__get_story_chapters( - lore_section.css_first("blz-accordion-section blz-accordion") - ), - } - - def __get_media(self, showcase_section: LexborNode) -> dict | None: - if video := showcase_section.css_first("blz-youtube-video"): - return { - "type": MediaType.VIDEO, - "link": f"https://youtu.be/{video.attributes['youtube-id']}", - } - - if button := showcase_section.css_first("blz-button"): - if not button.attributes["href"]: - logger.warning("Missing href attribute in button element") - return None - - return { - "type": ( - MediaType.SHORT_STORY - if button.attributes["analytics-label"] == "short-story" - else MediaType.COMIC - ), - "link": self._get_full_url(button.attributes["href"]), - } - - return None - - @staticmethod - def _get_full_url(url: str) -> str: - """Get full URL from extracted URL. If URL begins with /, we use the - blizzard host to get the full URL""" - return f"{settings.blizzard_host}{url}" if url.startswith("/") else url - - @staticmethod - def __get_story_chapters(accordion: LexborNode) -> list[dict]: - chapters_content = [ - ( - " ".join( - [paragraph.text() for paragraph in content_container.css("p,pr")], - ).strip() - ) - for content_container in accordion.css("div[slot=content]") - ] - chapters_picture = [ - picture.attributes["src"] for picture in accordion.css("blz-image") - ] - titles = [node for node in accordion.iter() if node.tag == "span"] - - return [ - { - "title": title_span.text().capitalize().strip(), - "content": chapters_content[title_index], - "picture": chapters_picture[title_index], - } - for title_index, title_span in enumerate(titles) - ] diff --git a/app/heroes/parsers/hero_stats_summary_parser.py b/app/heroes/parsers/hero_stats_summary_parser.py deleted file mode 100644 index a87009b1..00000000 --- a/app/heroes/parsers/hero_stats_summary_parser.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Hero Stats Summary Parser module""" - -from typing import ClassVar - -from fastapi import status - -from app.config import settings -from app.exceptions import ParserBlizzardError -from app.parsers import JSONParser -from app.players.enums import PlayerGamemode, PlayerPlatform - - -class HeroStatsSummaryParser(JSONParser): - """Hero Stats Summary Parser class""" - - request_headers_headers: ClassVar[dict] = JSONParser.request_headers | { - "X-Requested-With": "XMLHttpRequest" - } - - root_path: str = settings.hero_stats_path - - platform_mapping: ClassVar[dict[PlayerPlatform, str]] = { - PlayerPlatform.PC: "PC", - PlayerPlatform.CONSOLE: "Console", - } - - gamemode_mapping: ClassVar[dict[PlayerGamemode, str]] = { - PlayerGamemode.QUICKPLAY: "0", - PlayerGamemode.COMPETITIVE: "1", - } - - def __init__(self, **kwargs): - # Mandatory query params - self.platform_filter: str = self.platform_mapping[kwargs["platform"]] - self.gamemode = kwargs["gamemode"] - self.gamemode_filter: str = self.gamemode_mapping[self.gamemode] - self.region_filter: str = kwargs["region"].capitalize() - self.order_by: str = kwargs["order_by"] - - # Optional query params - self.map_filter: str = kwargs.get("map") or "all-maps" - self.role_filter: str | None = kwargs.get("role") - self.competitive_division_filter: str = ( - kwargs.get("competitive_division") or "all" - ).capitalize() - - super().__init__(**kwargs) - - async def parse_data(self) -> list[dict]: - """ - Parses and filters hero data from the loaded JSON based on the current filters. - - Returns: - list[dict]: A list of dictionaries containing hero stats data. - - Raises: - ParserBlizzardError: If the selected map does not match the gamemode in the filters. - """ - # Check if the data matches the expected map filter - self.check_data_validity() - - # Role filter isn't applied server-side, so we filter it here if provided. - hero_stats_data = self.filter_heroes() - - # Only retrieve needed data - hero_stats_data = self.apply_transformations(hero_stats_data) - - # Apply ordering before returning - return self.apply_ordering(hero_stats_data) - - def check_data_validity(self) -> None: - """ - Validates that the selected map matches the current gamemode. - - Raises: - ParserBlizzardError: If the selected map does not match the gamemode. - """ - if self.map_filter != self.json_data["selected"]["map"]: - raise ParserBlizzardError( - status_code=status.HTTP_400_BAD_REQUEST, - message=( - f"Selected map '{self.map_filter}' is not compatible with '{self.gamemode}' gamemode." - ), - ) - - def filter_heroes(self) -> list[dict]: - """ - Filters hero statistics based on the current role filter. - - Returns: - list[dict]: A list of hero entries matching the role filter (if provided). - """ - return [ - rate - for rate in self.json_data["rates"] - if ( - self.role_filter is None - or rate["hero"]["role"].lower() == self.role_filter - ) - ] - - def apply_transformations(self, hero_stats_data: list[dict]) -> list[dict]: - """ - Extracts and structures relevant hero statistics fields from the filtered data. - - Args: - hero_stats_data (list[dict]): The filtered list of hero data. - - Returns: - list[dict]: A list of dictionaries, each containing only the key statistics fields for a hero. - """ - return [ - { - "hero": rate["id"], - "pickrate": self.get_rate_value(rate["cells"]["pickrate"]), - "winrate": self.get_rate_value(rate["cells"]["winrate"]), - } - for rate in hero_stats_data - ] - - def apply_ordering(self, hero_stats_data: list[dict]) -> list[dict]: - """ - Orders the hero statistics data based on the specified field and direction. - - Returns: - list[dict]: The input list, sorted according to the provided ordering. - """ - order_field, order_arrangement = self.order_by.split(":") - hero_stats_data.sort( - key=lambda hero_stat: hero_stat[order_field], - reverse=order_arrangement == "desc", - ) - return hero_stats_data - - def get_blizzard_query_params(self, **kwargs) -> dict: - """ - Constructs a dictionary of query parameters for a Blizzard API request based - on the current filter attributes and additional keyword arguments. - - Args: - **kwargs: Additional keyword arguments that may influence the query parameters. - - gamemode (PlayerGamemode): The game mode being queried. If set to PlayerGamemode.COMPETITIVE, - the 'tier' parameter will be included based on the competitive division filter. - - Returns: - dict: A dictionary containing the Blizzard API query parameters, including platform, game mode, - region, and map filters. If the game mode is competitive, the competitive division tier is also included. - """ - blizzard_query_params = { - "input": self.platform_filter, - "rq": self.gamemode_filter, - "region": self.region_filter, - "map": self.map_filter, - } - - if kwargs["gamemode"] == PlayerGamemode.COMPETITIVE: - blizzard_query_params["tier"] = self.competitive_division_filter - - return blizzard_query_params - - @staticmethod - def get_rate_value(rate: float) -> float: - """ - Get the output rate value to display to users. - - Whenever data isn't available for some reason, data from Blizzard will be -1 - We're converting it to zero in order to be coherent regarding API return values. - """ - return rate if rate != -1 else 0.0 diff --git a/app/heroes/parsers/heroes_parser.py b/app/heroes/parsers/heroes_parser.py deleted file mode 100644 index b6c0ab42..00000000 --- a/app/heroes/parsers/heroes_parser.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Heroes page Parser module""" - -from typing import cast - -from app.config import settings -from app.exceptions import ParserParsingError -from app.parsers import HTMLParser - - -class HeroesParser(HTMLParser): - """Overwatch heroes list page Parser class""" - - root_path = settings.heroes_path - - async def parse_data(self) -> list[dict]: - heroes = [] - for hero in self.root_tag.css("div.heroIndexWrapper blz-media-gallery a"): - if not hero.attributes["href"]: - msg = "Invalid hero URL" - raise ParserParsingError(msg) - heroes.append( - { - "key": hero.attributes["href"].split("/")[-1], - "name": hero.css_first("blz-card blz-content-block h2").text(), - "portrait": hero.css_first("blz-card blz-image").attributes["src"], - "role": hero.attributes["data-role"], - } - ) - - return sorted(heroes, key=lambda hero: hero["key"]) - - def filter_request_using_query(self, **kwargs) -> list[dict]: - # Type assertion: we know parse_data returns list[dict] - data = cast("list[dict]", self.data) - role = kwargs.get("role") - return ( - data - if not role - else [hero_dict for hero_dict in data if hero_dict["role"] == role] - ) diff --git a/app/heroes/parsers/heroes_stats_parser.py b/app/heroes/parsers/heroes_stats_parser.py deleted file mode 100644 index 4def48e0..00000000 --- a/app/heroes/parsers/heroes_stats_parser.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Heroes Stats Parser module""" - -from typing import ClassVar - -from app.parsers import CSVParser - - -class HeroesStatsParser(CSVParser): - """Heroes stats (health, armor, shields) Parser class""" - - filename = "heroes" - hitpoints_keys: ClassVar[set[str]] = {"health", "armor", "shields"} - - def parse_data(self) -> dict: - return { - hero_stats["key"]: {"hitpoints": self.__get_hitpoints(hero_stats)} - for hero_stats in self.csv_data - } - - def __get_hitpoints(self, hero_stats: dict) -> dict: - hitpoints = {hp_key: int(hero_stats[hp_key]) for hp_key in self.hitpoints_keys} - hitpoints["total"] = sum(hitpoints.values()) - return hitpoints diff --git a/app/maps/data/maps.csv b/app/maps/data/maps.csv deleted file mode 100644 index 7fdb53ec..00000000 --- a/app/maps/data/maps.csv +++ /dev/null @@ -1,58 +0,0 @@ -key,name,gamemodes,location,country_code -aatlis,Aatlis,flashpoint,Morocco,MA -antarctic-peninsula,Antarctic Peninsula,control,Antarctica,AQ -anubis,Temple of Anubis,assault,"Giza Plateau, Egypt",EG -arena-victoriae,Arena Victoriae,control,"Colosseo, Rome, Italy",IT -ayutthaya,Ayutthaya,capture-the-flag,Thailand,TH -black-forest,Black Forest,elimination,Germany,DE -blizzard-world,Blizzard World,hybrid,"Irvine, California, United States",US -busan,Busan,control,South Korea,KR -castillo,Castillo,elimination,Mexico,MX -chateau-guillard,Château Guillard,"deathmatch,team-deathmatch","Annecy, France",FR -circuit-royal,Circuit Royal,escort,"Monte Carlo, Monaco",MC -colosseo,Colosseo,push,"Rome, Italy",IT -dorado,Dorado,escort,Mexico,MX -ecopoint-antarctica,Ecopoint: Antarctica,elimination,Antarctica,AQ -eichenwalde,Eichenwalde,hybrid,"Stuttgart, Germany",DE -esperanca,Esperança,push,Portugal,PT -gogadoro,Gogadoro,control,"Busan, South Korea",KR -hanamura,Hanamura,assault,"Tokyo, Japan",JP -hanaoka,Hanaoka,clash,"Tokyo, Japan",JP -havana,Havana,escort,"Havana, Cuba",CU -hollywood,Hollywood,hybrid,"Los Angeles, United States",US -horizon,Horizon Lunar Colony,assault,Earth's moon, -ilios,Ilios,control,Greece,GR -junkertown,Junkertown,escort,Central Australia,AU -lijiang-tower,Lijiang Tower,control,China,CN -kanezaka,Kanezaka,"deathmatch,team-deathmatch","Tokyo, Japan",JP -kings-row,King’s Row,hybrid,"London, United Kingdom",UK -malevento,Malevento,"deathmatch,team-deathmatch",Italy,IT -midtown,Midtown,hybrid,"New York, United States",US -necropolis,Necropolis,elimination,Egypt,EG -nepal,Nepal,control,Nepal,NP -new-junk-city,New Junk City,flashpoint,Central Australia,AU -new-queen-street,New Queen Street,push,"Toronto, Canada",CA -numbani,Numbani,hybrid,Numbani (near Nigeria), -oasis,Oasis,control,Iraq,IQ -paraiso,Paraíso,hybrid,"Rio de Janeiro, Brazil",BR -paris,Paris,assault,"Paris, France",FR -petra,Petra,"deathmatch,team-deathmatch",Southern Jordan,JO -place-lacroix,Place Lacroix,push,"Paris, France",FR -powder-keg-mine,Powder Keg Mine,payload-race,"Deadlock Gorge, Arizona, United States",US -practice-range,Practice Range,practice-range,Swiss HQ,CH -redwood-dam,Redwood Dam,push,Gibraltar,GI -rialto,Rialto,escort,"Venice, Italy",IT -route-66,Route 66,escort,"Albuquerque, New Mexico, United States",US -runasapi,Runasapi,push,Peru,PE -samoa,Samoa,control,Samoa,WS -shambali-monastery,Shambali Monastery,escort,Nepal,NP -suravasa,Suravasa,flashpoint,India,IN -thames-district,Thames District,payload-race,"London, United Kingdom",UK -throne-of-anubis,Throne of Anubis,clash,"Giza Plateau, Egypt",EG -volskaya,Volskaya Industries,assault,"St. Petersburg, Russia",RU -watchpoint-gibraltar,Watchpoint: Gibraltar,escort,Gibraltar,GI -workshop-chamber,Workshop Chamber,workshop,Earth, -workshop-expanse,Workshop Expanse,workshop,Earth, -workshop-green-screen,Workshop Green Screen,workshop,Earth, -workshop-island,Workshop Island,workshop,Earth, -wuxing-university,Wuxing University,control,"Chengdu, Sichuan, China",CN \ No newline at end of file diff --git a/app/maps/enums.py b/app/maps/enums.py index 10974755..bd37d5da 100644 --- a/app/maps/enums.py +++ b/app/maps/enums.py @@ -1,9 +1,9 @@ from enum import StrEnum -from app.helpers import read_csv_data_file +from app.adapters.csv import CSVReader # Dynamically create the MapKey enum by using the CSV File -maps_data = read_csv_data_file("maps") +maps_data = CSVReader.read_csv_file("maps") MapKey = StrEnum( "MapKey", { diff --git a/app/maps/parsers/__init__.py b/app/maps/parsers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/maps/parsers/maps_parser.py b/app/maps/parsers/maps_parser.py deleted file mode 100644 index 6a776a8b..00000000 --- a/app/maps/parsers/maps_parser.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Maps Parser module""" - -from typing import cast - -from app.parsers import CSVParser - - -class MapsParser(CSVParser): - """Overwatch maps list page Parser class""" - - filename = "maps" - - def parse_data(self) -> list[dict]: - return [ - { - "key": map_dict["key"], - "name": map_dict["name"], - "screenshot": self.get_static_url(map_dict["key"]), - "gamemodes": map_dict["gamemodes"].split(","), - "location": map_dict["location"], - "country_code": map_dict.get("country_code") or None, - } - for map_dict in self.csv_data - ] - - def filter_request_using_query(self, **kwargs) -> list: - # Type assertion: we know parse_data returns list[dict] - data = cast("list[dict]", self.data) - gamemode = kwargs.get("gamemode") - return ( - data - if not gamemode - else [map_dict for map_dict in data if gamemode in map_dict["gamemodes"]] - ) diff --git a/app/parsers.py b/app/parsers.py deleted file mode 100644 index e03c252b..00000000 --- a/app/parsers.py +++ /dev/null @@ -1,168 +0,0 @@ -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, ClassVar - -from fastapi import status -from selectolax.lexbor import LexborHTMLParser - -from .cache_manager import CacheManager -from .config import settings -from .enums import Locale -from .exceptions import ParserParsingError -from .helpers import read_csv_data_file -from .overfast_client import OverFastClient -from .overfast_logger import logger - -if TYPE_CHECKING: - import httpx - - -class AbstractParser(ABC): - """Abstract Parser class used to define generic behavior for parsers. - - A parser is meant to convert some input data into meaningful data - in dict/list format. The Parse Cache system is handled here. - """ - - cache_manager = CacheManager() - - def __init__(self, **_): - self.data: dict | list | None = None - - @abstractmethod - async def parse(self) -> None: - """Method used to retrieve data, parsing it and - storing it into self.data attribute. - """ - - def filter_request_using_query(self, **_) -> dict | list | None: - """If the route contains subroutes accessible using GET queries, this method - will filter data using the query data. This method should be - redefined in child classes if needed. The default behaviour is to return - the parsed data directly. - """ - return self.data - - -class CSVParser(AbstractParser): - """CSV Parser class used to define generic behavior for parsers used - to extract data from local CSV files. - """ - - # Name of CSV file to retrieve (without extension), also - # used as a sub-folder name for storing related static files - filename: str - - async def parse(self) -> None: - """Method used to retrieve data from CSV file and storing - it into self.data attribute - """ - - # Read the CSV file - self.csv_data = read_csv_data_file(self.filename) - - # Parse the data - self.data = self.parse_data() - - @abstractmethod - def parse_data(self) -> dict | list[dict]: - """Main submethod of the parser, mainly doing the parsing of CSV data and - returning a dict, which will be cached and used by the API. Can - raise an error if there is an issue when parsing the data. - """ - - def get_static_url(self, key: str, extension: str = "jpg") -> str: - """Method used to retrieve the URL of a local static file""" - return f"{settings.app_base_url}/static/{self.filename}/{key}.{extension}" - - -class APIParser(AbstractParser): - """Abstract API Parser class used to define generic behavior for parsers used - to extract data from Blizzard HTML pages. The blizzard URL call is handled here. - """ - - # List of valid HTTP codes when retrieving Blizzard pages - valid_http_codes: ClassVar[list] = [status.HTTP_200_OK] - - # Request headers to send while making the request - request_headers: ClassVar[dict] = {} - - def __init__(self, **kwargs): - self.blizzard_url = self.get_blizzard_url(**kwargs) - self.blizzard_query_params = self.get_blizzard_query_params(**kwargs) - self.overfast_client = OverFastClient() - super().__init__(**kwargs) - - @property - @abstractmethod - def root_path(self) -> str: - """Root path of the Blizzard URL containing the data (/en-us/career/, etc.""" - - @abstractmethod - def store_response_data(self, response: httpx.Response) -> None: - """Submethod to handle response data storage""" - - @abstractmethod - async def parse_data(self) -> dict | list[dict]: - """Main submethod of the parser, mainly doing the parsing of input data and - returning a dict, which will be cached and used by the API. Can - raise an error if there is an issue when parsing the data. - """ - - async def parse(self) -> None: - """Method used to retrieve data from Blizzard, parsing it - and storing it into self.data attribute. - """ - response = await self.overfast_client.get( - url=self.blizzard_url, - headers=self.request_headers, - params=self.blizzard_query_params, - ) - if response.status_code not in self.valid_http_codes: - raise self.overfast_client.blizzard_response_error_from_response(response) - - # Store associated request data - self.store_response_data(response) - - # Parse stored request data - await self.parse_response_data() - - def get_blizzard_url(self, **kwargs) -> str: - """URL used when requesting data to Blizzard. It usually is a concatenation - of root url and query data (kwargs) if the Controller supports it. - For example : single hero page (hero key), player career page - (player id, etc.). Default is just the blizzard root url. - """ - locale = kwargs.get("locale") or Locale.ENGLISH_US - return f"{settings.blizzard_host}/{locale}{self.root_path}" - - def get_blizzard_query_params(self, **_) -> dict: - """Query params to use when calling Blizzard URL. Defaults to empty dict""" - return {} - - async def parse_response_data(self) -> None: - logger.info("Parsing data...") - try: - self.data = await self.parse_data() - except (AttributeError, KeyError, IndexError, TypeError) as error: - raise ParserParsingError(repr(error)) from error - - -class HTMLParser(APIParser): - request_headers: ClassVar[dict] = {"Accept": "text/html"} - - def store_response_data(self, response: httpx.Response) -> None: - """Initialize parser tag with Blizzard response""" - self.create_parser_tag(response.text) - - def create_parser_tag(self, html_content: str) -> None: - self.root_tag = LexborHTMLParser(html_content).css_first( - "div.main-content,main" - ) - - -class JSONParser(APIParser): - request_headers: ClassVar[dict] = {"Accept": "application/json"} - - def store_response_data(self, response: httpx.Response) -> None: - """Initialize object with Blizzard response""" - self.json_data = response.json() diff --git a/app/players/helpers.py b/app/players/helpers.py index 9fdeb8c1..6aa352cf 100644 --- a/app/players/helpers.py +++ b/app/players/helpers.py @@ -2,7 +2,7 @@ import unicodedata from functools import cache -from app.helpers import read_csv_data_file +from app.adapters.csv import CSVReader from app.roles.enums import Role from .enums import CareerStatCategory, CompetitiveDivision, CompetitiveRole, HeroKey @@ -16,7 +16,7 @@ @cache def get_hero_name(hero_key: HeroKey) -> str: # ty: ignore[invalid-type-form] """Get a hero name based on the CSV file""" - heroes_data = read_csv_data_file("heroes") + heroes_data = CSVReader.read_csv_file("heroes") return next( ( hero_data["name"] @@ -160,7 +160,7 @@ def remove_accents(input_str: str) -> str: @cache def get_hero_role(hero_key: HeroKey) -> Role | None: # ty: ignore[invalid-type-form] """Get the role of a given hero based on the CSV file""" - heroes_data = read_csv_data_file("heroes") + heroes_data = CSVReader.read_csv_file("heroes") role_key = next( ( hero_data["role"] diff --git a/app/roles/parsers/__init__.py b/app/roles/parsers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/roles/parsers/roles_parser.py b/app/roles/parsers/roles_parser.py deleted file mode 100644 index 300229e2..00000000 --- a/app/roles/parsers/roles_parser.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Roles Parser module""" - -from app.config import settings -from app.parsers import HTMLParser - -from ..helpers import get_role_from_icon_url - - -class RolesParser(HTMLParser): - """Overwatch map gamemodes list page Parser class""" - - root_path = settings.home_path - - async def parse_data(self) -> list[dict]: - roles_container = self.root_tag.css_first( - "div.homepage-features-heroes blz-feature-carousel-section" - ) - - roles_icons = [ - role_icon_div.css_first("blz-image").attributes["src"] - for role_icon_div in roles_container.css_first("blz-tab-controls").css( - "blz-tab-control" - ) - ] - - return [ - { - "key": get_role_from_icon_url(roles_icons[role_index]), - "name": role_div.css_first("blz-header h3").text().capitalize(), - "icon": roles_icons[role_index], - "description": (role_div.css_first("blz-header div").text().strip()), - } - for role_index, role_div in list( - enumerate(roles_container.css("blz-feature")) - )[:3] - ] diff --git a/tests/gamemodes/parsers/conftest.py b/tests/gamemodes/parsers/conftest.py deleted file mode 100644 index 1e03bdbc..00000000 --- a/tests/gamemodes/parsers/conftest.py +++ /dev/null @@ -1,8 +0,0 @@ -import pytest - -from app.gamemodes.parsers.gamemodes_parser import GamemodesParser - - -@pytest.fixture(scope="package") -def gamemodes_parser() -> GamemodesParser: - return GamemodesParser() diff --git a/tests/gamemodes/parsers/test_gamemodes_parser.py b/tests/gamemodes/parsers/test_gamemodes_parser.py index 9fefd576..169a8f28 100644 --- a/tests/gamemodes/parsers/test_gamemodes_parser.py +++ b/tests/gamemodes/parsers/test_gamemodes_parser.py @@ -1,26 +1,16 @@ -from typing import TYPE_CHECKING +from app.adapters.blizzard.parsers.gamemodes import parse_gamemodes_csv +from app.gamemodes.enums import MapGamemode -import pytest -from app.exceptions import OverfastError +def test_parse_gamemodes_csv_returns_all_gamemodes(): + result = parse_gamemodes_csv() + assert isinstance(result, list) + assert len(result) > 0 + assert {g["key"] for g in result} == {str(g) for g in MapGamemode} -if TYPE_CHECKING: - from app.gamemodes.parsers.gamemodes_parser import GamemodesParser - -@pytest.mark.asyncio -async def test_gamemodes_page_parsing(gamemodes_parser: GamemodesParser): - try: - await gamemodes_parser.parse() - except OverfastError: - pytest.fail("Game modes list parsing failed") - - # Just check the format of the first gamemode in the list - assert isinstance(gamemodes_parser.data, list) - assert gamemodes_parser.data[0] == { - "key": "assault", - "name": "Assault", - "icon": "https://overfast-api.tekrop.fr/static/gamemodes/assault-icon.svg", - "description": "Teams fight to capture or defend two successive points against the enemy team. It's an inactive Overwatch 1 gamemode, also called 2CP.", - "screenshot": "https://overfast-api.tekrop.fr/static/gamemodes/assault.avif", - } +def test_parse_gamemodes_csv_entry_format(): + result = parse_gamemodes_csv() + first = result[0] + assert set(first.keys()) == {"key", "name", "icon", "description", "screenshot"} + assert first["key"] == "assault" diff --git a/tests/heroes/parsers/conftest.py b/tests/heroes/parsers/conftest.py deleted file mode 100644 index c89a4404..00000000 --- a/tests/heroes/parsers/conftest.py +++ /dev/null @@ -1,26 +0,0 @@ -import pytest - -from app.heroes.parsers.hero_parser import HeroParser -from app.heroes.parsers.hero_stats_summary_parser import HeroStatsSummaryParser -from app.heroes.parsers.heroes_parser import HeroesParser -from app.players.enums import PlayerGamemode, PlayerPlatform, PlayerRegion - - -@pytest.fixture(scope="package") -def hero_parser() -> HeroParser: - return HeroParser() - - -@pytest.fixture(scope="package") -def heroes_parser() -> HeroesParser: - return HeroesParser() - - -@pytest.fixture(scope="package") -def hero_stats_summary_parser() -> HeroStatsSummaryParser: - return HeroStatsSummaryParser( - platform=PlayerPlatform.PC, - gamemode=PlayerGamemode.COMPETITIVE, - region=PlayerRegion.EUROPE, - order_by="hero:asc", - ) diff --git a/tests/heroes/parsers/test_hero_parser.py b/tests/heroes/parsers/test_hero_parser.py deleted file mode 100644 index 3776b4e6..00000000 --- a/tests/heroes/parsers/test_hero_parser.py +++ /dev/null @@ -1,101 +0,0 @@ -from unittest.mock import Mock, patch - -import pytest -from fastapi import status - -from app.config import settings -from app.enums import Locale -from app.exceptions import OverfastError, ParserBlizzardError -from app.heroes.enums import HeroKey -from app.heroes.parsers.hero_parser import HeroParser - - -@pytest.mark.parametrize( - ("hero_key", "hero_html_data"), - [(h, h) for h in HeroKey], - indirect=["hero_html_data"], -) -@pytest.mark.asyncio -async def test_hero_page_parsing( - hero_parser: HeroParser, - hero_key: HeroKey, # ty: ignore[invalid-type-form] - hero_html_data: str, -): - if not hero_html_data: - pytest.skip("Hero HTML file not saved yet, skipping") - - with patch( - "httpx.AsyncClient.get", - return_value=Mock(status_code=status.HTTP_200_OK, text=hero_html_data), - ): - try: - await hero_parser.parse() - except OverfastError: - pytest.fail(f"Hero page parsing failed for '{hero_key}' hero") - - -@pytest.mark.parametrize("hero_html_data", ["unknown-hero"], indirect=True) -@pytest.mark.asyncio -async def test_not_released_hero_parser_blizzard_error( - hero_parser: HeroParser, hero_html_data: str -): - with ( - pytest.raises(ParserBlizzardError), - patch( - "httpx.AsyncClient.get", - return_value=Mock( - status_code=status.HTTP_404_NOT_FOUND, text=hero_html_data - ), - ), - ): - await hero_parser.parse() - - -@pytest.mark.parametrize( - ("url", "full_url"), - [ - ( - "https://www.youtube.com/watch?v=yzFWIw7wV8Q", - "https://www.youtube.com/watch?v=yzFWIw7wV8Q", - ), - ("/media/stories/bastet", f"{settings.blizzard_host}/media/stories/bastet"), - ], -) -def test_get_full_url(url: str, full_url: str): - assert HeroParser._get_full_url(url) == full_url - - -@pytest.mark.parametrize( - ("input_str", "locale", "result"), - [ - # Classic cases - ("Aug 19 (Age: 37)", Locale.ENGLISH_US, ("Aug 19", 37)), - ("May 9 (Age: 1)", Locale.ENGLISH_US, ("May 9", 1)), - # Specific unknown case (bastion) - ("Unknown (Age: 32)", Locale.ENGLISH_US, (None, 32)), - # Specific venture case (not the same spacing) - ("Aug 6 (Age : 26)", Locale.ENGLISH_US, ("Aug 6", 26)), - ("Aug 6 (Age : 26)", Locale.ENGLISH_EU, ("Aug 6", 26)), - # Other languages than english - ("6. Aug. (Alter: 26)", Locale.GERMAN, ("6. Aug.", 26)), - ("6 ago (Edad: 26)", Locale.SPANISH_EU, ("6 ago", 26)), - ("6 ago (Edad: 26)", Locale.SPANISH_LATIN, ("6 ago", 26)), - ("6 août (Âge : 26 ans)", Locale.FRENCH, ("6 août", 26)), - ("6 ago (Età: 26)", Locale.ITALIANO, ("6 ago", 26)), - ("8月6日 （年齢: 26）", Locale.JAPANESE, ("8月6日", 26)), - ("8월 6일 (나이: 26세)", Locale.KOREAN, ("8월 6일", 26)), - ("6 sie (Wiek: 26 lat)", Locale.POLISH, ("6 sie", 26)), - ("6 de ago. (Idade: 26)", Locale.PORTUGUESE_BRAZIL, ("6 de ago.", 26)), - ("6 авг. (Возраст: 26)", Locale.RUSSIAN, ("6 авг.", 26)), - ("8月6日 （年齡：26）", Locale.CHINESE_TAIWAN, ("8月6日", 26)), - # Invalid case - ("Unknown", Locale.ENGLISH_US, (None, None)), - ], -) -def test_get_birthday_and_age( - input_str: str, - locale: Locale, - result: tuple[str | None, int | None], -): - """Get birthday and age from text for a given hero""" - assert HeroParser._get_birthday_and_age(input_str, locale) == result diff --git a/tests/heroes/parsers/test_hero_stats_summary.py b/tests/heroes/parsers/test_hero_stats_summary.py index 128d3ba5..3903473e 100644 --- a/tests/heroes/parsers/test_hero_stats_summary.py +++ b/tests/heroes/parsers/test_hero_stats_summary.py @@ -2,8 +2,9 @@ import pytest -from app.exceptions import OverfastError, ParserBlizzardError -from app.heroes.parsers.hero_stats_summary_parser import HeroStatsSummaryParser +from app.adapters.blizzard.parsers.hero_stats_summary import parse_hero_stats_summary +from app.exceptions import ParserBlizzardError +from app.overfast_client import OverFastClient from app.players.enums import ( CompetitiveDivision, PlayerGamemode, @@ -14,31 +15,15 @@ @pytest.mark.asyncio @pytest.mark.parametrize( - ("parser_init_kwargs", "blizzard_query_params", "raises_error"), + ("extra_kwargs", "raises_error"), [ - # Nominal case - ( - {}, - {}, - False, - ), - # Specific filter (tier) - ( - {"competitive_division": CompetitiveDivision.DIAMOND}, - {"tier": "Diamond"}, - False, - ), - # Invalid map filter (not compatible with competitive) - ( - {"map": "hanaoka"}, - {"map": "hanaoka"}, - True, - ), + ({}, False), + ({"competitive_division": CompetitiveDivision.DIAMOND}, False), + ({"map_filter": "hanaoka"}, True), ], ) -async def test_hero_stats_summary_parser( - parser_init_kwargs: dict, - blizzard_query_params: dict, +async def test_parse_hero_stats_summary( + extra_kwargs: dict, raises_error: bool, hero_stats_response_mock: Mock, ): @@ -48,35 +33,16 @@ async def test_hero_stats_summary_parser( "region": PlayerRegion.EUROPE, "order_by": "hero:asc", } - init_kwargs = base_kwargs | parser_init_kwargs - # Instanciate with given kwargs - parser = HeroStatsSummaryParser(**init_kwargs) - - # Ensure running the parsing won't fail with patch("httpx.AsyncClient.get", return_value=hero_stats_response_mock): + client = OverFastClient() if raises_error: - with pytest.raises( - ParserBlizzardError, - match=( - f"Selected map '{init_kwargs['map']}' is not compatible " - f"with '{init_kwargs['gamemode']}' gamemode" - ), - ): - await parser.parse() + with pytest.raises(ParserBlizzardError): + await parse_hero_stats_summary(client, **base_kwargs, **extra_kwargs) # ty: ignore[invalid-argument-type] else: - try: - await parser.parse() - except OverfastError: - pytest.fail("Hero stats summary parsing failed") - - # Ensure we're sending the right parameters to Blibli - base_query_params = { - "input": "PC", - "rq": "1", - "region": "Europe", - "map": "all-maps", - "tier": "All", - } - query_params = base_query_params | blizzard_query_params - assert parser.get_blizzard_query_params(**init_kwargs) == query_params + result = await parse_hero_stats_summary( + client, **base_kwargs, **extra_kwargs + ) # ty: ignore[invalid-argument-type] + assert isinstance(result, list) + assert len(result) > 0 + assert "hero" in result[0] diff --git a/tests/heroes/parsers/test_heroes_parser.py b/tests/heroes/parsers/test_heroes_parser.py index 41ef036b..bb474a12 100644 --- a/tests/heroes/parsers/test_heroes_parser.py +++ b/tests/heroes/parsers/test_heroes_parser.py @@ -1,26 +1,55 @@ -from typing import TYPE_CHECKING from unittest.mock import Mock, patch import pytest from fastapi import status -from app.exceptions import OverfastError -from app.heroes.enums import HeroKey +from app.adapters.blizzard.parsers.heroes import ( + fetch_heroes_html, + filter_heroes, + parse_heroes_html, +) +from app.heroes.enums import HeroGamemode, HeroKey +from app.overfast_client import OverFastClient +from app.roles.enums import Role -if TYPE_CHECKING: - from app.heroes.parsers.heroes_parser import HeroesParser + +def test_parse_heroes_html_returns_all_heroes(heroes_html_data: str): + result = parse_heroes_html(heroes_html_data) + assert isinstance(result, list) + assert all(hero["key"] in iter(HeroKey) for hero in result) + + +def test_parse_heroes_html_entry_format(heroes_html_data: str): + result = parse_heroes_html(heroes_html_data) + first = result[0] + assert set(first.keys()) == {"key", "name", "portrait", "role", "gamemodes"} + + +def test_filter_heroes_by_role(heroes_html_data: str): + heroes = parse_heroes_html(heroes_html_data) + filtered = filter_heroes(heroes, role=Role.TANK, gamemode=None) + assert all(h["role"] == Role.TANK for h in filtered) + assert len(filtered) < len(heroes) + + +def test_filter_heroes_by_gamemode(heroes_html_data: str): + heroes = parse_heroes_html(heroes_html_data) + filtered = filter_heroes(heroes, role=None, gamemode=HeroGamemode.STADIUM) + assert len(filtered) <= len(heroes) + + +def test_filter_heroes_no_filter(heroes_html_data: str): + heroes = parse_heroes_html(heroes_html_data) + assert filter_heroes(heroes, role=None, gamemode=None) == heroes @pytest.mark.asyncio -async def test_heroes_page_parsing(heroes_parser: HeroesParser, heroes_html_data: str): +async def test_fetch_heroes_html_calls_blizzard(heroes_html_data: str): with patch( "httpx.AsyncClient.get", return_value=Mock(status_code=status.HTTP_200_OK, text=heroes_html_data), ): - try: - await heroes_parser.parse() - except OverfastError: - pytest.fail("Heroes list parsing failed") + client = OverFastClient() + html = await fetch_heroes_html(client) - assert isinstance(heroes_parser.data, list) - assert all(hero["key"] in iter(HeroKey) for hero in heroes_parser.data) + assert html == heroes_html_data diff --git a/tests/maps/parsers/conftest.py b/tests/maps/parsers/conftest.py deleted file mode 100644 index 45e88b9e..00000000 --- a/tests/maps/parsers/conftest.py +++ /dev/null @@ -1,8 +0,0 @@ -import pytest - -from app.maps.parsers.maps_parser import MapsParser - - -@pytest.fixture(scope="package") -def maps_parser() -> MapsParser: - return MapsParser() diff --git a/tests/maps/parsers/test_maps_parser.py b/tests/maps/parsers/test_maps_parser.py index 14bb2c31..c776e717 100644 --- a/tests/maps/parsers/test_maps_parser.py +++ b/tests/maps/parsers/test_maps_parser.py @@ -1,27 +1,24 @@ -from typing import TYPE_CHECKING +from app.adapters.blizzard.parsers.maps import parse_maps_csv +from app.maps.enums import MapKey -import pytest -from app.exceptions import OverfastError +def test_parse_maps_csv_returns_all_maps(): + result = parse_maps_csv() + assert isinstance(result, list) + assert len(result) > 0 + assert {m["key"] for m in result} == {str(m) for m in MapKey} -if TYPE_CHECKING: - from app.maps.parsers.maps_parser import MapsParser - -@pytest.mark.asyncio -async def test_maps_page_parsing(maps_parser: MapsParser): - try: - await maps_parser.parse() - except OverfastError: - pytest.fail("Maps list parsing failed") - - # Just check the format of the first map in the list - assert isinstance(maps_parser.data, list) - assert maps_parser.data[0] == { - "key": "aatlis", - "name": "Aatlis", - "screenshot": "https://overfast-api.tekrop.fr/static/maps/aatlis.jpg", - "gamemodes": ["flashpoint"], - "location": "Morocco", - "country_code": "MA", +def test_parse_maps_csv_entry_format(): + result = parse_maps_csv() + first = result[0] + assert set(first.keys()) == { + "key", + "name", + "screenshot", + "gamemodes", + "location", + "country_code", } + assert first["key"] == "aatlis" + assert isinstance(first["gamemodes"], list) diff --git a/tests/roles/parsers/conftest.py b/tests/roles/parsers/conftest.py deleted file mode 100644 index fe89a99b..00000000 --- a/tests/roles/parsers/conftest.py +++ /dev/null @@ -1,8 +0,0 @@ -import pytest - -from app.roles.parsers.roles_parser import RolesParser - - -@pytest.fixture(scope="package") -def roles_parser() -> RolesParser: - return RolesParser() diff --git a/tests/roles/parsers/test_roles_parser.py b/tests/roles/parsers/test_roles_parser.py index 8fd72321..f27b92a5 100644 --- a/tests/roles/parsers/test_roles_parser.py +++ b/tests/roles/parsers/test_roles_parser.py @@ -1,26 +1,32 @@ -from typing import TYPE_CHECKING from unittest.mock import Mock, patch import pytest from fastapi import status -from app.exceptions import OverfastError +from app.adapters.blizzard.parsers.roles import fetch_roles_html, parse_roles_html +from app.overfast_client import OverFastClient from app.roles.enums import Role -if TYPE_CHECKING: - from app.roles.parsers.roles_parser import RolesParser + +def test_parse_roles_html_returns_all_roles(home_html_data: str): + result = parse_roles_html(home_html_data) + assert isinstance(result, list) + assert {role["key"] for role in result} == {r.value for r in Role} + + +def test_parse_roles_html_entry_format(home_html_data: str): + result = parse_roles_html(home_html_data) + first = result[0] + assert set(first.keys()) == {"key", "name", "icon", "description"} @pytest.mark.asyncio -async def test_roles_page_parsing(roles_parser: RolesParser, home_html_data: str): +async def test_fetch_roles_html_calls_blizzard(home_html_data: str): with patch( "httpx.AsyncClient.get", return_value=Mock(status_code=status.HTTP_200_OK, text=home_html_data), ): - try: - await roles_parser.parse() - except OverfastError: - pytest.fail("Roles list parsing failed") + client = OverFastClient() + html = await fetch_roles_html(client) - assert isinstance(roles_parser.data, list) - assert {role["key"] for role in roles_parser.data} == {r.value for r in Role} + assert html == home_html_data From 9bc1406de0f2ceb0a8f1c4dc4929c50bf74c51ce Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sat, 21 Feb 2026 14:42:37 +0100 Subject: [PATCH 07/26] --wip-- [skip ci] --- app/adapters/tasks/asyncio_task_queue.py | 18 ++++++++++++++++++ .../heroes/parsers/test_hero_stats_summary.py | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/app/adapters/tasks/asyncio_task_queue.py b/app/adapters/tasks/asyncio_task_queue.py index 1dc8e99f..0eb6bf0b 100644 --- a/app/adapters/tasks/asyncio_task_queue.py +++ b/app/adapters/tasks/asyncio_task_queue.py @@ -1,8 +1,14 @@ """AsyncIO task queue adapter — Phase 4 in-process background tasks with deduplication""" import asyncio +import time from typing import TYPE_CHECKING, Any, ClassVar +from app.monitoring.metrics import ( + background_tasks_duration_seconds, + background_tasks_queue_size, + background_tasks_total, +) from app.overfast_logger import logger if TYPE_CHECKING: @@ -40,8 +46,11 @@ async def enqueue( return effective_id self._pending_jobs.add(effective_id) + background_tasks_queue_size.labels(task_type=task_name).inc() async def _run() -> None: + start = time.monotonic() + status = "success" try: logger.info( f"[TaskQueue] Running task '{task_name}' (job_id={effective_id})" @@ -49,10 +58,19 @@ async def _run() -> None: if coro is not None: await coro except Exception as exc: # noqa: BLE001 + status = "failure" logger.warning( f"[TaskQueue] Task '{task_name}' (job_id={effective_id}) failed: {exc}" ) finally: + elapsed = time.monotonic() - start + background_tasks_total.labels( + task_type=task_name, status=status + ).inc() + background_tasks_duration_seconds.labels(task_type=task_name).observe( + elapsed + ) + background_tasks_queue_size.labels(task_type=task_name).dec() self._pending_jobs.discard(effective_id) task = asyncio.create_task(_run(), name=effective_id) diff --git a/tests/heroes/parsers/test_hero_stats_summary.py b/tests/heroes/parsers/test_hero_stats_summary.py index 3903473e..cfcbc940 100644 --- a/tests/heroes/parsers/test_hero_stats_summary.py +++ b/tests/heroes/parsers/test_hero_stats_summary.py @@ -41,8 +41,8 @@ async def test_parse_hero_stats_summary( await parse_hero_stats_summary(client, **base_kwargs, **extra_kwargs) # ty: ignore[invalid-argument-type] else: result = await parse_hero_stats_summary( - client, **base_kwargs, **extra_kwargs - ) # ty: ignore[invalid-argument-type] + client, **base_kwargs, **extra_kwargs # ty: ignore[invalid-argument-type] + ) assert isinstance(result, list) assert len(result) > 0 assert "hero" in result[0] From 935376127ddc38f2ce05304a0a4e95a5fe645a8e Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sat, 21 Feb 2026 15:07:07 +0100 Subject: [PATCH 08/26] --wip-- [skip ci] --- app/adapters/tasks/asyncio_task_queue.py | 4 +- app/domain/services/player_service.py | 318 ++++-------------- .../heroes/parsers/test_hero_stats_summary.py | 4 +- 3 files changed, 71 insertions(+), 255 deletions(-) diff --git a/app/adapters/tasks/asyncio_task_queue.py b/app/adapters/tasks/asyncio_task_queue.py index 0eb6bf0b..66f27e06 100644 --- a/app/adapters/tasks/asyncio_task_queue.py +++ b/app/adapters/tasks/asyncio_task_queue.py @@ -64,9 +64,7 @@ async def _run() -> None: ) finally: elapsed = time.monotonic() - start - background_tasks_total.labels( - task_type=task_name, status=status - ).inc() + background_tasks_total.labels(task_type=task_name, status=status).inc() background_tasks_duration_seconds.labels(task_type=task_name).observe( elapsed ) diff --git a/app/domain/services/player_service.py b/app/domain/services/player_service.py index fa66be1a..347a3623 100644 --- a/app/domain/services/player_service.py +++ b/app/domain/services/player_service.py @@ -36,6 +36,8 @@ from app.overfast_logger import logger if TYPE_CHECKING: + from collections.abc import Callable + from app.players.enums import ( HeroKeyCareerFilter, PlayerGamemode, @@ -92,17 +94,12 @@ async def get_player_summary( player_id: str, cache_key: str, ) -> tuple[dict, bool]: - """Return player summary (name, avatar, competitive ranks, …). + """Return player summary (name, avatar, competitive ranks, …).""" - Returns: - (data, is_stale, age_seconds) - """ - return await self._execute_player_request( - player_id=player_id, - cache_key=cache_key, - summary=True, - stats=False, - ) + def extract(html: str, player_summary: dict) -> dict: + return parse_player_profile_html(html, player_summary).get("summary") or {} + + return await self._execute_player_request(player_id, cache_key, extract) # ------------------------------------------------------------------ # Player career (GET /players/{player_id}) @@ -115,19 +112,18 @@ async def get_player_career( platform: PlayerPlatform | None, cache_key: str, ) -> tuple[dict, bool]: - """Return full player data: summary + stats. + """Return full player data: summary + stats.""" - Returns: - (data, is_stale, age_seconds) - """ - return await self._execute_player_request( - player_id=player_id, - cache_key=cache_key, - summary=False, - stats=False, - gamemode=gamemode, - platform=platform, - ) + def extract(html: str, player_summary: dict) -> dict: + profile = parse_player_profile_html(html, player_summary) + return { + "summary": profile.get("summary") or {}, + "stats": filter_all_stats_data( + profile.get("stats") or {}, platform, gamemode + ), + } + + return await self._execute_player_request(player_id, cache_key, extract) # ------------------------------------------------------------------ # Player stats (GET /players/{player_id}/stats) @@ -141,20 +137,15 @@ async def get_player_stats( hero: HeroKeyCareerFilter | None, # ty: ignore[invalid-type-form] cache_key: str, ) -> tuple[dict, bool]: - """Return player stats with category labels. + """Return player stats with category labels.""" - Returns: - (data, is_stale, age_seconds) - """ - return await self._execute_player_request( - player_id=player_id, - cache_key=cache_key, - summary=False, - stats=True, - gamemode=gamemode, - platform=platform, - hero=hero, - ) + def extract(html: str, player_summary: dict) -> dict: + profile = parse_player_profile_html(html, player_summary) + return filter_stats_by_query( + profile.get("stats") or {}, platform, gamemode, hero + ) + + return await self._execute_player_request(player_id, cache_key, extract) # ------------------------------------------------------------------ # Player stats summary (GET /players/{player_id}/stats/summary) @@ -167,42 +158,14 @@ async def get_player_stats_summary( platform: PlayerPlatform | None, cache_key: str, ) -> tuple[dict, bool]: - """Return player statistics summary (winrate, kda, …). + """Return player statistics summary (winrate, kda, …).""" - Returns: - (data, is_stale, age_seconds) - """ - cache_key_player = player_id - battletag_input: str | None = None - player_summary: dict = {} - - try: - ( - blizzard_id, - player_summary, - cached_html, - battletag_input, - ) = await self._resolve_player_identity(player_id) - cache_key_player = blizzard_id or player_id - - data = await self._fetch_stats_summary_with_cache( - blizzard_id=cache_key_player, - player_summary=player_summary, - gamemode=gamemode, - platform=platform, - battletag_input=battletag_input, - cached_html=cached_html, - ) - except Exception as exc: # noqa: BLE001 - await self._handle_player_exceptions( - exc, cache_key_player, battletag_input, player_summary + def extract(html: str, player_summary: dict) -> dict: + return parse_player_stats_summary_from_html( + html, player_summary, gamemode, platform ) - is_stale = self._check_player_staleness(cache_key_player) - await self._update_api_cache( - cache_key, data, settings.career_path_cache_timeout - ) - return data, is_stale + return await self._execute_player_request(player_id, cache_key, extract) # ------------------------------------------------------------------ # Player career stats (GET /players/{player_id}/stats/career) @@ -216,62 +179,37 @@ async def get_player_career_stats( hero: HeroKeyCareerFilter | None, # ty: ignore[invalid-type-form] cache_key: str, ) -> tuple[dict, bool]: - """Return player career stats (no labels). - - Returns: - (data, is_stale, age_seconds) - """ - cache_key_player = player_id - battletag_input: str | None = None - player_summary: dict = {} - - try: - ( - blizzard_id, - player_summary, - cached_html, - battletag_input, - ) = await self._resolve_player_identity(player_id) - cache_key_player = blizzard_id or player_id + """Return player career stats (no labels).""" - data = await self._fetch_career_stats_with_cache( - blizzard_id=cache_key_player, - player_summary=player_summary, - platform=platform, - gamemode=gamemode, - hero=hero, - battletag_input=battletag_input, - cached_html=cached_html, - ) - except Exception as exc: # noqa: BLE001 - await self._handle_player_exceptions( - exc, cache_key_player, battletag_input, player_summary + def extract(html: str, player_summary: dict) -> dict: + return parse_player_career_stats_from_html( + html, player_summary, platform, gamemode, hero ) - is_stale = self._check_player_staleness(cache_key_player) - await self._update_api_cache( - cache_key, data, settings.career_path_cache_timeout - ) - return data, is_stale + return await self._execute_player_request(player_id, cache_key, extract) # ------------------------------------------------------------------ - # Core request execution (career + summary) + # Core request execution — universal scaffold # ------------------------------------------------------------------ async def _execute_player_request( self, player_id: str, cache_key: str, - summary: bool, - stats: bool, - gamemode: PlayerGamemode | None = None, - platform: PlayerPlatform | None = None, - hero: HeroKeyCareerFilter | None = None, # ty: ignore[invalid-type-form] + data_factory: Callable[[str, dict], dict], ) -> tuple[dict, bool]: - """Shared execution path for summary, career, and stats endpoints.""" + """Resolve identity → get HTML → compute data → update cache → return. + + Args: + player_id: BattleTag or Blizzard ID. + cache_key: Valkey API-cache key to write after serving. + data_factory: Pure function ``(html, player_summary) → dict`` that + extracts the endpoint-specific payload from the raw HTML. + """ cache_key_player = player_id battletag_input: str | None = None player_summary: dict = {} + data: dict = {} try: ( @@ -282,21 +220,10 @@ async def _execute_player_request( ) = await self._resolve_player_identity(player_id) cache_key_player = blizzard_id or player_id - _html, profile_data = await self._fetch_profile_with_cache( - blizzard_id=cache_key_player, - player_summary=player_summary, - battletag_input=battletag_input, - cached_html=cached_html, - ) - - data = self._filter_profile_data( - profile_data, - summary_filter=summary, - stats_filter=stats, - platform_filter=platform, - gamemode_filter=gamemode, - hero_filter=hero, + html = await self._get_player_html( + cache_key_player, player_summary, cached_html, battletag_input ) + data = data_factory(html, player_summary) except Exception as exc: # noqa: BLE001 await self._handle_player_exceptions( exc, cache_key_player, battletag_input, player_summary @@ -360,165 +287,54 @@ def _check_player_staleness(self, _player_id: str) -> bool: """ return False - async def _fetch_profile_with_cache( + async def _get_player_html( self, blizzard_id: str, player_summary: dict, + cached_html: str | None, battletag_input: str | None, - cached_html: str | None = None, - ) -> tuple[str, dict]: - """Fetch player profile with SQLite cache.""" + ) -> str: + """Return player HTML, always storing fresh HTML in SQLite. + + Priority order: + 1. ``cached_html`` — fetched during identity resolution; store and return. + 2. SQLite hit with matching ``lastUpdated`` — return cached HTML, backfilling + battletag if it was missing. + 3. Fetch from Blizzard, store, return. + """ if cached_html: - logger.info("Using cached HTML from identity resolution") - profile_data = parse_player_profile_html(cached_html, player_summary) name = extract_name_from_profile_html(cached_html) or player_summary.get( "name" ) await self.update_player_profile_cache( blizzard_id, player_summary, cached_html, battletag_input, name ) - return cached_html, profile_data + return cached_html player_cache = await self.get_player_profile_cache(blizzard_id) - if ( player_cache is not None and player_summary - and player_cache["summary"]["lastUpdated"] == player_summary["lastUpdated"] + and player_cache["summary"].get("lastUpdated") + == player_summary.get("lastUpdated") ): - logger.info("Player profile cache found and up-to-date") html = cast("str", player_cache["profile"]) - profile_data = parse_player_profile_html(html, player_summary) - if battletag_input and not player_cache.get("battletag"): - cached_name = player_cache.get("name") await self.update_player_profile_cache( blizzard_id, player_summary, html, battletag_input, - cached_name if isinstance(cached_name, str) else None, + player_cache.get("name"), ) - return html, profile_data + return html - logger.info("Player profile cache miss — fetching from Blizzard") html, _ = await fetch_player_html(self.blizzard_client, blizzard_id) - profile_data = parse_player_profile_html(html, player_summary) name = extract_name_from_profile_html(html) or player_summary.get("name") await self.update_player_profile_cache( blizzard_id, player_summary, html, battletag_input, name ) - return html, profile_data - - async def _fetch_stats_summary_with_cache( - self, - blizzard_id: str, - player_summary: dict, - gamemode: PlayerGamemode | None, - platform: PlayerPlatform | None, - battletag_input: str | None = None, - cached_html: str | None = None, - ) -> dict: - """Fetch player stats summary with SQLite cache.""" - if cached_html: - return parse_player_stats_summary_from_html( - cached_html, player_summary, gamemode, platform - ) - - if not blizzard_id: - msg = "Unable to resolve player identity" - raise ParserParsingError(msg) - - player_cache = await self.get_player_profile_cache(blizzard_id) - - if ( - player_cache is not None - and player_summary - and player_cache["summary"]["lastUpdated"] == player_summary["lastUpdated"] - ): - html = cast("str", player_cache["profile"]) - return parse_player_stats_summary_from_html( - html, player_summary, gamemode, platform - ) - - html, _ = await fetch_player_html(self.blizzard_client, blizzard_id) - name = extract_name_from_profile_html(html) - data = parse_player_stats_summary_from_html( - html, player_summary, gamemode, platform - ) - await self.update_player_profile_cache( - blizzard_id, player_summary, html, battletag_input, name - ) - return data - - async def _fetch_career_stats_with_cache( - self, - blizzard_id: str, - player_summary: dict, - platform: PlayerPlatform | None, - gamemode: PlayerGamemode | None, - hero: HeroKeyCareerFilter | None, # ty: ignore[invalid-type-form] - battletag_input: str | None = None, - cached_html: str | None = None, - ) -> dict: - """Fetch player career stats with SQLite cache.""" - if cached_html: - return parse_player_career_stats_from_html( - cached_html, player_summary, platform, gamemode, hero - ) - - if not blizzard_id: - msg = "Unable to resolve player identity" - raise ParserParsingError(msg) - - player_cache = await self.get_player_profile_cache(blizzard_id) - - if ( - player_cache is not None - and player_summary - and player_cache["summary"]["lastUpdated"] == player_summary["lastUpdated"] - ): - html = cast("str", player_cache["profile"]) - return parse_player_career_stats_from_html( - html, player_summary, platform, gamemode, hero - ) - - html, _ = await fetch_player_html(self.blizzard_client, blizzard_id) - name = extract_name_from_profile_html(html) - data = parse_player_career_stats_from_html( - html, player_summary, platform, gamemode, hero - ) - await self.update_player_profile_cache( - blizzard_id, player_summary, html, battletag_input, name - ) - return data - - @staticmethod - def _filter_profile_data( - profile_data: dict, - summary_filter: bool, - stats_filter: bool, - platform_filter: PlayerPlatform | None, - gamemode_filter: PlayerGamemode | None, - hero_filter: HeroKeyCareerFilter | None, # ty: ignore[invalid-type-form] - ) -> dict: - if summary_filter: - return profile_data.get("summary") or {} - if stats_filter: - return filter_stats_by_query( - profile_data.get("stats") or {}, - platform_filter, - gamemode_filter, - hero_filter, - ) - return { - "summary": profile_data.get("summary") or {}, - "stats": filter_all_stats_data( - profile_data.get("stats") or {}, - platform_filter, - gamemode_filter, - ), - } + return html # ------------------------------------------------------------------ # Identity resolution diff --git a/tests/heroes/parsers/test_hero_stats_summary.py b/tests/heroes/parsers/test_hero_stats_summary.py index cfcbc940..ecb51fc6 100644 --- a/tests/heroes/parsers/test_hero_stats_summary.py +++ b/tests/heroes/parsers/test_hero_stats_summary.py @@ -41,7 +41,9 @@ async def test_parse_hero_stats_summary( await parse_hero_stats_summary(client, **base_kwargs, **extra_kwargs) # ty: ignore[invalid-argument-type] else: result = await parse_hero_stats_summary( - client, **base_kwargs, **extra_kwargs # ty: ignore[invalid-argument-type] + client, + **base_kwargs, + **extra_kwargs, # ty: ignore[invalid-argument-type] ) assert isinstance(result, list) assert len(result) > 0 From 4a9813947fe6764393e6c54f87141dd8df3ad6df Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sat, 21 Feb 2026 18:25:55 +0100 Subject: [PATCH 09/26] --wip-- [skip ci] --- app/api/routers/gamemodes.py | 4 ++-- app/api/routers/heroes.py | 12 +++++------ app/api/routers/maps.py | 4 ++-- app/api/routers/players.py | 20 +++++++++---------- app/api/routers/roles.py | 4 ++-- app/domain/services/gamemode_service.py | 2 +- app/domain/services/hero_service.py | 8 ++++---- app/domain/services/map_service.py | 2 +- app/domain/services/player_service.py | 14 ++++++------- app/domain/services/role_service.py | 2 +- app/domain/services/static_data_service.py | 16 ++++++++++----- app/helpers.py | 14 ++++++++----- .../heroes/parsers/test_hero_stats_summary.py | 4 ++-- tests/heroes/test_hero_routes.py | 2 +- tests/heroes/test_hero_stats_route.py | 2 +- tests/heroes/test_heroes_route.py | 2 +- tests/players/test_player_career_route.py | 2 +- tests/players/test_player_stats_route.py | 1 + .../test_player_stats_summary_route.py | 1 + tests/players/test_player_summary_route.py | 2 +- tests/roles/test_roles_route.py | 2 +- 21 files changed, 66 insertions(+), 54 deletions(-) diff --git a/app/api/routers/gamemodes.py b/app/api/routers/gamemodes.py index 74ba0f0a..e94c422e 100644 --- a/app/api/routers/gamemodes.py +++ b/app/api/routers/gamemodes.py @@ -30,6 +30,6 @@ async def list_map_gamemodes( response: Response, service: GamemodeServiceDep, ) -> Any: - data, is_stale = await service.list_gamemodes(cache_key=build_cache_key(request)) - apply_swr_headers(response, settings.csv_cache_timeout, is_stale) + data, is_stale, age = await service.list_gamemodes(cache_key=build_cache_key(request)) + apply_swr_headers(response, settings.csv_cache_timeout, is_stale, age) return data diff --git a/app/api/routers/heroes.py b/app/api/routers/heroes.py index 23193027..679fbe07 100644 --- a/app/api/routers/heroes.py +++ b/app/api/routers/heroes.py @@ -50,10 +50,10 @@ async def list_heroes( ] = Locale.ENGLISH_US, gamemode: Annotated[HeroGamemode | None, Query(title="Gamemode filter")] = None, ) -> Any: - data, is_stale = await service.list_heroes( + data, is_stale, age = await service.list_heroes( locale=locale, role=role, gamemode=gamemode, cache_key=build_cache_key(request) ) - apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale) + apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale, age) return data @@ -120,7 +120,7 @@ async def get_hero_stats( ), ] = "hero:asc", ) -> Any: - data, is_stale = await service.get_hero_stats( + data, is_stale, age = await service.get_hero_stats( platform=platform, gamemode=gamemode, region=region, @@ -130,7 +130,7 @@ async def get_hero_stats( order_by=order_by, cache_key=build_cache_key(request), ) - apply_swr_headers(response, settings.hero_stats_cache_timeout, is_stale) + apply_swr_headers(response, settings.hero_stats_cache_timeout, is_stale, age) return data @@ -161,8 +161,8 @@ async def get_hero( Locale, Query(title="Locale to be displayed") ] = Locale.ENGLISH_US, ) -> Any: - data, is_stale = await service.get_hero( + data, is_stale, age = await service.get_hero( hero_key=str(hero_key), locale=locale, cache_key=build_cache_key(request) ) - apply_swr_headers(response, settings.hero_path_cache_timeout, is_stale) + apply_swr_headers(response, settings.hero_path_cache_timeout, is_stale, age) return data diff --git a/app/api/routers/maps.py b/app/api/routers/maps.py index da292874..afc4798c 100644 --- a/app/api/routers/maps.py +++ b/app/api/routers/maps.py @@ -38,8 +38,8 @@ async def list_maps( ), ] = None, ) -> Any: - data, is_stale = await service.list_maps( + data, is_stale, age = await service.list_maps( gamemode=gamemode, cache_key=build_cache_key(request) ) - apply_swr_headers(response, settings.csv_cache_timeout, is_stale) + apply_swr_headers(response, settings.csv_cache_timeout, is_stale, age) return data diff --git a/app/api/routers/players.py b/app/api/routers/players.py index fbf51d32..cccb6afe 100644 --- a/app/api/routers/players.py +++ b/app/api/routers/players.py @@ -158,11 +158,11 @@ async def get_player_summary( commons: CommonsPlayerDep, ) -> Any: cache_key = build_cache_key(request) - data, is_stale = await service.get_player_summary( + data, is_stale, age = await service.get_player_summary( player_id=commons["player_id"], cache_key=cache_key, ) - apply_swr_headers(response, settings.career_path_cache_timeout, is_stale) + apply_swr_headers(response, settings.career_path_cache_timeout, is_stale, age) return data @@ -214,13 +214,13 @@ async def get_player_stats_summary( ] = None, ) -> Any: cache_key = build_cache_key(request) - data, is_stale = await service.get_player_stats_summary( + data, is_stale, age = await service.get_player_stats_summary( player_id=commons["player_id"], gamemode=gamemode, platform=platform, cache_key=cache_key, ) - apply_swr_headers(response, settings.career_path_cache_timeout, is_stale) + apply_swr_headers(response, settings.career_path_cache_timeout, is_stale, age) return data @@ -247,14 +247,14 @@ async def get_player_career_stats( commons: CommonsPlayerCareerDep, ) -> Any: cache_key = build_cache_key(request) - data, is_stale = await service.get_player_career_stats( + data, is_stale, age = await service.get_player_career_stats( player_id=commons["player_id"], gamemode=commons.get("gamemode"), platform=commons.get("platform"), hero=commons.get("hero"), cache_key=cache_key, ) - apply_swr_headers(response, settings.career_path_cache_timeout, is_stale) + apply_swr_headers(response, settings.career_path_cache_timeout, is_stale, age) return data @@ -279,14 +279,14 @@ async def get_player_stats( commons: CommonsPlayerCareerDep, ) -> Any: cache_key = build_cache_key(request) - data, is_stale = await service.get_player_stats( + data, is_stale, age = await service.get_player_stats( player_id=commons["player_id"], gamemode=commons.get("gamemode"), platform=commons.get("platform"), hero=commons.get("hero"), cache_key=cache_key, ) - apply_swr_headers(response, settings.career_path_cache_timeout, is_stale) + apply_swr_headers(response, settings.career_path_cache_timeout, is_stale, age) return data @@ -325,11 +325,11 @@ async def get_player_career( ] = None, ) -> Any: cache_key = build_cache_key(request) - data, is_stale = await service.get_player_career( + data, is_stale, age = await service.get_player_career( player_id=commons["player_id"], gamemode=gamemode, platform=platform, cache_key=cache_key, ) - apply_swr_headers(response, settings.career_path_cache_timeout, is_stale) + apply_swr_headers(response, settings.career_path_cache_timeout, is_stale, age) return data diff --git a/app/api/routers/roles.py b/app/api/routers/roles.py index 5e890600..2a1881a6 100644 --- a/app/api/routers/roles.py +++ b/app/api/routers/roles.py @@ -33,8 +33,8 @@ async def list_roles( Locale, Query(title="Locale to be displayed") ] = Locale.ENGLISH_US, ) -> Any: - data, is_stale = await service.list_roles( + data, is_stale, age = await service.list_roles( locale=locale, cache_key=build_cache_key(request) ) - apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale) + apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale, age) return data diff --git a/app/domain/services/gamemode_service.py b/app/domain/services/gamemode_service.py index a7ecc331..fa1a83ac 100644 --- a/app/domain/services/gamemode_service.py +++ b/app/domain/services/gamemode_service.py @@ -11,7 +11,7 @@ class GamemodeService(StaticDataService): async def list_gamemodes( self, cache_key: str, - ) -> tuple[list[dict], bool]: + ) -> tuple[list[dict], bool, int]: """Return the gamemodes list.""" async def _fetch() -> list[dict]: diff --git a/app/domain/services/hero_service.py b/app/domain/services/hero_service.py index 130f4b66..41838fc2 100644 --- a/app/domain/services/hero_service.py +++ b/app/domain/services/hero_service.py @@ -44,7 +44,7 @@ async def list_heroes( role: Role | None, gamemode: HeroGamemode | None, cache_key: str, - ) -> tuple[list[dict], bool]: + ) -> tuple[list[dict], bool, int]: """Return the heroes list (with optional role/gamemode filters). Stores the *full* (unfiltered) heroes list per locale in SQLite so @@ -77,7 +77,7 @@ async def get_hero( hero_key: str, locale: Locale, cache_key: str, - ) -> tuple[dict, bool]: + ) -> tuple[dict, bool, int]: """Return full hero details merged with portrait and hitpoints. Single-hero data is not stored persistently; the Valkey API cache is @@ -100,7 +100,7 @@ async def get_hero( raise overfast_internal_error(blizzard_url, exc) from exc await self._update_api_cache(cache_key, data, settings.hero_path_cache_timeout) - return data, False + return data, False, 0 # ------------------------------------------------------------------ # Hero stats summary (GET /heroes/stats) @@ -116,7 +116,7 @@ async def get_hero_stats( competitive_division: CompetitiveDivisionFilter | None, # ty: ignore[invalid-type-form] order_by: str, cache_key: str, - ) -> tuple[list[dict], bool]: + ) -> tuple[list[dict], bool, int]: """Return hero usage statistics with SWR.""" storage_key = _build_hero_stats_storage_key( platform, gamemode, region, map_filter, competitive_division diff --git a/app/domain/services/map_service.py b/app/domain/services/map_service.py index 23af340f..063a904f 100644 --- a/app/domain/services/map_service.py +++ b/app/domain/services/map_service.py @@ -12,7 +12,7 @@ async def list_maps( self, gamemode: str | None, cache_key: str, - ) -> tuple[list[dict], bool]: + ) -> tuple[list[dict], bool, int]: """Return the maps list (with optional gamemode filter). Stores the full (unfiltered) maps list in SQLite. diff --git a/app/domain/services/player_service.py b/app/domain/services/player_service.py index 347a3623..e2c30ca4 100644 --- a/app/domain/services/player_service.py +++ b/app/domain/services/player_service.py @@ -93,7 +93,7 @@ async def get_player_summary( self, player_id: str, cache_key: str, - ) -> tuple[dict, bool]: + ) -> tuple[dict, bool, int]: """Return player summary (name, avatar, competitive ranks, …).""" def extract(html: str, player_summary: dict) -> dict: @@ -111,7 +111,7 @@ async def get_player_career( gamemode: PlayerGamemode | None, platform: PlayerPlatform | None, cache_key: str, - ) -> tuple[dict, bool]: + ) -> tuple[dict, bool, int]: """Return full player data: summary + stats.""" def extract(html: str, player_summary: dict) -> dict: @@ -136,7 +136,7 @@ async def get_player_stats( platform: PlayerPlatform | None, hero: HeroKeyCareerFilter | None, # ty: ignore[invalid-type-form] cache_key: str, - ) -> tuple[dict, bool]: + ) -> tuple[dict, bool, int]: """Return player stats with category labels.""" def extract(html: str, player_summary: dict) -> dict: @@ -157,7 +157,7 @@ async def get_player_stats_summary( gamemode: PlayerGamemode | None, platform: PlayerPlatform | None, cache_key: str, - ) -> tuple[dict, bool]: + ) -> tuple[dict, bool, int]: """Return player statistics summary (winrate, kda, …).""" def extract(html: str, player_summary: dict) -> dict: @@ -178,7 +178,7 @@ async def get_player_career_stats( platform: PlayerPlatform | None, hero: HeroKeyCareerFilter | None, # ty: ignore[invalid-type-form] cache_key: str, - ) -> tuple[dict, bool]: + ) -> tuple[dict, bool, int]: """Return player career stats (no labels).""" def extract(html: str, player_summary: dict) -> dict: @@ -197,7 +197,7 @@ async def _execute_player_request( player_id: str, cache_key: str, data_factory: Callable[[str, dict], dict], - ) -> tuple[dict, bool]: + ) -> tuple[dict, bool, int]: """Resolve identity → get HTML → compute data → update cache → return. Args: @@ -233,7 +233,7 @@ async def _execute_player_request( await self._update_api_cache( cache_key, data, settings.career_path_cache_timeout ) - return data, is_stale + return data, is_stale, 0 # ------------------------------------------------------------------ # Profile caching helpers diff --git a/app/domain/services/role_service.py b/app/domain/services/role_service.py index 2f66e6b3..50c3c297 100644 --- a/app/domain/services/role_service.py +++ b/app/domain/services/role_service.py @@ -15,7 +15,7 @@ async def list_roles( self, locale: Locale, cache_key: str, - ) -> tuple[list[dict], bool]: + ) -> tuple[list[dict], bool, int]: """Return the roles list.""" async def _fetch() -> list[dict]: diff --git a/app/domain/services/static_data_service.py b/app/domain/services/static_data_service.py index 1bd1ebf0..e866d274 100644 --- a/app/domain/services/static_data_service.py +++ b/app/domain/services/static_data_service.py @@ -37,7 +37,7 @@ async def get_or_fetch( entity_type: str, parser: Callable[[Any], Any] | None = None, result_filter: Callable[[Any], Any] | None = None, - ) -> tuple[Any, bool]: + ) -> tuple[Any, bool, int]: """SWR orchestration for static data. Args: @@ -54,7 +54,9 @@ async def get_or_fetch( returning. Not persisted — re-applied on every request. Returns: - ``(data, is_stale)`` tuple. + ``(data, is_stale, age_seconds)`` tuple. ``age_seconds`` is the + number of seconds since the data was last stored in SQLite (0 on + a cold-start fetch). """ stored = await self._load_from_storage(storage_key) @@ -77,18 +79,22 @@ async def get_or_fetch( ) stale_responses_total.inc() background_refresh_triggered_total.labels(entity_type=entity_type).inc() + # Do NOT write stale data to Valkey: all requests during the + # stale window must reach FastAPI so they get correct headers. + # The background refresh updates SQLite; the next FastAPI request + # will see fresh data and repopulate Valkey naturally. else: logger.info( f"[SWR] {entity_type} fresh (age={age}s) — serving from SQLite" ) + await self._update_api_cache(cache_key, data if result_filter is None else result_filter(data), cache_ttl) storage_hits_total.labels(result="hit").inc() if result_filter is not None: data = result_filter(data) - await self._update_api_cache(cache_key, data, cache_ttl) - return data, is_stale + return data, is_stale, age # Cold start — fetch synchronously logger.info(f"[SWR] {entity_type} not in SQLite — fetching from source") @@ -100,7 +106,7 @@ async def get_or_fetch( filtered = result_filter(data) if result_filter is not None else data await self._update_api_cache(cache_key, filtered, cache_ttl) - return filtered, False + return filtered, False, 0 async def _refresh_static( self, diff --git a/app/helpers.py b/app/helpers.py index 8b6ad492..adf54d73 100644 --- a/app/helpers.py +++ b/app/helpers.py @@ -230,17 +230,21 @@ def apply_swr_headers( response: Response, cache_ttl: int, is_stale: bool, + age_seconds: int = 0, ) -> None: """Add standard SWR and cache metadata headers to the response. - Always sets ``X-Cache-TTL``. + Always sets ``X-Cache-TTL`` and ``Age`` (when known). When ``is_stale`` is True, additionally sets RFC-5861 ``Cache-Control`` - and ``X-Cache-Status`` so downstream proxies (nginx/Lua) can handle stale - content correctly. The ``Age`` header is intentionally left to nginx, which - knows the actual time the response has been in its cache. + and ``X-Cache-Status``. + + Note: nginx/Lua only sets ``X-Cache-TTL`` (remaining Valkey TTL) for + Valkey-served responses; all other headers are FastAPI-only and reflect + the actual SQLite data age. """ response.headers[settings.cache_ttl_header] = str(cache_ttl) - + if age_seconds > 0: + response.headers["Age"] = str(age_seconds) if is_stale: response.headers["Cache-Control"] = ( f"max-age={cache_ttl}, stale-while-revalidate={cache_ttl * 2}" diff --git a/tests/heroes/parsers/test_hero_stats_summary.py b/tests/heroes/parsers/test_hero_stats_summary.py index ecb51fc6..5a9876bf 100644 --- a/tests/heroes/parsers/test_hero_stats_summary.py +++ b/tests/heroes/parsers/test_hero_stats_summary.py @@ -42,8 +42,8 @@ async def test_parse_hero_stats_summary( else: result = await parse_hero_stats_summary( client, - **base_kwargs, - **extra_kwargs, # ty: ignore[invalid-argument-type] + **base_kwargs, # ty: ignore[invalid-argument-type] + **extra_kwargs, ) assert isinstance(result, list) assert len(result) > 0 diff --git a/tests/heroes/test_hero_routes.py b/tests/heroes/test_hero_routes.py index 3ad1c61d..ce96fa81 100644 --- a/tests/heroes/test_hero_routes.py +++ b/tests/heroes/test_hero_routes.py @@ -73,7 +73,7 @@ def test_get_hero_blizzard_error(client: TestClient): def test_get_hero_internal_error(client: TestClient): with patch( "app.domain.services.hero_service.HeroService.get_hero", - return_value=({"invalid_key": "invalid_value"}, False), + return_value=({"invalid_key": "invalid_value"}, False, 0), ): response = client.get(f"/heroes/{HeroKey.ANA}") # ty: ignore[unresolved-attribute] assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/tests/heroes/test_hero_stats_route.py b/tests/heroes/test_hero_stats_route.py index dd0af1b7..408e8b16 100644 --- a/tests/heroes/test_hero_stats_route.py +++ b/tests/heroes/test_hero_stats_route.py @@ -97,7 +97,7 @@ def test_get_hero_stats_blizzard_error(client: TestClient): def test_get_heroes_internal_error(client: TestClient): with patch( "app.domain.services.hero_service.HeroService.get_hero_stats", - return_value=([{"invalid_key": "invalid_value"}], False), + return_value=([{"invalid_key": "invalid_value"}], False, 0), ): response = client.get( "/heroes/stats", diff --git a/tests/heroes/test_heroes_route.py b/tests/heroes/test_heroes_route.py index 2d68a8ba..5fd2cfdc 100644 --- a/tests/heroes/test_heroes_route.py +++ b/tests/heroes/test_heroes_route.py @@ -70,7 +70,7 @@ def test_get_heroes_blizzard_error(client: TestClient): def test_get_heroes_internal_error(client: TestClient): with patch( "app.domain.services.hero_service.HeroService.list_heroes", - return_value=([{"invalid_key": "invalid_value"}], False), + return_value=([{"invalid_key": "invalid_value"}], False, 0), ): response = client.get("/heroes") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/tests/players/test_player_career_route.py b/tests/players/test_player_career_route.py index 8f8a72ec..70845fa3 100644 --- a/tests/players/test_player_career_route.py +++ b/tests/players/test_player_career_route.py @@ -98,7 +98,7 @@ def test_get_player_career_blizzard_remote_protocol_error(client: TestClient): def test_get_player_career_internal_error(client: TestClient): with patch( "app.domain.services.player_service.PlayerService.get_player_career", - return_value=({"invalid_key": "invalid_value"}, False), + return_value=({"invalid_key": "invalid_value"}, False, 0), ): response = client.get("/players/TeKrop-2217") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/tests/players/test_player_stats_route.py b/tests/players/test_player_stats_route.py index 64bfee7f..4ee57608 100644 --- a/tests/players/test_player_stats_route.py +++ b/tests/players/test_player_stats_route.py @@ -218,6 +218,7 @@ def test_get_player_stats_internal_error( "ana": [{"category": "invalid_value", "stats": [{"key": "test"}]}], }, False, + 0, ), ): response = client.get( diff --git a/tests/players/test_player_stats_summary_route.py b/tests/players/test_player_stats_summary_route.py index 3c05fc66..cb00df54 100644 --- a/tests/players/test_player_stats_summary_route.py +++ b/tests/players/test_player_stats_summary_route.py @@ -136,6 +136,7 @@ def test_get_player_stats_summary_internal_error(client: TestClient): "general": [{"category": "invalid_value", "stats": [{"key": "test"}]}], }, False, + 0, ), ): response = client.get( diff --git a/tests/players/test_player_summary_route.py b/tests/players/test_player_summary_route.py index d877d33d..03795b95 100644 --- a/tests/players/test_player_summary_route.py +++ b/tests/players/test_player_summary_route.py @@ -75,7 +75,7 @@ def test_get_player_summary_blizzard_timeout(client: TestClient): def test_get_player_summary_internal_error(client: TestClient): with patch( "app.domain.services.player_service.PlayerService.get_player_summary", - return_value=({"invalid_key": "invalid_value"}, False), + return_value=({"invalid_key": "invalid_value"}, False, 0), ): response = client.get("/players/TeKrop-2217/summary") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/tests/roles/test_roles_route.py b/tests/roles/test_roles_route.py index e924348d..4a225740 100644 --- a/tests/roles/test_roles_route.py +++ b/tests/roles/test_roles_route.py @@ -37,7 +37,7 @@ def test_get_roles_blizzard_error(client: TestClient): def test_get_roles_internal_error(client: TestClient): with patch( "app.domain.services.role_service.RoleService.list_roles", - return_value=([{"invalid_key": "invalid_value"}], False), + return_value=([{"invalid_key": "invalid_value"}], False, 0), ): response = client.get("/roles") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR From 944e61786fa4cb8ddf2214cfa68ab5b0dc26a961 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 08:51:10 +0100 Subject: [PATCH 10/26] --wip-- [skip ci] --- .env.dist | 2 + app/cache_manager.py | 10 ----- app/config.py | 6 +++ app/domain/services/static_data_service.py | 37 +++++++++++++------ app/heroes/commands/check_new_hero.py | 2 +- app/main.py | 2 +- app/overfast_client.py | 10 ----- tests/conftest.py | 2 +- .../heroes/parsers/test_hero_stats_summary.py | 2 +- tests/heroes/parsers/test_heroes_parser.py | 2 +- tests/roles/parsers/test_roles_parser.py | 2 +- tests/test_cache_manager.py | 2 +- 12 files changed, 40 insertions(+), 39 deletions(-) delete mode 100644 app/cache_manager.py delete mode 100644 app/overfast_client.py diff --git a/.env.dist b/.env.dist index 6a1caa33..ba3ea741 100644 --- a/.env.dist +++ b/.env.dist @@ -52,6 +52,8 @@ CAREER_PATH_CACHE_TIMEOUT=600 SEARCH_ACCOUNT_PATH_CACHE_TIMEOUT=600 HERO_STATS_CACHE_TIMEOUT=3600 PLAYER_PROFILE_MAX_AGE=259200 +# TTL (seconds) for stale responses written to Valkey while background refresh is in-flight +STALE_CACHE_TIMEOUT=60 # Critical error Discord webhook DISCORD_WEBHOOK_ENABLED=false diff --git a/app/cache_manager.py b/app/cache_manager.py deleted file mode 100644 index f419846e..00000000 --- a/app/cache_manager.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -Cache manager module - DEPRECATED, moved to app/adapters/cache/valkey_cache.py - -This module is kept for backward compatibility during the migration to v4. -Import from app.adapters.cache instead. -""" - -from app.adapters.cache import CacheManager - -__all__ = ["CacheManager"] diff --git a/app/config.py b/app/config.py index 388f6f56..0cd48817 100644 --- a/app/config.py +++ b/app/config.py @@ -166,6 +166,12 @@ class Settings(BaseSettings): # Age (seconds) after which a player profile is considered stale. player_staleness_threshold: int = 1800 # 30 min + # TTL (seconds) for stale responses written to Valkey API cache. + # Short enough that background refresh (typically seconds) will overwrite it + # with fresh data before it expires; long enough to absorb burst traffic + # while the refresh is in-flight. + stale_cache_timeout: int = 60 + ############ # UNKNOWN PLAYERS SYSTEM ############ diff --git a/app/domain/services/static_data_service.py b/app/domain/services/static_data_service.py index e866d274..929ddf32 100644 --- a/app/domain/services/static_data_service.py +++ b/app/domain/services/static_data_service.py @@ -2,6 +2,7 @@ import time from typing import TYPE_CHECKING, Any +from app.config import settings from app.domain.services import BaseService from app.monitoring.metrics import ( background_refresh_triggered_total, @@ -74,27 +75,33 @@ async def get_or_fetch( entity_type, storage_key, refresh_coro=self._refresh_static( - storage_key, fetcher, parser, entity_type + storage_key, + fetcher, + parser, + entity_type, + cache_key=cache_key, + cache_ttl=cache_ttl, + result_filter=result_filter, ), ) stale_responses_total.inc() background_refresh_triggered_total.labels(entity_type=entity_type).inc() - # Do NOT write stale data to Valkey: all requests during the - # stale window must reach FastAPI so they get correct headers. - # The background refresh updates SQLite; the next FastAPI request - # will see fresh data and repopulate Valkey naturally. + # Write stale data to Valkey with a short TTL so nginx can absorb + # burst traffic while the background refresh is in-flight. + # The refresh will overwrite this entry with fresh data + full TTL. + filtered = result_filter(data) if result_filter is not None else data + await self._update_api_cache( + cache_key, filtered, settings.stale_cache_timeout + ) else: logger.info( f"[SWR] {entity_type} fresh (age={age}s) — serving from SQLite" ) - await self._update_api_cache(cache_key, data if result_filter is None else result_filter(data), cache_ttl) + filtered = result_filter(data) if result_filter is not None else data + await self._update_api_cache(cache_key, filtered, cache_ttl) storage_hits_total.labels(result="hit").inc() - - if result_filter is not None: - data = result_filter(data) - - return data, is_stale, age + return filtered, is_stale, age # Cold start — fetch synchronously logger.info(f"[SWR] {entity_type} not in SQLite — fetching from source") @@ -114,13 +121,19 @@ async def _refresh_static( fetcher: Callable[[], Any], parser: Callable[[Any], Any] | None, entity_type: str, + *, + cache_key: str, + cache_ttl: int, + result_filter: Callable[[Any], Any] | None = None, ) -> None: - """Fetch fresh data and persist it — executed as a background task.""" + """Fetch fresh data, persist to SQLite and update Valkey with full TTL.""" logger.info(f"[SWR] Background refresh started for {entity_type}/{storage_key}") try: raw = await fetcher() data = parser(raw) if parser is not None else raw await self._store_in_storage(storage_key, data) + filtered = result_filter(data) if result_filter is not None else data + await self._update_api_cache(cache_key, filtered, cache_ttl) logger.info( f"[SWR] Background refresh complete for {entity_type}/{storage_key}" ) diff --git a/app/heroes/commands/check_new_hero.py b/app/heroes/commands/check_new_hero.py index 6104fc83..0acb3fae 100644 --- a/app/heroes/commands/check_new_hero.py +++ b/app/heroes/commands/check_new_hero.py @@ -7,11 +7,11 @@ from fastapi import HTTPException +from app.adapters.blizzard import OverFastClient from app.adapters.blizzard.parsers.heroes import fetch_heroes_html, parse_heroes_html from app.config import settings from app.exceptions import ParserParsingError from app.helpers import send_discord_webhook_message -from app.overfast_client import OverFastClient from app.overfast_logger import logger from ..enums import HeroKey diff --git a/app/main.py b/app/main.py index 40f8dde0..4bfaee24 100644 --- a/app/main.py +++ b/app/main.py @@ -13,6 +13,7 @@ from fastapi.staticfiles import StaticFiles from starlette.exceptions import HTTPException as StarletteHTTPException +from .adapters.blizzard import OverFastClient from .adapters.cache import CacheManager from .adapters.storage import SQLiteStorage from .api import gamemodes, heroes, maps, players, roles @@ -27,7 +28,6 @@ TraceMallocMiddleware, ) from .monitoring.middleware import register_prometheus_middleware -from .overfast_client import OverFastClient from .overfast_logger import logger if TYPE_CHECKING: diff --git a/app/overfast_client.py b/app/overfast_client.py deleted file mode 100644 index f6a1784f..00000000 --- a/app/overfast_client.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -OverFast HTTP client - DEPRECATED, moved to app/adapters/blizzard/client.py - -This module is kept for backward compatibility during the migration to v4. -Import from app.adapters.blizzard instead. -""" - -from app.adapters.blizzard import OverFastClient - -__all__ = ["OverFastClient"] diff --git a/tests/conftest.py b/tests/conftest.py index 942905cb..3d3528f3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -52,7 +52,7 @@ async def _patch_before_every_test( patch("app.helpers.settings.discord_webhook_enabled", False), patch("app.helpers.settings.profiler", None), patch( - "app.cache_manager.CacheManager.valkey_server", + "app.adapters.cache.valkey_cache.CacheManager.valkey_server", valkey_server, ), ): diff --git a/tests/heroes/parsers/test_hero_stats_summary.py b/tests/heroes/parsers/test_hero_stats_summary.py index 5a9876bf..e5eee3bf 100644 --- a/tests/heroes/parsers/test_hero_stats_summary.py +++ b/tests/heroes/parsers/test_hero_stats_summary.py @@ -2,9 +2,9 @@ import pytest +from app.adapters.blizzard import OverFastClient from app.adapters.blizzard.parsers.hero_stats_summary import parse_hero_stats_summary from app.exceptions import ParserBlizzardError -from app.overfast_client import OverFastClient from app.players.enums import ( CompetitiveDivision, PlayerGamemode, diff --git a/tests/heroes/parsers/test_heroes_parser.py b/tests/heroes/parsers/test_heroes_parser.py index bb474a12..2fae076a 100644 --- a/tests/heroes/parsers/test_heroes_parser.py +++ b/tests/heroes/parsers/test_heroes_parser.py @@ -3,13 +3,13 @@ import pytest from fastapi import status +from app.adapters.blizzard import OverFastClient from app.adapters.blizzard.parsers.heroes import ( fetch_heroes_html, filter_heroes, parse_heroes_html, ) from app.heroes.enums import HeroGamemode, HeroKey -from app.overfast_client import OverFastClient from app.roles.enums import Role diff --git a/tests/roles/parsers/test_roles_parser.py b/tests/roles/parsers/test_roles_parser.py index f27b92a5..8ece049d 100644 --- a/tests/roles/parsers/test_roles_parser.py +++ b/tests/roles/parsers/test_roles_parser.py @@ -3,8 +3,8 @@ import pytest from fastapi import status +from app.adapters.blizzard import OverFastClient from app.adapters.blizzard.parsers.roles import fetch_roles_html, parse_roles_html -from app.overfast_client import OverFastClient from app.roles.enums import Role diff --git a/tests/test_cache_manager.py b/tests/test_cache_manager.py index 900ad9dc..09b4824a 100644 --- a/tests/test_cache_manager.py +++ b/tests/test_cache_manager.py @@ -5,7 +5,7 @@ import pytest from valkey.exceptions import ValkeyError -from app.cache_manager import CacheManager +from app.adapters.cache import CacheManager from app.config import settings from app.enums import Locale From 3f6cf1832f837029923a3730c00b0c9bdc9a851d Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 09:04:10 +0100 Subject: [PATCH 11/26] --wip-- [skip ci] --- app/domain/services/static_data_service.py | 174 +++++++++++++++------ 1 file changed, 122 insertions(+), 52 deletions(-) diff --git a/app/domain/services/static_data_service.py b/app/domain/services/static_data_service.py index 929ddf32..4dc62504 100644 --- a/app/domain/services/static_data_service.py +++ b/app/domain/services/static_data_service.py @@ -60,60 +60,125 @@ async def get_or_fetch( a cold-start fetch). """ stored = await self._load_from_storage(storage_key) - if stored is not None: - data = stored["data"] - age = int(time.time()) - stored["updated_at"] - is_stale = age >= staleness_threshold - - if is_stale: - logger.info( - f"[SWR] {entity_type} stale (age={age}s, " - f"threshold={staleness_threshold}s) — serving + triggering refresh" - ) - await self._enqueue_refresh( - entity_type, + storage_hits_total.labels(result="hit").inc() + return await self._serve_from_storage( + stored, + storage_key=storage_key, + fetcher=fetcher, + parser=parser, + cache_key=cache_key, + cache_ttl=cache_ttl, + staleness_threshold=staleness_threshold, + entity_type=entity_type, + result_filter=result_filter, + ) + + storage_hits_total.labels(result="miss").inc() + return await self._cold_fetch( + storage_key=storage_key, + fetcher=fetcher, + parser=parser, + cache_key=cache_key, + cache_ttl=cache_ttl, + entity_type=entity_type, + result_filter=result_filter, + ) + + async def _serve_from_storage( + self, + stored: dict[str, Any], + *, + storage_key: str, + fetcher: Callable[[], Any], + parser: Callable[[Any], Any] | None, + cache_key: str, + cache_ttl: int, + staleness_threshold: int, + entity_type: str, + result_filter: Callable[[Any], Any] | None, + ) -> tuple[Any, bool, int]: + """Serve data from a SQLite hit, triggering a background refresh if stale.""" + data = stored["data"] + age = int(time.time()) - stored["updated_at"] + is_stale = age >= staleness_threshold + filtered = self._apply_filter(data, result_filter) + + if is_stale: + logger.info( + f"[SWR] {entity_type} stale (age={age}s, " + f"threshold={staleness_threshold}s) — serving + triggering refresh" + ) + await self._enqueue_refresh( + entity_type, + storage_key, + refresh_coro=self._refresh_static( storage_key, - refresh_coro=self._refresh_static( - storage_key, - fetcher, - parser, - entity_type, - cache_key=cache_key, - cache_ttl=cache_ttl, - result_filter=result_filter, - ), - ) - stale_responses_total.inc() - background_refresh_triggered_total.labels(entity_type=entity_type).inc() - # Write stale data to Valkey with a short TTL so nginx can absorb - # burst traffic while the background refresh is in-flight. - # The refresh will overwrite this entry with fresh data + full TTL. - filtered = result_filter(data) if result_filter is not None else data - await self._update_api_cache( - cache_key, filtered, settings.stale_cache_timeout - ) - else: - logger.info( - f"[SWR] {entity_type} fresh (age={age}s) — serving from SQLite" - ) - filtered = result_filter(data) if result_filter is not None else data - await self._update_api_cache(cache_key, filtered, cache_ttl) + fetcher, + parser, + entity_type, + cache_key=cache_key, + cache_ttl=cache_ttl, + result_filter=result_filter, + ), + ) + stale_responses_total.inc() + background_refresh_triggered_total.labels(entity_type=entity_type).inc() + # Short TTL absorbs burst traffic while the background refresh is in-flight. + await self._update_api_cache( + cache_key, filtered, settings.stale_cache_timeout + ) + else: + logger.info(f"[SWR] {entity_type} fresh (age={age}s) — serving from SQLite") + await self._update_api_cache(cache_key, filtered, cache_ttl) - storage_hits_total.labels(result="hit").inc() - return filtered, is_stale, age + return filtered, is_stale, age - # Cold start — fetch synchronously + async def _cold_fetch( + self, + *, + storage_key: str, + fetcher: Callable[[], Any], + parser: Callable[[Any], Any] | None, + cache_key: str, + cache_ttl: int, + entity_type: str, + result_filter: Callable[[Any], Any] | None, + ) -> tuple[Any, bool, int]: + """Fetch from source on cold start, persist to SQLite and Valkey.""" logger.info(f"[SWR] {entity_type} not in SQLite — fetching from source") - storage_hits_total.labels(result="miss").inc() + filtered = await self._fetch_and_store( + storage_key=storage_key, + fetcher=fetcher, + parser=parser, + cache_key=cache_key, + cache_ttl=cache_ttl, + result_filter=result_filter, + ) + return filtered, False, 0 + async def _fetch_and_store( + self, + *, + storage_key: str, + fetcher: Callable[[], Any], + parser: Callable[[Any], Any] | None, + cache_key: str, + cache_ttl: int, + result_filter: Callable[[Any], Any] | None, + ) -> Any: + """Fetch from source, persist to SQLite, update Valkey, return filtered data.""" raw = await fetcher() data = parser(raw) if parser is not None else raw await self._store_in_storage(storage_key, data) - - filtered = result_filter(data) if result_filter is not None else data + filtered = self._apply_filter(data, result_filter) await self._update_api_cache(cache_key, filtered, cache_ttl) - return filtered, False, 0 + return filtered + + @staticmethod + def _apply_filter(data: Any, result_filter: Callable[[Any], Any] | None) -> Any: + """Apply ``result_filter`` to ``data`` if provided, otherwise return as-is.""" + return result_filter(data) if result_filter is not None else data async def _refresh_static( self, @@ -129,11 +194,14 @@ async def _refresh_static( """Fetch fresh data, persist to SQLite and update Valkey with full TTL.""" logger.info(f"[SWR] Background refresh started for {entity_type}/{storage_key}") try: - raw = await fetcher() - data = parser(raw) if parser is not None else raw - await self._store_in_storage(storage_key, data) - filtered = result_filter(data) if result_filter is not None else data - await self._update_api_cache(cache_key, filtered, cache_ttl) + await self._fetch_and_store( + storage_key=storage_key, + fetcher=fetcher, + parser=parser, + cache_key=cache_key, + cache_ttl=cache_ttl, + result_filter=result_filter, + ) logger.info( f"[SWR] Background refresh complete for {entity_type}/{storage_key}" ) @@ -145,12 +213,14 @@ async def _refresh_static( async def _load_from_storage(self, storage_key: str) -> dict[str, Any] | None: """Load data from the ``static_data`` SQLite table. Returns ``None`` on miss.""" result = await self.storage.get_static_data(storage_key) - if result: - return { + return ( + { "data": json.loads(result["data"]), "updated_at": result["updated_at"], } - return None + if result + else None + ) async def _store_in_storage(self, storage_key: str, data: Any) -> None: """Persist data to the ``static_data`` SQLite table as JSON.""" From 314abe745267214d73c3ce803bb2fb48f6713431 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 09:11:17 +0100 Subject: [PATCH 12/26] --wip-- [skip ci] --- app/api/routers/gamemodes.py | 4 +- app/domain/services/gamemode_service.py | 16 +- app/domain/services/hero_service.py | 34 +-- app/domain/services/map_service.py | 18 +- app/domain/services/role_service.py | 16 +- app/domain/services/static_data_service.py | 232 +++++++-------------- 6 files changed, 130 insertions(+), 190 deletions(-) diff --git a/app/api/routers/gamemodes.py b/app/api/routers/gamemodes.py index e94c422e..7d9bde9b 100644 --- a/app/api/routers/gamemodes.py +++ b/app/api/routers/gamemodes.py @@ -30,6 +30,8 @@ async def list_map_gamemodes( response: Response, service: GamemodeServiceDep, ) -> Any: - data, is_stale, age = await service.list_gamemodes(cache_key=build_cache_key(request)) + data, is_stale, age = await service.list_gamemodes( + cache_key=build_cache_key(request) + ) apply_swr_headers(response, settings.csv_cache_timeout, is_stale, age) return data diff --git a/app/domain/services/gamemode_service.py b/app/domain/services/gamemode_service.py index fa1a83ac..e0b710af 100644 --- a/app/domain/services/gamemode_service.py +++ b/app/domain/services/gamemode_service.py @@ -2,7 +2,7 @@ from app.adapters.blizzard.parsers.gamemodes import parse_gamemodes_csv from app.config import settings -from app.domain.services.static_data_service import StaticDataService +from app.domain.services.static_data_service import StaticDataService, StaticFetchConfig class GamemodeService(StaticDataService): @@ -18,10 +18,12 @@ async def _fetch() -> list[dict]: return parse_gamemodes_csv() return await self.get_or_fetch( - storage_key="gamemodes:all", - fetcher=_fetch, - cache_key=cache_key, - cache_ttl=settings.csv_cache_timeout, - staleness_threshold=settings.gamemodes_staleness_threshold, - entity_type="gamemodes", + StaticFetchConfig( + storage_key="gamemodes:all", + fetcher=_fetch, + cache_key=cache_key, + cache_ttl=settings.csv_cache_timeout, + staleness_threshold=settings.gamemodes_staleness_threshold, + entity_type="gamemodes", + ) ) diff --git a/app/domain/services/hero_service.py b/app/domain/services/hero_service.py index 41838fc2..65d2db04 100644 --- a/app/domain/services/hero_service.py +++ b/app/domain/services/hero_service.py @@ -13,7 +13,7 @@ ) from app.adapters.blizzard.parsers.heroes_hitpoints import parse_heroes_hitpoints from app.config import settings -from app.domain.services.static_data_service import StaticDataService +from app.domain.services.static_data_service import StaticDataService, StaticFetchConfig from app.enums import Locale from app.exceptions import ParserBlizzardError, ParserParsingError from app.helpers import overfast_internal_error @@ -59,13 +59,15 @@ def _filter(data: list[dict]) -> list[dict]: return filter_heroes(data, role, gamemode) return await self.get_or_fetch( - storage_key=f"heroes:{locale}", - fetcher=_fetch, - result_filter=_filter, - cache_key=cache_key, - cache_ttl=settings.heroes_path_cache_timeout, - staleness_threshold=settings.heroes_staleness_threshold, - entity_type="heroes", + StaticFetchConfig( + storage_key=f"heroes:{locale}", + fetcher=_fetch, + result_filter=_filter, + cache_key=cache_key, + cache_ttl=settings.heroes_path_cache_timeout, + staleness_threshold=settings.heroes_staleness_threshold, + entity_type="heroes", + ) ) # ------------------------------------------------------------------ @@ -143,13 +145,15 @@ def _filter(data: list[dict]) -> list[dict]: return _filter_hero_stats(data, role, order_by) return await self.get_or_fetch( - storage_key=storage_key, - fetcher=_fetch, - result_filter=_filter, - cache_key=cache_key, - cache_ttl=settings.hero_stats_cache_timeout, - staleness_threshold=settings.hero_stats_staleness_threshold, - entity_type="hero_stats", + StaticFetchConfig( + storage_key=storage_key, + fetcher=_fetch, + result_filter=_filter, + cache_key=cache_key, + cache_ttl=settings.hero_stats_cache_timeout, + staleness_threshold=settings.hero_stats_staleness_threshold, + entity_type="hero_stats", + ) ) diff --git a/app/domain/services/map_service.py b/app/domain/services/map_service.py index 063a904f..c07b10c0 100644 --- a/app/domain/services/map_service.py +++ b/app/domain/services/map_service.py @@ -2,7 +2,7 @@ from app.adapters.blizzard.parsers.maps import parse_maps_csv from app.config import settings -from app.domain.services.static_data_service import StaticDataService +from app.domain.services.static_data_service import StaticDataService, StaticFetchConfig class MapService(StaticDataService): @@ -28,11 +28,13 @@ def _filter(data: list[dict]) -> list[dict]: return [m for m in data if gamemode_val in m.get("gamemodes", [])] return await self.get_or_fetch( - storage_key="maps:all", - fetcher=_fetch, - result_filter=_filter, - cache_key=cache_key, - cache_ttl=settings.csv_cache_timeout, - staleness_threshold=settings.maps_staleness_threshold, - entity_type="maps", + StaticFetchConfig( + storage_key="maps:all", + fetcher=_fetch, + result_filter=_filter, + cache_key=cache_key, + cache_ttl=settings.csv_cache_timeout, + staleness_threshold=settings.maps_staleness_threshold, + entity_type="maps", + ) ) diff --git a/app/domain/services/role_service.py b/app/domain/services/role_service.py index 50c3c297..fe7d4868 100644 --- a/app/domain/services/role_service.py +++ b/app/domain/services/role_service.py @@ -2,7 +2,7 @@ from app.adapters.blizzard.parsers.roles import fetch_roles_html, parse_roles_html from app.config import settings -from app.domain.services.static_data_service import StaticDataService +from app.domain.services.static_data_service import StaticDataService, StaticFetchConfig from app.enums import Locale from app.exceptions import ParserParsingError from app.helpers import overfast_internal_error @@ -27,10 +27,12 @@ async def _fetch() -> list[dict]: raise overfast_internal_error(blizzard_url, exc) from exc return await self.get_or_fetch( - storage_key=f"roles:{locale}", - fetcher=_fetch, - cache_key=cache_key, - cache_ttl=settings.heroes_path_cache_timeout, - staleness_threshold=settings.roles_staleness_threshold, - entity_type="roles", + StaticFetchConfig( + storage_key=f"roles:{locale}", + fetcher=_fetch, + cache_key=cache_key, + cache_ttl=settings.heroes_path_cache_timeout, + staleness_threshold=settings.roles_staleness_threshold, + entity_type="roles", + ) ) diff --git a/app/domain/services/static_data_service.py b/app/domain/services/static_data_service.py index 4dc62504..14728161 100644 --- a/app/domain/services/static_data_service.py +++ b/app/domain/services/static_data_service.py @@ -1,5 +1,6 @@ import json import time +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any from app.config import settings @@ -15,212 +16,139 @@ from collections.abc import Callable +@dataclass +class StaticFetchConfig: + """Parameter object grouping all inputs needed for a static SWR fetch. + + Pass a single ``StaticFetchConfig`` to ``StaticDataService.get_or_fetch`` + instead of passing each field as a separate keyword argument. + """ + + storage_key: str + fetcher: Callable[[], Any] + cache_key: str + cache_ttl: int + staleness_threshold: int + entity_type: str + parser: Callable[[Any], Any] | None = field(default=None) + result_filter: Callable[[Any], Any] | None = field(default=None) + + class StaticDataService(BaseService): """SWR orchestration for static content backed by the ``static_data`` SQLite table. Staleness is determined by a configurable time threshold. Concrete static - services (heroes, maps, gamemodes, roles) call ``get_or_fetch`` with - domain-specific ``fetcher``, ``parser``, and optional ``result_filter`` - callables — no subclass-level overrides are needed for the storage layer. + services (heroes, maps, gamemodes, roles) call ``get_or_fetch`` with a + ``StaticFetchConfig`` — no subclass-level overrides are needed for the + storage layer. Note: Valkey API-cache *reads* happen at the Nginx/Lua layer before FastAPI is reached; this service only ever *writes* to the API cache. """ - async def get_or_fetch( - self, - *, - storage_key: str, - fetcher: Callable[[], Any], - cache_key: str, - cache_ttl: int, - staleness_threshold: int, - entity_type: str, - parser: Callable[[Any], Any] | None = None, - result_filter: Callable[[Any], Any] | None = None, - ) -> tuple[Any, bool, int]: + async def get_or_fetch(self, config: StaticFetchConfig) -> tuple[Any, bool, int]: """SWR orchestration for static data. - Args: - storage_key: Key in the ``static_data`` SQLite table. - fetcher: Async callable that retrieves raw data from the upstream - source (Blizzard HTML/JSON or a local CSV file). - cache_key: Valkey API-cache key to update after serving data. - cache_ttl: TTL in seconds for the Valkey API-cache entry. - staleness_threshold: Seconds after which stored data is considered stale. - entity_type: Human-readable label used in metrics and logs. - parser: Optional callable that converts raw fetcher output into the - stored/returned format. Defaults to identity (raw data as-is). - result_filter: Optional callable applied to the stored data before - returning. Not persisted — re-applied on every request. - Returns: ``(data, is_stale, age_seconds)`` tuple. ``age_seconds`` is the number of seconds since the data was last stored in SQLite (0 on a cold-start fetch). """ - stored = await self._load_from_storage(storage_key) + stored = await self._load_from_storage(config.storage_key) if stored is not None: storage_hits_total.labels(result="hit").inc() - return await self._serve_from_storage( - stored, - storage_key=storage_key, - fetcher=fetcher, - parser=parser, - cache_key=cache_key, - cache_ttl=cache_ttl, - staleness_threshold=staleness_threshold, - entity_type=entity_type, - result_filter=result_filter, - ) + return await self._serve_from_storage(stored, config) storage_hits_total.labels(result="miss").inc() - return await self._cold_fetch( - storage_key=storage_key, - fetcher=fetcher, - parser=parser, - cache_key=cache_key, - cache_ttl=cache_ttl, - entity_type=entity_type, - result_filter=result_filter, + return await self._cold_fetch(config) + + async def _load_from_storage(self, storage_key: str) -> dict[str, Any] | None: + """Load data from the ``static_data`` SQLite table. Returns ``None`` on miss.""" + result = await self.storage.get_static_data(storage_key) + return ( + { + "data": json.loads(result["data"]), + "updated_at": result["updated_at"], + } + if result + else None ) async def _serve_from_storage( - self, - stored: dict[str, Any], - *, - storage_key: str, - fetcher: Callable[[], Any], - parser: Callable[[Any], Any] | None, - cache_key: str, - cache_ttl: int, - staleness_threshold: int, - entity_type: str, - result_filter: Callable[[Any], Any] | None, + self, stored: dict[str, Any], config: StaticFetchConfig ) -> tuple[Any, bool, int]: """Serve data from a SQLite hit, triggering a background refresh if stale.""" data = stored["data"] age = int(time.time()) - stored["updated_at"] - is_stale = age >= staleness_threshold - filtered = self._apply_filter(data, result_filter) + is_stale = age >= config.staleness_threshold + filtered = self._apply_filter(data, config.result_filter) if is_stale: logger.info( - f"[SWR] {entity_type} stale (age={age}s, " - f"threshold={staleness_threshold}s) — serving + triggering refresh" + f"[SWR] {config.entity_type} stale (age={age}s, " + f"threshold={config.staleness_threshold}s) — serving + triggering refresh" ) await self._enqueue_refresh( - entity_type, - storage_key, - refresh_coro=self._refresh_static( - storage_key, - fetcher, - parser, - entity_type, - cache_key=cache_key, - cache_ttl=cache_ttl, - result_filter=result_filter, - ), + config.entity_type, + config.storage_key, + refresh_coro=self._refresh_static(config), ) stale_responses_total.inc() - background_refresh_triggered_total.labels(entity_type=entity_type).inc() + background_refresh_triggered_total.labels( + entity_type=config.entity_type + ).inc() # Short TTL absorbs burst traffic while the background refresh is in-flight. await self._update_api_cache( - cache_key, filtered, settings.stale_cache_timeout + config.cache_key, filtered, settings.stale_cache_timeout ) else: - logger.info(f"[SWR] {entity_type} fresh (age={age}s) — serving from SQLite") - await self._update_api_cache(cache_key, filtered, cache_ttl) + logger.info( + f"[SWR] {config.entity_type} fresh (age={age}s) — serving from SQLite" + ) + await self._update_api_cache(config.cache_key, filtered, config.cache_ttl) return filtered, is_stale, age - async def _cold_fetch( - self, - *, - storage_key: str, - fetcher: Callable[[], Any], - parser: Callable[[Any], Any] | None, - cache_key: str, - cache_ttl: int, - entity_type: str, - result_filter: Callable[[Any], Any] | None, - ) -> tuple[Any, bool, int]: - """Fetch from source on cold start, persist to SQLite and Valkey.""" - logger.info(f"[SWR] {entity_type} not in SQLite — fetching from source") - filtered = await self._fetch_and_store( - storage_key=storage_key, - fetcher=fetcher, - parser=parser, - cache_key=cache_key, - cache_ttl=cache_ttl, - result_filter=result_filter, - ) - return filtered, False, 0 - - async def _fetch_and_store( - self, - *, - storage_key: str, - fetcher: Callable[[], Any], - parser: Callable[[Any], Any] | None, - cache_key: str, - cache_ttl: int, - result_filter: Callable[[Any], Any] | None, - ) -> Any: - """Fetch from source, persist to SQLite, update Valkey, return filtered data.""" - raw = await fetcher() - data = parser(raw) if parser is not None else raw - await self._store_in_storage(storage_key, data) - filtered = self._apply_filter(data, result_filter) - await self._update_api_cache(cache_key, filtered, cache_ttl) - return filtered - @staticmethod def _apply_filter(data: Any, result_filter: Callable[[Any], Any] | None) -> Any: """Apply ``result_filter`` to ``data`` if provided, otherwise return as-is.""" return result_filter(data) if result_filter is not None else data - async def _refresh_static( - self, - storage_key: str, - fetcher: Callable[[], Any], - parser: Callable[[Any], Any] | None, - entity_type: str, - *, - cache_key: str, - cache_ttl: int, - result_filter: Callable[[Any], Any] | None = None, - ) -> None: + async def _refresh_static(self, config: StaticFetchConfig) -> None: """Fetch fresh data, persist to SQLite and update Valkey with full TTL.""" - logger.info(f"[SWR] Background refresh started for {entity_type}/{storage_key}") + logger.info( + f"[SWR] Background refresh started for" + f" {config.entity_type}/{config.storage_key}" + ) try: - await self._fetch_and_store( - storage_key=storage_key, - fetcher=fetcher, - parser=parser, - cache_key=cache_key, - cache_ttl=cache_ttl, - result_filter=result_filter, - ) + await self._fetch_and_store(config) logger.info( - f"[SWR] Background refresh complete for {entity_type}/{storage_key}" + f"[SWR] Background refresh complete for" + f" {config.entity_type}/{config.storage_key}" ) except Exception as exc: # noqa: BLE001 logger.warning( - f"[SWR] Background refresh failed for {entity_type}/{storage_key}: {exc}" + f"[SWR] Background refresh failed for" + f" {config.entity_type}/{config.storage_key}: {exc}" ) - async def _load_from_storage(self, storage_key: str) -> dict[str, Any] | None: - """Load data from the ``static_data`` SQLite table. Returns ``None`` on miss.""" - result = await self.storage.get_static_data(storage_key) - return ( - { - "data": json.loads(result["data"]), - "updated_at": result["updated_at"], - } - if result - else None - ) + async def _fetch_and_store(self, config: StaticFetchConfig) -> Any: + """Fetch from source, persist to SQLite, update Valkey, return filtered data.""" + raw = await config.fetcher() + + data = config.parser(raw) if config.parser is not None else raw + await self._store_in_storage(config.storage_key, data) + + filtered = self._apply_filter(data, config.result_filter) + await self._update_api_cache(config.cache_key, filtered, config.cache_ttl) + + return filtered + + async def _cold_fetch(self, config: StaticFetchConfig) -> tuple[Any, bool, int]: + """Fetch from source on cold start, persist to SQLite and Valkey.""" + logger.info(f"[SWR] {config.entity_type} not in SQLite — fetching from source") + filtered = await self._fetch_and_store(config) + return filtered, False, 0 async def _store_in_storage(self, storage_key: str, data: Any) -> None: """Persist data to the ``static_data`` SQLite table as JSON.""" From 7a011bebfca71b85243d0979e280dc3ea98edc80 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 09:25:37 +0100 Subject: [PATCH 13/26] --wip-- [skip ci] --- app/adapters/tasks/asyncio_task_queue.py | 8 +- app/domain/ports/task_queue.py | 9 +- app/domain/services/base_service.py | 18 ++ app/domain/services/static_data_service.py | 18 +- app/monitoring/metrics.py | 12 + .../dashboards/overfast-api-tasks.json | 273 +++++++++++++++++- 6 files changed, 318 insertions(+), 20 deletions(-) diff --git a/app/adapters/tasks/asyncio_task_queue.py b/app/adapters/tasks/asyncio_task_queue.py index 66f27e06..42f449bd 100644 --- a/app/adapters/tasks/asyncio_task_queue.py +++ b/app/adapters/tasks/asyncio_task_queue.py @@ -12,7 +12,7 @@ from app.overfast_logger import logger if TYPE_CHECKING: - from collections.abc import Coroutine + from collections.abc import Awaitable, Callable, Coroutine class AsyncioTaskQueue: @@ -35,6 +35,8 @@ async def enqueue( *_args: Any, job_id: str | None = None, coro: Coroutine[Any, Any, Any] | None = None, + on_complete: Callable[[str], Awaitable[None]] | None = None, + on_failure: Callable[[str, Exception], Awaitable[None]] | None = None, **_kwargs: Any, ) -> str: """Schedule a background task if not already pending.""" @@ -57,11 +59,15 @@ async def _run() -> None: ) if coro is not None: await coro + if on_complete is not None: + await on_complete(effective_id) except Exception as exc: # noqa: BLE001 status = "failure" logger.warning( f"[TaskQueue] Task '{task_name}' (job_id={effective_id}) failed: {exc}" ) + if on_failure is not None: + await on_failure(effective_id, exc) finally: elapsed = time.monotonic() - start background_tasks_total.labels(task_type=task_name, status=status).inc() diff --git a/app/domain/ports/task_queue.py b/app/domain/ports/task_queue.py index 26e70379..df47f9fc 100644 --- a/app/domain/ports/task_queue.py +++ b/app/domain/ports/task_queue.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Any, Protocol if TYPE_CHECKING: - from collections.abc import Coroutine + from collections.abc import Awaitable, Callable, Coroutine class TaskQueuePort(Protocol): @@ -15,6 +15,8 @@ async def enqueue( *args: Any, job_id: str | None = None, coro: Coroutine[Any, Any, Any] | None = None, + on_complete: Callable[[str], Awaitable[None]] | None = None, + on_failure: Callable[[str, Exception], Awaitable[None]] | None = None, **kwargs: Any, ) -> str: """Enqueue a background task. @@ -22,6 +24,11 @@ async def enqueue( ``coro``, when provided, is executed immediately (Phase 4 asyncio) or dispatched to a worker process (Phase 5 arq). ``task_name`` is kept for arq compatibility and logging. + + ``on_complete(job_id)`` is awaited when the task finishes successfully. + ``on_failure(job_id, exc)`` is awaited when the task raises an exception. + Both callbacks are optional and intended for domain-level monitoring. + Returns the effective job ID. """ ... diff --git a/app/domain/services/base_service.py b/app/domain/services/base_service.py index 8c52bc29..620415d9 100644 --- a/app/domain/services/base_service.py +++ b/app/domain/services/base_service.py @@ -16,6 +16,10 @@ from enum import StrEnum from typing import TYPE_CHECKING, Any +from app.monitoring.metrics import ( + background_refresh_completed_total, + background_refresh_failed_total, +) from app.overfast_logger import logger if TYPE_CHECKING: @@ -76,8 +80,20 @@ async def _enqueue_refresh( ``refresh_coro``, when provided, is passed to the task queue and executed as the actual refresh work (Phase 4: asyncio, Phase 5: arq). + Completion and failure are reported via domain-level metrics. """ job_id = f"refresh:{entity_type}:{entity_id}" + + async def _on_complete(_job_id: str) -> None: + logger.info( + f"[SWR] Background refresh complete for {entity_type}/{entity_id}" + ) + background_refresh_completed_total.labels(entity_type=entity_type).inc() + + async def _on_failure(_job_id: str, exc: Exception) -> None: + logger.warning(f"[SWR] Refresh failed for {entity_type}/{entity_id}: {exc}") + background_refresh_failed_total.labels(entity_type=entity_type).inc() + try: if not await self.task_queue.is_job_pending_or_running(job_id): await self.task_queue.enqueue( @@ -85,6 +101,8 @@ async def _enqueue_refresh( entity_id, job_id=job_id, coro=refresh_coro, + on_complete=_on_complete, + on_failure=_on_failure, ) except Exception as exc: # noqa: BLE001 logger.warning( diff --git a/app/domain/services/static_data_service.py b/app/domain/services/static_data_service.py index 14728161..340668bf 100644 --- a/app/domain/services/static_data_service.py +++ b/app/domain/services/static_data_service.py @@ -115,22 +115,16 @@ def _apply_filter(data: Any, result_filter: Callable[[Any], Any] | None) -> Any: return result_filter(data) if result_filter is not None else data async def _refresh_static(self, config: StaticFetchConfig) -> None: - """Fetch fresh data, persist to SQLite and update Valkey with full TTL.""" + """Fetch fresh data, persist to SQLite and update Valkey with full TTL. + + Exceptions are intentionally not caught here — they propagate to the + task queue adapter which calls the ``on_failure`` callback for metrics. + """ logger.info( f"[SWR] Background refresh started for" f" {config.entity_type}/{config.storage_key}" ) - try: - await self._fetch_and_store(config) - logger.info( - f"[SWR] Background refresh complete for" - f" {config.entity_type}/{config.storage_key}" - ) - except Exception as exc: # noqa: BLE001 - logger.warning( - f"[SWR] Background refresh failed for" - f" {config.entity_type}/{config.storage_key}: {exc}" - ) + await self._fetch_and_store(config) async def _fetch_and_store(self, config: StaticFetchConfig) -> Any: """Fetch from source, persist to SQLite, update Valkey, return filtered data.""" diff --git a/app/monitoring/metrics.py b/app/monitoring/metrics.py index 2678740f..1a9b3cb9 100644 --- a/app/monitoring/metrics.py +++ b/app/monitoring/metrics.py @@ -73,6 +73,18 @@ ["entity_type"], # "heroes", "maps", "gamemodes", "roles", "player", "hero_stats" ) +background_refresh_completed_total = Counter( + "background_refresh_completed_total", + "Background refresh tasks that completed successfully", + ["entity_type"], +) + +background_refresh_failed_total = Counter( + "background_refresh_failed_total", + "Background refresh tasks that raised an exception", + ["entity_type"], +) + ######################## # Background Task Metrics (Phase 5) ######################## diff --git a/build/grafana/provisioning/dashboards/overfast-api-tasks.json b/build/grafana/provisioning/dashboards/overfast-api-tasks.json index 8651b7b4..e503631b 100644 --- a/build/grafana/provisioning/dashboards/overfast-api-tasks.json +++ b/build/grafana/provisioning/dashboards/overfast-api-tasks.json @@ -418,7 +418,7 @@ }, { "type": "row", - "title": "\ud83d\uded1 Rate Limiting", + "title": "\u2705 Refresh Outcomes by Entity", "gridPos": { "h": 1, "w": 24, @@ -428,6 +428,267 @@ "collapsed": false, "panels": [] }, + { + "type": "stat", + "title": "Refreshes Completed (24h)", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 22 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "round(sum(increase(background_refresh_completed_total{job=\"fastapi\"}[1d])))", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "fixed", + "fixedColor": "green" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "textMode": "auto" + } + }, + { + "type": "stat", + "title": "Refreshes Failed (24h)", + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 22 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "round(sum(increase(background_refresh_failed_total{job=\"fastapi\"}[1d])))", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1, + "color": "red" + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "textMode": "auto" + } + }, + { + "type": "stat", + "title": "Refresh Success Rate (24h)", + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 22 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "round(100 * sum(increase(background_refresh_completed_total{job=\"fastapi\"}[1d])) / clamp_min(sum(increase(background_refresh_triggered_total{job=\"fastapi\"}[1d])), 1))", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "red" + }, + { + "value": 90, + "color": "yellow" + }, + { + "value": 99, + "color": "green" + } + ] + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "textMode": "auto" + } + }, + { + "type": "timeseries", + "title": "Refresh Completion Rate by Entity", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "sum by (entity_type) (rate(background_refresh_completed_total{job=\"fastapi\"}[5m]))", + "legendFormat": "{{entity_type}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "type": "timeseries", + "title": "Refresh Failure Rate by Entity", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "expr": "sum by (entity_type) (rate(background_refresh_failed_total{job=\"fastapi\"}[5m]))", + "legendFormat": "{{entity_type}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "type": "row", + "title": "\ud83d\uded1 Rate Limiting", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "collapsed": false, + "panels": [] + }, { "type": "stat", "title": "AIMD Current Rate", @@ -435,7 +696,7 @@ "h": 4, "w": 8, "x": 0, - "y": 22 + "y": 35 }, "datasource": { "type": "prometheus", @@ -485,7 +746,7 @@ "h": 4, "w": 8, "x": 8, - "y": 22 + "y": 35 }, "datasource": { "type": "prometheus", @@ -544,7 +805,7 @@ "h": 4, "w": 8, "x": 16, - "y": 22 + "y": 35 }, "datasource": { "type": "prometheus", @@ -602,7 +863,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 39 }, "datasource": { "type": "prometheus", @@ -643,7 +904,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 39 }, "datasource": { "type": "prometheus", From 0789b6552285518824a89baef267715b84c604ec Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 11:43:07 +0100 Subject: [PATCH 14/26] --wip-- [skip ci] --- app/domain/services/player_service.py | 203 +++++++++++++++++--------- 1 file changed, 132 insertions(+), 71 deletions(-) diff --git a/app/domain/services/player_service.py b/app/domain/services/player_service.py index e2c30ca4..7fad3419 100644 --- a/app/domain/services/player_service.py +++ b/app/domain/services/player_service.py @@ -1,6 +1,7 @@ """Player domain service — career, stats, summary, and search""" import time +from dataclasses import dataclass, field from typing import TYPE_CHECKING, cast from fastapi import HTTPException, status @@ -45,6 +46,33 @@ ) +@dataclass +class PlayerIdentity: + """Result of player identity resolution. + + Groups the four fields that travel together after resolving a + BattleTag or Blizzard ID to a canonical identity. + """ + + blizzard_id: str | None = field(default=None) + player_summary: dict = field(default_factory=dict) + cached_html: str | None = field(default=None) + battletag_input: str | None = field(default=None) + + +@dataclass +class PlayerRequest: + """Parameter object for a player data request. + + Pass a single ``PlayerRequest`` to ``PlayerService._execute_player_request`` + instead of passing each field as a separate keyword argument. + """ + + player_id: str + cache_key: str + data_factory: Callable[[str, dict], dict] + + class PlayerService(BaseService): """Domain service for all player-related endpoints. @@ -99,7 +127,11 @@ async def get_player_summary( def extract(html: str, player_summary: dict) -> dict: return parse_player_profile_html(html, player_summary).get("summary") or {} - return await self._execute_player_request(player_id, cache_key, extract) + return await self._execute_player_request( + PlayerRequest( + player_id=player_id, cache_key=cache_key, data_factory=extract + ) + ) # ------------------------------------------------------------------ # Player career (GET /players/{player_id}) @@ -123,7 +155,11 @@ def extract(html: str, player_summary: dict) -> dict: ), } - return await self._execute_player_request(player_id, cache_key, extract) + return await self._execute_player_request( + PlayerRequest( + player_id=player_id, cache_key=cache_key, data_factory=extract + ) + ) # ------------------------------------------------------------------ # Player stats (GET /players/{player_id}/stats) @@ -145,7 +181,11 @@ def extract(html: str, player_summary: dict) -> dict: profile.get("stats") or {}, platform, gamemode, hero ) - return await self._execute_player_request(player_id, cache_key, extract) + return await self._execute_player_request( + PlayerRequest( + player_id=player_id, cache_key=cache_key, data_factory=extract + ) + ) # ------------------------------------------------------------------ # Player stats summary (GET /players/{player_id}/stats/summary) @@ -165,7 +205,11 @@ def extract(html: str, player_summary: dict) -> dict: html, player_summary, gamemode, platform ) - return await self._execute_player_request(player_id, cache_key, extract) + return await self._execute_player_request( + PlayerRequest( + player_id=player_id, cache_key=cache_key, data_factory=extract + ) + ) # ------------------------------------------------------------------ # Player career stats (GET /players/{player_id}/stats/career) @@ -186,52 +230,40 @@ def extract(html: str, player_summary: dict) -> dict: html, player_summary, platform, gamemode, hero ) - return await self._execute_player_request(player_id, cache_key, extract) + return await self._execute_player_request( + PlayerRequest( + player_id=player_id, cache_key=cache_key, data_factory=extract + ) + ) # ------------------------------------------------------------------ # Core request execution — universal scaffold # ------------------------------------------------------------------ async def _execute_player_request( - self, - player_id: str, - cache_key: str, - data_factory: Callable[[str, dict], dict], + self, request: PlayerRequest ) -> tuple[dict, bool, int]: """Resolve identity → get HTML → compute data → update cache → return. Args: - player_id: BattleTag or Blizzard ID. - cache_key: Valkey API-cache key to write after serving. - data_factory: Pure function ``(html, player_summary) → dict`` that - extracts the endpoint-specific payload from the raw HTML. + request: ``PlayerRequest`` holding ``player_id``, ``cache_key``, + and the endpoint-specific ``data_factory``. """ - cache_key_player = player_id - battletag_input: str | None = None - player_summary: dict = {} + identity = PlayerIdentity() + effective_id = request.player_id data: dict = {} try: - ( - blizzard_id, - player_summary, - cached_html, - battletag_input, - ) = await self._resolve_player_identity(player_id) - cache_key_player = blizzard_id or player_id - - html = await self._get_player_html( - cache_key_player, player_summary, cached_html, battletag_input - ) - data = data_factory(html, player_summary) + identity = await self._resolve_player_identity(request.player_id) + effective_id = identity.blizzard_id or request.player_id + html = await self._get_player_html(effective_id, identity) + data = request.data_factory(html, identity.player_summary) except Exception as exc: # noqa: BLE001 - await self._handle_player_exceptions( - exc, cache_key_player, battletag_input, player_summary - ) + await self._handle_player_exceptions(exc, request.player_id, identity) - is_stale = self._check_player_staleness(cache_key_player) + is_stale = self._check_player_staleness(effective_id) await self._update_api_cache( - cache_key, data, settings.career_path_cache_timeout + request.cache_key, data, settings.career_path_cache_timeout ) return data, is_stale, 0 @@ -289,50 +321,58 @@ def _check_player_staleness(self, _player_id: str) -> bool: async def _get_player_html( self, - blizzard_id: str, - player_summary: dict, - cached_html: str | None, - battletag_input: str | None, + effective_id: str, + identity: PlayerIdentity, ) -> str: """Return player HTML, always storing fresh HTML in SQLite. Priority order: - 1. ``cached_html`` — fetched during identity resolution; store and return. + 1. ``identity.cached_html`` — fetched during identity resolution; store and return. 2. SQLite hit with matching ``lastUpdated`` — return cached HTML, backfilling battletag if it was missing. 3. Fetch from Blizzard, store, return. """ - if cached_html: - name = extract_name_from_profile_html(cached_html) or player_summary.get( - "name" - ) + if identity.cached_html: + name = extract_name_from_profile_html( + identity.cached_html + ) or identity.player_summary.get("name") await self.update_player_profile_cache( - blizzard_id, player_summary, cached_html, battletag_input, name + effective_id, + identity.player_summary, + identity.cached_html, + identity.battletag_input, + name, ) - return cached_html + return identity.cached_html - player_cache = await self.get_player_profile_cache(blizzard_id) + player_cache = await self.get_player_profile_cache(effective_id) if ( player_cache is not None - and player_summary + and identity.player_summary and player_cache["summary"].get("lastUpdated") - == player_summary.get("lastUpdated") + == identity.player_summary.get("lastUpdated") ): html = cast("str", player_cache["profile"]) - if battletag_input and not player_cache.get("battletag"): + if identity.battletag_input and not player_cache.get("battletag"): await self.update_player_profile_cache( - blizzard_id, - player_summary, + effective_id, + identity.player_summary, html, - battletag_input, + identity.battletag_input, player_cache.get("name"), ) return html - html, _ = await fetch_player_html(self.blizzard_client, blizzard_id) - name = extract_name_from_profile_html(html) or player_summary.get("name") + html, _ = await fetch_player_html(self.blizzard_client, effective_id) + name = extract_name_from_profile_html(html) or identity.player_summary.get( + "name" + ) await self.update_player_profile_cache( - blizzard_id, player_summary, html, battletag_input, name + effective_id, + identity.player_summary, + html, + identity.battletag_input, + name, ) return html @@ -340,16 +380,18 @@ async def _get_player_html( # Identity resolution # ------------------------------------------------------------------ - async def _resolve_player_identity( - self, player_id: str - ) -> tuple[str | None, dict, str | None, str | None]: - """Resolve BattleTag or Blizzard ID to a canonical (blizzard_id, summary, html, battletag).""" + async def _resolve_player_identity(self, player_id: str) -> PlayerIdentity: + """Resolve BattleTag or Blizzard ID to a canonical ``PlayerIdentity``.""" logger.info("Retrieving Player Summary...") if is_blizzard_id(player_id): logger.info("Player ID is a Blizzard ID — attempting reverse enrichment") player_summary, html = await self._enrich_from_blizzard_id(player_id) - return player_id, player_summary, html, None + return PlayerIdentity( + blizzard_id=player_id, + player_summary=player_summary, + cached_html=html, + ) battletag_input = player_id search_json = await fetch_player_summary_json(self.blizzard_client, player_id) @@ -357,8 +399,11 @@ async def _resolve_player_identity( if player_summary: logger.info("Player Summary retrieved!") - blizzard_id = player_summary.get("url") - return blizzard_id, player_summary, None, battletag_input + return PlayerIdentity( + blizzard_id=player_summary.get("url"), + player_summary=player_summary, + battletag_input=battletag_input, + ) logger.info( "Player not found in search — checking SQLite for cached Blizzard ID" @@ -374,7 +419,11 @@ async def _resolve_player_identity( search_json, player_id, cached_blizzard_id ) if player_summary: - return cached_blizzard_id, player_summary, None, battletag_input + return PlayerIdentity( + blizzard_id=cached_blizzard_id, + player_summary=player_summary, + battletag_input=battletag_input, + ) elif settings.prometheus_enabled: sqlite_battletag_lookup_total.labels(result="miss").inc() @@ -386,9 +435,18 @@ async def _resolve_player_identity( search_json, player_id, blizzard_id ) if player_summary: - return blizzard_id, player_summary, html, battletag_input + return PlayerIdentity( + blizzard_id=blizzard_id, + player_summary=player_summary, + cached_html=html, + battletag_input=battletag_input, + ) - return blizzard_id, {}, html, battletag_input + return PlayerIdentity( + blizzard_id=blizzard_id, + cached_html=html, + battletag_input=battletag_input, + ) async def _enrich_from_blizzard_id( self, blizzard_id: str @@ -460,16 +518,19 @@ async def _mark_player_unknown( async def _handle_player_exceptions( self, error: Exception, - cache_key: str, - battletag_input: str | None, - player_summary: dict, + player_id: str, + identity: PlayerIdentity, ) -> None: """Translate all player exceptions to HTTPException and always raise.""" + effective_id = identity.blizzard_id or player_id + battletag_input = identity.battletag_input + player_summary = identity.player_summary + if isinstance(error, ParserBlizzardError): exc = HTTPException(status_code=error.status_code, detail=error.message) if error.status_code == status.HTTP_404_NOT_FOUND: await self._mark_player_unknown( - cache_key, exc, battletag=battletag_input + effective_id, exc, battletag=battletag_input ) raise exc from error @@ -480,20 +541,20 @@ async def _handle_player_exceptions( detail="Player not found", ) await self._mark_player_unknown( - cache_key, exc, battletag=battletag_input + effective_id, exc, battletag=battletag_input ) raise exc from error blizzard_url = ( f"{settings.blizzard_host}{settings.career_path}/" - f"{player_summary.get('url', cache_key) if player_summary else cache_key}/" + f"{player_summary.get('url', effective_id) if player_summary else effective_id}/" ) raise overfast_internal_error(blizzard_url, error) from error if isinstance(error, HTTPException): if error.status_code == status.HTTP_404_NOT_FOUND: await self._mark_player_unknown( - cache_key, error, battletag=battletag_input + effective_id, error, battletag=battletag_input ) raise error From fa343758854ce81bfe35393435d8cc893114042f Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 12:08:04 +0100 Subject: [PATCH 15/26] --wip-- [skip ci] --- app/adapters/cache/valkey_cache.py | 27 ++++++++-- app/api/routers/gamemodes.py | 2 +- app/api/routers/heroes.py | 4 +- app/api/routers/maps.py | 2 +- app/api/routers/roles.py | 2 +- app/domain/ports/cache.py | 29 +++++++++-- app/domain/services/base_service.py | 16 +++++- app/domain/services/static_data_service.py | 20 ++++++-- app/helpers.py | 57 ++++++++++++++++++--- build/nginx/lua/valkey_handler.lua.template | 33 ++++++++++-- 10 files changed, 162 insertions(+), 30 deletions(-) diff --git a/app/adapters/cache/valkey_cache.py b/app/adapters/cache/valkey_cache.py index f59ab393..82f5550f 100644 --- a/app/adapters/cache/valkey_cache.py +++ b/app/adapters/cache/valkey_cache.py @@ -20,6 +20,7 @@ """ import json +import time from compression import zstd from functools import wraps from typing import TYPE_CHECKING, Any @@ -128,19 +129,35 @@ async def exists(self, key: str) -> bool: # Application-specific cache methods @handle_valkey_error(default_return=None) async def get_api_cache(self, cache_key: str) -> dict | list | None: - """Get the API Cache value associated with a given cache key""" + """Get the API Cache value associated with a given cache key.""" api_cache_key = f"{settings.api_cache_key_prefix}:{cache_key}" api_cache = await self.valkey_server.get(api_cache_key) if not api_cache or not isinstance(api_cache, bytes): return None - return self._decompress_json_value(api_cache) + envelope = self._decompress_json_value(api_cache) + if isinstance(envelope, dict) and "data" in envelope: + return envelope["data"] + return envelope @handle_valkey_error(default_return=None) async def update_api_cache( - self, cache_key: str, value: dict | list, expire: int + self, + cache_key: str, + value: dict | list, + expire: int, + *, + stored_at: int | None = None, + staleness_threshold: int | None = None, + stale_while_revalidate: int = 0, ) -> None: - """Update or set an API Cache value with an expiration value (in seconds)""" - bytes_value = self._compress_json_value(value) + """Wrap value in a metadata envelope, compress, and store with TTL.""" + envelope: dict = { + "data": value, + "stored_at": stored_at if stored_at is not None else int(time.time()), + "staleness_threshold": staleness_threshold if staleness_threshold is not None else expire, + "stale_while_revalidate": stale_while_revalidate, + } + bytes_value = self._compress_json_value(envelope) await self.valkey_server.set( f"{settings.api_cache_key_prefix}:{cache_key}", bytes_value, diff --git a/app/api/routers/gamemodes.py b/app/api/routers/gamemodes.py index 7d9bde9b..14312e6d 100644 --- a/app/api/routers/gamemodes.py +++ b/app/api/routers/gamemodes.py @@ -33,5 +33,5 @@ async def list_map_gamemodes( data, is_stale, age = await service.list_gamemodes( cache_key=build_cache_key(request) ) - apply_swr_headers(response, settings.csv_cache_timeout, is_stale, age) + apply_swr_headers(response, settings.csv_cache_timeout, is_stale, age, staleness_threshold=settings.gamemodes_staleness_threshold) return data diff --git a/app/api/routers/heroes.py b/app/api/routers/heroes.py index 679fbe07..7497c727 100644 --- a/app/api/routers/heroes.py +++ b/app/api/routers/heroes.py @@ -53,7 +53,7 @@ async def list_heroes( data, is_stale, age = await service.list_heroes( locale=locale, role=role, gamemode=gamemode, cache_key=build_cache_key(request) ) - apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale, age) + apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale, age, staleness_threshold=settings.heroes_staleness_threshold) return data @@ -130,7 +130,7 @@ async def get_hero_stats( order_by=order_by, cache_key=build_cache_key(request), ) - apply_swr_headers(response, settings.hero_stats_cache_timeout, is_stale, age) + apply_swr_headers(response, settings.hero_stats_cache_timeout, is_stale, age, staleness_threshold=settings.hero_stats_staleness_threshold) return data diff --git a/app/api/routers/maps.py b/app/api/routers/maps.py index afc4798c..2ebf21fb 100644 --- a/app/api/routers/maps.py +++ b/app/api/routers/maps.py @@ -41,5 +41,5 @@ async def list_maps( data, is_stale, age = await service.list_maps( gamemode=gamemode, cache_key=build_cache_key(request) ) - apply_swr_headers(response, settings.csv_cache_timeout, is_stale, age) + apply_swr_headers(response, settings.csv_cache_timeout, is_stale, age, staleness_threshold=settings.maps_staleness_threshold) return data diff --git a/app/api/routers/roles.py b/app/api/routers/roles.py index 2a1881a6..45f13c4e 100644 --- a/app/api/routers/roles.py +++ b/app/api/routers/roles.py @@ -36,5 +36,5 @@ async def list_roles( data, is_stale, age = await service.list_roles( locale=locale, cache_key=build_cache_key(request) ) - apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale, age) + apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale, age, staleness_threshold=settings.roles_staleness_threshold) return data diff --git a/app/domain/ports/cache.py b/app/domain/ports/cache.py index 4ab43065..930dcae2 100644 --- a/app/domain/ports/cache.py +++ b/app/domain/ports/cache.py @@ -47,12 +47,31 @@ async def get_api_cache(self, cache_key: str) -> dict | list | None: ... async def update_api_cache( - self, cache_key: str, value: dict | list, expire: int + self, + cache_key: str, + value: dict | list, + expire: int, + *, + stored_at: int | None = None, + staleness_threshold: int | None = None, + stale_while_revalidate: int = 0, ) -> None: - """ - Update or set an API Cache value with an expiration value (in seconds). - - Value is JSON-serialized and compressed before storage. + """Update or set an API Cache value with an expiration value (in seconds). + + Value is wrapped in a metadata envelope before JSON-serialization and + compression. The envelope allows nginx/Lua to set standard ``Age`` + and ``Cache-Control: stale-while-revalidate`` headers without calling + FastAPI. + + Args: + cache_key: Valkey key suffix (after ``api-cache:``). + value: Data payload to cache. + expire: Valkey key TTL in seconds. + stored_at: Unix timestamp when the data was generated. Defaults to now. + staleness_threshold: Seconds after which the payload is considered stale. + Used for ``Cache-Control: max-age``. Defaults to ``expire``. + stale_while_revalidate: Seconds nginx may serve stale while revalidating. + 0 means no SWR window (omits the directive). """ ... diff --git a/app/domain/services/base_service.py b/app/domain/services/base_service.py index 620415d9..2e0f7acb 100644 --- a/app/domain/services/base_service.py +++ b/app/domain/services/base_service.py @@ -62,11 +62,23 @@ def __init__( self.task_queue = task_queue async def _update_api_cache( - self, cache_key: str, data: Any, cache_ttl: int + self, + cache_key: str, + data: Any, + cache_ttl: int, + *, + staleness_threshold: int | None = None, + stale_while_revalidate: int = 0, ) -> None: """Write data to Valkey API cache, swallowing errors.""" try: - await self.cache.update_api_cache(cache_key, data, cache_ttl) + await self.cache.update_api_cache( + cache_key, + data, + cache_ttl, + staleness_threshold=staleness_threshold, + stale_while_revalidate=stale_while_revalidate, + ) except Exception as exc: # noqa: BLE001 logger.warning(f"[SWR] Valkey write failed for {cache_key}: {exc}") diff --git a/app/domain/services/static_data_service.py b/app/domain/services/static_data_service.py index 340668bf..a5456678 100644 --- a/app/domain/services/static_data_service.py +++ b/app/domain/services/static_data_service.py @@ -99,13 +99,22 @@ async def _serve_from_storage( ).inc() # Short TTL absorbs burst traffic while the background refresh is in-flight. await self._update_api_cache( - config.cache_key, filtered, settings.stale_cache_timeout + config.cache_key, + filtered, + settings.stale_cache_timeout, + staleness_threshold=config.staleness_threshold, + stale_while_revalidate=settings.stale_cache_timeout, ) else: logger.info( f"[SWR] {config.entity_type} fresh (age={age}s) — serving from SQLite" ) - await self._update_api_cache(config.cache_key, filtered, config.cache_ttl) + await self._update_api_cache( + config.cache_key, + filtered, + config.cache_ttl, + staleness_threshold=config.staleness_threshold, + ) return filtered, is_stale, age @@ -134,7 +143,12 @@ async def _fetch_and_store(self, config: StaticFetchConfig) -> Any: await self._store_in_storage(config.storage_key, data) filtered = self._apply_filter(data, config.result_filter) - await self._update_api_cache(config.cache_key, filtered, config.cache_ttl) + await self._update_api_cache( + config.cache_key, + filtered, + config.cache_ttl, + staleness_threshold=config.staleness_threshold, + ) return filtered diff --git a/app/helpers.py b/app/helpers.py index adf54d73..2624b7f9 100644 --- a/app/helpers.py +++ b/app/helpers.py @@ -32,6 +32,42 @@ "example": "600", }, }, + "Age": { + "description": ( + "Number of seconds since the response payload was generated " + "(RFC 7234 §5.1). Present on FastAPI-served responses; " + "also set by nginx for Valkey cache hits." + ), + "schema": { + "type": "string", + "example": "42", + }, + }, + "Cache-Control": { + "description": ( + "Standard caching directives (RFC 7234 + RFC 5861). " + "``max-age`` reflects the staleness threshold in seconds. " + "``stale-while-revalidate`` is present when a background " + "refresh is in-flight, indicating how long stale data may " + "still be served." + ), + "schema": { + "type": "string", + "example": "public, max-age=86400, stale-while-revalidate=60", + }, + }, + "X-Cache-Status": { + "description": ( + "Indicates whether the response was served from a fresh " + "cache entry (``hit``) or a stale one while a background " + "refresh is in-flight (``stale``)." + ), + "schema": { + "type": "string", + "enum": ["hit", "stale"], + "example": "hit", + }, + }, }, }, } @@ -231,22 +267,29 @@ def apply_swr_headers( cache_ttl: int, is_stale: bool, age_seconds: int = 0, + *, + staleness_threshold: int | None = None, ) -> None: """Add standard SWR and cache metadata headers to the response. - Always sets ``X-Cache-TTL`` and ``Age`` (when known). - When ``is_stale`` is True, additionally sets RFC-5861 ``Cache-Control`` - and ``X-Cache-Status``. + Sets ``Cache-Control`` (RFC 5861), ``Age`` (RFC 7234), ``X-Cache-Status``, + and the non-standard ``X-Cache-TTL`` on every FastAPI-served response. - Note: nginx/Lua only sets ``X-Cache-TTL`` (remaining Valkey TTL) for - Valkey-served responses; all other headers are FastAPI-only and reflect - the actual SQLite data age. + ``staleness_threshold`` is used for ``Cache-Control: max-age``; it defaults + to ``cache_ttl`` for endpoints that have no SWR (e.g. player, search). + ``stale-while-revalidate`` is included only on stale responses, using the + configured ``stale_cache_timeout`` as the revalidation window. """ + max_age = staleness_threshold if staleness_threshold is not None else cache_ttl response.headers[settings.cache_ttl_header] = str(cache_ttl) if age_seconds > 0: response.headers["Age"] = str(age_seconds) if is_stale: response.headers["Cache-Control"] = ( - f"max-age={cache_ttl}, stale-while-revalidate={cache_ttl * 2}" + f"public, max-age={max_age}," + f" stale-while-revalidate={settings.stale_cache_timeout}" ) response.headers["X-Cache-Status"] = "stale" + else: + response.headers["Cache-Control"] = f"public, max-age={max_age}" + response.headers["X-Cache-Status"] = "hit" diff --git a/build/nginx/lua/valkey_handler.lua.template b/build/nginx/lua/valkey_handler.lua.template index adefe382..6e6bebe3 100644 --- a/build/nginx/lua/valkey_handler.lua.template +++ b/build/nginx/lua/valkey_handler.lua.template @@ -1,4 +1,5 @@ local zstd = require "zstd" +local cjson = require "cjson" local valkey = require "resty.redis" local EXCLUDED_PATHS = { ["/"] = true, ["/docs"] = true, ["/openapi.json"] = true } @@ -96,15 +97,41 @@ local function handle_valkey_request() return ngx.exec("@fallback") end - local ok_decomp, value = pcall(zstd.decompress, compressed_value) - if not ok_decomp or not value or value == ngx.null then + local ok_decomp, decompressed = pcall(zstd.decompress, compressed_value) + if not ok_decomp or not decompressed or decompressed == ngx.null then ngx.log(ngx.ERR, "Cache decompression error for key: ", cache_key) release(valk) return ngx.exec("@fallback") end + -- Parse metadata envelope: {"data":..., "stored_at":N, "staleness_threshold":N, "stale_while_revalidate":N} + local ok_parse, envelope = pcall(cjson.decode, decompressed) + if not ok_parse or type(envelope) ~= "table" or envelope.data == nil then + ngx.log(ngx.ERR, "Cache envelope parse error for key: ", cache_key) + release(valk) + return ngx.exec("@fallback") + end + + -- Age (RFC 7234 §5.1): seconds since the payload was generated + local stored_at = tonumber(envelope.stored_at) or 0 + local age = math.max(0, ngx.time() - stored_at) + ngx.header["Age"] = age + + -- Cache-Control with SWR directives (RFC 5861) + local max_age = tonumber(envelope.staleness_threshold) or cache_ttl + local swr = tonumber(envelope.stale_while_revalidate) or 0 + if swr > 0 then + ngx.header["Cache-Control"] = "public, max-age=" .. max_age .. ", stale-while-revalidate=" .. swr + ngx.header["X-Cache-Status"] = "stale" + else + ngx.header["Cache-Control"] = "public, max-age=" .. max_age + ngx.header["X-Cache-Status"] = "hit" + end + + -- X-Cache-TTL: remaining Valkey TTL (non-standard, kept for backward compat) ngx.header["${CACHE_TTL_HEADER}"] = cache_ttl - ngx.print(value) + + ngx.print(cjson.encode(envelope.data)) release(valk) end From abeee2f314217a61dfa1e4291f7ec8a29dbf6a7c Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 13:26:58 +0100 Subject: [PATCH 16/26] fix: typing update --- app/domain/services/player_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/domain/services/player_service.py b/app/domain/services/player_service.py index 7fad3419..4e67a8d7 100644 --- a/app/domain/services/player_service.py +++ b/app/domain/services/player_service.py @@ -2,7 +2,7 @@ import time from dataclasses import dataclass, field -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Never, cast from fastapi import HTTPException, status @@ -520,7 +520,7 @@ async def _handle_player_exceptions( error: Exception, player_id: str, identity: PlayerIdentity, - ) -> None: + ) -> Never: """Translate all player exceptions to HTTPException and always raise.""" effective_id = identity.blizzard_id or player_id battletag_input = identity.battletag_input From 68938008a1fce9033b4c13d98cd49c8dae1e5345 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 13:49:24 +0100 Subject: [PATCH 17/26] fix: adjusted hero endpoints cache behaviour --- .env.dist | 1 - app/adapters/cache/valkey_cache.py | 4 +- app/api/routers/gamemodes.py | 8 +- app/api/routers/heroes.py | 24 ++++- app/api/routers/maps.py | 8 +- app/api/routers/roles.py | 8 +- app/domain/services/hero_service.py | 136 +++++++++++----------------- 7 files changed, 100 insertions(+), 89 deletions(-) diff --git a/.env.dist b/.env.dist index ba3ea741..41902cc7 100644 --- a/.env.dist +++ b/.env.dist @@ -52,7 +52,6 @@ CAREER_PATH_CACHE_TIMEOUT=600 SEARCH_ACCOUNT_PATH_CACHE_TIMEOUT=600 HERO_STATS_CACHE_TIMEOUT=3600 PLAYER_PROFILE_MAX_AGE=259200 -# TTL (seconds) for stale responses written to Valkey while background refresh is in-flight STALE_CACHE_TIMEOUT=60 # Critical error Discord webhook diff --git a/app/adapters/cache/valkey_cache.py b/app/adapters/cache/valkey_cache.py index 82f5550f..d5b6f717 100644 --- a/app/adapters/cache/valkey_cache.py +++ b/app/adapters/cache/valkey_cache.py @@ -154,7 +154,9 @@ async def update_api_cache( envelope: dict = { "data": value, "stored_at": stored_at if stored_at is not None else int(time.time()), - "staleness_threshold": staleness_threshold if staleness_threshold is not None else expire, + "staleness_threshold": ( + staleness_threshold if staleness_threshold is not None else expire + ), "stale_while_revalidate": stale_while_revalidate, } bytes_value = self._compress_json_value(envelope) diff --git a/app/api/routers/gamemodes.py b/app/api/routers/gamemodes.py index 14312e6d..4f63030f 100644 --- a/app/api/routers/gamemodes.py +++ b/app/api/routers/gamemodes.py @@ -33,5 +33,11 @@ async def list_map_gamemodes( data, is_stale, age = await service.list_gamemodes( cache_key=build_cache_key(request) ) - apply_swr_headers(response, settings.csv_cache_timeout, is_stale, age, staleness_threshold=settings.gamemodes_staleness_threshold) + apply_swr_headers( + response, + settings.csv_cache_timeout, + is_stale, + age, + staleness_threshold=settings.gamemodes_staleness_threshold, + ) return data diff --git a/app/api/routers/heroes.py b/app/api/routers/heroes.py index 7497c727..90ccb454 100644 --- a/app/api/routers/heroes.py +++ b/app/api/routers/heroes.py @@ -53,7 +53,13 @@ async def list_heroes( data, is_stale, age = await service.list_heroes( locale=locale, role=role, gamemode=gamemode, cache_key=build_cache_key(request) ) - apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale, age, staleness_threshold=settings.heroes_staleness_threshold) + apply_swr_headers( + response, + settings.heroes_path_cache_timeout, + is_stale, + age, + staleness_threshold=settings.heroes_staleness_threshold, + ) return data @@ -130,7 +136,13 @@ async def get_hero_stats( order_by=order_by, cache_key=build_cache_key(request), ) - apply_swr_headers(response, settings.hero_stats_cache_timeout, is_stale, age, staleness_threshold=settings.hero_stats_staleness_threshold) + apply_swr_headers( + response, + settings.hero_stats_cache_timeout, + is_stale, + age, + staleness_threshold=settings.hero_stats_staleness_threshold, + ) return data @@ -164,5 +176,11 @@ async def get_hero( data, is_stale, age = await service.get_hero( hero_key=str(hero_key), locale=locale, cache_key=build_cache_key(request) ) - apply_swr_headers(response, settings.hero_path_cache_timeout, is_stale, age) + apply_swr_headers( + response, + settings.hero_path_cache_timeout, + is_stale, + age, + staleness_threshold=settings.heroes_staleness_threshold, + ) return data diff --git a/app/api/routers/maps.py b/app/api/routers/maps.py index 2ebf21fb..55766c7c 100644 --- a/app/api/routers/maps.py +++ b/app/api/routers/maps.py @@ -41,5 +41,11 @@ async def list_maps( data, is_stale, age = await service.list_maps( gamemode=gamemode, cache_key=build_cache_key(request) ) - apply_swr_headers(response, settings.csv_cache_timeout, is_stale, age, staleness_threshold=settings.maps_staleness_threshold) + apply_swr_headers( + response, + settings.csv_cache_timeout, + is_stale, + age, + staleness_threshold=settings.maps_staleness_threshold, + ) return data diff --git a/app/api/routers/roles.py b/app/api/routers/roles.py index 45f13c4e..b54f686e 100644 --- a/app/api/routers/roles.py +++ b/app/api/routers/roles.py @@ -36,5 +36,11 @@ async def list_roles( data, is_stale, age = await service.list_roles( locale=locale, cache_key=build_cache_key(request) ) - apply_swr_headers(response, settings.heroes_path_cache_timeout, is_stale, age, staleness_threshold=settings.roles_staleness_threshold) + apply_swr_headers( + response, + settings.heroes_path_cache_timeout, + is_stale, + age, + staleness_threshold=settings.roles_staleness_threshold, + ) return data diff --git a/app/domain/services/hero_service.py b/app/domain/services/hero_service.py index 65d2db04..f10620f6 100644 --- a/app/domain/services/hero_service.py +++ b/app/domain/services/hero_service.py @@ -17,7 +17,6 @@ from app.enums import Locale from app.exceptions import ParserBlizzardError, ParserParsingError from app.helpers import overfast_internal_error -from app.overfast_logger import logger if TYPE_CHECKING: from app.heroes.enums import HeroGamemode @@ -82,27 +81,37 @@ async def get_hero( ) -> tuple[dict, bool, int]: """Return full hero details merged with portrait and hitpoints. - Single-hero data is not stored persistently; the Valkey API cache is - still updated on every fetch. + Stores the merged hero data per ``hero_key:locale`` in SQLite so that + subsequent requests benefit from the SWR cache and background refresh. """ - try: - hero_data = await parse_hero(self.blizzard_client, hero_key, locale) - heroes_html = await fetch_heroes_html(self.blizzard_client, locale) - heroes_list = parse_heroes_html(heroes_html) - heroes_hitpoints = parse_heroes_hitpoints() - data = _merge_hero_data(hero_data, heroes_list, heroes_hitpoints, hero_key) - except ParserBlizzardError as exc: - raise HTTPException( - status_code=exc.status_code, detail=exc.message - ) from exc - except ParserParsingError as exc: - blizzard_url = ( - f"{settings.blizzard_host}/{locale}{settings.heroes_path}{hero_key}/" - ) - raise overfast_internal_error(blizzard_url, exc) from exc - await self._update_api_cache(cache_key, data, settings.hero_path_cache_timeout) - return data, False, 0 + async def _fetch() -> dict: + try: + hero_data = await parse_hero(self.blizzard_client, hero_key, locale) + heroes_html = await fetch_heroes_html(self.blizzard_client, locale) + heroes_list = parse_heroes_html(heroes_html) + heroes_hitpoints = parse_heroes_hitpoints() + return _merge_hero_data( + hero_data, heroes_list, heroes_hitpoints, hero_key + ) + except ParserBlizzardError as exc: + raise HTTPException( + status_code=exc.status_code, detail=exc.message + ) from exc + except ParserParsingError as exc: + blizzard_url = f"{settings.blizzard_host}/{locale}{settings.heroes_path}{hero_key}/" + raise overfast_internal_error(blizzard_url, exc) from exc + + return await self.get_or_fetch( + StaticFetchConfig( + storage_key=f"hero:{hero_key}:{locale}", + fetcher=_fetch, + cache_key=cache_key, + cache_ttl=settings.hero_path_cache_timeout, + staleness_threshold=settings.heroes_staleness_threshold, + entity_type="hero", + ) + ) # ------------------------------------------------------------------ # Hero stats summary (GET /heroes/stats) @@ -119,42 +128,35 @@ async def get_hero_stats( order_by: str, cache_key: str, ) -> tuple[list[dict], bool, int]: - """Return hero usage statistics with SWR.""" - storage_key = _build_hero_stats_storage_key( - platform, gamemode, region, map_filter, competitive_division - ) - - async def _fetch() -> list[dict]: - try: - return await parse_hero_stats_summary( - self.blizzard_client, - platform=platform, - gamemode=gamemode, - region=region, - role=role, - map_filter=map_filter, - competitive_division=competitive_division, - order_by=order_by, - ) - except ParserBlizzardError as exc: - raise HTTPException( - status_code=exc.status_code, detail=exc.message - ) from exc - - def _filter(data: list[dict]) -> list[dict]: - return _filter_hero_stats(data, role, order_by) + """Return hero usage statistics — Valkey-only cache, no persistent storage. - return await self.get_or_fetch( - StaticFetchConfig( - storage_key=storage_key, - fetcher=_fetch, - result_filter=_filter, - cache_key=cache_key, - cache_ttl=settings.hero_stats_cache_timeout, - staleness_threshold=settings.hero_stats_staleness_threshold, - entity_type="hero_stats", + Stats change frequently and have too many parameter combinations to + store in SQLite. The Valkey API cache (populated here, served by nginx) + is sufficient. + """ + try: + data = await parse_hero_stats_summary( + self.blizzard_client, + platform=platform, + gamemode=gamemode, + region=region, + role=role, + map_filter=map_filter, + competitive_division=competitive_division, + order_by=order_by, ) + except ParserBlizzardError as exc: + raise HTTPException( + status_code=exc.status_code, detail=exc.message + ) from exc + + await self._update_api_cache( + cache_key, + data, + settings.hero_stats_cache_timeout, + staleness_threshold=settings.hero_stats_staleness_threshold, ) + return data, False, 0 # --------------------------------------------------------------------------- @@ -205,31 +207,3 @@ def dict_insert_value_before_key( items = list(data.items()) items.insert(pos, (new_key, new_value)) return dict(items) - - -def _build_hero_stats_storage_key( - platform: Any, - gamemode: Any, - region: Any, - map_filter: Any, - competitive_division: Any, -) -> str: - map_val = map_filter.value if map_filter else "all-maps" - tier_val = competitive_division.value if competitive_division else "null" - return ( - f"hero_stats:{platform.value}:{gamemode.value}:{region.value}" - f":{map_val}:{tier_val}" - ) - - -def _filter_hero_stats( - data: list[dict], - role: Any, - order_by: str, -) -> list[dict]: - """Re-apply role filter and ordering (used both on stale data and cold-start).""" - logger.debug("[SWR] Applying hero_stats filters") - if role: - data = [h for h in data if h.get("role") == role.value] - field, direction = order_by.split(":") - return sorted(data, key=lambda h: h.get(field, ""), reverse=(direction == "desc")) From a2223ef5ef080d77703e5d89026b3c3bc071e421 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 14:18:33 +0100 Subject: [PATCH 18/26] --wip-- [skip ci] --- .env.dist | 7 +++++++ app/api/routers/heroes.py | 1 - app/config.py | 1 - app/domain/services/hero_service.py | 1 - 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.env.dist b/.env.dist index 41902cc7..ed527761 100644 --- a/.env.dist +++ b/.env.dist @@ -52,6 +52,13 @@ CAREER_PATH_CACHE_TIMEOUT=600 SEARCH_ACCOUNT_PATH_CACHE_TIMEOUT=600 HERO_STATS_CACHE_TIMEOUT=3600 PLAYER_PROFILE_MAX_AGE=259200 + +# SWR staleness thresholds +HEROES_STALENESS_THRESHOLD=86400 # 24 hours +MAPS_STALENESS_THRESHOLD=86400 +GAMEMODES_STALENESS_THRESHOLD=86400 +ROLES_STALENESS_THRESHOLD=86400 +PLAYER_STALENESS_THRESHOLD=3600 # 1 hour STALE_CACHE_TIMEOUT=60 # Critical error Discord webhook diff --git a/app/api/routers/heroes.py b/app/api/routers/heroes.py index 90ccb454..73aac41c 100644 --- a/app/api/routers/heroes.py +++ b/app/api/routers/heroes.py @@ -141,7 +141,6 @@ async def get_hero_stats( settings.hero_stats_cache_timeout, is_stale, age, - staleness_threshold=settings.hero_stats_staleness_threshold, ) return data diff --git a/app/config.py b/app/config.py index 0cd48817..c01d05fc 100644 --- a/app/config.py +++ b/app/config.py @@ -161,7 +161,6 @@ class Settings(BaseSettings): maps_staleness_threshold: int = 86400 gamemodes_staleness_threshold: int = 86400 roles_staleness_threshold: int = 86400 - hero_stats_staleness_threshold: int = 3600 # 1 hour (same as cache TTL) # Age (seconds) after which a player profile is considered stale. player_staleness_threshold: int = 1800 # 30 min diff --git a/app/domain/services/hero_service.py b/app/domain/services/hero_service.py index f10620f6..4be734cc 100644 --- a/app/domain/services/hero_service.py +++ b/app/domain/services/hero_service.py @@ -154,7 +154,6 @@ async def get_hero_stats( cache_key, data, settings.hero_stats_cache_timeout, - staleness_threshold=settings.hero_stats_staleness_threshold, ) return data, False, 0 From f2d807ac670a6f8a7269af1611d54c76d771cbb2 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 14:39:49 +0100 Subject: [PATCH 19/26] --wip-- [skip ci] --- app/api/routers/players.py | 4 +--- app/helpers.py | 9 +++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/api/routers/players.py b/app/api/routers/players.py index cccb6afe..b770036e 100644 --- a/app/api/routers/players.py +++ b/app/api/routers/players.py @@ -133,9 +133,7 @@ async def search_players( limit=limit, cache_key=cache_key, ) - response.headers[settings.cache_ttl_header] = str( - settings.search_account_path_cache_timeout - ) + apply_swr_headers(response, settings.search_account_path_cache_timeout, False, 0) return data diff --git a/app/helpers.py b/app/helpers.py index 2624b7f9..ba7eb99c 100644 --- a/app/helpers.py +++ b/app/helpers.py @@ -257,8 +257,13 @@ def get_human_readable_duration(duration: int) -> str: def build_cache_key(request: Request) -> str: - """Build a canonical cache key from the request URL path + query string.""" - qs = str(request.query_params) + """Build a canonical cache key from the request URL path + query string. + + Uses the raw query string (``request.url.query``) to preserve the original + percent-encoding, so the key matches what nginx stores in + ``api-cache:`` exactly. + """ + qs = request.url.query return f"{request.url.path}?{qs}" if qs else request.url.path From c1788778a4dc821a31a648226526913de2c3bdfa Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 14:56:21 +0100 Subject: [PATCH 20/26] fix: fixed issue with api-cache and ordering --- app/adapters/cache/valkey_cache.py | 13 +++++++++---- app/domain/ports/cache.py | 12 ++++++++++-- build/nginx/lua/valkey_handler.lua.template | 6 +++--- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/app/adapters/cache/valkey_cache.py b/app/adapters/cache/valkey_cache.py index d5b6f717..61e2bb2b 100644 --- a/app/adapters/cache/valkey_cache.py +++ b/app/adapters/cache/valkey_cache.py @@ -135,8 +135,8 @@ async def get_api_cache(self, cache_key: str) -> dict | list | None: if not api_cache or not isinstance(api_cache, bytes): return None envelope = self._decompress_json_value(api_cache) - if isinstance(envelope, dict) and "data" in envelope: - return envelope["data"] + if isinstance(envelope, dict) and "data_json" in envelope: + return json.loads(envelope["data_json"]) return envelope @handle_valkey_error(default_return=None) @@ -150,9 +150,14 @@ async def update_api_cache( staleness_threshold: int | None = None, stale_while_revalidate: int = 0, ) -> None: - """Wrap value in a metadata envelope, compress, and store with TTL.""" + """Wrap value in a metadata envelope, compress, and store with TTL. + + ``data_json`` is the pre-serialized JSON string (key order preserved by + Python's ``json.dumps``), so nginx/Lua can print it verbatim without + re-encoding through cjson (which does not guarantee key ordering). + """ envelope: dict = { - "data": value, + "data_json": json.dumps(value, separators=(",", ":")), "stored_at": stored_at if stored_at is not None else int(time.time()), "staleness_threshold": ( staleness_threshold if staleness_threshold is not None else expire diff --git a/app/domain/ports/cache.py b/app/domain/ports/cache.py index 930dcae2..5e5ba150 100644 --- a/app/domain/ports/cache.py +++ b/app/domain/ports/cache.py @@ -58,8 +58,16 @@ async def update_api_cache( ) -> None: """Update or set an API Cache value with an expiration value (in seconds). - Value is wrapped in a metadata envelope before JSON-serialization and - compression. The envelope allows nginx/Lua to set standard ``Age`` + Value is wrapped in a metadata envelope before compression:: + + {"data_json": "", + "stored_at": , + "staleness_threshold": , + "stale_while_revalidate": } + + ``data_json`` is a pre-serialized JSON string so nginx/Lua can print it + verbatim without re-encoding through cjson, preserving key ordering. + The envelope allows nginx/Lua to set standard ``Age`` and ``Cache-Control: stale-while-revalidate`` headers without calling FastAPI. diff --git a/build/nginx/lua/valkey_handler.lua.template b/build/nginx/lua/valkey_handler.lua.template index 6e6bebe3..a8414b72 100644 --- a/build/nginx/lua/valkey_handler.lua.template +++ b/build/nginx/lua/valkey_handler.lua.template @@ -104,9 +104,9 @@ local function handle_valkey_request() return ngx.exec("@fallback") end - -- Parse metadata envelope: {"data":..., "stored_at":N, "staleness_threshold":N, "stale_while_revalidate":N} + -- Parse metadata envelope: {"data_json":"...", "stored_at":N, "staleness_threshold":N, "stale_while_revalidate":N} local ok_parse, envelope = pcall(cjson.decode, decompressed) - if not ok_parse or type(envelope) ~= "table" or envelope.data == nil then + if not ok_parse or type(envelope) ~= "table" or envelope.data_json == nil then ngx.log(ngx.ERR, "Cache envelope parse error for key: ", cache_key) release(valk) return ngx.exec("@fallback") @@ -131,7 +131,7 @@ local function handle_valkey_request() -- X-Cache-TTL: remaining Valkey TTL (non-standard, kept for backward compat) ngx.header["${CACHE_TTL_HEADER}"] = cache_ttl - ngx.print(cjson.encode(envelope.data)) + ngx.print(envelope.data_json) release(valk) end From fe4aa1f492bd3402622925c1336fa747be33d318 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 15:05:09 +0100 Subject: [PATCH 21/26] fix: fixed human readable timeout --- app/api/routers/gamemodes.py | 9 +++++++-- app/api/routers/heroes.py | 13 +++++++++---- app/api/routers/maps.py | 9 +++++++-- app/api/routers/players.py | 14 +++++++------- app/api/routers/roles.py | 9 +++++++-- uv.lock | 2 +- 6 files changed, 38 insertions(+), 18 deletions(-) diff --git a/app/api/routers/gamemodes.py b/app/api/routers/gamemodes.py index 4f63030f..53aae701 100644 --- a/app/api/routers/gamemodes.py +++ b/app/api/routers/gamemodes.py @@ -8,7 +8,12 @@ from app.config import settings from app.enums import RouteTag from app.gamemodes.models import GamemodeDetails -from app.helpers import apply_swr_headers, build_cache_key, success_responses +from app.helpers import ( + apply_swr_headers, + build_cache_key, + get_human_readable_duration, + success_responses, +) router = APIRouter() @@ -20,7 +25,7 @@ summary="Get a list of gamemodes", description=( "Get a list of Overwatch gamemodes : Assault, Escort, Flashpoint, Hybrid, etc." - f"
**Cache TTL : {settings.csv_cache_timeout} seconds.**" + f"
**Cache TTL : {get_human_readable_duration(settings.csv_cache_timeout)}.**" ), operation_id="list_map_gamemodes", response_model=list[GamemodeDetails], diff --git a/app/api/routers/heroes.py b/app/api/routers/heroes.py index 73aac41c..d18e00d8 100644 --- a/app/api/routers/heroes.py +++ b/app/api/routers/heroes.py @@ -7,7 +7,12 @@ from app.api.dependencies import HeroServiceDep from app.config import settings from app.enums import Locale, RouteTag -from app.helpers import apply_swr_headers, build_cache_key, routes_responses +from app.helpers import ( + apply_swr_headers, + build_cache_key, + get_human_readable_duration, + routes_responses, +) from app.heroes.enums import HeroGamemode, HeroKey from app.heroes.models import ( BadRequestErrorMessage, @@ -35,7 +40,7 @@ summary="Get a list of heroes", description=( "Get a list of Overwatch heroes, which can be filtered using roles or gamemodes. " - f"
**Cache TTL : {settings.heroes_path_cache_timeout} seconds.**" + f"
**Cache TTL : {get_human_readable_duration(settings.heroes_path_cache_timeout)}.**" ), operation_id="list_heroes", response_model=list[HeroShort], @@ -77,7 +82,7 @@ async def list_heroes( description=( "Get hero statistics usage, filtered by platform, region, role, etc." "Only Role Queue gamemodes are concerned." - f"
**Cache TTL : {settings.hero_stats_cache_timeout} seconds.**" + f"
**Cache TTL : {get_human_readable_duration(settings.hero_stats_cache_timeout)}.**" ), operation_id="get_hero_stats", response_model=list[HeroStatsSummary], @@ -158,7 +163,7 @@ async def get_hero_stats( summary="Get hero data", description=( "Get data about an Overwatch hero : description, abilities, stadium powers, story, etc. " - f"
**Cache TTL : {settings.hero_path_cache_timeout} seconds.**" + f"
**Cache TTL : {get_human_readable_duration(settings.hero_path_cache_timeout)}.**" ), operation_id="get_hero", response_model=Hero, diff --git a/app/api/routers/maps.py b/app/api/routers/maps.py index 55766c7c..1e6e2083 100644 --- a/app/api/routers/maps.py +++ b/app/api/routers/maps.py @@ -8,7 +8,12 @@ from app.config import settings from app.enums import RouteTag from app.gamemodes.enums import MapGamemode -from app.helpers import apply_swr_headers, build_cache_key, success_responses +from app.helpers import ( + apply_swr_headers, + build_cache_key, + get_human_readable_duration, + success_responses, +) from app.maps.models import Map router = APIRouter() @@ -21,7 +26,7 @@ summary="Get a list of maps", description=( "Get a list of Overwatch maps : Hanamura, King's Row, Dorado, etc." - f"
**Cache TTL : {settings.csv_cache_timeout} seconds.**" + f"
**Cache TTL : {get_human_readable_duration(settings.csv_cache_timeout)}.**" ), operation_id="list_maps", response_model=list[Map], diff --git a/app/api/routers/players.py b/app/api/routers/players.py index b770036e..e14a7f22 100644 --- a/app/api/routers/players.py +++ b/app/api/routers/players.py @@ -7,7 +7,7 @@ from app.api.dependencies import PlayerServiceDep from app.config import settings from app.enums import RouteTag -from app.helpers import apply_swr_headers, build_cache_key +from app.helpers import apply_swr_headers, build_cache_key, get_human_readable_duration from app.helpers import routes_responses as common_routes_responses from app.players.enums import ( HeroKeyCareerFilter, @@ -99,7 +99,7 @@ async def get_player_career_common_parameters( description=( "Search for a given player by using its username or BattleTag (with # replaced by -). " "
You should be able to find the associated player_id to use in order to request career data." - f"
**Cache TTL : {settings.search_account_path_cache_timeout} seconds.**" + f"
**Cache TTL : {get_human_readable_duration(settings.search_account_path_cache_timeout)}.**" ), operation_id="search_players", response_model=PlayerSearchResult, @@ -144,7 +144,7 @@ async def search_players( summary="Get player summary", description=( "Get player summary : name, avatar, competitive ranks, etc. " - f"
**Cache TTL : {settings.career_path_cache_timeout} seconds.**" + f"
**Cache TTL : {get_human_readable_duration(settings.career_path_cache_timeout)}.**" ), operation_id="get_player_summary", response_model=PlayerSummary, @@ -178,7 +178,7 @@ async def get_player_summary( "
Depending on filters, data from both competitive and quickplay, " "and/or pc and console will be merged." "
Default behaviour : all gamemodes and platforms are taken in account." - f"
**Cache TTL : {settings.career_path_cache_timeout} seconds.**" + f"
**Cache TTL : {get_human_readable_duration(settings.career_path_cache_timeout)}.**" ), operation_id="get_player_stats_summary", response_model=PlayerStatsSummary, @@ -233,7 +233,7 @@ async def get_player_stats_summary( "(combat, game, best, hero specific, average, etc.). Filter them on " "specific platform and gamemode (mandatory). You can even retrieve " "data about a specific hero of your choice." - f"
**Cache TTL : {settings.career_path_cache_timeout} seconds.**" + f"
**Cache TTL : {get_human_readable_duration(settings.career_path_cache_timeout)}.**" ), operation_id="get_player_career_stats", response_model=PlayerCareerStats, @@ -265,7 +265,7 @@ async def get_player_career_stats( description=( "This endpoint exposes the same data as the previous one, except it also " "exposes labels of the categories and statistics." - f"
**Cache TTL : {settings.career_path_cache_timeout} seconds.**" + f"
**Cache TTL : {get_human_readable_duration(settings.career_path_cache_timeout)}.**" ), operation_id="get_player_stats", response_model=CareerStats, @@ -295,7 +295,7 @@ async def get_player_stats( summary="Get all player data", description=( "Get all player data : summary and statistics with labels." - f"
**Cache TTL : {settings.career_path_cache_timeout} seconds.**" + f"
**Cache TTL : {get_human_readable_duration(settings.career_path_cache_timeout)}.**" ), operation_id="get_player_career", response_model=Player, diff --git a/app/api/routers/roles.py b/app/api/routers/roles.py index b54f686e..dc3c41e1 100644 --- a/app/api/routers/roles.py +++ b/app/api/routers/roles.py @@ -7,7 +7,12 @@ from app.api.dependencies import RoleServiceDep from app.config import settings from app.enums import Locale, RouteTag -from app.helpers import apply_swr_headers, build_cache_key, routes_responses +from app.helpers import ( + apply_swr_headers, + build_cache_key, + get_human_readable_duration, + routes_responses, +) from app.roles.models import RoleDetail router = APIRouter() @@ -20,7 +25,7 @@ summary="Get a list of roles", description=( "Get a list of available Overwatch roles." - f"
**Cache TTL : {settings.heroes_path_cache_timeout} seconds.**" + f"
**Cache TTL : {get_human_readable_duration(settings.heroes_path_cache_timeout)}.**" ), operation_id="list_roles", response_model=list[RoleDetail], diff --git a/uv.lock b/uv.lock index 5a6c718e..f6ba5356 100644 --- a/uv.lock +++ b/uv.lock @@ -604,7 +604,7 @@ wheels = [ [[package]] name = "overfast-api" -version = "3.42.1" +version = "3.43.0" source = { virtual = "." } dependencies = [ { name = "aiosqlite" }, From 5bd452777e51f7ae5e0549faa566f21a85a9f6b2 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 17:58:59 +0100 Subject: [PATCH 22/26] fix: review --- app/api/dependencies.py | 2 +- app/domain/services/player_service.py | 10 ++-- tests/conftest.py | 2 +- .../heroes/parsers/test_hero_stats_summary.py | 58 ++++++++++++++++++- 4 files changed, 64 insertions(+), 8 deletions(-) diff --git a/app/api/dependencies.py b/app/api/dependencies.py index bf6a5b0e..c4cd2dce 100644 --- a/app/api/dependencies.py +++ b/app/api/dependencies.py @@ -23,7 +23,7 @@ def get_blizzard_client() -> BlizzardClientPort: - """Dependency for Blizzard HTTP client (Singleton).""" + """Dependency for Blizzard HTTP client.""" return BlizzardClient() diff --git a/app/domain/services/player_service.py b/app/domain/services/player_service.py index 4e67a8d7..8383579b 100644 --- a/app/domain/services/player_service.py +++ b/app/domain/services/player_service.py @@ -261,7 +261,7 @@ async def _execute_player_request( except Exception as exc: # noqa: BLE001 await self._handle_player_exceptions(exc, request.player_id, identity) - is_stale = self._check_player_staleness(effective_id) + is_stale = self._check_player_staleness() await self._update_api_cache( request.cache_key, data, settings.career_path_cache_timeout ) @@ -311,11 +311,11 @@ async def update_player_profile_cache( name=name, ) - def _check_player_staleness(self, _player_id: str) -> bool: - """Return is_stale based purely on time — best effort. + def _check_player_staleness(self) -> bool: + """Return is_stale — stub always returning False until Phase 5. - Phase 5 will perform an actual async storage lookup; for now always - returns False (fresh) and lets the background refresh handle real staleness. + Phase 5 will perform a real async storage lookup comparing + ``player_profiles.updated_at`` against ``player_staleness_threshold``. """ return False diff --git a/tests/conftest.py b/tests/conftest.py index 3d3528f3..c73e937d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -52,7 +52,7 @@ async def _patch_before_every_test( patch("app.helpers.settings.discord_webhook_enabled", False), patch("app.helpers.settings.profiler", None), patch( - "app.adapters.cache.valkey_cache.CacheManager.valkey_server", + "app.adapters.cache.valkey_cache.ValkeyCache.valkey_server", valkey_server, ), ): diff --git a/tests/heroes/parsers/test_hero_stats_summary.py b/tests/heroes/parsers/test_hero_stats_summary.py index e5eee3bf..2eb4dffd 100644 --- a/tests/heroes/parsers/test_hero_stats_summary.py +++ b/tests/heroes/parsers/test_hero_stats_summary.py @@ -3,7 +3,11 @@ import pytest from app.adapters.blizzard import OverFastClient -from app.adapters.blizzard.parsers.hero_stats_summary import parse_hero_stats_summary +from app.adapters.blizzard.parsers.hero_stats_summary import ( + GAMEMODE_MAPPING, + PLATFORM_MAPPING, + parse_hero_stats_summary, +) from app.exceptions import ParserBlizzardError from app.players.enums import ( CompetitiveDivision, @@ -48,3 +52,55 @@ async def test_parse_hero_stats_summary( assert isinstance(result, list) assert len(result) > 0 assert "hero" in result[0] + + +@pytest.mark.asyncio +async def test_parse_hero_stats_summary_query_params(hero_stats_response_mock: Mock): + """Verify the exact Blizzard query parameters built by the parser.""" + platform = PlayerPlatform.PC + gamemode = PlayerGamemode.COMPETITIVE + region = PlayerRegion.EUROPE + division = CompetitiveDivision.DIAMOND + map_key = "all-maps" + + with patch("httpx.AsyncClient.get", return_value=hero_stats_response_mock) as mock_get: + client = OverFastClient() + await parse_hero_stats_summary( + client, + platform=platform, + gamemode=gamemode, + region=region, + competitive_division=division, + map_filter=map_key, + order_by="hero:asc", + ) + + mock_get.assert_called_once() + _, kwargs = mock_get.call_args + params = kwargs.get("params", {}) + + assert params["input"] == PLATFORM_MAPPING[platform] + assert params["rq"] == GAMEMODE_MAPPING[gamemode] + assert params["region"] == region.capitalize() + assert params["map"] == map_key + assert params["tier"] == division.capitalize() + + +@pytest.mark.asyncio +async def test_parse_hero_stats_summary_invalid_map_error_message( + hero_stats_response_mock: Mock, +): + """ParserBlizzardError message should name the incompatible map.""" + with patch("httpx.AsyncClient.get", return_value=hero_stats_response_mock): + client = OverFastClient() + with pytest.raises(ParserBlizzardError) as exc_info: + await parse_hero_stats_summary( + client, + platform=PlayerPlatform.PC, + gamemode=PlayerGamemode.COMPETITIVE, + region=PlayerRegion.EUROPE, + map_filter="hanaoka", + ) + + assert "hanaoka" in exc_info.value.message + assert "compatible" in exc_info.value.message.lower() From 0b104ffadd05a1c49df5d730773ef3137d753fdf Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 18:01:48 +0100 Subject: [PATCH 23/26] fix: sonar issues --- app/adapters/tasks/asyncio_task_queue.py | 4 ++-- app/domain/services/base_service.py | 4 ++-- app/domain/services/gamemode_service.py | 2 +- app/domain/services/map_service.py | 2 +- app/domain/services/static_data_service.py | 6 +++++- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/app/adapters/tasks/asyncio_task_queue.py b/app/adapters/tasks/asyncio_task_queue.py index 42f449bd..769788c6 100644 --- a/app/adapters/tasks/asyncio_task_queue.py +++ b/app/adapters/tasks/asyncio_task_queue.py @@ -29,7 +29,7 @@ class AsyncioTaskQueue: _pending_jobs: ClassVar[set[str]] = set() - async def enqueue( + async def enqueue( # NOSONAR self, task_name: str, *_args: Any, @@ -81,6 +81,6 @@ async def _run() -> None: task.add_done_callback(lambda _: None) return effective_id - async def is_job_pending_or_running(self, job_id: str) -> bool: + async def is_job_pending_or_running(self, job_id: str) -> bool: # NOSONAR """Return True if a job with this ID is already in-flight.""" return job_id in self._pending_jobs diff --git a/app/domain/services/base_service.py b/app/domain/services/base_service.py index 2e0f7acb..869ecec5 100644 --- a/app/domain/services/base_service.py +++ b/app/domain/services/base_service.py @@ -96,13 +96,13 @@ async def _enqueue_refresh( """ job_id = f"refresh:{entity_type}:{entity_id}" - async def _on_complete(_job_id: str) -> None: + async def _on_complete(_job_id: str) -> None: # NOSONAR logger.info( f"[SWR] Background refresh complete for {entity_type}/{entity_id}" ) background_refresh_completed_total.labels(entity_type=entity_type).inc() - async def _on_failure(_job_id: str, exc: Exception) -> None: + async def _on_failure(_job_id: str, exc: Exception) -> None: # NOSONAR logger.warning(f"[SWR] Refresh failed for {entity_type}/{entity_id}: {exc}") background_refresh_failed_total.labels(entity_type=entity_type).inc() diff --git a/app/domain/services/gamemode_service.py b/app/domain/services/gamemode_service.py index e0b710af..bbdb325b 100644 --- a/app/domain/services/gamemode_service.py +++ b/app/domain/services/gamemode_service.py @@ -14,7 +14,7 @@ async def list_gamemodes( ) -> tuple[list[dict], bool, int]: """Return the gamemodes list.""" - async def _fetch() -> list[dict]: + def _fetch() -> list[dict]: return parse_gamemodes_csv() return await self.get_or_fetch( diff --git a/app/domain/services/map_service.py b/app/domain/services/map_service.py index c07b10c0..4efb3053 100644 --- a/app/domain/services/map_service.py +++ b/app/domain/services/map_service.py @@ -18,7 +18,7 @@ async def list_maps( Stores the full (unfiltered) maps list in SQLite. """ - async def _fetch() -> list[dict]: + def _fetch() -> list[dict]: return parse_maps_csv() def _filter(data: list[dict]) -> list[dict]: diff --git a/app/domain/services/static_data_service.py b/app/domain/services/static_data_service.py index a5456678..e27b2502 100644 --- a/app/domain/services/static_data_service.py +++ b/app/domain/services/static_data_service.py @@ -1,3 +1,4 @@ +import inspect import json import time from dataclasses import dataclass, field @@ -137,7 +138,10 @@ async def _refresh_static(self, config: StaticFetchConfig) -> None: async def _fetch_and_store(self, config: StaticFetchConfig) -> Any: """Fetch from source, persist to SQLite, update Valkey, return filtered data.""" - raw = await config.fetcher() + if inspect.iscoroutinefunction(config.fetcher): + raw = await config.fetcher() + else: + raw = config.fetcher() data = config.parser(raw) if config.parser is not None else raw await self._store_in_storage(config.storage_key, data) From 759b43727df68508f7fbc7b31222504e4a425a8b Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 18:02:03 +0100 Subject: [PATCH 24/26] fix: ruff format --- tests/heroes/parsers/test_hero_stats_summary.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/heroes/parsers/test_hero_stats_summary.py b/tests/heroes/parsers/test_hero_stats_summary.py index 2eb4dffd..e35b2301 100644 --- a/tests/heroes/parsers/test_hero_stats_summary.py +++ b/tests/heroes/parsers/test_hero_stats_summary.py @@ -63,7 +63,9 @@ async def test_parse_hero_stats_summary_query_params(hero_stats_response_mock: M division = CompetitiveDivision.DIAMOND map_key = "all-maps" - with patch("httpx.AsyncClient.get", return_value=hero_stats_response_mock) as mock_get: + with patch( + "httpx.AsyncClient.get", return_value=hero_stats_response_mock + ) as mock_get: client = OverFastClient() await parse_hero_stats_summary( client, From f28e5a4c50c04e7a3cd9a58a862c2d47fdd8ec14 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 18:04:32 +0100 Subject: [PATCH 25/26] fix: review --- app/api/dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/dependencies.py b/app/api/dependencies.py index c4cd2dce..bf6a5b0e 100644 --- a/app/api/dependencies.py +++ b/app/api/dependencies.py @@ -23,7 +23,7 @@ def get_blizzard_client() -> BlizzardClientPort: - """Dependency for Blizzard HTTP client.""" + """Dependency for Blizzard HTTP client (Singleton).""" return BlizzardClient() From ada87988ca85dae97bd71160cec66691c2c96ae2 Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 22 Feb 2026 18:44:50 +0100 Subject: [PATCH 26/26] fix: review --- app/adapters/blizzard/parsers/heroes.py | 1 - app/adapters/tasks/asyncio_task_queue.py | 3 +- app/config.py | 2 +- tests/test_asyncio_task_queue.py | 264 +++++++++++++++++++++++ 4 files changed, 267 insertions(+), 3 deletions(-) create mode 100644 tests/test_asyncio_task_queue.py diff --git a/app/adapters/blizzard/parsers/heroes.py b/app/adapters/blizzard/parsers/heroes.py index f5fbe875..283f2ce6 100644 --- a/app/adapters/blizzard/parsers/heroes.py +++ b/app/adapters/blizzard/parsers/heroes.py @@ -9,7 +9,6 @@ validate_response_status, ) from app.config import settings -from app.domain.ports import BlizzardClientPort from app.enums import Locale from app.exceptions import ParserParsingError from app.heroes.enums import HeroGamemode diff --git a/app/adapters/tasks/asyncio_task_queue.py b/app/adapters/tasks/asyncio_task_queue.py index 769788c6..4cdae23f 100644 --- a/app/adapters/tasks/asyncio_task_queue.py +++ b/app/adapters/tasks/asyncio_task_queue.py @@ -4,6 +4,7 @@ import time from typing import TYPE_CHECKING, Any, ClassVar +from app.metaclasses import Singleton from app.monitoring.metrics import ( background_tasks_duration_seconds, background_tasks_queue_size, @@ -15,7 +16,7 @@ from collections.abc import Awaitable, Callable, Coroutine -class AsyncioTaskQueue: +class AsyncioTaskQueue(metaclass=Singleton): """In-process task queue backed by asyncio.create_task(). Uses a class-level set for deduplication so concurrent requests don't diff --git a/app/config.py b/app/config.py index c01d05fc..63059fa9 100644 --- a/app/config.py +++ b/app/config.py @@ -163,7 +163,7 @@ class Settings(BaseSettings): roles_staleness_threshold: int = 86400 # Age (seconds) after which a player profile is considered stale. - player_staleness_threshold: int = 1800 # 30 min + player_staleness_threshold: int = 3600 # 1 hour # TTL (seconds) for stale responses written to Valkey API cache. # Short enough that background refresh (typically seconds) will overwrite it diff --git a/tests/test_asyncio_task_queue.py b/tests/test_asyncio_task_queue.py new file mode 100644 index 00000000..29388477 --- /dev/null +++ b/tests/test_asyncio_task_queue.py @@ -0,0 +1,264 @@ +"""Tests for AsyncioTaskQueue adapter""" + +import asyncio + +import pytest + +from app.adapters.tasks.asyncio_task_queue import AsyncioTaskQueue +from app.metaclasses import Singleton +from app.monitoring.metrics import ( + background_tasks_queue_size, + background_tasks_total, +) + + +@pytest.fixture(autouse=True) +def _reset_task_queue(): + """Reset Singleton and pending jobs before each test for isolation.""" + Singleton._instances.pop(AsyncioTaskQueue, None) + AsyncioTaskQueue._pending_jobs.clear() + yield + AsyncioTaskQueue._pending_jobs.clear() + Singleton._instances.pop(AsyncioTaskQueue, None) + + +class TestSingleton: + def test_same_instance_returned(self): + q1 = AsyncioTaskQueue() + q2 = AsyncioTaskQueue() + assert q1 is q2 + + def test_shared_pending_jobs_set(self): + q1 = AsyncioTaskQueue() + q2 = AsyncioTaskQueue() + assert q1._pending_jobs is q2._pending_jobs + + +class TestDeduplication: + @pytest.mark.asyncio + async def test_duplicate_job_skipped(self): + queue = AsyncioTaskQueue() + completed = [] + + async def work(): + await asyncio.sleep(0.05) + completed.append(1) + + await queue.enqueue("refresh", job_id="job-1", coro=work()) + await queue.enqueue("refresh", job_id="job-1") # duplicate — no coro + + await asyncio.sleep(0.1) + assert len(completed) == 1 + + @pytest.mark.asyncio + async def test_duplicate_coro_closed(self): + """The second coroutine must be closed to prevent 'never awaited' warning.""" + queue = AsyncioTaskQueue() + + async def work(): + await asyncio.sleep(0.05) + + async def second(): + pass # never should run + + await queue.enqueue("refresh", job_id="job-1", coro=work()) + + # Enqueue with a coro while job-1 is still running + coro = second() + await queue.enqueue("refresh", job_id="job-1", coro=coro) + + # coro should be closed (not just pending) + assert coro.cr_frame is None # closed coroutine has no frame + + await asyncio.sleep(0.1) + + @pytest.mark.asyncio + async def test_same_job_id_requeued_after_completion(self): + """After a job finishes, the same job_id can be enqueued again.""" + queue = AsyncioTaskQueue() + completed = [] + + async def work(): + completed.append(1) + + await queue.enqueue("refresh", job_id="job-1", coro=work()) + await asyncio.sleep(0.05) + assert len(completed) == 1 + + await queue.enqueue("refresh", job_id="job-1", coro=work()) + await asyncio.sleep(0.05) + assert len(completed) == 2 # noqa: PLR2004 + + @pytest.mark.asyncio + async def test_different_job_ids_both_run(self): + queue = AsyncioTaskQueue() + completed = [] + + async def work(name: str): + completed.append(name) + + await queue.enqueue("refresh", job_id="job-a", coro=work("a")) + await queue.enqueue("refresh", job_id="job-b", coro=work("b")) + await asyncio.sleep(0.05) + + assert sorted(completed) == ["a", "b"] + + +class TestCallbacks: + @pytest.mark.asyncio + async def test_on_complete_called_on_success(self): + queue = AsyncioTaskQueue() + completed_ids = [] + + async def work(): + pass + + async def on_complete(job_id: str) -> None: + completed_ids.append(job_id) + + await queue.enqueue( + "refresh", job_id="job-1", coro=work(), on_complete=on_complete + ) + await asyncio.sleep(0.05) + + assert completed_ids == ["job-1"] + + @pytest.mark.asyncio + async def test_on_failure_called_on_error(self): + queue = AsyncioTaskQueue() + failures: list[tuple[str, Exception]] = [] + + async def failing_work(): + msg = "boom" + raise ValueError(msg) + + async def on_failure(job_id: str, exc: Exception) -> None: + failures.append((job_id, exc)) + + await queue.enqueue( + "refresh", job_id="job-1", coro=failing_work(), on_failure=on_failure + ) + await asyncio.sleep(0.05) + + assert len(failures) == 1 + assert failures[0][0] == "job-1" + assert isinstance(failures[0][1], ValueError) + + @pytest.mark.asyncio + async def test_on_complete_not_called_on_failure(self): + queue = AsyncioTaskQueue() + completed_ids = [] + + async def failing_work(): + msg = "fail" + raise RuntimeError(msg) + + async def on_complete(job_id: str) -> None: + completed_ids.append(job_id) + + await queue.enqueue( + "refresh", job_id="job-1", coro=failing_work(), on_complete=on_complete + ) + await asyncio.sleep(0.05) + + assert completed_ids == [] + + @pytest.mark.asyncio + async def test_no_callbacks_no_error(self): + """Enqueue without callbacks should run cleanly with no exception.""" + queue = AsyncioTaskQueue() + + async def work(): + pass + + await queue.enqueue("refresh", job_id="job-1", coro=work()) + await asyncio.sleep(0.05) + + +class TestMetrics: + @pytest.mark.asyncio + async def test_queue_size_increments_then_decrements(self): + queue = AsyncioTaskQueue() + sizes_during: list[float] = [] + + async def work(): + val = background_tasks_queue_size.labels(task_type="refresh")._value.get() + sizes_during.append(val) + await asyncio.sleep(0.01) + + await queue.enqueue("refresh", job_id="job-1", coro=work()) + await asyncio.sleep(0.05) + + assert sizes_during[0] == 1.0 + after = background_tasks_queue_size.labels(task_type="refresh")._value.get() + assert after == 0.0 + + @pytest.mark.asyncio + async def test_tasks_total_success(self): + queue = AsyncioTaskQueue() + + async def work(): + pass + + before = background_tasks_total.labels( + task_type="refresh", status="success" + )._value.get() + await queue.enqueue("refresh", job_id="job-1", coro=work()) + await asyncio.sleep(0.05) + after = background_tasks_total.labels( + task_type="refresh", status="success" + )._value.get() + + assert after == before + 1 + + @pytest.mark.asyncio + async def test_tasks_total_failure(self): + queue = AsyncioTaskQueue() + + async def failing_work(): + msg = "fail" + raise ValueError(msg) + + before = background_tasks_total.labels( + task_type="refresh", status="failure" + )._value.get() + await queue.enqueue("refresh", job_id="job-1", coro=failing_work()) + await asyncio.sleep(0.05) + after = background_tasks_total.labels( + task_type="refresh", status="failure" + )._value.get() + + assert after == before + 1 + + +class TestIsJobPendingOrRunning: + @pytest.mark.asyncio + async def test_pending_while_running(self): + queue = AsyncioTaskQueue() + is_pending_during: list[bool] = [] + + async def work(): + is_pending_during.append(await queue.is_job_pending_or_running("job-1")) + await asyncio.sleep(0.02) + + await queue.enqueue("refresh", job_id="job-1", coro=work()) + await asyncio.sleep(0.05) + + assert is_pending_during == [True] + + @pytest.mark.asyncio + async def test_not_pending_after_completion(self): + queue = AsyncioTaskQueue() + + async def work(): + pass + + await queue.enqueue("refresh", job_id="job-1", coro=work()) + await asyncio.sleep(0.05) + + assert not await queue.is_job_pending_or_running("job-1") + + @pytest.mark.asyncio + async def test_not_pending_for_unknown_job(self): + queue = AsyncioTaskQueue() + assert not await queue.is_job_pending_or_running("unknown-job")