diff --git a/.gitignore b/.gitignore index 3d7cde3..7f975d5 100644 --- a/.gitignore +++ b/.gitignore @@ -47,10 +47,8 @@ htmlcov/ collection.db collection.db.bak* -# Sync drift/diff reports (written to cwd by sync; regenerated on demand) +# Sync diff reports (only written when `sync --diff-report PATH` is passed) sync-diff-*.md -sync-schema-diff-*.md -sync-schema-observed-*.json # Claude Code code-graph index cache .codegraph/ diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..4bebb0b --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,86 @@ +# KARDS Collection Manager + +Local-first manager for a player's KARDS collection and saved decks: +syncs the official card set into a local database, tracks owned +quantities, and exports collection/deck data. + +## Language + +**Collection**: +The full set of KARDS cards together with the player's owned quantity per +card — a replica of the in-game collection, whose only added value is +richer filtering and management than the game offers. A card the player +owns zero copies of is still part of the Collection. There is no separate +word for the card set without quantities. +_Avoid_: Catalog + +**Faction**: +The power a card belongs to (Soviet, USA, Britain, Germany, Japan, France, +Italy, Poland, Finland), named the way the game's data and API name it. +Use Faction everywhere internal: code, database, GraphQL, business logic. + +**Nation**: +The user-facing label for a Faction — what the player sees on screen and in +exports (the "Nation" column, localized per language). Same concept as +Faction, different layer: Nation is presentation, Faction is the internal +code. A KARDS client TXT deck file also labels its sections by Nation. +Do not use Nation in internal code; do not rename user-facing "Nation" to +"Faction". + +**Deck**: +A saved deck brought in from a KARDS client TXT file. Because a deck comes +from the game client, it is evidence of ownership: the player owns at +least as many copies of each card as the deck uses. A Collection quantity +below a deck's count means the Collection is stale — never that the deck +is invalid. A deck using fewer copies than owned is normal. + +**Quantity**: +The number of copies of a card the player owns. Managed by the player, +never touched by sync. Capped by rarity exactly as in the game +(Standard 4, Limited 3, Special 2, Elite 1) — a quantity above the cap is +a data error, not player freedom, and every write path enforces the cap. + +**Baseline**: +The committed snapshot of the API contract shape that sync checks the +live response against. Drift is measured relative to the Baseline; +accepting drift promotes the observed shape to become the new Baseline. + +**Spawnable**: +A card that cannot be obtained in packs or crafted — it only appears +in-game when another card creates it. Spawnable cards are part of the +Collection as reference material: the player looks up what a spawned card +does. They are hidden by default behind a view toggle; owning them is +meaningless but nothing enforces a zero quantity — the Collection mirrors +the game, it does not police it. + +**Exile**: +A cross-faction link on a card: the card belongs to one faction but may be +played in decks of another faction (its exile faction), reflecting the +game's exile-forces mechanic. Deck import falls back to the exile link when +a card is not found under its own faction. + +**Diff**: +The comparison of card content between the local database and a fresh API +pull: new cards, changed stats/text, reserve transitions, removed cards. +The player reviews and approves a Diff before it is applied. +_Avoid_: Drift (that word is for contract shape changes) + +**Drift**: +A change in the *shape* of the API contract against the committed baseline: +a field added or removed, a new faction/type/rarity/ability value. Drift +halts sync until the player reviews and accepts the new baseline. +_Avoid_: Diff (that word is for card content changes) + +**Reserved**: +A card state set by the game (not the player): the card has been moved to +the reserve pool. Cards transition into and out of reserve over time, so +sync reports these transitions as their own category rather than as a +generic field change. + +**Ability**: +A named game mechanic a card has (guard, blitz, smokescreen, …). One concept +regardless of source: regular abilities come from the card JSON in the +GraphQL API; extra abilities are manually curated for mechanics the API does +not expose. The split may be revisited — the game recently introduced its own +categorization — but for now source is the only distinction. +_Avoid_: Attribute diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2f70b37..6616e5c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,7 +20,7 @@ make help # list available make targets make sync # install runtime dependencies make sync-dev # install runtime/dev dependencies make run # show kardscm CLI help -make sync-diff # preview catalog sync without DB changes +make sync-diff # preview card-set sync without DB changes make web # start the local web UI make web-admin # start admin web UI with DB backup make test # pytest with coverage @@ -102,7 +102,7 @@ collection.db ```text KARDS deck TXT -> importing.parser - -> commands.add_deck (or) commands.import_deck + -> commands.add_deck -> storage.database -> collection.db ``` @@ -142,8 +142,9 @@ Schema initialization also handles: ## Sync And API Drift -`kardscm sync` fetches the catalog, computes a diff, asks for category approval, -and writes only after approval. Rejected syncs leave the DB unchanged. +`kardscm sync` fetches the card set, computes a diff, asks for category approval, +and writes only after approval. Rejected syncs leave the DB unchanged. The diff +is shown on screen; nothing is written unless `--diff-report PATH` is passed. The API baseline lives at: @@ -153,13 +154,10 @@ kardscm/data/api_baseline.json During sync, the raw GraphQL response *shape* is compared with this committed baseline. A contract change **halts the sync** (it raises -`ApiContractDriftError` before any DB write) and produces local files for -review: - -```text -sync-schema-diff-*.md -sync-schema-observed-*.json -``` +`ApiContractDriftError` before any DB write). The halt prints the drift to the +terminal (CLI) or renders it in the sync modal (web); no files are written. The +observed shape is stashed in the database under the `drift_observed_snapshot` +metadata key so it can be promoted later. A contract change means: a top-level or JSON key added or removed, a key becoming sparse, a new `faction`/`type`/`rarity`/ability value, or a sharp drop @@ -168,9 +166,10 @@ drift and never halts. Workflow when a sync halts on drift: -1. Review the generated schema diff and observed snapshot. +1. Review the drift printed by the halt. 2. Update code/constants/locales if the new shape needs handling. -3. Run `uv run kardscm baseline accept` to promote the latest observed snapshot. +3. Run `uv run kardscm baseline accept` to promote the stashed observed + snapshot — exactly the shape you reviewed — and clear it. 4. Commit the baseline update with the related code or data change, then re-run the sync. @@ -182,9 +181,8 @@ data-derived contract snapshot. ## Extra Abilities -Some KARDS mechanics are visible in the game client but are not exposed as -official GraphQL attributes. Those are tracked as manually curated -extra-ability flags. +Some KARDS abilities are visible in the game client but are not exposed by the +official GraphQL API. Those are tracked as manually curated extra-ability flags. Relevant files: diff --git a/README.md b/README.md index ecc5188..9daa38c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ `kardscm` is a local collection and deck manager for [KARDS](https://www.kards.com/). -It syncs the official card catalog into a local SQLite database, lets you keep +It syncs the official card set into a local SQLite database, lets you keep your owned card quantities up to date, saves decks from KARDS client TXT files, and exports collection or deck data to XLSX or JSON. @@ -29,13 +29,13 @@ responsible for complying with the [KARDS Terms of Use](https://www.kards.com/terms-of-use). This repository ships no card data, card art, or pre-built database. The local -SQLite catalog is built on the user's own machine. +SQLite collection is built on the user's own machine. ## What It Does -- Syncs the full KARDS card catalog, including reserved and spawnable cards. +- Syncs the full KARDS card set, including reserved and spawnable cards. - Preserves user-managed card quantities across syncs. -- Shows catalog changes before applying them: new cards, changed stats/text, +- Shows card-set changes before applying them: new cards, changed stats/text, reserve transitions, and removed cards. - Exports the collection to XLSX or JSON. - Updates card quantities from an edited XLSX export. @@ -77,7 +77,7 @@ repo and ran `make sync`, prefix each command with `uv run` (or activate `.venv` first) so the console script is found: ```bash -# 1. Sync the card catalog into collection.db. +# 1. Sync the card set into collection.db. uv run kardscm sync # 2. Open the local browser UI and edit quantities. @@ -118,22 +118,26 @@ kardscm sync --yes kardscm sync --diff-report ./sync-report.md ``` -`sync` fetches the official catalog, compares it with the local database, and +`sync` fetches the official card set, compares it with the local database, and shows a diff before writing changes. Any rejected prompt aborts the sync and -leaves the database unchanged. A Markdown report is written whenever there are -changes. +leaves the database unchanged. The diff is shown on screen; nothing is written +to disk unless you pass `--diff-report`. -`--diff-only` writes the report without modifying the database. `--yes` +`--diff-only` prints the diff without modifying the database. `--yes` (short: `-y`) auto-approves every category for scripted runs. +The diff is shown on screen and no file is written. Pass `--diff-report PATH` +to also save it as Markdown — useful for scripted runs where nobody is +watching the terminal. + Sync also checks the live GraphQL response *shape* against the committed baseline `kardscm/data/api_baseline.json`. A genuine contract change — a field added or removed, a field becoming sparse, a new `faction`/`type`/`rarity`/ -ability value, or a sharp drop in card count — **halts the sync** and writes -`sync-schema-diff-*.md` and `sync-schema-observed-*.json` to the current -directory. Normal content growth (new card sets, more cards) is not a contract -change and never blocks. After reviewing a halt, run `kardscm baseline accept` -to adopt the new shape, then sync again. +ability value, or a sharp drop in card count — **halts the sync** and prints +what changed. Normal content growth (new card sets, more cards) is not a +contract change and never blocks. The observed shape is stored in the database, +so after reviewing a halt you can run `kardscm baseline accept` to adopt +exactly the shape you reviewed, then sync again. ## Collection Export And Update @@ -215,7 +219,7 @@ kardscm web --admin # short: -A kardscm --lang ru web --admin ``` -Admin mode is for trusted local correction of catalog data. It exposes editable +Admin mode is for trusted local correction of collection data. It exposes editable card stats, categories, ability flags, extra-ability flags, reserved state, and localized title/text for the active locale. @@ -242,11 +246,12 @@ kardscm deck export -f json -o deck.json back to exile links when needed, checks collection quantities, and can update or replace existing data: -- `--update` / `-u`: raise collection quantities to match the deck +- `--update` / `-u`: raise collection quantities to the deck's counts - `--replace` / `-r`: overwrite an existing saved deck with the same name -`deck import` still exists as a simpler single-file import path, but day-to-day -use should prefer `deck add`. +`deck add` only reports a quantity shortfall when the deck needs more copies +than the collection records — a deck using fewer copies than you own is fine. +Quantities are capped per rarity exactly as in the game. ## Deck File Format @@ -280,26 +285,25 @@ Rules: ## API Baseline The committed baseline at `kardscm/data/api_baseline.json` is the contract that -sync drift is checked against. A contract change halts the sync; after reviewing -the drift report, promote the new shape (run from a clone; pipx users can drop -the `uv run` prefix): +sync drift is checked against. A contract change halts the sync and prints what +changed; the observed shape is stored in the database. After reviewing the drift +and updating any constants or translations, promote the reviewed shape (run from +a clone; pipx users can drop the `uv run` prefix): ```bash uv run kardscm baseline accept ``` -`baseline accept` adopts the latest `sync-schema-observed-*.json` from the sync -drift report — after you have reviewed it and updated any constants or -translations. A from-scratch baseline is created automatically on the first sync +`baseline accept` promotes the shape from the last halted sync — exactly what +you reviewed, not whatever the API happens to serve at accept time — and then +clears it. A from-scratch baseline is created automatically on the first sync when none exists. ## Output Files - `collection.db`: local SQLite database - `collection.db.bak*`: local backups -- `sync-diff-*.md`: sync reports -- `sync-schema-diff-*.md`: API drift reports -- `sync-schema-observed-*.json`: observed API snapshots +- `sync-diff-*.md`: sync reports, only when `--diff-report` is passed - export files: whatever path you pass with `-o` Generated local data is intentionally not part of the repository. diff --git a/kardscm/cli.py b/kardscm/cli.py index b024436..df4168c 100644 --- a/kardscm/cli.py +++ b/kardscm/cli.py @@ -16,7 +16,6 @@ baseline_accept, export_collection, export_deck, - import_deck, remove_deck, sync_collection, update_collection, @@ -115,7 +114,7 @@ def sync( bool, typer.Option( "--diff-only", - help="Print diff and write the Markdown report; do not modify the DB.", + help="Print the diff; do not modify the DB.", ), ] = False, yes: Annotated[ @@ -130,7 +129,7 @@ def sync( Path | None, typer.Option( "--diff-report", - help="Markdown diff report path (default: ./sync-diff-TIMESTAMP.md).", + help="Also write the diff as Markdown to this path.", resolve_path=True, ), ] = None, @@ -140,8 +139,8 @@ def sync( Fetches all cards via GraphQL, computes a diff against the local DB, and prompts approval per non-empty category (new cards / changed characteristics / reserve transitions / removed cards). Any - rejection aborts the sync; the DB is left untouched. A Markdown - diff report is always written when the diff is non-empty. + rejection aborts the sync; the DB is left untouched. The diff is + shown on screen; pass --diff-report to also save it as Markdown. """ sync_collection( lang=_lang(ctx), @@ -203,33 +202,6 @@ def update( update_collection(str(file), lang=_lang(ctx)) -@deck_app.command( - "import", - epilog="Examples:\n\n* `kards deck import -i deck.txt`", -) -def deck_import( - ctx: typer.Context, - file: Annotated[ - Path, - typer.Option( - "--file", - "-i", - help="Deck TXT file to import", - exists=True, - readable=True, - resolve_path=True, - ), - ], -) -> None: - """Import a deck from a TXT file. - - Parses the deck file and saves it to the local database. - Cards in the file must already exist in the collection. - """ - _validate_extension(file, ".txt") - import_deck(str(file), lang=_lang(ctx)) - - @deck_app.command( "add", epilog="Examples:\n\n" @@ -250,7 +222,7 @@ def deck_add( ], update: Annotated[ bool, - typer.Option("--update", "-u", help="Update collection quantities to match deck"), + typer.Option("--update", "-u", help="Raise collection quantities to the deck's counts"), ] = False, replace: Annotated[ bool, @@ -260,7 +232,8 @@ def deck_add( """Add deck(s) from TXT file(s), with exile card support. Looks up cards by faction first, then falls back to the exile field. - Checks collection quantities; use --update to fix mismatches. + Fails when a deck needs more copies than the collection records; + use --update to raise them. Use --replace to overwrite an existing deck with the same name. On error, continues with remaining files and prints a summary at the end. """ @@ -354,10 +327,11 @@ def web( @baseline_app.command("accept") def baseline_accept_cmd() -> None: - """Promote the latest `sync-schema-observed-*.json` to baseline. + """Adopt the API shape from the last halted sync. - After reviewing a `sync-schema-diff-*.md` report and updating any - constants/translations, run this to acknowledge the new API shape. + After reviewing the drift a halted sync printed and updating any + constants/translations it calls for, run this to acknowledge the new + API shape, then sync again. """ baseline_accept() diff --git a/kardscm/commands/__init__.py b/kardscm/commands/__init__.py index 88d6d06..61e7921 100644 --- a/kardscm/commands/__init__.py +++ b/kardscm/commands/__init__.py @@ -3,7 +3,7 @@ from __future__ import annotations from kardscm.commands.baseline import baseline_accept -from kardscm.commands.decks import add_deck, add_decks, export_deck, import_deck, remove_deck +from kardscm.commands.decks import add_deck, add_decks, export_deck, remove_deck from kardscm.commands.export import export_collection, update_collection from kardscm.commands.sync import apply_sync_changes, fetch_and_compute_diff, sync_collection from kardscm.commands.utils import _emit_locale_warnings @@ -13,7 +13,6 @@ "add_deck", "add_decks", "export_deck", - "import_deck", "remove_deck", "export_collection", "update_collection", diff --git a/kardscm/commands/baseline.py b/kardscm/commands/baseline.py index c8a1768..55e925a 100644 --- a/kardscm/commands/baseline.py +++ b/kardscm/commands/baseline.py @@ -4,49 +4,63 @@ import json import logging -import shutil -from pathlib import Path +from typing import cast -from kardscm.scraping import baseline +from kardscm.commands.sync import OBSERVED_SNAPSHOT_KEY +from kardscm.constants import DEFAULT_DB_PATH +from kardscm.scraping.baseline import Snapshot, save_baseline +from kardscm.storage import ( + delete_metadata, + get_connection, + get_metadata, + initialize_schema, +) logger = logging.getLogger(__name__) _BASELINE_REQUIRED_KEYS = ("card_count", "node_keys", "json_keys", "enum_values") +_REQUIRED_KEY_TYPES: dict[str, tuple[type, str]] = { + "card_count": (int, "an int"), + "node_keys": (list, "a list"), + "json_keys": (dict, "a dict"), + "enum_values": (dict, "a dict"), +} -def baseline_accept() -> None: - """Promote the most recent ``sync-schema-observed-*.json`` to baseline. +def baseline_accept(db_path: str = DEFAULT_DB_PATH) -> None: + """Promote the snapshot stashed by the last halted sync to baseline. - Looks for files matching the pattern in cwd, picks the lexicographically - latest (timestamps are ISO-like and sort correctly), validates its - structural shape (must be a dict with all required snapshot keys of - the correct types), and copies it to the committed baseline location. + The snapshot is the shape the user reviewed when the sync halted, so + accepting adopts exactly that — not whatever the API serves right now. + + Args: + db_path: SQLite database path. """ - candidates = sorted(Path.cwd().glob("sync-schema-observed-*.json")) - if not candidates: + with get_connection(db_path) as conn: + initialize_schema(conn, db_path) + raw = get_metadata(conn, OBSERVED_SNAPSHOT_KEY) + + if raw is None: raise SystemExit( - "No sync-schema-observed-*.json files found in current directory. " - "Run `kardscm sync` first." + "No drifted API shape is waiting to be accepted. " + "Run `kardscm sync` first — accept only applies after a sync halts on drift." ) - latest = candidates[-1] + try: - parsed = json.loads(latest.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: - raise SystemExit(f"Cannot parse {latest}: {exc}") from exc + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + raise SystemExit(f"Stored snapshot is not valid JSON: {exc}") from exc if not isinstance(parsed, dict): - raise SystemExit(f"{latest} is not a snapshot object (got {type(parsed).__name__}).") + raise SystemExit(f"Stored snapshot is not an object (got {type(parsed).__name__}).") missing = [k for k in _BASELINE_REQUIRED_KEYS if k not in parsed] if missing: - raise SystemExit(f"{latest} is missing required keys: {', '.join(missing)}") - if not isinstance(parsed["card_count"], int): - raise SystemExit(f"{latest}: card_count must be an int") - if not isinstance(parsed["node_keys"], list): - raise SystemExit(f"{latest}: node_keys must be a list") - if not isinstance(parsed["json_keys"], dict): - raise SystemExit(f"{latest}: json_keys must be a dict") - if not isinstance(parsed["enum_values"], dict): - raise SystemExit(f"{latest}: enum_values must be a dict") - - shutil.copy2(latest, baseline.BASELINE_PATH) - logger.info("Baseline updated from %s.", latest.name) + raise SystemExit(f"Stored snapshot is missing required keys: {', '.join(missing)}") + for key, (expected, label) in _REQUIRED_KEY_TYPES.items(): + if not isinstance(parsed[key], expected): + raise SystemExit(f"Stored snapshot: {key} must be {label}") + + save_baseline(cast(Snapshot, parsed)) + with get_connection(db_path) as conn: + delete_metadata(conn, OBSERVED_SNAPSHOT_KEY) + logger.info("Baseline updated from the last halted sync.") diff --git a/kardscm/commands/decks.py b/kardscm/commands/decks.py index 1587231..3f3a026 100644 --- a/kardscm/commands/decks.py +++ b/kardscm/commands/decks.py @@ -63,58 +63,6 @@ def _select_deck(conn: sqlite3.Connection) -> dict: return decks[choice - 1] -def import_deck( - filename: str, - db_path: str = DEFAULT_DB_PATH, - *, - lang: str | None = None, -) -> None: - """Import a deck from TXT file into the database. - - Args: - filename: Path to deck TXT file. - db_path: SQLite database path. - lang: Active language code (e.g. "en", "ru"). Defaults to English. - """ - lang_config = get_language_config(lang) - _emit_locale_warnings(lang_config) - logger.info("Importing deck from file: %s", filename) - - try: - deck = parse_deck_file(filename) - except (FileNotFoundError, ValueError) as e: - raise SystemExit(f"Failed to parse deck file: {e}") from e - - with get_connection(db_path) as conn: - initialize_schema(conn, db_path) - - existing = find_deck_by_name(conn, deck["name"]) - if existing: - raise SystemExit(f"Deck '{deck['name']}' already exists (id={existing['deck_id']})") - - not_found = [] - for card in deck["cards"]: - faction = DECK_NATION_TO_DB.get(card["nation"], card["nation"]) - card_id = find_card_id(conn, faction, card["name"], lang_config.locale_key) - if card_id is None: - not_found.append(f"{faction} / {card['name']}") - - if not_found: - lines = "\n".join(f" - {entry}" for entry in not_found) - raise SystemExit(f"Cards not found in collection:\n{lines}") - - deck_id = insert_deck(conn, deck) - insert_deck_cards( - conn, - deck_id, - deck["cards"], - lang_config.locale_key, - ) - conn.commit() - - logger.info("Deck '%s' imported (%d cards)", deck["name"], len(deck["cards"])) - - def add_deck( filename: str, update: bool = False, @@ -130,7 +78,7 @@ def add_deck( Args: filename: Path to deck TXT file. - update: If True, update collection quantities to match deck. + update: If True, raise collection quantities to the deck's counts. replace: If True, replace existing deck with same name. db_path: SQLite database path. lang: Active language code (e.g. "en", "ru"). Defaults to English. @@ -177,23 +125,24 @@ def add_deck( lines = "\n".join(f" - {entry}" for entry in not_found) raise RuntimeError(f"Cards not found in collection:\n{lines}") - # Quantity check - mismatches: list[tuple[DeckCardEntry, str, int, int]] = [] + # A deck comes from the game client, so it is evidence of ownership: + # a deck using more copies than the collection records means the + # collection is stale. Using fewer copies than owned is normal. + shortfalls: list[tuple[DeckCardEntry, str, int, int]] = [] for card, card_id in resolved: - faction = DECK_NATION_TO_DB.get(card["nation"], card["nation"]) collection_qty = get_card_quantity_by_id(conn, card_id) - if card["quantity"] != collection_qty: - mismatches.append((card, card_id, card["quantity"], collection_qty)) + if card["quantity"] > collection_qty: + shortfalls.append((card, card_id, card["quantity"], collection_qty)) - if mismatches and not update: + if shortfalls and not update: lines = "\n".join( f" - {DECK_NATION_TO_DB.get(c['nation'], c['nation'])} / {c['name']}:" f" deck={deck_qty}, collection={col_qty}" - for c, _, deck_qty, col_qty in mismatches + for c, _, deck_qty, col_qty in shortfalls ) raise RuntimeError( f"Card quantity mismatch:\n{lines}\n" - "Re-run with --update (-u) to update collection quantities." + "Re-run with --update (-u) to raise collection quantities." ) deck_id = insert_deck(conn, deck) @@ -206,8 +155,8 @@ def add_deck( ) conn.commit() - if update and mismatches: - for _, card_id, deck_qty, _ in mismatches: + if update and shortfalls: + for _, card_id, deck_qty, _ in shortfalls: update_card_quantity_by_id(conn, card_id, deck_qty) conn.commit() @@ -286,7 +235,7 @@ def add_decks( Args: filenames: List of paths to deck TXT files. - update: If True, update collection quantities to match deck. + update: If True, raise collection quantities to the deck's counts. replace: If True, replace existing decks with same name. db_path: SQLite database path. lang: Active language code (e.g. "en", "ru"). Defaults to English. diff --git a/kardscm/commands/sync.py b/kardscm/commands/sync.py index 74dc6a0..57d0b46 100644 --- a/kardscm/commands/sync.py +++ b/kardscm/commands/sync.py @@ -2,13 +2,13 @@ from __future__ import annotations +import json import logging from pathlib import Path import typer from kardscm.commands.utils import ( - _default_diff_report_path, _emit_locale_warnings, _safe_timestamp, _utc_timestamp, @@ -23,6 +23,7 @@ ) from kardscm.models import CardDict, DiffReport from kardscm.scraping import ApiContractDriftError, scrape_cards +from kardscm.scraping.baseline import Snapshot, format_drift_report_md from kardscm.storage import ( apply_extra_abilities_seed, delete_cards, @@ -35,6 +36,11 @@ logger = logging.getLogger(__name__) +# Metadata key holding the observed API snapshot from the last halted sync. +# `baseline accept` promotes exactly this shape, so the user adopts what they +# reviewed rather than whatever the API happens to serve at accept time. +OBSERVED_SNAPSHOT_KEY = "drift_observed_snapshot" + _APPROVAL_CATEGORIES = ( ("new", "Apply"), @@ -74,9 +80,9 @@ def fetch_and_compute_diff( ) -> tuple[DiffReport, list[CardDict], str]: """Fetch fresh cards from the website and compute the DB diff. - Pure read path: no DB writes, no console echo, no markdown report. - Web flows call this to drive the preview modal; the CLI orchestrator - composes it with `apply_sync_changes`. + Read path for the collection: no card writes, no console echo, no + markdown report. Web flows call this to drive the preview modal; the + CLI orchestrator composes it with `apply_sync_changes`. Args: db_path: SQLite database path. @@ -86,8 +92,17 @@ def fetch_and_compute_diff( Tuple of (DiffReport, fetched cards, filesystem-safe UTC timestamp). The timestamp is generated once here so every report path the caller writes shares the same identifier. + + Raises: + ApiContractDriftError: The API shape drifted. The observed snapshot + is stashed in metadata first, so `baseline accept` can promote + the very shape the user is about to review. """ - new_cards = scrape_cards(language=lang_config.code, lang_config=lang_config) + try: + new_cards = scrape_cards(language=lang_config.code) + except ApiContractDriftError as exc: + _stash_observed_snapshot(db_path, exc.observed) + raise with get_connection(db_path) as conn: initialize_schema(conn, db_path) old_cards = fetch_cards(conn) @@ -95,6 +110,13 @@ def fetch_and_compute_diff( return report, new_cards, _safe_timestamp() +def _stash_observed_snapshot(db_path: str, observed: Snapshot) -> None: + """Persist the drifted API shape for a later `baseline accept`.""" + with get_connection(db_path) as conn: + initialize_schema(conn, db_path) + set_metadata(conn, OBSERVED_SNAPSHOT_KEY, json.dumps(observed, sort_keys=True)) + + def apply_sync_changes( db_path: str, new_cards: list[CardDict], @@ -111,12 +133,13 @@ def apply_sync_changes( report: Diff report bucketed by category. lang_config: Active language configuration. timestamp: Filesystem-safe UTC timestamp from `fetch_and_compute_diff`. - diff_report_path: Optional override for the markdown report path. - Defaults to `./sync-diff-.md` when a report is written. + diff_report_path: Where to write the markdown report. No report is + written when None — the user has already reviewed the diff on + screen, so a file is only produced on explicit request. Returns: - Path to the markdown report when a non-empty diff was applied, - otherwise None (empty diff → metadata-only update). + Path to the markdown report when one was requested and written, + otherwise None. """ with get_connection(db_path) as conn: initialize_schema(conn, db_path) @@ -132,9 +155,10 @@ def apply_sync_changes( set_metadata(conn, "last_sync", _utc_timestamp()) set_metadata(conn, "language", lang_config.code) - report_path = diff_report_path or _default_diff_report_path() - _write_diff_report(report_path, report, lang_config, timestamp) - return report_path + if diff_report_path is None: + return None + _write_diff_report(diff_report_path, report, lang_config, timestamp) + return diff_report_path def sync_collection( @@ -149,17 +173,16 @@ def sync_collection( Computes a diff between the current DB state and the fresh API pull, prints it, and asks the user to bulk-approve each non-empty category. - Any rejection aborts the sync — the DB is left untouched. The - Markdown diff report is written whenever the diff is non-empty. + Any rejection aborts the sync — the DB is left untouched. Args: db_path: SQLite database path. lang: Active language code (e.g. "en", "ru"). Defaults to English. - diff_only: If True, write the report and return without prompting + diff_only: If True, print the diff and return without prompting or modifying the DB. Useful for previews and CI. yes: If True, auto-approve every category without prompting. - diff_report_path: Override the default report path - (`./sync-diff-.md`). + diff_report_path: Write a Markdown report to this path. Nothing is + written when None; the diff is shown on screen either way. """ lang_config = get_language_config(lang) _emit_locale_warnings(lang_config) @@ -168,12 +191,13 @@ def sync_collection( try: report, new_cards, timestamp = fetch_and_compute_diff(db_path, lang_config) except ApiContractDriftError as exc: - location = exc.report_path or "the current directory" + typer.echo(format_drift_report_md(exc.report, lang_config), err=True) raise SystemExit( f"API contract drift detected ({exc.report.count()} change(s)). " "Sync halted to avoid silently corrupting data.\n" - f"Review {location}, then run `kardscm baseline accept` to adopt the new " - "shape (or fix normalization), and re-run the sync." + "Review the drift above, update any constants or translations it " + "calls for, then run `kardscm baseline accept` to adopt the new " + "shape and re-run the sync." ) from exc if is_empty(report): @@ -182,17 +206,19 @@ def sync_collection( return typer.echo(format_console_report(report, lang_config)) - report_path = diff_report_path or _default_diff_report_path() if diff_only: - _write_diff_report(report_path, report, lang_config, timestamp) - logger.info("Diff report written to %s. No DB changes.", report_path) + if diff_report_path is not None: + _write_diff_report(diff_report_path, report, lang_config, timestamp) + logger.info("Diff report written to %s.", diff_report_path) + logger.info("No DB changes.") return if not yes and not _approve_all_categories(report, lang_config): - _write_diff_report(report_path, report, lang_config, timestamp) - logger.info("Sync aborted by user. Diff report written to %s.", report_path) + logger.info("Sync aborted by user. No DB changes.") return - written = apply_sync_changes(db_path, new_cards, report, lang_config, timestamp, report_path) + written = apply_sync_changes( + db_path, new_cards, report, lang_config, timestamp, diff_report_path + ) logger.info("Sync completed. Stored %s cards. Report: %s", len(new_cards), written) diff --git a/kardscm/commands/utils.py b/kardscm/commands/utils.py index a2eb475..057402a 100644 --- a/kardscm/commands/utils.py +++ b/kardscm/commands/utils.py @@ -3,7 +3,6 @@ from __future__ import annotations from datetime import UTC, datetime -from pathlib import Path import typer @@ -29,7 +28,3 @@ def _emit_locale_warnings(cfg: LanguageConfig) -> None: f"Locale '{cfg.code}': {len(keys)} key(s) fell back to English ({summary}{suffix}).", err=True, ) - - -def _default_diff_report_path() -> Path: - return Path.cwd() / f"sync-diff-{_safe_timestamp()}.md" diff --git a/kardscm/constants.py b/kardscm/constants.py index 3c90111..ff76b60 100644 --- a/kardscm/constants.py +++ b/kardscm/constants.py @@ -76,8 +76,8 @@ "title", "type", "rarity", - "attributes", - "extra_attributes", + "abilities", + "extra_abilities", "set", "quantity", "kredits", @@ -93,8 +93,8 @@ "title": 35, "type": 18, "rarity": 15, - "attributes": 18, - "extra_attributes": 18, + "abilities": 18, + "extra_abilities": 18, "set": 20, "quantity": 10, "kredits": 10, diff --git a/kardscm/diff.py b/kardscm/diff.py index f4fb329..92054a5 100644 --- a/kardscm/diff.py +++ b/kardscm/diff.py @@ -109,7 +109,7 @@ def _card_changes(old: dict, new: CardDict, locale_key: str) -> list[FieldChange if old_abilities != new_abilities: changes.append( { - "field": "attributes", + "field": "abilities", "old": sorted(old_abilities), "new": sorted(new_abilities), } @@ -146,7 +146,7 @@ def _group_by_faction(cards: list[CardDict] | list[dict]) -> dict[str, list]: def _format_value(field: str, value: object, lang_config: LanguageConfig) -> str: """Render an old/new value for display in console + Markdown reports.""" - if field == "attributes": + if field == "abilities": if isinstance(value, list): translated = [lang_config.ability_names.get(str(v), str(v)) for v in value] return "[" + ", ".join(translated) + "]" diff --git a/kardscm/export/collection.py b/kardscm/export/collection.py index 9fed250..39c470f 100644 --- a/kardscm/export/collection.py +++ b/kardscm/export/collection.py @@ -76,7 +76,7 @@ def translate_card_for_export(card: dict, lang_config: LanguageConfig) -> dict: ) # Format curated extra abilities from binary columns - extra_attributes = sanitize_text( + extra_abilities = sanitize_text( ", ".join( lang_config.extra_ability_names.get(a, a) for a in KNOWN_EXTRA_ABILITIES @@ -89,8 +89,8 @@ def translate_card_for_export(card: dict, lang_config: LanguageConfig) -> dict: "title": sanitize_text(title), "type": sanitize_text(type_name), "rarity": sanitize_text(rarity), - "attributes": abilities, - "extra_attributes": extra_attributes, + "abilities": abilities, + "extra_abilities": extra_abilities, "set": sanitize_text(set_name), "quantity": card.get("quantity", 0), "kredits": card.get("kredits", 0), @@ -123,8 +123,8 @@ def build_collection_headers(lang_config: LanguageConfig) -> list[str]: "title": h[1], "type": h[2], "rarity": h[3], - "attributes": h[4], - "extra_attributes": ui["filter_extra_abilities"], + "abilities": h[4], + "extra_abilities": ui["filter_extra_abilities"], "set": h[5], "quantity": h[6], "kredits": h[7], diff --git a/kardscm/scraping/__init__.py b/kardscm/scraping/__init__.py index 9f0831d..7444c24 100644 --- a/kardscm/scraping/__init__.py +++ b/kardscm/scraping/__init__.py @@ -3,45 +3,30 @@ from __future__ import annotations import logging -from datetime import UTC, datetime -from pathlib import Path -from typing import TYPE_CHECKING -from kardscm.locales import LANGUAGE_EN from kardscm.models import CardDict from kardscm.scraping.baseline import ( ApiContractDriftError, build_snapshot, diff_snapshots, - format_drift_report_md, load_baseline, save_baseline, - write_observed, ) from kardscm.scraping.fetcher import fetch_all_cards from kardscm.scraping.normalizer import normalize_card from kardscm.scraping.probe import build_static_probe -if TYPE_CHECKING: - from kardscm.locales import LanguageConfig - logger = logging.getLogger(__name__) -def _drift_report_paths() -> tuple[Path, Path]: - ts = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%SZ") - cwd = Path.cwd() - return cwd / f"sync-schema-diff-{ts}.md", cwd / f"sync-schema-observed-{ts}.json" - - -def _check_api_drift(raw_cards: list[dict], lang_config: LanguageConfig) -> None: +def _check_api_drift(raw_cards: list[dict]) -> None: """Compare observed snapshot vs committed baseline; halt on contract drift. On first run (no baseline file) — initialize the baseline silently and - return. On any contract change, write the drift report + observed snapshot - (a disk-write failure is logged, not fatal) and raise ApiContractDriftError - so the sync stops and the user decides. Benign content growth (new sets, - more cards) is not a contract change and does not reach this raise. + return. On any contract change, raise ApiContractDriftError carrying the + drift and the observed snapshot, so the sync stops and the user decides. + Benign content growth (new sets, more cards) is not a contract change and + does not reach this raise. """ observed = build_snapshot(raw_cards) baseline = load_baseline() @@ -58,46 +43,27 @@ def _check_api_drift(raw_cards: list[dict], lang_config: LanguageConfig) -> None drift = diff_snapshots(baseline, observed) if not drift.has_changes(): return - report_path: Path | None - observed_path: Path | None - report_path, observed_path = _drift_report_paths() - try: - report_path.write_text(format_drift_report_md(drift, lang_config), encoding="utf-8") - write_observed(observed, observed_path) - logger.warning( - "API contract drift detected (%d items). Report: %s | Observed: %s", - drift.count(), - report_path, - observed_path, - ) - except OSError as exc: - logger.warning( - "API contract drift detected (%d items) but report write failed (%s).", - drift.count(), - exc, - ) - report_path = observed_path = None - raise ApiContractDriftError(drift, report_path, observed_path) - - -def scrape_cards(language: str = "en", lang_config: LanguageConfig | None = None) -> list[CardDict]: + logger.warning("API contract drift detected (%d items).", drift.count()) + raise ApiContractDriftError(drift, observed) + + +def scrape_cards(language: str = "en") -> list[CardDict]: """Scrape all cards via probe + GraphQL fetch + normalize. Args: language: GraphQL `$language` value (short code). - lang_config: Optional LanguageConfig for the drift report. If None, - falls back to English (drift detection is silent on missing locale). Returns: List of normalized CardDict objects. + + Raises: + ApiContractDriftError: The API shape diverged from the baseline. """ logger.info("Starting card scrape (language=%s)...", language) probe = build_static_probe(language) raw_cards = fetch_all_cards(probe) - if lang_config is None: - lang_config = LANGUAGE_EN - _check_api_drift(raw_cards, lang_config) + _check_api_drift(raw_cards) cards: list[CardDict] = [] for node in raw_cards: diff --git a/kardscm/scraping/baseline.py b/kardscm/scraping/baseline.py index bb5e062..8073004 100644 --- a/kardscm/scraping/baseline.py +++ b/kardscm/scraping/baseline.py @@ -91,20 +91,15 @@ def count(self) -> int: class ApiContractDriftError(Exception): """Raised when the observed API shape diverges from the committed baseline. - Carries the categorised ``DriftReport`` and the paths of the written drift - report / observed snapshot (``None`` if the disk write failed). The sync - layer catches this to halt and hand the decision to the user. + Carries the categorised ``DriftReport`` and the ``Snapshot`` that produced + it, so the sync layer can both show the drift and stash the exact reviewed + shape for a later ``baseline accept``. Scraping stays free of storage + concerns: persisting the snapshot is the caller's job. """ - def __init__( - self, - report: DriftReport, - report_path: Path | None, - observed_path: Path | None, - ) -> None: + def __init__(self, report: DriftReport, observed: Snapshot) -> None: self.report = report - self.report_path = report_path - self.observed_path = observed_path + self.observed = observed super().__init__(f"API contract drift detected ({report.count()} change(s)).") @@ -274,11 +269,3 @@ def save_baseline(snapshot: Snapshot) -> None: json.dumps(snapshot, indent=2, ensure_ascii=False, sort_keys=True) + "\n", encoding="utf-8", ) - - -def write_observed(snapshot: Snapshot, path: Path) -> None: - """Write a per-sync observed snapshot (for later promotion to baseline).""" - path.write_text( - json.dumps(snapshot, indent=2, ensure_ascii=False, sort_keys=True) + "\n", - encoding="utf-8", - ) diff --git a/kardscm/storage/__init__.py b/kardscm/storage/__init__.py index 100a0e5..cc77fda 100644 --- a/kardscm/storage/__init__.py +++ b/kardscm/storage/__init__.py @@ -8,6 +8,7 @@ delete_all_decks, delete_cards, delete_deck, + delete_metadata, fetch_all_decks, fetch_cards, fetch_deck_cards, @@ -16,6 +17,7 @@ find_deck_by_name, get_card_quantity_by_id, get_connection, + get_metadata, initialize_schema, insert_deck, insert_deck_cards, @@ -34,6 +36,7 @@ "delete_all_decks", "delete_cards", "delete_deck", + "delete_metadata", "fetch_all_decks", "fetch_cards", "fetch_deck_cards", @@ -42,6 +45,7 @@ "find_deck_by_name", "get_card_quantity_by_id", "get_connection", + "get_metadata", "initialize_schema", "insert_deck", "insert_deck_cards", diff --git a/kardscm/storage/cards.py b/kardscm/storage/cards.py index 0afecc2..6d823d0 100644 --- a/kardscm/storage/cards.py +++ b/kardscm/storage/cards.py @@ -2,12 +2,39 @@ from __future__ import annotations +import logging import sqlite3 from collections.abc import Iterable -from kardscm.constants import KNOWN_ABILITIES, KNOWN_EXTRA_ABILITIES +from kardscm.constants import KNOWN_ABILITIES, KNOWN_EXTRA_ABILITIES, RARITY_MAX_QUANTITY from kardscm.models import CardDict +logger = logging.getLogger(__name__) + +_DEFAULT_RARITY_CAP = 4 + + +def _cap_quantity(quantity: int, rarity: str, card_label: str) -> int: + """Clamp a quantity to the rarity's maximum, as the game does. + + Owning more copies than the rarity allows is impossible in-game, so a + higher value is a data error (a mistyped spreadsheet cell, say) rather + than player intent. Clamping keeps the write path forgiving while the + warning tells the user their input was not taken literally. + """ + cap = RARITY_MAX_QUANTITY.get(rarity, _DEFAULT_RARITY_CAP) + if quantity <= cap: + return quantity + logger.warning( + "%s: quantity %d exceeds the %s cap of %d — storing %d.", + card_label, + quantity, + rarity or "unknown rarity", + cap, + cap, + ) + return cap + def upsert_cards(conn: sqlite3.Connection, cards: Iterable[CardDict]) -> None: """Insert or update cards in the database. @@ -158,6 +185,8 @@ def update_quantity( updates: Iterable of (faction_display, localized_title, quantity) tuples. locale_key: Locale key for JSON title extraction (e.g. "en-EN"). + Quantities above the card's rarity cap are clamped (see `_cap_quantity`). + Returns: Tuple of (updated_count, not_found_list). """ @@ -170,17 +199,21 @@ def update_quantity( if qty is None: continue - cursor = conn.execute( - "UPDATE cards SET quantity = ? " + rows = conn.execute( + "SELECT cardId, rarity FROM cards " "WHERE faction = ? " "AND sanitize_text(json_extract(title, ?)) = sanitize_text(?)", - (qty, faction, f'$."{locale_key}"', title), - ) + (faction, f'$."{locale_key}"', title), + ).fetchall() - if cursor.rowcount > 0: - updated += 1 - else: + if not rows: not_found.append(f"{faction} / {title}") + continue + + for card_id, rarity in rows: + capped = _cap_quantity(qty, rarity or "", f"{faction} / {title}") + conn.execute("UPDATE cards SET quantity = ? WHERE cardId = ?", (capped, card_id)) + updated += 1 conn.commit() return updated, not_found @@ -255,9 +288,14 @@ def get_card_quantity_by_id(conn: sqlite3.Connection, card_id: str) -> int: def update_card_quantity_by_id(conn: sqlite3.Connection, card_id: str, quantity: int) -> None: """Update quantity for a card by its ID. + Quantities above the card's rarity cap are clamped (see `_cap_quantity`). + Args: conn: SQLite connection instance. card_id: cardId to update. quantity: New quantity value. """ + row = conn.execute("SELECT rarity FROM cards WHERE cardId = ?", (card_id,)).fetchone() + if row is not None: + quantity = _cap_quantity(quantity, row[0] or "", card_id) conn.execute("UPDATE cards SET quantity = ? WHERE cardId = ?", (quantity, card_id)) diff --git a/kardscm/storage/database.py b/kardscm/storage/database.py index d0a7277..5f24ab3 100644 --- a/kardscm/storage/database.py +++ b/kardscm/storage/database.py @@ -26,7 +26,11 @@ insert_deck, insert_deck_cards, ) -from kardscm.storage.metadata import set_metadata # noqa: F401 +from kardscm.storage.metadata import ( # noqa: F401 + delete_metadata, + get_metadata, + set_metadata, +) from kardscm.storage.schema import ( # noqa: F401 SCHEMA_SQL, _ensure_extra_ability_columns, diff --git a/kardscm/storage/metadata.py b/kardscm/storage/metadata.py index 8e3653a..0a5f535 100644 --- a/kardscm/storage/metadata.py +++ b/kardscm/storage/metadata.py @@ -19,3 +19,28 @@ def set_metadata(conn: sqlite3.Connection, key: str, value: str) -> None: (key, value), ) conn.commit() + + +def get_metadata(conn: sqlite3.Connection, key: str) -> str | None: + """Read a metadata key value. + + Args: + conn: SQLite connection instance. + key: Metadata key. + + Returns: + The stored value, or None if the key is not set. + """ + row = conn.execute("SELECT value FROM metadata WHERE key = ?", (key,)).fetchone() + return row[0] if row else None + + +def delete_metadata(conn: sqlite3.Connection, key: str) -> None: + """Remove a metadata key. + + Args: + conn: SQLite connection instance. + key: Metadata key. + """ + conn.execute("DELETE FROM metadata WHERE key = ?", (key,)) + conn.commit() diff --git a/kardscm/web/routes_collection.py b/kardscm/web/routes_collection.py index 04a3e63..0773fe6 100644 --- a/kardscm/web/routes_collection.py +++ b/kardscm/web/routes_collection.py @@ -13,7 +13,6 @@ from kardscm.constants import ( KNOWN_ABILITIES, KNOWN_EXTRA_ABILITIES, - RARITY_MAX_QUANTITY, ) from kardscm.helpers import extract_locale from kardscm.storage.database import ( @@ -198,8 +197,6 @@ def update_quantity( if row is None: raise HTTPException(status_code=404, detail="card not found") rarity_raw = row[0] or "" - max_qty = RARITY_MAX_QUANTITY.get(rarity_raw, 4) - quantity = min(quantity, max_qty) update_card_quantity_by_id(conn, card_id, quantity) conn.commit() persisted = get_card_quantity_by_id(conn, card_id) diff --git a/kardscm/web/routes_sync.py b/kardscm/web/routes_sync.py index 99baa3d..342b574 100644 --- a/kardscm/web/routes_sync.py +++ b/kardscm/web/routes_sync.py @@ -51,7 +51,7 @@ async def sync_start(request: Request) -> HTMLResponse: "_sync_drift.html", { "ui": cfg.ui_strings, - "report_path": str(exc.report_path) if exc.report_path else None, + "drift": exc.report, "drift_count": exc.report.count(), }, ) diff --git a/kardscm/web/static/main.css b/kardscm/web/static/main.css index e7c8b0a..920b09e 100644 --- a/kardscm/web/static/main.css +++ b/kardscm/web/static/main.css @@ -488,6 +488,23 @@ body.admin-mode .cards-table th.sort-desc { background: #991b1b; } max-height: 80vh; overflow-y: auto; } +.sync-drift-content { + max-width: 640px; + max-height: 80vh; + overflow-y: auto; +} +.sync-drift-details h3 { + margin: 1rem 0 0.35rem; + font-size: 0.95rem; +} +.sync-drift-details ul { + margin: 0; + padding-left: 1.25rem; + color: #374151; +} +.sync-drift-details p { + margin: 0.5rem 0 0.25rem; +} .sync-confirm-body, .sync-empty-body { margin: 0.5rem 0 1rem; diff --git a/kardscm/web/templates/_sync_drift.html b/kardscm/web/templates/_sync_drift.html index 231b177..68020eb 100644 --- a/kardscm/web/templates/_sync_drift.html +++ b/kardscm/web/templates/_sync_drift.html @@ -1,5 +1,5 @@