From 0a29e8cd9bfb5bc4a984235233bc0223bdeaf3ec Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:12:59 +0000 Subject: [PATCH 1/5] Initial plan From e94abe5063ba917afab78fa4ee7d428194d680c4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:17:23 +0000 Subject: [PATCH 2/5] Add database restoration from gzip before processing new auctions Co-authored-by: UltimateBoi <63293118+UltimateBoi@users.noreply.github.com> --- __main__.py | 58 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/__main__.py b/__main__.py index e741de633..8e8745d9b 100644 --- a/__main__.py +++ b/__main__.py @@ -10,6 +10,9 @@ import asyncio import traceback from datetime import datetime +import gzip +import subprocess +from pathlib import Path DECODE_ERROR_LOG = 'decode_errors.log' @@ -32,6 +35,36 @@ def log_decode_error(context, exc): except Exception as log_exc: print(f"Logging failure (ignored): {log_exc}") +def restore_database_from_gzip(db_name): + """Restore a database from its .sql.gz file if it exists and the .db doesn't.""" + db_path = Path(db_name) + gz_path = db_path.with_suffix('.sql.gz') + + # Only restore if gzip exists and db doesn't exist + if gz_path.exists() and not db_path.exists(): + try: + print(f"Restoring {db_name} from {gz_path.name}...") + # Decompress and pipe to sqlite3 + sql_bytes = gzip.decompress(gz_path.read_bytes()) + # Use subprocess to restore the database + result = subprocess.run( + ['sqlite3', str(db_path)], + input=sql_bytes, + capture_output=True, + check=True + ) + print(f"Successfully restored {db_name} from {gz_path.name}") + return True + except Exception as e: + print(f"Warning: Failed to restore {db_name} from {gz_path.name}: {e}") + return False + elif db_path.exists(): + print(f"{db_name} already exists, skipping restoration") + return True + else: + print(f"No backup found for {db_name}, will create fresh database") + return False + def decode_item_bytes(b, context=None): """Decode base64 NBT item bytes into a Python structure; returns None if fails.""" try: @@ -52,7 +85,14 @@ def unpack(tag): def main(): print("Starting...") - # 1. Load config + + # 1. Restore databases from gzip if they exist (before any operations) + print("\n=== Checking for existing database backups ===") + restore_database_from_gzip('database.db') + restore_database_from_gzip('database2.db') + print("=== Database restoration check complete ===\n") + + # 2. Load config with open('options.json') as f: options = json.load(f) @@ -61,7 +101,7 @@ def json_default(o): return base64.b64encode(o).decode('ascii') return str(o) - # 2. Fetch auctions + # 3. Fetch auctions print("Getting auctions...") async def fetch_auctions(): async with aiohttp.ClientSession() as session: @@ -78,7 +118,7 @@ async def fetch_auctions(): auctions = data0.get('auctions', []) auctions = [x for x in auctions if x.get('bin') and x.get('buyer')] - # 3. Decode NBT + # 4. Decode NBT decoded = [] failures = 0 for x in auctions: @@ -101,7 +141,7 @@ async def fetch_auctions(): except Exception as e: print("Error: Failed to write auctions.json: ", e) - # 4. Extract detail.i[0] + # 5. Extract detail.i[0] filtered = [] missing_detail = 0 for x in auctions: @@ -114,7 +154,7 @@ async def fetch_auctions(): print(f"Warning: {missing_detail} decoded item(s) lacked expected structure (logged).") auctions = filtered - # 5. Build processed list + # 6. Build processed list processed = [] for x in auctions: try: @@ -144,7 +184,7 @@ async def fetch_auctions(): log_decode_error({'stage': 'process_record'}, e) auctions = processed - # 6. Create composite + base keys + # 7. Create composite + base keys for a in auctions: parts = [] if a.get('ench2'): @@ -171,7 +211,7 @@ async def fetch_auctions(): a['key'] = a.get('id', 'UNKNOWN') + '.' + '+'.join(parts) a['base_key'] = a.get('id') - # 7. Dump processed JSON snapshots + # 8. Dump processed JSON snapshots try: with open('auctions2.json', 'w') as f: json.dump(auctions, f, indent=4, default=json_default) @@ -181,7 +221,7 @@ async def fetch_auctions(): with open('auctions3.json', 'w') as f: json.dump(auctions3, f, indent=4, default=json_default) - # 8. Insert legacy DB + # 9. Insert legacy DB sql = "INSERT INTO prices (timestamp, itemkey, price) VALUES (?, ?, ?)" conn = sqlite3.connect('database.db') cursor = conn.cursor() @@ -190,7 +230,7 @@ async def fetch_auctions(): cursor.execute(sql, (auction['timestamp'], auction['key'], auction['unitprice'])) conn.commit(); cursor.close(); conn.close() - # 9. Insert new detailed DB + # 10. Insert new detailed DB conn2 = sqlite3.connect('database2.db') c2 = conn2.cursor() c2.execute(""" From 9ddf28a7fda4e84bc09ade208bee1d3d13459917 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:19:09 +0000 Subject: [PATCH 3/5] Improve database restoration security and efficiency Co-authored-by: UltimateBoi <63293118+UltimateBoi@users.noreply.github.com> --- __main__.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/__main__.py b/__main__.py index 8e8745d9b..2e1fe6fd7 100644 --- a/__main__.py +++ b/__main__.py @@ -11,7 +11,6 @@ import traceback from datetime import datetime import gzip -import subprocess from pathlib import Path DECODE_ERROR_LOG = 'decode_errors.log' @@ -44,15 +43,13 @@ def restore_database_from_gzip(db_name): if gz_path.exists() and not db_path.exists(): try: print(f"Restoring {db_name} from {gz_path.name}...") - # Decompress and pipe to sqlite3 - sql_bytes = gzip.decompress(gz_path.read_bytes()) - # Use subprocess to restore the database - result = subprocess.run( - ['sqlite3', str(db_path)], - input=sql_bytes, - capture_output=True, - check=True - ) + # Use streaming decompression for better memory efficiency + with gzip.open(gz_path, 'rt') as gz_file: + sql_script = gz_file.read() + # Use sqlite3 module for safer database restoration + conn = sqlite3.connect(str(db_path)) + conn.executescript(sql_script) + conn.close() print(f"Successfully restored {db_name} from {gz_path.name}") return True except Exception as e: From 79c5adb72580aea4d5e8a0136a7b7b05cb40fc2f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:20:41 +0000 Subject: [PATCH 4/5] Use context manager for database connection in restoration Co-authored-by: UltimateBoi <63293118+UltimateBoi@users.noreply.github.com> --- __main__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/__main__.py b/__main__.py index 2e1fe6fd7..eeab82f3b 100644 --- a/__main__.py +++ b/__main__.py @@ -43,13 +43,12 @@ def restore_database_from_gzip(db_name): if gz_path.exists() and not db_path.exists(): try: print(f"Restoring {db_name} from {gz_path.name}...") - # Use streaming decompression for better memory efficiency + # Decompress and read SQL script with gzip.open(gz_path, 'rt') as gz_file: sql_script = gz_file.read() - # Use sqlite3 module for safer database restoration - conn = sqlite3.connect(str(db_path)) - conn.executescript(sql_script) - conn.close() + # Use sqlite3 module with context manager for safe database restoration + with sqlite3.connect(str(db_path)) as conn: + conn.executescript(sql_script) print(f"Successfully restored {db_name} from {gz_path.name}") return True except Exception as e: From 459d3dd7c3b708c21ce53eeb07a5437eb7966a2c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:22:30 +0000 Subject: [PATCH 5/5] Add comprehensive docstring to restore_database_from_gzip Co-authored-by: UltimateBoi <63293118+UltimateBoi@users.noreply.github.com> --- __main__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/__main__.py b/__main__.py index eeab82f3b..97baf6627 100644 --- a/__main__.py +++ b/__main__.py @@ -35,7 +35,14 @@ def log_decode_error(context, exc): print(f"Logging failure (ignored): {log_exc}") def restore_database_from_gzip(db_name): - """Restore a database from its .sql.gz file if it exists and the .db doesn't.""" + """Restore a database from its .sql.gz file if it exists and the .db doesn't. + + Args: + db_name: Name of the database file to restore (e.g., 'database.db') + + Returns: + True if database was restored or already exists, False if restoration failed + """ db_path = Path(db_name) gz_path = db_path.with_suffix('.sql.gz')