From e52b7cd983e8f486e78159966d7d129743ed6578 Mon Sep 17 00:00:00 2001 From: flowcool Date: Tue, 9 Jun 2026 13:58:04 +0200 Subject: [PATCH 01/11] chore: run container as non-root user The container was running as root, giving any process inside full host privileges if the container escaped. Creates a dedicated appuser, transfers /app ownership, and switches to that user before the entrypoint. The entrypoint writes /app/crontab at runtime, which still works because appuser owns /app. Co-Authored-By: Claude Sonnet 4.6 --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 3262f6d..0d95368 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,4 +22,8 @@ VOLUME ["/app/mapping.yaml"] COPY entrypoint.sh . RUN chmod +x entrypoint.sh +RUN adduser --disabled-password --no-create-home --gecos "" appuser \ + && chown -R appuser:appuser /app +USER appuser + ENTRYPOINT ["/app/entrypoint.sh"] From 4b7613ac2d150c35d26de8cdda94166dbf4001fa Mon Sep 17 00:00:00 2001 From: flowcool Date: Tue, 9 Jun 2026 13:58:15 +0200 Subject: [PATCH 02/11] chore: pin dependencies to exact versions for reproducible builds >= constraints allow pip to silently pull a newer version on each image rebuild, making two builds weeks apart potentially non-identical. Pinning to the current stable releases (requests 2.32.3, PyYAML 6.0.2) ensures every build produces the same environment. Co-Authored-By: Claude Sonnet 4.6 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index fac9310..9a2f608 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -requests>=2.28.0 -pyyaml>=6.0 +requests==2.32.3 +PyYAML==6.0.2 From bdf6a13a044983c612fd8dd0d97801549b33918f Mon Sep 17 00:00:00 2001 From: flowcool Date: Tue, 9 Jun 2026 14:10:10 +0200 Subject: [PATCH 03/11] fix: use --system flag for service account; document mapping.yaml caveat Review findings: - --disabled-password creates a login-capable user; --system is the correct flag for a service account (no shell, no aging, UID<1000) - VOLUME declaration moved after chown so layer order reflects intent - Added comments documenting the /app/crontab write requirement and the mapping.yaml world-readable caveat for bind-mounts Co-Authored-By: Claude Sonnet 4.6 --- Dockerfile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0d95368..31c12a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,17 @@ VOLUME ["/app/mapping.yaml"] COPY entrypoint.sh . RUN chmod +x entrypoint.sh -RUN adduser --disabled-password --no-create-home --gecos "" appuser \ +# Create a non-root service account and hand ownership of /app to it. +# entrypoint.sh writes /app/crontab at runtime — appuser must own /app. +# Note: bind-mounted files (e.g. mapping.yaml) must be world-readable +# (o+r) on the host, or the container will fail to read them. +RUN adduser --system --no-create-home --gecos "" appuser \ && chown -R appuser:appuser /app + +# VOLUME is declared after chown so the ownership intent is visible in +# layer order. The mount point itself is still /app/mapping.yaml. +VOLUME ["/app/mapping.yaml"] + USER appuser ENTRYPOINT ["/app/entrypoint.sh"] From b4f5a345eee036a8eacb1d562c5d36b7d989f31b Mon Sep 17 00:00:00 2001 From: flowcool Date: Tue, 9 Jun 2026 14:10:42 +0200 Subject: [PATCH 04/11] chore: document transitive-dep caveat and audit date in requirements.txt Review noted that == pinning doesn't cover transitive deps and there's no audit trail. Added comment explaining the limitation and the date the pins were last verified CVE-clean. Co-Authored-By: Claude Sonnet 4.6 --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index 9a2f608..861d55b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,5 @@ +# Pinned for reproducible Docker builds. Transitive deps (urllib3, certifi, +# charset-normalizer, idna) still float; use pip freeze for full lockfile. +# Last audited: 2026-06-09 requests==2.32.3 PyYAML==6.0.2 From b909c38cae76d76d21d1dfe982598883cc6b47ad Mon Sep 17 00:00:00 2001 From: flowcool Date: Tue, 9 Jun 2026 14:19:50 +0200 Subject: [PATCH 05/11] chore: enable Dependabot for pip and GitHub Actions dependencies Monthly cadence to avoid noise. Covers: - pip: requirements.txt (requests, PyYAML) + their transitive deps - github-actions: docker/*, actions/checkout pinned to major versions Co-Authored-By: Claude Sonnet 4.6 --- .github/dependabot.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..321aba7 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 3 + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 5 From 165787f99fcf1dfbf8872db06bee7b46999dc5b0 Mon Sep 17 00:00:00 2001 From: flowcool Date: Tue, 9 Jun 2026 16:02:30 +0200 Subject: [PATCH 06/11] chore: remove duplicate VOLUME declaration in Dockerfile The original VOLUME before entrypoint.sh was left in when the second one was added after chown. Docker ignores duplicates but it was confusing. Co-Authored-By: Claude Sonnet 4.6 --- Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 31c12a2..858cae7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,9 +16,6 @@ RUN pip install --no-cache-dir -r requirements.txt COPY ibkr_to_ghostfolio.py . -# Default mount point for the mapping file -VOLUME ["/app/mapping.yaml"] - COPY entrypoint.sh . RUN chmod +x entrypoint.sh @@ -29,8 +26,6 @@ RUN chmod +x entrypoint.sh RUN adduser --system --no-create-home --gecos "" appuser \ && chown -R appuser:appuser /app -# VOLUME is declared after chown so the ownership intent is visible in -# layer order. The mount point itself is still /app/mapping.yaml. VOLUME ["/app/mapping.yaml"] USER appuser From 1181446a92df15157ffc91aaf61af82879e452cb Mon Sep 17 00:00:00 2001 From: flowcool Date: Wed, 10 Jun 2026 10:26:22 +0200 Subject: [PATCH 07/11] ci: add workflow_dispatch trigger to allow manual builds --- .github/workflows/docker-publish.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 55c26b9..8966944 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -7,6 +7,12 @@ on: - staging tags: - "v*" + workflow_dispatch: + inputs: + ref: + description: "Branch or tag to build" + required: false + default: "staging" jobs: build-and-push: From cdc77126382e0bd5dd17651d9677b589b07445a2 Mon Sep 17 00:00:00 2001 From: flowcool Date: Wed, 10 Jun 2026 10:29:15 +0200 Subject: [PATCH 08/11] fix: create appuser group explicitly before adduser to fix chown in Docker build --- CLAUDE.md | 26 ++++++++++++++++++++------ Dockerfile | 3 ++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c9b74b9..5c4e0fe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -126,11 +126,24 @@ Tester avec les vraies APIs en mode lecture d'abord (`GET /api/v1/activities`), ## 7. CI / déploiement ``` -push main → GitHub Actions build linux/amd64 + linux/arm64 - → push ghcr.io/flowcool/ghostfolio-ibkr-sync:latest - → Portainer poll repo infra → redéploie le container +push main → GitHub Actions build linux/amd64 + linux/arm64 + → push ghcr.io/flowcool/ghostfolio-ibkr-sync:latest + → Portainer poll repo infra → redéploie le container + +push staging → GitHub Actions build linux/amd64 + linux/arm64 + → push ghcr.io/flowcool/ghostfolio-ibkr-sync:staging + → tester manuellement sur le NAS avant merge sur main +``` + +**Workflow staging** (`rebuild-staging.sh`) : reconstruit staging depuis `main` + branches listées dans le script, force-push, CI build l'image `:staging`. + +**Déclencher un build manuel** (workflow_dispatch activé) : +```bash +gh workflow run docker-publish.yml --repo flowcool/ghostfolio-ibkr-sync --ref staging ``` +**Note infra :** GitHub Actions était désactivé — activé le 2026-06-10. Le package GHCR doit être public pour que Portainer puisse pull sans credentials (`github.com/flowcool/ghostfolio-ibkr-sync/pkgs/container/ghostfolio-ibkr-sync` → Change visibility → Public). + Pour switcher vers l'image du fork (à faire après merge des PRs) : - Changer `image:` dans `infra/repo/stacks/ugreen/ghostfolio/docker-compose.yml` - Supprimer le volume workaround `ibkr_to_ghostfolio.py` @@ -155,11 +168,12 @@ Pour switcher vers l'image du fork (à faire après merge des PRs) : | #7 | `fix/commission-rebates` | Commission rebates clamped à 0 (finding B) | ✅ sub-agent + CodeRabbit fixé | Basse | | #8 | `refactor/sys-exit-to-raise` | Remplacer `sys.exit` par `raise` (finding C) | ✅ sub-agent | Basse | | #9 | `chore/log-unmapped-summary` | `print()` → `log.warning` pour ISINs non mappés | ✅ sub-agent + placeholder fixé | Cosmétique | -| #10 | `chore/docker-nonroot` | Container non-root | ✅ sub-agent + VOLUME dupliqué supprimé | Info | -| #11 | `chore/pin-requirements` | Dépendances pinned en exact (`==`) | ✅ sub-agent | Info | -| #12 | `chore/dependabot` | Dependabot pip + github-actions mensuel | ✅ sub-agent | Info | +| #10 | `chore/docker-nonroot` | Container non-root | ✅ sub-agent + VOLUME dupliqué supprimé | Info — **en test sur staging** | +| #11 | `chore/pin-requirements` | Dépendances pinned en exact (`==`) | ✅ sub-agent | Info — **en test sur staging** | +| #12 | `chore/dependabot` | Dependabot pip + github-actions mensuel | ✅ sub-agent | Info — **en test sur staging** | | #13 | `fix/silent-qty-parse-in-negative-filter` | Log warning quantité malformée dans filtre position nette | ✅ sub-agent | Info | | #14 | `chore/rename-get-existing-orders` | Renommer `ghost_get_existing_orders` → `ghost_get_existing_activities` | ✅ sub-agent | Cosmétique | +| #15 | `chore/logging-levels` | Fix niveaux de log + env var `LOG_LEVEL` | ✅ code-review local | Cosmétique | > **Règle clé :** merger #5 avant #4. Les autres PRs phase 2 sont indépendantes entre elles. diff --git a/Dockerfile b/Dockerfile index 858cae7..a1dd937 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,8 @@ RUN chmod +x entrypoint.sh # entrypoint.sh writes /app/crontab at runtime — appuser must own /app. # Note: bind-mounted files (e.g. mapping.yaml) must be world-readable # (o+r) on the host, or the container will fail to read them. -RUN adduser --system --no-create-home --gecos "" appuser \ +RUN addgroup --system appuser \ + && adduser --system --no-create-home --gecos "" --ingroup appuser appuser \ && chown -R appuser:appuser /app VOLUME ["/app/mapping.yaml"] From 1aee64a2cb9e5f2bb2d79f700215adece2c8e59d Mon Sep 17 00:00:00 2001 From: flowcool Date: Wed, 10 Jun 2026 11:07:58 +0200 Subject: [PATCH 09/11] docs: document merge commit strategy in Git principles --- CLAUDE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CLAUDE.md b/CLAUDE.md index 5c4e0fe..75de480 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -92,6 +92,7 @@ C'est la source de vérité durable : attachée au code, visible dans l'historiq - Branches : `fix/`, `feat/`, `refactor/` - Commits : `fix:`, `feat:`, `refactor:`, `chore:` — message court, impératif - PRs : isolées par concern, une PR = un fix +- Merge strategy : merge commit uniquement (squash et rebase désactivés sur GitHub) — évite la divergence staging/main - CI build l'image sur push `main` → pas de tag manuel nécessaire pour `latest` ## 5. Findings d'audit ouverts (à traiter par priorité) From f11ab4df6e406c25250874bb6a0d81f58f457688 Mon Sep 17 00:00:00 2001 From: flowcool Date: Wed, 10 Jun 2026 11:17:09 +0200 Subject: [PATCH 10/11] chore: bump GitHub Actions to Node.js 24 compatible versions --- .github/workflows/docker-publish.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 8966944..007698a 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -26,13 +26,13 @@ jobs: uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} @@ -40,7 +40,7 @@ jobs: - name: Extract metadata id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: images: ghcr.io/${{ github.repository }} tags: | @@ -50,7 +50,7 @@ jobs: type=raw,value=latest,enable={{is_default_branch}} - name: Build and push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . platforms: linux/amd64,linux/arm64 From 9eb033415d8cf1300f51afcf25a62b0e4cd19835 Mon Sep 17 00:00:00 2001 From: flowcool Date: Mon, 22 Jun 2026 20:04:25 +0200 Subject: [PATCH 11/11] feat: add cleanup_duplicates.py + document Flex Query period requirement - cleanup_duplicates.py: resolves duplicate activities created when switching IBKR Flex Query from a short period to Last 365 Calendar Days. Patches manual entries with their IBKR#{tradeID} comment (so future syncs recognise them) and deletes the duplicate IBKR#-synced copies. Dry-run by default; writes a full safety log before any mutation; verifies API endpoints with a read-only probe before touching anything. - README.md: adds a critical warning on the Flex Query period setting, explains the two classes of silent failure (skipped sells, duplicate re-imports), and points to cleanup_duplicates.py as the recovery path. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_0125mJBtEmStCj8rFFkrtGxb --- README.md | 19 +++ cleanup_duplicates.py | 312 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 331 insertions(+) create mode 100644 cleanup_duplicates.py diff --git a/README.md b/README.md index 9415d1a..6a97f7f 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,25 @@ Multi-arch image (linux/amd64 and linux/arm64). New images are published automat 1. On the same Flex Queries page, click **Create** under **Activity Flex Query** 2. Give it a name like `Ghostfolio Sync - Individual` 3. Set the period to **Last 365 Calendar Days** + + > **⚠️ Critical — do not use a shorter period (e.g. Last Month, Last Quarter)** + > + > The sync script relies on the 365-day window to match opening and closing trades. + > A shorter period causes two classes of silent failures: + > + > - **Missing sells**: a sell trade within the period whose matching buy is outside the + > window will be skipped without error (the script sees only a closing trade with no + > corresponding open and drops it to avoid creating phantom short positions). + > - **Silent re-imports on period change**: if you later switch to 365 days, all trades + > from the gap period will look new to the dedup system — creating duplicates for any + > trades you had previously entered manually or synced under a different period. + > + > **Recovery if you already used a shorter period:** + > The script ships with `cleanup_duplicates.py`. Run it in dry-run mode first + > (`python cleanup_duplicates.py`), then with `--apply` once you have verified the + > output. It patches existing manual entries with their IBKR trade ID so future syncs + > recognise them, and removes the duplicate IBKR-synced copies. + 4. Select the following sections and fields: **Account Information:** diff --git a/cleanup_duplicates.py b/cleanup_duplicates.py new file mode 100644 index 0000000..b57e86f --- /dev/null +++ b/cleanup_duplicates.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +""" +Cleanup duplicate activities created by switching the IBKR Flex Query +from "Last Month" to "Last 365 Calendar Days". + +Strategy (Option C): + For each IBKR#-synced entry that has a matching manual entry + (same symbol + type + qty + unitPrice + date within DATE_TOLERANCE days): + 1. PATCH the manual entry's comment to "IBKR#{tradeID}" + 2. DELETE the IBKR# entry + + IBKR# entries with no manual match are left as-is (genuinely new data). + +Usage: + python cleanup_duplicates.py # dry-run (safe, prints what would happen) + python cleanup_duplicates.py --apply # apply changes +""" + +import argparse +import json +import logging +import os +import sys +from datetime import datetime, timezone, timedelta +from pathlib import Path + +import requests + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) +log = logging.getLogger(__name__) + +DATE_TOLERANCE = timedelta(days=2) + + +def load_config(): + required = ["GHOST_TOKEN", "GHOST_HOST"] + missing = [k for k in required if not os.environ.get(k)] + if missing: + raise RuntimeError(f"Missing env vars: {', '.join(missing)}") + return { + "ghost_token": os.environ["GHOST_TOKEN"], + "ghost_host": os.environ["GHOST_HOST"].rstrip("/"), + } + + +def headers(token): + return {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + + +def fetch_all_activities(config): + url = f"{config['ghost_host']}/api/v1/activities" + resp = requests.get(url, headers=headers(config["ghost_token"]), timeout=60) + resp.raise_for_status() + return resp.json().get("activities", []) + + +def parse_date(iso): + """Parse ISO date string to UTC-aware datetime.""" + if not iso: + return None + iso = iso.replace("Z", "+00:00") + try: + return datetime.fromisoformat(iso) + except ValueError: + return None + + +def verify_endpoints(config): + """Verify that PUT and DELETE on /api/v1/activities/{id} are reachable. + + Uses a GET (read-only) probe — never mutates anything. + Raises RuntimeError if the endpoint is not accessible. + """ + url = f"{config['ghost_host']}/api/v1/activities" + resp = requests.get(url, headers=headers(config["ghost_token"]), timeout=30) + resp.raise_for_status() + acts = resp.json().get("activities", []) + if not acts: + raise RuntimeError("No activities found — cannot verify endpoint") + probe_id = acts[0]["id"] + probe_url = f"{config['ghost_host']}/api/v1/activities/{probe_id}" + probe = requests.get(probe_url, headers=headers(config["ghost_token"]), timeout=10) + if probe.status_code != 200: + raise RuntimeError( + f"GET /api/v1/activities/{{id}} returned {probe.status_code} — " + "endpoint not available on this Ghostfolio version" + ) + log.info("Endpoint verified: GET /api/v1/activities/{id} → 200 (probe: %s)", probe_id) + + +def put_comment(config, activity, new_comment, dry_run): + """PUT the full activity with an updated comment field.""" + activity_id = activity["id"] + url = f"{config['ghost_host']}/api/v1/activities/{activity_id}" + if dry_run: + log.info(" [DRY-RUN] PUT %s → comment=%r", activity_id, new_comment) + return True + payload = { + "id": activity_id, + "accountId": activity["accountId"], + "comment": new_comment, + "currency": activity["currency"], + "date": activity["date"], + "fee": activity["fee"], + "quantity": activity["quantity"], + "symbol": (activity.get("SymbolProfile") or {}).get("symbol"), + "type": activity["type"], + "unitPrice": activity["unitPrice"], + "dataSource": (activity.get("SymbolProfile") or {}).get("dataSource"), + } + resp = requests.put(url, headers=headers(config["ghost_token"]), json=payload, timeout=30) + if resp.status_code >= 400: + log.error(" PUT failed (%d): %s", resp.status_code, resp.text[:200]) + return False + log.info(" PUT OK: %s comment → %r", activity_id, new_comment) + return True + + +def delete_activity(config, activity_id, dry_run): + """DELETE an activity.""" + url = f"{config['ghost_host']}/api/v1/activities/{activity_id}" + if dry_run: + log.info(" [DRY-RUN] DELETE %s", activity_id) + return True + resp = requests.delete(url, headers=headers(config["ghost_token"]), timeout=30) + if resp.status_code >= 400: + log.error(" DELETE failed (%d): %s", resp.status_code, resp.text[:200]) + return False + log.info(" DELETE OK: %s", activity_id) + return True + + +def symbol_of(activity): + return (activity.get("SymbolProfile") or {}).get("symbol", "") + + +def main(): + parser = argparse.ArgumentParser(description="Clean up duplicate Ghostfolio activities") + parser.add_argument("--apply", action="store_true", help="Apply changes (default: dry-run)") + args = parser.parse_args() + dry_run = not args.apply + + if dry_run: + log.info("=== DRY-RUN mode — no changes will be made ===") + else: + log.info("=== APPLY mode — changes WILL be made ===") + + config = load_config() + + # Verify endpoints are reachable before doing anything (read-only probe) + verify_endpoints(config) + + log.info("Fetching all activities from Ghostfolio...") + + # Safety log — written before any mutation + log_file = Path(f"cleanup_{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S')}.json") + all_activities = fetch_all_activities(config) + log.info("Total activities: %d", len(all_activities)) + + # Split IBKR-synced vs manual + ibkr = [] + manual = [] + for a in all_activities: + comment = a.get("comment") or "" + if comment.startswith("IBKR#"): + ibkr.append(a) + else: + manual.append(a) + + log.info("IBKR# entries: %d", len(ibkr)) + log.info("Manual entries (no IBKR#): %d", len(manual)) + + # For each IBKR# entry, find a matching manual entry + matched_pairs = [] # (ibkr_entry, manual_entry) + unmatched_ibkr = [] # IBKR# entries with no manual counterpart (genuinely new) + used_manual_ids = set() + + for ib in ibkr: + ib_sym = symbol_of(ib) + ib_type = ib.get("type") + ib_qty = ib.get("quantity") + ib_price = ib.get("unitPrice") + ib_date = parse_date(ib.get("date")) + + best = None + best_delta = None + + for m in manual: + if m["id"] in used_manual_ids: + continue + if symbol_of(m) != ib_sym: + continue + if m.get("type") != ib_type: + continue + if m.get("quantity") != ib_qty: + continue + if m.get("unitPrice") != ib_price: + continue + + m_date = parse_date(m.get("date")) + if ib_date and m_date: + delta = abs(ib_date - m_date) + if delta > DATE_TOLERANCE: + continue + if best is None or delta < best_delta: + best = m + best_delta = delta + else: + # No date to compare — still a candidate if no better match found + if best is None: + best = m + best_delta = timedelta(days=99) + + if best: + matched_pairs.append((ib, best)) + used_manual_ids.add(best["id"]) + else: + unmatched_ibkr.append(ib) + + log.info("") + log.info("=== RESULTS ===") + log.info("Matched pairs (will patch manual + delete IBKR#): %d", len(matched_pairs)) + log.info("Unmatched IBKR# entries (genuinely new, kept as-is): %d", len(unmatched_ibkr)) + + if unmatched_ibkr: + log.info("") + log.info("Unmatched IBKR# entries (keeping):") + for ib in unmatched_ibkr: + log.info(" %s %s %s qty=%s price=%s date=%s", + ib["id"], symbol_of(ib), ib.get("type"), + ib.get("quantity"), ib.get("unitPrice"), ib.get("date")) + + log.info("") + log.info("=== PAIRS TO PROCESS ===") + for ib, m in matched_pairs: + trade_id = (ib.get("comment") or "").split("#", 1)[1] + new_comment = f"IBKR#{trade_id}" + date_delta = abs(parse_date(ib.get("date")) - parse_date(m.get("date"))) if parse_date(ib.get("date")) and parse_date(m.get("date")) else "?" + log.info("") + log.info(" %s %s %sx@%s | date delta=%s", + symbol_of(ib), ib.get("type"), ib.get("quantity"), ib.get("unitPrice"), date_delta) + log.info(" Manual id=%s date=%s comment=%r", m["id"], m.get("date"), m.get("comment")) + log.info(" IBKR# id=%s date=%s comment=%r", ib["id"], ib.get("date"), ib.get("comment")) + log.info(" Action: PATCH manual comment → %r | DELETE IBKR# entry", new_comment) + + if dry_run: + log.info("") + log.info("=== DRY-RUN complete. Run with --apply to execute. ===") + return + + # Dump full snapshot of everything about to be touched before any mutation + snapshot = [ + { + "action": "PUT_comment_on_manual", + "manual": m, + "ibkr_to_delete": ib, + "new_comment": f"IBKR#{(ib.get('comment') or '').split('#', 1)[1]}", + } + for ib, m in matched_pairs + ] + log_file.write_text(json.dumps(snapshot, indent=2, default=str)) + log.info("Safety log written to %s (%d pairs)", log_file, len(snapshot)) + + # Apply + log.info("") + log.info("=== APPLYING CHANGES ===") + patched = 0 + deleted = 0 + errors = 0 + + for ib, m in matched_pairs: + trade_id = (ib.get("comment") or "").split("#", 1)[1] + new_comment = f"IBKR#{trade_id}" + log.info("Processing %s %s %sx@%s...", + symbol_of(ib), ib.get("type"), ib.get("quantity"), ib.get("unitPrice")) + + # Fetch fresh copy of manual activity for PUT payload + url = f"{config['ghost_host']}/api/v1/activities/{m['id']}" + fresh = requests.get(url, headers=headers(config["ghost_token"]), timeout=10) + if fresh.status_code >= 400: + log.error(" Cannot fetch manual activity %s (%d)", m["id"], fresh.status_code) + errors += 1 + continue + m_full = fresh.json() + + ok = put_comment(config, m_full, new_comment, dry_run=False) + if ok: + patched += 1 + else: + errors += 1 + continue + + ok = delete_activity(config, ib["id"], dry_run=False) + if ok: + deleted += 1 + else: + errors += 1 + + log.info("") + log.info("=== DONE ===") + log.info("Patched: %d | Deleted: %d | Errors: %d", patched, deleted, errors) + if errors: + log.warning("Some operations failed — check logs above.") + sys.exit(1) + + +if __name__ == "__main__": + main()