diff --git a/README.rst b/README.rst index 918e42d..767bbc6 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,118 @@ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Plugin for reading statements of Lloyds UK bank -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +ofxstatement-lloyds +=================== -This is a plugin for ofxstatement to convert CSV statements issued by Lloyds UK bank. \ No newline at end of file +Plugin for `ofxstatement `_ to convert +CSV statements exported from Lloyds Bank (UK) into OFX format for import into +accounting software such as Odoo, GnuCash, and others. + +Features +-------- + +Payee extraction + Parses Lloyds transaction descriptions to produce a clean payee name in the + OFX ```` field. Downstream importers use this as the primary + transaction label for display and auto-reconciliation. + +Transaction type preservation + Prefixes the OFX ```` field with the original Lloyds transaction type + code (DD, FPI, FPO, DEB, BGC etc.), preserving detail that is lost in the + standard OFX TRNTYPE mapping. + +Structured memo + Splits the description into payee and remainder, so importers that display + ``NAME : MEMO`` show useful detail rather than duplication. + +Balance tracking + Calculates opening and closing balances from the reverse-chronological CSV, + included in the OFX output for statement validation. + +Supported transaction patterns +------------------------------ + +Card purchases + ``MERCHANT CD nnnn [ddMMMYY]`` extracts the merchant name. + +FX purchases + ``MERCHANT [ref] [CURRENCY] amount VISAXR rate CD nnnn`` extracts the + merchant, with FX details in the memo. + +FX fees + ``NON-GBP TRANS FEE n.nn% CD nnnn`` normalised to + ``Non-GBP Transaction Fee``. + +Faster payments (FPI/FPO/BGC) + ``PAYEE reference sortcode ddMMMYY HH:MM`` extracts the payer or payee + name. + +Direct debits and standing orders (DD/SO) + ``PAYEE mandate-reference`` extracts the payee name. + +Service charges (PAY) + ``SERVICE CHARGES REF : number`` extracts the charge description. + +Unrecognised formats fall back to the full description as payee with the +transaction type code as memo. + +Installation +------------ + +.. code-block:: bash + + pip install ofxstatement-lloyds + +Or with pipx: + +.. code-block:: bash + + pipx inject ofxstatement ofxstatement-lloyds + +Configuration +------------- + +Edit the ofxstatement configuration: + +.. code-block:: bash + + ofxstatement edit-config + +Add a section for your Lloyds account: + +.. code-block:: ini + + [lloyds] + plugin = lloyds + currency = GBP + +Usage +----- + +Export a CSV statement from Lloyds online banking, then convert: + +.. code-block:: bash + + ofxstatement convert -t lloyds statement.csv statement.ofx + +The CSV is expected in the format provided by Lloyds online banking with +columns: date, type, sort code, account number, description, debit, credit, +balance. + +Odoo reconciliation +------------------- + +The payee and memo split is designed to work with Odoo's reconciliation models +and the OCA ``account_statement_completion_label_simple`` module. + +The ``Label`` field in Odoo matches against the OFX ```` (clean payee). +The ``Note`` field matches against ```` (prefixed with the Lloyds +transaction type code). + +Example reconciliation model rules: + +================= ============== ========================================= +Label Contains Note Contains Action +================= ============== ========================================= +``Non-GBP`` Counterpart: bank charges expense account +``DIRECT LINE`` ``DD`` Partner: Direct Line Insurance +\ ``FPI`` Filter: incoming faster payments only +``HMRC`` ``FPO`` Partner: HMRC +================= ============== ========================================= diff --git a/pyproject.toml b/pyproject.toml index 8158a56..5c7b589 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "ofxstatement-lloyds" -version = "1.0.1.dev0" +version = "1.1.2" authors = [ { name="Victoria Lebedeva", email="victoria@lebedev.lt" }, ] @@ -30,4 +30,4 @@ dependencies = [ Homepage = "https://github.com/Metasaura/ofxstatement-lloyds" [project.entry-points."ofxstatement"] -lloyds = "ofxstatement_lloyds.plugin:LloydsPlugin" \ No newline at end of file +lloyds = "ofxstatement_lloyds.plugin:LloydsPlugin" diff --git a/src/ofxstatement_lloyds/plugin.py b/src/ofxstatement_lloyds/plugin.py index 183bd8c..43965b5 100644 --- a/src/ofxstatement_lloyds/plugin.py +++ b/src/ofxstatement_lloyds/plugin.py @@ -1,24 +1,152 @@ -from datetime import date +import datetime +import re from decimal import Decimal from typing import Iterable, Iterator, Optional, TextIO, cast +from ofxstatement.parser import CsvStatementParser from ofxstatement.plugin import Plugin -from ofxstatement.parser import StatementParser from ofxstatement.statement import ( Statement, StatementLine, generate_unique_transaction_id, ) -from ofxstatement.parser import CsvStatementParser + +# Lloyds transaction type to OFX TRNTYPE mapping +TRNTYPE_MAP = { + "BGC": "CREDIT", # Bank Giro Credit + "BP": "DEBIT", # Bill Payment + "CD": "DEBIT", # Card Payment + "CHQ": "CHECK", # Cheque + "COR": "OTHER", # Correction + "CPT": "ATM", # Cashpoint/ATM + "CR": "CREDIT", # Credit + "DD": "DIRECTDEBIT", # Direct Debit + "DEB": "DEBIT", # Debit + "DEP": "DEP", # Deposit + "FEE": "SRVCHG", # Fee + "FPI": "CREDIT", # Faster Payment In + "FPO": "DEBIT", # Faster Payment Out + "PAY": "PAYMENT", # Payment + "SO": "REPEATPMT", # Standing Order + "TFR": "XFER", # Transfer +} + +# Patterns for extracting payee from Lloyds description text. +# Each returns (payee, memo) where memo is the useful remainder. +# Order matters: more specific patterns must come first. + +# FX fee: "NON-GBP TRANS FEE n.nn% CD nnnn [ddMMMYY]" +FX_FEE_RE = re.compile( + r"^(NON-GBP TRANS FEE)\s+([\d.]+%\s+CD\s+\d{4}(?:\s+\d{2}[A-Z]{3}\d{2})?)\s*$" +) + +# FX purchase: merchant then amount VISAXR rate CD nnnn [date] +# Anchors on VISAXR. Optional country/currency code before amount. +FX_PURCHASE_RE = re.compile( + r"^(.+?)\s+(?:(?:[A-Z]{2,5})\s+)?" + r"(\d[\d.]+\s+VISAXR\s+[\d.]+\s+CD\s+\d{4}" + r"(?:\s+\d{2}[A-Z]{3}\d{2})?)\s*$" +) + +# Card payment: "MERCHANT CD nnnn [ddMMMYY]" +CARD_PAYMENT_RE = re.compile(r"^(.+?)\s+(CD\s+\d{4}(?:\s+\d{2}[A-Z]{3}\d{2})?)\s*$") + +# Faster payment (in/out): name then long numeric ref then ddMMMYY HH:MM +# Reference may have RP/FP prefix +FASTER_PAYMENT_RE = re.compile( + r"^(.+?)\s+((?:RP|FP)?\d{9,}[\d\s/A-Z-]*\d{2}[A-Z]{3}\d{2}" r"\s+\d{2}:\d{2})\s*$" +) + +# Service charge: "SERVICE CHARGES REF : number" +SERVICE_CHARGE_RE = re.compile(r"^(.+?)\s+(REF\s*:\s*\d+)\s*$") + +# Direct debit / standing order: name then alphanumeric reference +DD_REF_RE = re.compile(r"^(.+?)\s+([\dA-Z][\dA-Z/-]{7,})\s*$") + + +def extract_payee(description: str, trtype: str) -> tuple[str, str]: + """Extract a clean payee name from the Lloyds description. + + Returns (payee, memo) where payee is the cleaned name and + memo is the useful remainder (reference numbers, FX details). + When no pattern matches, memo is empty to avoid duplication. + """ + desc = description.strip() + + # FX fee (before card payment - both have CD suffix) + m = FX_FEE_RE.match(desc) + if m: + return ("Non-GBP Transaction Fee", m.group(2).strip()) + + # FX purchase (before card payment - both have CD suffix) + m = FX_PURCHASE_RE.match(desc) + if m: + return (m.group(1).strip(), m.group(2).strip()) + + # Card payment + m = CARD_PAYMENT_RE.match(desc) + if m: + return (m.group(1).strip(), m.group(2).strip()) + + # Faster payment (in and out) + if trtype in ("FPI", "FPO", "BGC"): + m = FASTER_PAYMENT_RE.match(desc) + if m: + return (m.group(1).strip(), m.group(2).strip()) + + # Service charge + m = SERVICE_CHARGE_RE.match(desc) + if m: + return (m.group(1).strip(), m.group(2).strip()) + + # Direct debit / standing order + if trtype in ("DD", "SO"): + m = DD_REF_RE.match(desc) + if m: + return (m.group(1).strip(), m.group(2).strip()) + + # Fallback: full description as payee, empty memo + return (desc, "") + + +def parse_amount(debit_str: str, credit_str: str) -> tuple[Decimal, Decimal, Decimal]: + """Parse the separate debit/credit columns into amounts. + + Returns (amount, debit, credit) where amount is the signed + transaction value (negative for debits). + """ + debit = Decimal(debit_str) if debit_str else Decimal("0") + credit = Decimal(credit_str) if credit_str else Decimal("0") + amount = credit - debit + return (amount, debit, credit) + + +def determine_trntype(trtype: str, debit: Decimal, credit: Decimal) -> str: + """Map Lloyds transaction type code to OFX TRNTYPE. + + Falls back to generic DEBIT/CREDIT based on amount direction. + """ + if trtype in TRNTYPE_MAP: + return TRNTYPE_MAP[trtype] + if credit: + return "CREDIT" + return "DEBIT" + + +def clean_sort_code(raw: str) -> str: + """Strip the leading quote that Lloyds adds to prevent + Excel formula interpretation. + """ + return raw.lstrip("'").strip() class LloydsPlugin(Plugin): - """Lloyds plugin (for developers only)""" + """Lloyds UK bank CSV statement plugin""" def get_parser(self, filename: str) -> "LloydsParser": f = open(filename, "r") - abc = LloydsParser(f) # create an instanse of lloyds parser + parser = LloydsParser(f) if "currency" not in self.settings: self.ui.warning("Currency is not set") self.ui.status("") @@ -27,90 +155,80 @@ def get_parser(self, filename: str) -> "LloydsParser": self.ui.status("[lloyds]") self.ui.status("plugin = lloyds") self.ui.status("currency = GBP") - abc.statement.currency = self.settings.get("currency") - return abc + parser.statement.currency = self.settings.get("currency") + return parser class LloydsParser(CsvStatementParser): - mappings = {"date": 0, "memo": 4} + mappings = {"date": 0, "payee": 4, "memo": 4} date_format = "%d/%m/%Y" - start_balance: Optional[Decimal] = None - end_balance: Optional[str] = None - start_date = None - end_date = None def __init__(self, fin: TextIO) -> None: super().__init__(fin) self.uids: set[str] = set() + self.account_id: Optional[str] = None + self.start_balance: Optional[Decimal] = None + self.end_balance: Optional[Decimal] = None + self.start_date: Optional[datetime.datetime] = None + self.end_date: Optional[datetime.datetime] = None def parse(self) -> Statement: stmt = super().parse() stmt.start_date = self.start_date - if self.start_balance is not None: - stmt.start_balance = Decimal(self.start_balance) stmt.end_date = self.end_date + if self.start_balance is not None: + stmt.start_balance = self.start_balance if self.end_balance is not None: - stmt.end_balance = Decimal(self.end_balance) - stmt.account_id = self.account_id + stmt.end_balance = self.end_balance + if self.account_id is not None: + stmt.account_id = self.account_id return stmt def parse_record(self, line: list[str]) -> Optional[StatementLine]: sline = super().parse_record(line) if sline is None: return None - sline.id = generate_unique_transaction_id(sline, self.uids) - account_id = line[3] - debit_str = line[5] - credit_str = line[6] - balance_str = line[7] - self.account_id = account_id - self.start_date = sline.date - - if debit_str == "": - debit = Decimal("0") - else: - debit = self.parse_decimal(debit_str) - - if credit_str == "": - credit = Decimal("0") - else: - credit = self.parse_decimal(credit_str) - sline.amount = -debit + credit + sline.id = generate_unique_transaction_id(sline, self.uids) - if self.end_balance == None: - self.end_balance = balance_str + # Set account_id once from first record + if self.account_id is None: + self.account_id = line[3].strip() + + # Parse amounts from separate debit/credit columns + amount, debit, credit = parse_amount(line[5].strip(), line[6].strip()) + sline.amount = amount + + # Map transaction type + trtype = line[1].strip() + sline.trntype = determine_trntype(trtype, debit, credit) + + # Extract clean payee from description + description = line[4].strip() + payee, memo = extract_payee(description, trtype) + sline.payee = payee + # Prefix memo with Lloyds type code for reconciliation matching + # via the Note field (e.g. "DD 701956574-33938726") + sline.memo = f"{trtype} {memo}".strip() if memo else trtype + + # Track balance and dates. + # Lloyds CSV is reverse chronological (newest first), + # so first record has the end balance/date and last + # record yields the start balance/date. + balance_str = line[7].strip() + balance = self.parse_decimal(balance_str) - if self.end_date == None: + if self.end_balance is None: + self.end_balance = balance self.end_date = sline.date - if debit: - sline.trntype = "DEBIT" - - if credit: - sline.trntype = "CREDIT" - - typemap = dict( - DD="DIRECTDEBIT", - FPI="CREDIT", - BGC="CREDIT", - FPO="DEBIT", - PAY="PAYMENT", - DEB="DEBIT", - SO="REPEATPMT", - COR="OTHER", - TFR="XFER", - ) - trtype = line[1] - if trtype in typemap: - sline.trntype = typemap[trtype] - - balance = self.parse_decimal(balance_str) + # Overwritten each record; final value is the earliest self.start_balance = balance + debit - credit + self.start_date = sline.date return sline def split_records(self) -> Iterable[list[str]]: reader = cast(Iterator[list[str]], super().split_records()) - next(reader) # Skip first line + next(reader) # Skip CSV header row return reader diff --git a/tests/sample-statement.csv b/tests/sample-statement.csv index 48a4936..629d99a 100644 --- a/tests/sample-statement.csv +++ b/tests/sample-statement.csv @@ -1,7 +1,13 @@ Transaction Date,Transaction Type,Sort Code,Account Number,Transaction Description,Debit Amount,Credit Amount,Balance, -15/01/2024,DEB,'99-88-74,1515152252,Ebay*xxxx CD xxxx 14JAN24 ,8.99,,2040.59 -12/01/2024,DEB,'99-88-74,1515152252,NON-GBP TRANS FEE 2.75% CD 1417 ,4.79,,2049.58 -12/01/2024,DEB,'99-88-74,1515152252,OUiog dollaros 202.40 VISAXR 1.16168 CD 1417 ,5975.12,,2054.37 -01/12/2023,FPI,'99-88-74,1515152252,HHHHH LTD RP010110001110 207348 10 01DEC23 16:05 ,,2000,8029.49 -01/12/2023,DD,'99-88-74,1515152252,INS2 01000011/010011110010 ,5.97,,6029.49 -01/12/2023,DD,'99-88-74,1515152252,INS1 1010100110101-10101010 ,4.12,,6039.58 +15/01/2024,DEB,'99-88-74,1515152252,ACME STORE CD 1417 14JAN24,8.99,,2040.59 +12/01/2024,DEB,'99-88-74,1515152252,NON-GBP TRANS FEE 2.75% CD 1417,4.79,,2049.58 +12/01/2024,DEB,'99-88-74,1515152252,OUiog dollaros 202.40 VISAXR 1.16168 CD 1417,5975.12,,2054.37 +01/12/2023,FPI,'99-88-74,1515152252,HHHHH LTD RP555000111222333 207348 10 01DEC23 16:05,,2000,8029.49 +01/12/2023,DD,'99-88-74,1515152252,INS2 01000011/010011110010,5.97,,6029.49 +01/12/2023,DD,'99-88-74,1515152252,INS1 1010100110101-10101010,4.12,,6039.58 +01/12/2023,FPO,'99-88-74,1515152252,HMRC - TAXES 700000009988776655 8833445566B 083210 10 01DEC23 14:49,250.00,,6043.70 +01/12/2023,DEB,'99-88-74,1515152252,HOTEL MARAIS EUROS 78.80 VISAXR 1.0724 CD 1425,73.48,,6293.70 +01/12/2023,PAY,'99-88-74,1515152252,SERVICE CHARGES REF : 998877,12.50,,6367.18 +01/12/2023,DEB,'99-88-74,1515152252,NON-GBP TRANS FEE 2.75% CD 1425 30NOV23,2.02,,6379.68 +01/12/2023,DD,'99-88-74,1515152252,ACME PENSIONS 112233A44556677889,45.00,,6381.70 +01/12/2023,BGC,'99-88-74,1515152252,CLIENT CO LTD 400000005566778899 306364 10 01DEC23 09:15,,500,6426.70 diff --git a/tests/test_sample.py b/tests/test_sample.py index b175c88..71919bc 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -17,35 +17,91 @@ def test_sample() -> None: statement = parser.parse() assert statement is not None - assert len(statement.lines) == 6 + assert len(statement.lines) == 12 - assert statement.lines[0].amount == Decimal("-8.99") - assert statement.lines[3].amount == Decimal("2000") - assert statement.lines[5].date == datetime.datetime(2023, 12, 1) - assert ( - statement.lines[2].memo - == "OUiog dollaros 202.40 VISAXR 1.16168 CD 1417 " - ) - - assert statement.start_balance == Decimal("6043.70") + # Statement-level fields + assert statement.currency == "EUR" + assert statement.account_id == "1515152252" + assert statement.start_balance == Decimal("5926.70") assert statement.end_balance == Decimal("2040.59") assert statement.start_date == datetime.datetime(2023, 12, 1) assert statement.end_date == datetime.datetime(2024, 1, 15) - assert statement.currency == "EUR" - assert statement.account_id == "1515152252" + # [0] Card payment: ACME STORE CD 1417 14JAN24 + assert statement.lines[0].amount == Decimal("-8.99") + assert statement.lines[0].trntype == "DEBIT" + assert statement.lines[0].payee == "ACME STORE" + assert statement.lines[0].memo == "DEB CD 1417 14JAN24" + # [1] FX fee: NON-GBP TRANS FEE 2.75% CD 1417 + assert statement.lines[1].amount == Decimal("-4.79") assert statement.lines[1].trntype == "DEBIT" - assert statement.lines[5].trntype == "DIRECTDEBIT" + assert statement.lines[1].payee == "Non-GBP Transaction Fee" + assert statement.lines[1].memo == "DEB 2.75% CD 1417" + + # [2] FX purchase: OUiog dollaros 202.40 VISAXR 1.16168 CD 1417 + assert statement.lines[2].amount == Decimal("-5975.12") + assert statement.lines[2].trntype == "DEBIT" + assert statement.lines[2].payee == "OUiog dollaros" + assert statement.lines[2].memo == "DEB 202.40 VISAXR 1.16168 CD 1417" + # [3] Faster payment in: HHHHH LTD RP... 01DEC23 16:05 + assert statement.lines[3].amount == Decimal("2000") + assert statement.lines[3].trntype == "CREDIT" + assert statement.lines[3].payee == "HHHHH LTD" + assert ( + statement.lines[3].memo == "FPI RP555000111222333 207348 10 01DEC23 16:05" + ) -def sum2num(x, y): - return x + y + # [4] Direct debit: INS2 01000011/010011110010 + assert statement.lines[4].amount == Decimal("-5.97") + assert statement.lines[4].trntype == "DIRECTDEBIT" + assert statement.lines[4].payee == "INS2" + assert statement.lines[4].memo == "DD 01000011/010011110010" + # [5] Direct debit: INS1 1010100110101-10101010 + assert statement.lines[5].amount == Decimal("-4.12") + assert statement.lines[5].trntype == "DIRECTDEBIT" + assert statement.lines[5].payee == "INS1" + assert statement.lines[5].memo == "DD 1010100110101-10101010" -def test_iop(): - h = 2 - 5 - h = h + 5 * 19 + 8 - assert h == 100 + # [6] Faster payment out: HMRC - TAXES 700... 01DEC23 14:49 + assert statement.lines[6].amount == Decimal("-250.00") + assert statement.lines[6].trntype == "DEBIT" + assert statement.lines[6].payee == "HMRC - TAXES" + assert ( + statement.lines[6].memo + == "FPO 700000009988776655 8833445566B 083210 10 01DEC23 14:49" + ) - assert sum2num(5, 9) == 14 + # [7] FX purchase with country code: HOTEL MARAIS EUROS 78.80 VISAXR... + assert statement.lines[7].amount == Decimal("-73.48") + assert statement.lines[7].trntype == "DEBIT" + assert statement.lines[7].payee == "HOTEL MARAIS" + assert statement.lines[7].memo == "DEB 78.80 VISAXR 1.0724 CD 1425" + + # [8] Service charge: SERVICE CHARGES REF : 998877 + assert statement.lines[8].amount == Decimal("-12.50") + assert statement.lines[8].trntype == "PAYMENT" + assert statement.lines[8].payee == "SERVICE CHARGES" + assert statement.lines[8].memo == "PAY REF : 998877" + + # [9] FX fee with date: NON-GBP TRANS FEE 2.75% CD 1425 30NOV23 + assert statement.lines[9].amount == Decimal("-2.02") + assert statement.lines[9].trntype == "DEBIT" + assert statement.lines[9].payee == "Non-GBP Transaction Fee" + assert statement.lines[9].memo == "DEB 2.75% CD 1425 30NOV23" + + # [10] Direct debit with alphanumeric ref: ACME PENSIONS 112233... + assert statement.lines[10].amount == Decimal("-45.00") + assert statement.lines[10].trntype == "DIRECTDEBIT" + assert statement.lines[10].payee == "ACME PENSIONS" + assert statement.lines[10].memo == "DD 112233A44556677889" + + # [11] Bank giro credit: CLIENT CO LTD 400... 01DEC23 09:15 + assert statement.lines[11].amount == Decimal("500") + assert statement.lines[11].trntype == "CREDIT" + assert statement.lines[11].payee == "CLIENT CO LTD" + assert ( + statement.lines[11].memo == "BGC 400000005566778899 306364 10 01DEC23 09:15" + )