From 2d8dfd2be95e87c93414854764572f80a6f99d10 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Mon, 16 Mar 2026 16:14:29 +0530 Subject: [PATCH 01/88] feat: add Parser Benchmark module --- transaction_parser/modules.txt | 3 ++- transaction_parser/parser_benchmark/__init__.py | 0 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 transaction_parser/parser_benchmark/__init__.py diff --git a/transaction_parser/modules.txt b/transaction_parser/modules.txt index c672949..d31c756 100644 --- a/transaction_parser/modules.txt +++ b/transaction_parser/modules.txt @@ -1 +1,2 @@ -Transaction Parser \ No newline at end of file +Transaction Parser +Parser Benchmark \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/__init__.py b/transaction_parser/parser_benchmark/__init__.py new file mode 100644 index 0000000..e69de29 From 23983f7acedcf47570a7a57e9c66317edaef6c26 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Mon, 16 Mar 2026 17:31:52 +0530 Subject: [PATCH 02/88] feat: add Parser Benchmark module with dataset, log, and settings functionality --- .../parser_benchmark/doctype/__init__.py | 0 .../parser_benchmark_dataset/__init__.py | 0 .../parser_benchmark_dataset.js | 41 +++ .../parser_benchmark_dataset.json | 133 ++++++++++ .../parser_benchmark_dataset.py | 59 +++++ .../doctype/parser_benchmark_log/__init__.py | 0 .../parser_benchmark_log.js | 8 + .../parser_benchmark_log.json | 240 ++++++++++++++++++ .../parser_benchmark_log.py | 37 +++ .../test_parser_benchmark_log.py | 9 + .../parser_benchmark_settings/__init__.py | 0 .../parser_benchmark_settings.json | 39 +++ .../parser_benchmark_settings.py | 23 ++ .../parser_benchmark_token_cost/__init__.py | 0 .../parser_benchmark_token_cost.json | 55 ++++ .../parser_benchmark_token_cost.py | 25 ++ transaction_parser/parser_benchmark/runner.py | 169 ++++++++++++ .../transaction_parser_settings.py | 9 + .../transaction_parser/utils/pdf_processor.py | 6 + 19 files changed, 853 insertions(+) create mode 100644 transaction_parser/parser_benchmark/doctype/__init__.py create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/__init__.py create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_log/__init__.py create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.js create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_log/test_parser_benchmark_log.py create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/__init__.py create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/__init__.py create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py create mode 100644 transaction_parser/parser_benchmark/runner.py diff --git a/transaction_parser/parser_benchmark/doctype/__init__.py b/transaction_parser/parser_benchmark/doctype/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/__init__.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js new file mode 100644 index 0000000..cffdf79 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js @@ -0,0 +1,41 @@ +// Copyright (c) 2026, Resilient Tech and contributors +// For license information, please see license.txt + +frappe.ui.form.on("Parser Benchmark Dataset", { + refresh(frm) { + if (!frm.is_new()) { + frm.add_custom_button(__("Run Benchmark"), () => run_benchmark(frm), __("Actions")); + } + + set_pdf_processor_options(frm); + }, +}); + +function run_benchmark(frm) { + frappe.call({ + method: "transaction_parser.parser_benchmark.doctype.parser_benchmark_dataset.parser_benchmark_dataset.run_benchmark", + args: { dataset_name: frm.doc.name }, + freeze: true, + freeze_message: __("Queuing benchmark..."), + callback(r) { + if (r.message) { + frappe.msgprint({ + message: __("Benchmark queued. Redirecting to log..."), + alert: true, + }); + frappe.set_route("Form", "Parser Benchmark Log", r.message); + } + }, + }); +} + +function set_pdf_processor_options(frm) { + frappe.call({ + method: "transaction_parser.transaction_parser.doctype.transaction_parser_settings.transaction_parser_settings.get_pdf_processors", + callback(r) { + if (r.message) { + frm.set_df_property("pdf_processor", "options", r.message); + } + }, + }); +} diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json new file mode 100644 index 0000000..a1ffe84 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -0,0 +1,133 @@ +{ + "actions": [], + "autoname": "format:PBD-{#####}", + "creation": "2026-03-16 00:00:00.000000", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "title", + "column_break_title", + "enabled", + "section_break_file", + "file", + "column_break_file", + "transaction_type", + "country", + "processing_section", + "ai_model", + "pdf_processor", + "column_break_processing", + "company", + "page_limit" + ], + "fields": [ + { + "fieldname": "title", + "fieldtype": "Data", + "label": "Title", + "reqd": 1 + }, + { + "fieldname": "column_break_title", + "fieldtype": "Column Break" + }, + { + "default": "1", + "fieldname": "enabled", + "fieldtype": "Check", + "label": "Enabled" + }, + { + "fieldname": "section_break_file", + "fieldtype": "Section Break" + }, + { + "fieldname": "file", + "fieldtype": "Attach", + "label": "File", + "reqd": 1 + }, + { + "fieldname": "column_break_file", + "fieldtype": "Column Break" + }, + { + "fieldname": "transaction_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Transaction Type", + "options": "Sales Order\nExpense", + "reqd": 1 + }, + { + "default": "Other", + "fieldname": "country", + "fieldtype": "Select", + "label": "Country", + "options": "India\nOther", + "reqd": 1 + }, + { + "fieldname": "processing_section", + "fieldtype": "Section Break", + "label": "Processing Configuration" + }, + { + "fieldname": "ai_model", + "fieldtype": "Select", + "in_list_view": 1, + "label": "AI Model", + "options": "DeepSeek Chat\nDeepSeek Reasoner\nOpenAI gpt-4o\nOpenAI gpt-4o-mini\nOpenAI gpt-5\nOpenAI gpt-5-mini\nGoogle Gemini Pro\nGoogle Gemini Flash", + "reqd": 1 + }, + { + "fieldname": "pdf_processor", + "fieldtype": "Select", + "label": "PDF Processor", + "options": "OCRMyPDF\nDocling" + }, + { + "fieldname": "column_break_processing", + "fieldtype": "Column Break" + }, + { + "fieldname": "company", + "fieldtype": "Link", + "label": "Company", + "options": "Company", + "reqd": 1 + }, + { + "fieldname": "page_limit", + "fieldtype": "Int", + "label": "Page Limit" + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2026-03-16 00:00:00.000000", + "modified_by": "Administrator", + "module": "Parser Benchmark", + "name": "Parser Benchmark Dataset", + "naming_rule": "Expression", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "title_field": "title", + "show_title_field_in_link": 1 +} diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py new file mode 100644 index 0000000..e514ba8 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -0,0 +1,59 @@ +# Copyright (c) 2026, Resilient Tech and contributors +# For license information, please see license.txt + +import frappe +from frappe.model.document import Document + + +class ParserBenchmarkDataset(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + ai_model: DF.Select + company: DF.Link + country: DF.Select + enabled: DF.Check + file: DF.Attach + page_limit: DF.Int + pdf_processor: DF.Select | None + title: DF.Data + transaction_type: DF.Select + # end: auto-generated types + + pass + + +@frappe.whitelist() +def run_benchmark(dataset_name: str): + """Create a Benchmark Log and enqueue the benchmark run.""" + frappe.has_permission("Parser Benchmark Dataset", "write", throw=True) + + log = frappe.get_doc( + { + "doctype": "Parser Benchmark Log", + "dataset": dataset_name, + "status": "Queued", + } + ).insert(ignore_permissions=True) + + frappe.db.commit() + + frappe.enqueue( + _run_benchmark, + log_name=log.name, + queue="long", + now=frappe.conf.developer_mode, + ) + + return log.name + + +def _run_benchmark(log_name: str): + from transaction_parser.parser_benchmark.runner import BenchmarkRunner + + BenchmarkRunner(log_name).run() diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/__init__.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.js b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.js new file mode 100644 index 0000000..e21eaee --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.js @@ -0,0 +1,8 @@ +// Copyright (c) 2026, Resilient Tech and contributors +// For license information, please see license.txt + +// frappe.ui.form.on("Parser Benchmark Log", { +// refresh(frm) { + +// }, +// }); diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json new file mode 100644 index 0000000..3d1bc46 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -0,0 +1,240 @@ +{ + "actions": [], + "autoname": "format:PBL-{#####}", + "creation": "2026-03-16 00:00:00", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "dataset", + "status", + "column_break_summary", + "ai_model", + "pdf_processor", + "total_time", + "file_parsing_tab", + "file_parse_time", + "column_break_file_metrics", + "file_parse_memory", + "section_break_file_content", + "file_content", + "ai_response_tab", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "ai_parse_time", + "column_break_ai", + "input_cost", + "output_cost", + "total_cost", + "section_break_ai_content", + "ai_response", + "result_tab", + "document_type", + "column_break_wdyy", + "document_name", + "section_break_error", + "error" + ], + "fields": [ + { + "fieldname": "dataset", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Dataset", + "options": "Parser Benchmark Dataset", + "read_only": 1, + "reqd": 1 + }, + { + "default": "Queued", + "fieldname": "status", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Status", + "options": "Queued\nRunning\nCompleted\nFailed", + "read_only": 1 + }, + { + "fieldname": "column_break_summary", + "fieldtype": "Column Break" + }, + { + "fieldname": "ai_model", + "fieldtype": "Data", + "in_list_view": 1, + "label": "AI Model", + "read_only": 1 + }, + { + "fieldname": "pdf_processor", + "fieldtype": "Data", + "label": "PDF Processor", + "read_only": 1 + }, + { + "fieldname": "total_time", + "fieldtype": "Float", + "in_list_view": 1, + "label": "Total Time (s)", + "read_only": 1 + }, + { + "fieldname": "file_parsing_tab", + "fieldtype": "Tab Break", + "label": "File Parsing" + }, + { + "fieldname": "file_parse_time", + "fieldtype": "Float", + "label": "Parse Time (s)", + "read_only": 1 + }, + { + "fieldname": "column_break_file_metrics", + "fieldtype": "Column Break" + }, + { + "fieldname": "file_parse_memory", + "fieldtype": "Float", + "label": "Memory Delta (MB)", + "read_only": 1 + }, + { + "fieldname": "section_break_file_content", + "fieldtype": "Section Break", + "label": "Extracted Content" + }, + { + "fieldname": "file_content", + "fieldtype": "Code", + "label": "File Content", + "options": "Markdown", + "read_only": 1 + }, + { + "fieldname": "ai_response_tab", + "fieldtype": "Tab Break", + "label": "AI Response" + }, + { + "fieldname": "prompt_tokens", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Prompt Tokens", + "read_only": 1 + }, + { + "fieldname": "completion_tokens", + "fieldtype": "Int", + "label": "Completion Tokens", + "read_only": 1 + }, + { + "fieldname": "total_tokens", + "fieldtype": "Int", + "label": "Total Tokens", + "read_only": 1 + }, + { + "fieldname": "column_break_ai", + "fieldtype": "Column Break" + }, + { + "fieldname": "ai_parse_time", + "fieldtype": "Float", + "label": "AI Parse Time (s)", + "read_only": 1 + }, + { + "fieldname": "input_cost", + "fieldtype": "Float", + "label": "Input Cost", + "read_only": 1 + }, + { + "fieldname": "output_cost", + "fieldtype": "Float", + "label": "Output Cost", + "read_only": 1 + }, + { + "fieldname": "total_cost", + "fieldtype": "Float", + "in_list_view": 1, + "label": "Total Cost", + "read_only": 1 + }, + { + "fieldname": "section_break_ai_content", + "fieldtype": "Section Break", + "label": "AI Output" + }, + { + "fieldname": "ai_response", + "fieldtype": "JSON", + "label": "AI Response", + "read_only": 1 + }, + { + "fieldname": "result_tab", + "fieldtype": "Tab Break", + "label": "Result" + }, + { + "fieldname": "document_type", + "fieldtype": "Link", + "label": "Document Type", + "options": "DocType", + "read_only": 1 + }, + { + "fieldname": "document_name", + "fieldtype": "Dynamic Link", + "label": "Document", + "options": "document_type", + "read_only": 1 + }, + { + "depends_on": "eval: doc.status === 'Failed'", + "fieldname": "section_break_error", + "fieldtype": "Section Break", + "label": "Error" + }, + { + "fieldname": "error", + "fieldtype": "Code", + "label": "Error Traceback", + "read_only": 1 + }, + { + "fieldname": "column_break_wdyy", + "fieldtype": "Column Break" + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2026-03-16 12:52:18.204234", + "modified_by": "Administrator", + "module": "Parser Benchmark", + "name": "Parser Benchmark Log", + "naming_rule": "Expression (old style)", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "row_format": "Dynamic", + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py new file mode 100644 index 0000000..59f4cf3 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -0,0 +1,37 @@ +# Copyright (c) 2026, Resilient Tech and contributors +# For license information, please see license.txt + +from frappe.model.document import Document + + +class ParserBenchmarkLog(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + ai_model: DF.Data | None + ai_parse_time: DF.Float + ai_response: DF.JSON | None + completion_tokens: DF.Int + dataset: DF.Link + document_name: DF.DynamicLink | None + document_type: DF.Link | None + error: DF.Code | None + file_content: DF.Code | None + file_parse_memory: DF.Float + file_parse_time: DF.Float + input_cost: DF.Float + output_cost: DF.Float + pdf_processor: DF.Data | None + prompt_tokens: DF.Int + status: DF.Select | None + total_cost: DF.Float + total_time: DF.Float + total_tokens: DF.Int + # end: auto-generated types + + pass diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/test_parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/test_parser_benchmark_log.py new file mode 100644 index 0000000..a04dc86 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/test_parser_benchmark_log.py @@ -0,0 +1,9 @@ +# Copyright (c) 2026, Resilient Tech and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestParserBenchmarkLog(FrappeTestCase): + pass diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/__init__.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json new file mode 100644 index 0000000..3081b83 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json @@ -0,0 +1,39 @@ +{ + "actions": [], + "creation": "2026-03-16 00:00:00.000000", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "token_costs" + ], + "fields": [ + { + "fieldname": "token_costs", + "fieldtype": "Table", + "label": "Token Costs", + "options": "Parser Benchmark Token Cost" + } + ], + "issingle": 1, + "links": [], + "modified": "2026-03-16 00:00:00.000000", + "modified_by": "Administrator", + "module": "Parser Benchmark", + "name": "Parser Benchmark Settings", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py new file mode 100644 index 0000000..9ee4fae --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py @@ -0,0 +1,23 @@ +# Copyright (c) 2026, Resilient Tech and contributors +# For license information, please see license.txt + +from frappe.model.document import Document + + +class ParserBenchmarkSettings(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + from transaction_parser.parser_benchmark.doctype.parser_benchmark_token_cost.parser_benchmark_token_cost import ( + ParserBenchmarkTokenCost, + ) + + token_costs: DF.Table[ParserBenchmarkTokenCost] + # end: auto-generated types + + pass diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/__init__.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json new file mode 100644 index 0000000..9c7f0ee --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json @@ -0,0 +1,55 @@ +{ + "actions": [], + "creation": "2026-03-16 00:00:00.000000", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "ai_model", + "currency", + "input_cost_per_million", + "output_cost_per_million" + ], + "fields": [ + { + "fieldname": "ai_model", + "fieldtype": "Select", + "in_list_view": 1, + "label": "AI Model", + "options": "DeepSeek Chat\nDeepSeek Reasoner\nOpenAI gpt-4o\nOpenAI gpt-4o-mini\nOpenAI gpt-5\nOpenAI gpt-5-mini\nGoogle Gemini Pro\nGoogle Gemini Flash", + "reqd": 1 + }, + { + "default": "USD", + "fieldname": "currency", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Currency", + "options": "Currency" + }, + { + "fieldname": "input_cost_per_million", + "fieldtype": "Float", + "in_list_view": 1, + "label": "Input Cost / 1M Tokens", + "reqd": 1 + }, + { + "fieldname": "output_cost_per_million", + "fieldtype": "Float", + "in_list_view": 1, + "label": "Output Cost / 1M Tokens", + "reqd": 1 + } + ], + "istable": 1, + "links": [], + "modified": "2026-03-16 00:00:00.000000", + "modified_by": "Administrator", + "module": "Parser Benchmark", + "name": "Parser Benchmark Token Cost", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py new file mode 100644 index 0000000..9a06e21 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py @@ -0,0 +1,25 @@ +# Copyright (c) 2026, Resilient Tech and contributors +# For license information, please see license.txt + +from frappe.model.document import Document + + +class ParserBenchmarkTokenCost(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + ai_model: DF.Select + currency: DF.Link | None + input_cost_per_million: DF.Float + output_cost_per_million: DF.Float + parent: DF.Data + parentfield: DF.Data + parenttype: DF.Data + # end: auto-generated types + + pass diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py new file mode 100644 index 0000000..1f7587e --- /dev/null +++ b/transaction_parser/parser_benchmark/runner.py @@ -0,0 +1,169 @@ +import resource +from timeit import default_timer + +import frappe + +from transaction_parser.transaction_parser.ai_integration.parser import AIParser +from transaction_parser.transaction_parser.controllers import get_controller +from transaction_parser.transaction_parser.utils.file_processor import FileProcessor +from transaction_parser.transaction_parser.utils.pdf_processor import get_pdf_processor + + +class BenchmarkRunner: + """ + Wraps the existing Transaction Parser flow to capture + performance metrics (time, memory, tokens, cost) for benchmarking. + + Flow: + 1. File parsing → time, memory, extracted content + 2. AI parsing → time, tokens, cost, AI response + 3. Doc generation → linked document (Sales Order / Purchase Invoice) + """ + + def __init__(self, log_name: str): + self.log = frappe.get_doc("Parser Benchmark Log", log_name) + self.dataset = frappe.get_doc("Parser Benchmark Dataset", self.log.dataset) + + # intermediate state shared between steps + self._file_content = None + self._ai_content = None + self._controller = None + + def run(self): + self.log.status = "Running" + self.log.ai_model = self.dataset.ai_model + self.log.pdf_processor = self.dataset.pdf_processor + self.log.save(ignore_permissions=True) + frappe.db.commit() + + total_start = default_timer() + + try: + file_doc = self._get_file_doc() + self._run_file_parsing(file_doc) + self._run_ai_parsing(file_doc) + self._run_document_generation(file_doc) + self._calculate_cost() + + self.log.status = "Completed" + + except Exception: + self.log.status = "Failed" + self.log.error = frappe.get_traceback() + + finally: + self.log.total_time = round(default_timer() - total_start, 4) + self.log.save(ignore_permissions=True) + frappe.db.commit() + + return self.log.name + + # ── helpers ────────────────────────────────────────────── + + def _get_file_doc(self): + return frappe.get_last_doc("File", filters={"file_url": self.dataset.file}) + + def _get_controller(self): + cls = get_controller(self.dataset.country, self.dataset.transaction_type) + controller = cls(company=self.dataset.company) + controller.initialize() + return controller + + # ── step 1: file parsing ──────────────────────────────── + + def _run_file_parsing(self, file_doc): + pdf_processor = None + if file_doc.file_type == "PDF" and self.dataset.pdf_processor: + pdf_processor = get_pdf_processor(self.dataset.pdf_processor) + + mem_before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + start = default_timer() + + content = FileProcessor().get_content( + file_doc, + self.dataset.page_limit or None, + pdf_processor, + ) + + self.log.file_parse_time = round(default_timer() - start, 4) + + mem_after = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + # ru_maxrss is in KB on Linux + self.log.file_parse_memory = round((mem_after - mem_before) / 1024, 2) + + self.log.file_content = content + self._file_content = content + + # ── step 2: AI parsing ────────────────────────────────── + + def _run_ai_parsing(self, file_doc): + self._controller = self._get_controller() + self._controller.file = file_doc + + schema = self._controller.get_schema() + + parser = AIParser(self.dataset.ai_model) + messages = parser._build_messages( + self._controller.DOCTYPE, schema, self._file_content + ) + + start = default_timer() + response = parser.send_message(messages=messages, file_doc_name=file_doc.name) + self.log.ai_parse_time = round(default_timer() - start, 4) + + # token usage + usage = response.get("usage", {}) + self.log.prompt_tokens = usage.get("prompt_tokens", 0) + self.log.completion_tokens = usage.get("completion_tokens", 0) + self.log.total_tokens = usage.get("total_tokens", 0) + + # parsed content + ai_content = parser.get_content(response) + self.log.ai_response = frappe.as_json(ai_content, indent=2) + + self._ai_content = ai_content + + # ── step 3: document generation ───────────────────────── + + def _run_document_generation(self, file_doc): + c = self._controller + c.data = self._ai_content + c.doc = frappe.get_doc({"doctype": c.DOCTYPE}) + c.doc.is_created_by_transaction_parser = 1 + + c.set_details() + c.set_missing_values() + c._set_flags() + c.doc.insert() + c._attach_file() + + self.log.document_type = c.DOCTYPE + self.log.document_name = c.doc.name + + # ── step 4: cost calculation ──────────────────────────── + + def _calculate_cost(self): + try: + settings = frappe.get_cached_doc("Parser Benchmark Settings") + except Exception: + return + + cost_row = None + for row in settings.token_costs: + if row.ai_model == self.dataset.ai_model: + cost_row = row + break + + if not cost_row: + return + + prompt = self.log.prompt_tokens or 0 + completion = self.log.completion_tokens or 0 + + self.log.input_cost = round( + prompt * cost_row.input_cost_per_million / 1_000_000, 6 + ) + self.log.output_cost = round( + completion * cost_row.output_cost_per_million / 1_000_000, 6 + ) + self.log.total_cost = round(self.log.input_cost + self.log.output_cost, 6) diff --git a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py index 95b4c43..8b66d37 100644 --- a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py +++ b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py @@ -95,3 +95,12 @@ def get_ai_models(): "default_model": default_model, "supported_models": supported_models, } + + +@frappe.whitelist() +def get_pdf_processors(): + from transaction_parser.transaction_parser.utils.pdf_processor import ( + get_available_pdf_processors, + ) + + return get_available_pdf_processors() diff --git a/transaction_parser/transaction_parser/utils/pdf_processor.py b/transaction_parser/transaction_parser/utils/pdf_processor.py index a87237b..c50a207 100644 --- a/transaction_parser/transaction_parser/utils/pdf_processor.py +++ b/transaction_parser/transaction_parser/utils/pdf_processor.py @@ -195,3 +195,9 @@ def get_pdf_processor(name: str | None = None) -> PDFProcessor: ) return frappe.get_attr(class_path)() + + +def get_available_pdf_processors() -> list[str]: + """Return names of all registered PDF processors from hooks.""" + processors = frappe.get_hooks("pdf_processors") or {} + return list(processors.keys()) From f41abe52e2c50d20073fce9c92ffd19f8f898626 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Mon, 16 Mar 2026 18:00:09 +0530 Subject: [PATCH 03/88] chore: Minor changes --- transaction_parser/hooks.py | 2 ++ .../parser_benchmark_dataset/parser_benchmark_dataset.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/transaction_parser/hooks.py b/transaction_parser/hooks.py index a944370..c7f1d6e 100644 --- a/transaction_parser/hooks.py +++ b/transaction_parser/hooks.py @@ -32,3 +32,5 @@ "OCRMyPDF": "transaction_parser.transaction_parser.utils.pdf_processor.OCRMyPDFProcessor", "Docling": "transaction_parser.transaction_parser.utils.pdf_processor.DoclingPDFProcessor", } + +export_python_type_annotations = True diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index e514ba8..7819a79 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -41,7 +41,7 @@ def run_benchmark(dataset_name: str): } ).insert(ignore_permissions=True) - frappe.db.commit() + frappe.db.commit() # Ensure the log is saved before the background job picks it up frappe.enqueue( _run_benchmark, From 045024123028f8d18c747f8e4aaf63067536ad88 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 17 Mar 2026 10:21:52 +0530 Subject: [PATCH 04/88] chore: update Parser Benchmark Dataset naming --- .../parser_benchmark_dataset.json | 31 ++++++++++++++----- .../parser_benchmark_dataset.py | 20 +++++++++--- .../test_parser_benchmark_dataset.py | 9 ++++++ 3 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/test_parser_benchmark_dataset.py diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index a1ffe84..fd022a7 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -1,7 +1,7 @@ { "actions": [], - "autoname": "format:PBD-{#####}", - "creation": "2026-03-16 00:00:00.000000", + "autoname": "naming_series:", + "creation": "2026-03-16 00:00:00", "doctype": "DocType", "engine": "InnoDB", "field_order": [ @@ -18,7 +18,9 @@ "pdf_processor", "column_break_processing", "company", - "page_limit" + "page_limit", + "section_break_bhsg", + "naming_series" ], "fields": [ { @@ -84,7 +86,8 @@ "fieldname": "pdf_processor", "fieldtype": "Select", "label": "PDF Processor", - "options": "OCRMyPDF\nDocling" + "options": "OCRMyPDF\nDocling", + "reqd": 1 }, { "fieldname": "column_break_processing", @@ -101,15 +104,26 @@ "fieldname": "page_limit", "fieldtype": "Int", "label": "Page Limit" + }, + { + "fieldname": "section_break_bhsg", + "fieldtype": "Section Break" + }, + { + "default": "Parser-Dataset-", + "fieldname": "naming_series", + "fieldtype": "Select", + "label": "Naming Series", + "options": "Parser-Dataset-" } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-16 00:00:00.000000", + "modified": "2026-03-16 13:33:28.707264", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", - "naming_rule": "Expression", + "naming_rule": "By \"Naming Series\" field", "owner": "Administrator", "permissions": [ { @@ -125,9 +139,10 @@ "write": 1 } ], + "row_format": "Dynamic", + "show_title_field_in_link": 1, "sort_field": "modified", "sort_order": "DESC", "states": [], - "title_field": "title", - "show_title_field_in_link": 1 + "title_field": "title" } diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 7819a79..f3f2a20 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -14,15 +14,25 @@ class ParserBenchmarkDataset(Document): if TYPE_CHECKING: from frappe.types import DF - ai_model: DF.Select + ai_model: DF.Literal[ + "DeepSeek Chat", + "DeepSeek Reasoner", + "OpenAI gpt-4o", + "OpenAI gpt-4o-mini", + "OpenAI gpt-5", + "OpenAI gpt-5-mini", + "Google Gemini Pro", + "Google Gemini Flash", + ] company: DF.Link - country: DF.Select + country: DF.Literal["India", "Other"] enabled: DF.Check file: DF.Attach + naming_series: DF.Literal["PBD-#####"] page_limit: DF.Int - pdf_processor: DF.Select | None + pdf_processor: DF.Literal["OCRMyPDF", "Docling"] title: DF.Data - transaction_type: DF.Select + transaction_type: DF.Literal["Sales Order", "Expense"] # end: auto-generated types pass @@ -41,7 +51,7 @@ def run_benchmark(dataset_name: str): } ).insert(ignore_permissions=True) - frappe.db.commit() # Ensure the log is saved before the background job picks it up + frappe.db.commit() # Ensure the log is saved before the background job picks it up frappe.enqueue( _run_benchmark, diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/test_parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/test_parser_benchmark_dataset.py new file mode 100644 index 0000000..1ae2c88 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/test_parser_benchmark_dataset.py @@ -0,0 +1,9 @@ +# Copyright (c) 2026, Resilient Tech and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestParserBenchmarkDataset(FrappeTestCase): + pass From f3fb150bd9092c761898f392fe34df8caa51787d Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 17 Mar 2026 12:12:10 +0530 Subject: [PATCH 05/88] fix: update autoname format and add naming series field in parser benchmark log --- .../parser_benchmark_log.json | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index 3d1bc46..3648bc1 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -1,10 +1,11 @@ { "actions": [], - "autoname": "format:PBL-{#####}", + "autoname": "naming_series:", "creation": "2026-03-16 00:00:00", "doctype": "DocType", "engine": "InnoDB", "field_order": [ + "naming_series", "dataset", "status", "column_break_summary", @@ -36,6 +37,14 @@ "error" ], "fields": [ + { + "default": "PBL-", + "fieldname": "naming_series", + "fieldtype": "Select", + "hidden": 1, + "label": "Naming Series", + "options": "PBL-" + }, { "fieldname": "dataset", "fieldtype": "Link", @@ -217,7 +226,7 @@ "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Log", - "naming_rule": "Expression (old style)", + "naming_rule": "By \"Naming Series\" field", "owner": "Administrator", "permissions": [ { @@ -237,4 +246,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} \ No newline at end of file +} From 957eaffff6d069165c9e160c1ec6dd2255087228 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 17 Mar 2026 12:26:15 +0530 Subject: [PATCH 06/88] fix: replace resource usage tracking with tracemalloc for memory measurement in file parsing --- transaction_parser/parser_benchmark/runner.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index 1f7587e..1c91bd4 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -1,4 +1,4 @@ -import resource +import tracemalloc from timeit import default_timer import frappe @@ -76,7 +76,7 @@ def _run_file_parsing(self, file_doc): if file_doc.file_type == "PDF" and self.dataset.pdf_processor: pdf_processor = get_pdf_processor(self.dataset.pdf_processor) - mem_before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + tracemalloc.start() start = default_timer() content = FileProcessor().get_content( @@ -87,9 +87,9 @@ def _run_file_parsing(self, file_doc): self.log.file_parse_time = round(default_timer() - start, 4) - mem_after = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - # ru_maxrss is in KB on Linux - self.log.file_parse_memory = round((mem_after - mem_before) / 1024, 2) + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + self.log.file_parse_memory = round(peak / 1024 / 1024, 2) # bytes → MB self.log.file_content = content self._file_content = content From c5e309b475ba460747ce5a595f0f394a4254b5e0 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 17 Mar 2026 12:33:48 +0530 Subject: [PATCH 07/88] fix: add parse_with_response method to AIParser for enhanced diagnostics and benchmarking --- transaction_parser/parser_benchmark/runner.py | 13 ++++++------- .../ai_integration/parser.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index 1c91bd4..82f02f7 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -101,14 +101,15 @@ def _run_ai_parsing(self, file_doc): self._controller.file = file_doc schema = self._controller.get_schema() - parser = AIParser(self.dataset.ai_model) - messages = parser._build_messages( - self._controller.DOCTYPE, schema, self._file_content - ) start = default_timer() - response = parser.send_message(messages=messages, file_doc_name=file_doc.name) + ai_content, response = parser.parse_with_response( + document_type=self._controller.DOCTYPE, + document_schema=schema, + document_data=self._file_content, + file_doc_name=file_doc.name, + ) self.log.ai_parse_time = round(default_timer() - start, 4) # token usage @@ -118,9 +119,7 @@ def _run_ai_parsing(self, file_doc): self.log.total_tokens = usage.get("total_tokens", 0) # parsed content - ai_content = parser.get_content(response) self.log.ai_response = frappe.as_json(ai_content, indent=2) - self._ai_content = ai_content # ── step 3: document generation ───────────────────────── diff --git a/transaction_parser/transaction_parser/ai_integration/parser.py b/transaction_parser/transaction_parser/ai_integration/parser.py index f74eec2..04f1007 100644 --- a/transaction_parser/transaction_parser/ai_integration/parser.py +++ b/transaction_parser/transaction_parser/ai_integration/parser.py @@ -40,6 +40,25 @@ def parse( response = self.send_message(messages=messages, file_doc_name=file_doc_name) return self.get_content(response) + def parse_with_response( + self, + document_type: str, + document_schema: dict, + document_data: str, + file_doc_name: str | None = None, + ) -> tuple[dict, dict]: + """Parse document and return both the parsed content and the raw API response. + + Useful for benchmarking and diagnostics where token usage and other + response metadata are needed alongside the parsed output. + + Returns: + tuple: (parsed_content, raw_response) + """ + messages = self._build_messages(document_type, document_schema, document_data) + response = self.send_message(messages=messages, file_doc_name=file_doc_name) + return self.get_content(response), response + def _build_messages( self, document_type: str, document_schema: dict, document_data: str ) -> tuple: From b5925298947a9ebb37fb1aee2ff6feca3b038826 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 17 Mar 2026 14:31:59 +0530 Subject: [PATCH 08/88] fix: add 'is_created_by_benchmark' field to custom fields for Sales Order and Purchase Invoice --- transaction_parser/install.py | 18 ++++++++++++++++-- transaction_parser/parser_benchmark/runner.py | 1 + transaction_parser/uninstall.py | 2 ++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/transaction_parser/install.py b/transaction_parser/install.py index a8ef3fd..f8221e6 100644 --- a/transaction_parser/install.py +++ b/transaction_parser/install.py @@ -11,7 +11,14 @@ "label": "Is Created By Transaction Parser", "read_only": 1, "insert_after": "is_internal_customer", - } + }, + { + "fieldname": "is_created_by_benchmark", + "fieldtype": "Check", + "label": "Is Created By Benchmark", + "read_only": 1, + "insert_after": "is_created_by_transaction_parser", + }, ], "Purchase Invoice": [ { @@ -20,7 +27,14 @@ "label": "Is Created By Transaction Parser", "read_only": 1, "insert_after": "is_internal_supplier", - } + }, + { + "fieldname": "is_created_by_benchmark", + "fieldtype": "Check", + "label": "Is Created By Benchmark", + "read_only": 1, + "insert_after": "is_created_by_transaction_parser", + }, ], "Communication": [ { diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index 82f02f7..c31c22a 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -129,6 +129,7 @@ def _run_document_generation(self, file_doc): c.data = self._ai_content c.doc = frappe.get_doc({"doctype": c.DOCTYPE}) c.doc.is_created_by_transaction_parser = 1 + c.doc.is_created_by_benchmark = 1 c.set_details() c.set_missing_values() diff --git a/transaction_parser/uninstall.py b/transaction_parser/uninstall.py index bb326e2..d54f2b3 100644 --- a/transaction_parser/uninstall.py +++ b/transaction_parser/uninstall.py @@ -2,6 +2,8 @@ FIELDS_TO_DELETE = { "Transaction Parser Settings": ["in_auto_create_supplier"], + "Sales Order": ["is_created_by_transaction_parser", "is_created_by_benchmark"], + "Purchase Invoice": ["is_created_by_transaction_parser", "is_created_by_benchmark"], } From fa1446f945904074de615ef28083b56ac54f44ef Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 17 Mar 2026 14:34:31 +0530 Subject: [PATCH 09/88] fix: ensure file parsing time is logged correctly with error handling --- transaction_parser/parser_benchmark/runner.py | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index c31c22a..eeca6cc 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -78,19 +78,18 @@ def _run_file_parsing(self, file_doc): tracemalloc.start() start = default_timer() + try: + content = FileProcessor().get_content( + file_doc, + self.dataset.page_limit or None, + pdf_processor, + ) + finally: + self.log.file_parse_time = round(default_timer() - start, 4) + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() - content = FileProcessor().get_content( - file_doc, - self.dataset.page_limit or None, - pdf_processor, - ) - - self.log.file_parse_time = round(default_timer() - start, 4) - - _, peak = tracemalloc.get_traced_memory() - tracemalloc.stop() self.log.file_parse_memory = round(peak / 1024 / 1024, 2) # bytes → MB - self.log.file_content = content self._file_content = content From 060905b627f9aef4a7cf2b6aed327896def79b74 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 17 Mar 2026 15:02:35 +0530 Subject: [PATCH 10/88] fix: streamline document creation process in BenchmarkRunner and Transaction classes --- transaction_parser/parser_benchmark/runner.py | 11 ++--------- .../transaction_parser/controllers/transaction.py | 5 +++++ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index eeca6cc..ba2189d 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -126,15 +126,8 @@ def _run_ai_parsing(self, file_doc): def _run_document_generation(self, file_doc): c = self._controller c.data = self._ai_content - c.doc = frappe.get_doc({"doctype": c.DOCTYPE}) - c.doc.is_created_by_transaction_parser = 1 - c.doc.is_created_by_benchmark = 1 - - c.set_details() - c.set_missing_values() - c._set_flags() - c.doc.insert() - c._attach_file() + c.create_document() + c.doc.db_set("is_created_by_benchmark", 1) self.log.document_type = c.DOCTYPE self.log.document_name = c.doc.name diff --git a/transaction_parser/transaction_parser/controllers/transaction.py b/transaction_parser/transaction_parser/controllers/transaction.py index 05a59c0..178ffed 100644 --- a/transaction_parser/transaction_parser/controllers/transaction.py +++ b/transaction_parser/transaction_parser/controllers/transaction.py @@ -36,6 +36,11 @@ def generate( self.file = file self.ai_model = ai_model self.data = self._parse_file_content(ai_model, page_limit) + + return self.create_document() + + def create_document(self): + """Create, populate, and insert the transaction document.""" self.doc = frappe.get_doc({"doctype": self.DOCTYPE}) self.doc.is_created_by_transaction_parser = 1 From 58f4ff93f8af173522ab3f87e55f4f8e9186acd0 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 17 Mar 2026 16:07:01 +0530 Subject: [PATCH 11/88] fix: some fixes in parser benchmark dataset --- .../parser_benchmark_dataset.js | 13 +++++++++---- .../parser_benchmark_dataset.json | 4 ++-- .../parser_benchmark_dataset.py | 13 ++++++++++++- .../transaction_parser_settings.py | 9 --------- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js index cffdf79..80522dc 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js @@ -3,7 +3,7 @@ frappe.ui.form.on("Parser Benchmark Dataset", { refresh(frm) { - if (!frm.is_new()) { + if (!frm.is_new() && frm.doc.enabled) { frm.add_custom_button(__("Run Benchmark"), () => run_benchmark(frm), __("Actions")); } @@ -19,11 +19,16 @@ function run_benchmark(frm) { freeze_message: __("Queuing benchmark..."), callback(r) { if (r.message) { - frappe.msgprint({ + frappe.show_alert({ message: __("Benchmark queued. Redirecting to log..."), - alert: true, + indicator: "green", }); frappe.set_route("Form", "Parser Benchmark Log", r.message); + } else { + frappe.show_alert({ + message: __("Failed to queue benchmark. Please try again."), + indicator: "red", + }); } }, }); @@ -31,7 +36,7 @@ function run_benchmark(frm) { function set_pdf_processor_options(frm) { frappe.call({ - method: "transaction_parser.transaction_parser.doctype.transaction_parser_settings.transaction_parser_settings.get_pdf_processors", + method: "transaction_parser.parser_benchmark.doctype.parser_benchmark_dataset.parser_benchmark_dataset.get_pdf_processors", callback(r) { if (r.message) { frm.set_df_property("pdf_processor", "options", r.message); diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index fd022a7..a805917 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -119,7 +119,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-16 13:33:28.707264", + "modified": "2026-03-17 11:04:01.862839", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", @@ -145,4 +145,4 @@ "sort_order": "DESC", "states": [], "title_field": "title" -} +} \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index f3f2a20..ba9d0f3 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -28,7 +28,7 @@ class ParserBenchmarkDataset(Document): country: DF.Literal["India", "Other"] enabled: DF.Check file: DF.Attach - naming_series: DF.Literal["PBD-#####"] + naming_series: DF.Literal["Parser-Dataset-"] page_limit: DF.Int pdf_processor: DF.Literal["OCRMyPDF", "Docling"] title: DF.Data @@ -67,3 +67,14 @@ def _run_benchmark(log_name: str): from transaction_parser.parser_benchmark.runner import BenchmarkRunner BenchmarkRunner(log_name).run() + + +@frappe.whitelist() +def get_pdf_processors(): + frappe.has_permission("Parser Benchmark Dataset", "write", throw=True) + + from transaction_parser.transaction_parser.utils.pdf_processor import ( + get_available_pdf_processors, + ) + + return get_available_pdf_processors() diff --git a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py index 8b66d37..95b4c43 100644 --- a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py +++ b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py @@ -95,12 +95,3 @@ def get_ai_models(): "default_model": default_model, "supported_models": supported_models, } - - -@frappe.whitelist() -def get_pdf_processors(): - from transaction_parser.transaction_parser.utils.pdf_processor import ( - get_available_pdf_processors, - ) - - return get_available_pdf_processors() From 164f5182b05de81d1d66d39b508118fbe8b14b71 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 17 Mar 2026 16:50:37 +0530 Subject: [PATCH 12/88] refactor: enhance parser benchmark dataset and log with currency and cost fields --- .../parser_benchmark_dataset.json | 9 ++- .../parser_benchmark_dataset.py | 2 +- .../parser_benchmark_log.json | 74 ++++++++++++++++--- .../parser_benchmark_log.py | 12 ++- transaction_parser/parser_benchmark/runner.py | 4 + 5 files changed, 82 insertions(+), 19 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index a805917..aad1886 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -103,23 +103,24 @@ { "fieldname": "page_limit", "fieldtype": "Int", - "label": "Page Limit" + "label": "Page Limit", + "non_negative": 1 }, { "fieldname": "section_break_bhsg", "fieldtype": "Section Break" }, { - "default": "Parser-Dataset-", + "default": "PAR-BM-DTS-", "fieldname": "naming_series", "fieldtype": "Select", "label": "Naming Series", - "options": "Parser-Dataset-" + "options": "PAR-BM-DTS-" } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-17 11:04:01.862839", + "modified": "2026-03-17 12:17:35.880185", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index ba9d0f3..6af1df6 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -28,7 +28,7 @@ class ParserBenchmarkDataset(Document): country: DF.Literal["India", "Other"] enabled: DF.Check file: DF.Attach - naming_series: DF.Literal["Parser-Dataset-"] + naming_series: DF.Literal["PAR-BM-DTS-"] page_limit: DF.Int pdf_processor: DF.Literal["OCRMyPDF", "Docling"] title: DF.Data diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index 3648bc1..98d0c83 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -5,13 +5,15 @@ "doctype": "DocType", "engine": "InnoDB", "field_order": [ - "naming_series", "dataset", "status", + "total_time", "column_break_summary", "ai_model", "pdf_processor", - "total_time", + "section_break_nmcv", + "naming_series", + "column_break_ubhs", "file_parsing_tab", "file_parse_time", "column_break_file_metrics", @@ -19,6 +21,11 @@ "section_break_file_content", "file_content", "ai_response_tab", + "input_token_cost", + "currency", + "column_break_ncdl", + "output_token_cost", + "section_break_yqhs", "prompt_tokens", "completion_tokens", "total_tokens", @@ -38,12 +45,12 @@ ], "fields": [ { - "default": "PBL-", + "default": "PAR-BM-LOG-", "fieldname": "naming_series", "fieldtype": "Select", "hidden": 1, "label": "Naming Series", - "options": "PBL-" + "options": "PAR-BM-LOG-" }, { "fieldname": "dataset", @@ -103,9 +110,10 @@ "fieldtype": "Column Break" }, { + "description": "Peak memory allocated during file parsing, measured using tracemalloc", "fieldname": "file_parse_memory", "fieldtype": "Float", - "label": "Memory Delta (MB)", + "label": "Peak Memory (MB)", "read_only": 1 }, { @@ -156,21 +164,32 @@ }, { "fieldname": "input_cost", - "fieldtype": "Float", + "fieldtype": "Currency", "label": "Input Cost", + "options": "currency", "read_only": 1 }, { "fieldname": "output_cost", - "fieldtype": "Float", + "fieldtype": "Currency", "label": "Output Cost", + "options": "currency", "read_only": 1 }, { "fieldname": "total_cost", - "fieldtype": "Float", + "fieldtype": "Currency", "in_list_view": 1, "label": "Total Cost", + "options": "currency", + "read_only": 1 + }, + { + "fieldname": "currency", + "fieldtype": "Link", + "hidden": 1, + "label": "Currency", + "options": "Currency", "read_only": 1 }, { @@ -218,11 +237,46 @@ { "fieldname": "column_break_wdyy", "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_nmcv", + "fieldtype": "Section Break" + }, + { + "fieldname": "column_break_ubhs", + "fieldtype": "Column Break" + }, + { + "description": "Cost per 1M input tokens at the time of this run", + "fieldname": "input_token_cost", + "fieldtype": "Currency", + "label": "Input Token Cost", + "non_negative": 1, + "options": "currency", + "read_only": 1 + }, + { + "description": "Cost per 1M output tokens at the time of this run", + "fieldname": "output_token_cost", + "fieldtype": "Currency", + "label": "Output Token Cost", + "non_negative": 1, + "options": "currency", + "read_only": 1 + }, + { + "fieldname": "column_break_ncdl", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_yqhs", + "fieldtype": "Section Break" } ], + "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-16 12:52:18.204234", + "modified": "2026-03-17 12:19:10.850540", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Log", @@ -246,4 +300,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} +} \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py index 59f4cf3..0c6e306 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -17,6 +17,7 @@ class ParserBenchmarkLog(Document): ai_parse_time: DF.Float ai_response: DF.JSON | None completion_tokens: DF.Int + currency: DF.Link | None dataset: DF.Link document_name: DF.DynamicLink | None document_type: DF.Link | None @@ -24,12 +25,15 @@ class ParserBenchmarkLog(Document): file_content: DF.Code | None file_parse_memory: DF.Float file_parse_time: DF.Float - input_cost: DF.Float - output_cost: DF.Float + input_cost: DF.Currency + input_token_cost: DF.Currency + naming_series: DF.Literal["PAR-BM-LOG-"] + output_cost: DF.Currency + output_token_cost: DF.Currency pdf_processor: DF.Data | None prompt_tokens: DF.Int - status: DF.Select | None - total_cost: DF.Float + status: DF.Literal["Queued", "Running", "Completed", "Failed"] + total_cost: DF.Currency total_time: DF.Float total_tokens: DF.Int # end: auto-generated types diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index ba2189d..79c6292 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -149,6 +149,10 @@ def _calculate_cost(self): if not cost_row: return + self.log.currency = cost_row.currency + self.log.input_token_cost = cost_row.input_cost_per_million + self.log.output_token_cost = cost_row.output_cost_per_million + prompt = self.log.prompt_tokens or 0 completion = self.log.completion_tokens or 0 From a35f60e38011b9c007f167f503cd315bab5ba5e8 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 17 Mar 2026 17:11:22 +0530 Subject: [PATCH 13/88] refactor: refactor parser benchmark settings and token cost configurations --- .../parser_benchmark_settings.js | 8 +++++ .../parser_benchmark_settings.json | 10 ++++--- .../test_parser_benchmark_settings.py | 9 ++++++ .../parser_benchmark_token_cost.json | 30 ++++++++++++++----- .../parser_benchmark_token_cost.py | 15 ++++++++-- 5 files changed, 57 insertions(+), 15 deletions(-) create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.js create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/test_parser_benchmark_settings.py diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.js b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.js new file mode 100644 index 0000000..bb53144 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.js @@ -0,0 +1,8 @@ +// Copyright (c) 2026, Resilient Tech and contributors +// For license information, please see license.txt + +// frappe.ui.form.on("Parser Benchmark Settings", { +// refresh(frm) { + +// }, +// }); diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json index 3081b83..ea894cf 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json @@ -1,6 +1,6 @@ { "actions": [], - "creation": "2026-03-16 00:00:00.000000", + "creation": "2026-03-16 00:00:00", "doctype": "DocType", "engine": "InnoDB", "field_order": [ @@ -11,12 +11,13 @@ "fieldname": "token_costs", "fieldtype": "Table", "label": "Token Costs", - "options": "Parser Benchmark Token Cost" + "options": "Parser Benchmark Token Cost", + "reqd": 1 } ], "issingle": 1, "links": [], - "modified": "2026-03-16 00:00:00.000000", + "modified": "2026-03-17 12:28:36.279956", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Settings", @@ -33,7 +34,8 @@ "write": 1 } ], + "row_format": "Dynamic", "sort_field": "modified", "sort_order": "DESC", "states": [] -} +} \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/test_parser_benchmark_settings.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/test_parser_benchmark_settings.py new file mode 100644 index 0000000..d9bd1ae --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/test_parser_benchmark_settings.py @@ -0,0 +1,9 @@ +# Copyright (c) 2026, Resilient Tech and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestParserBenchmarkSettings(FrappeTestCase): + pass diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json index 9c7f0ee..b34524c 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json @@ -1,12 +1,13 @@ { "actions": [], - "creation": "2026-03-16 00:00:00.000000", + "creation": "2026-03-16 00:00:00", "doctype": "DocType", "engine": "InnoDB", "field_order": [ "ai_model", - "currency", "input_cost_per_million", + "column_break_jvcd", + "currency", "output_cost_per_million" ], "fields": [ @@ -20,36 +21,49 @@ }, { "default": "USD", + "description": "Recommend using USD", "fieldname": "currency", "fieldtype": "Link", "in_list_view": 1, "label": "Currency", - "options": "Currency" + "options": "Currency", + "reqd": 1 }, { + "description": "Per 1M Tokens", "fieldname": "input_cost_per_million", - "fieldtype": "Float", + "fieldtype": "Currency", "in_list_view": 1, - "label": "Input Cost / 1M Tokens", + "label": "Input Cost", + "non_negative": 1, + "options": "currency", "reqd": 1 }, { + "description": "Per 1M Tokens", "fieldname": "output_cost_per_million", "fieldtype": "Float", "in_list_view": 1, - "label": "Output Cost / 1M Tokens", + "label": "Output Cost", + "non_negative": 1, + "options": "currency", "reqd": 1 + }, + { + "fieldname": "column_break_jvcd", + "fieldtype": "Column Break" } ], "istable": 1, "links": [], - "modified": "2026-03-16 00:00:00.000000", + "modified": "2026-03-17 12:33:28.293555", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Token Cost", "owner": "Administrator", "permissions": [], + "row_format": "Dynamic", "sort_field": "modified", "sort_order": "DESC", "states": [] -} +} \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py index 9a06e21..7c2f236 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py @@ -13,9 +13,18 @@ class ParserBenchmarkTokenCost(Document): if TYPE_CHECKING: from frappe.types import DF - ai_model: DF.Select - currency: DF.Link | None - input_cost_per_million: DF.Float + ai_model: DF.Literal[ + "DeepSeek Chat", + "DeepSeek Reasoner", + "OpenAI gpt-4o", + "OpenAI gpt-4o-mini", + "OpenAI gpt-5", + "OpenAI gpt-5-mini", + "Google Gemini Pro", + "Google Gemini Flash", + ] + currency: DF.Link + input_cost_per_million: DF.Currency output_cost_per_million: DF.Float parent: DF.Data parentfield: DF.Data From 5834e2e69c4c335e1e2269f342a18c33d32b8477 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 17 Mar 2026 17:13:37 +0530 Subject: [PATCH 14/88] fix: update patch number --- transaction_parser/patches.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transaction_parser/patches.txt b/transaction_parser/patches.txt index 27002f4..ea34737 100644 --- a/transaction_parser/patches.txt +++ b/transaction_parser/patches.txt @@ -4,5 +4,5 @@ [post_model_sync] # Patches added in this section will be executed after doctypes are migrated -execute:from transaction_parser.install import after_install; after_install() #2 +execute:from transaction_parser.install import after_install; after_install() #3 transaction_parser.patches.set_default_pdf_processor #1 From b59ab2f0e44a08ea07168f766fa1b201573ea830 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 17 Mar 2026 17:37:38 +0530 Subject: [PATCH 15/88] refactor: simplify AIParser's parse method and remove parse_with_response --- transaction_parser/parser_benchmark/runner.py | 4 ++-- .../ai_integration/parser.py | 23 ++----------------- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index 79c6292..cf576b9 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -103,7 +103,7 @@ def _run_ai_parsing(self, file_doc): parser = AIParser(self.dataset.ai_model) start = default_timer() - ai_content, response = parser.parse_with_response( + ai_content = parser.parse( document_type=self._controller.DOCTYPE, document_schema=schema, document_data=self._file_content, @@ -112,7 +112,7 @@ def _run_ai_parsing(self, file_doc): self.log.ai_parse_time = round(default_timer() - start, 4) # token usage - usage = response.get("usage", {}) + usage = parser.ai_response.get("usage", {}) self.log.prompt_tokens = usage.get("prompt_tokens", 0) self.log.completion_tokens = usage.get("completion_tokens", 0) self.log.total_tokens = usage.get("total_tokens", 0) diff --git a/transaction_parser/transaction_parser/ai_integration/parser.py b/transaction_parser/transaction_parser/ai_integration/parser.py index 39281c7..169fcb9 100644 --- a/transaction_parser/transaction_parser/ai_integration/parser.py +++ b/transaction_parser/transaction_parser/ai_integration/parser.py @@ -37,27 +37,8 @@ def parse( file_doc_name: str | None = None, ) -> dict: messages = self._build_messages(document_type, document_schema, document_data) - response = self.send_message(messages=messages, file_doc_name=file_doc_name) - return self.get_content(response) - - def parse_with_response( - self, - document_type: str, - document_schema: dict, - document_data: str, - file_doc_name: str | None = None, - ) -> tuple[dict, dict]: - """Parse document and return both the parsed content and the raw API response. - - Useful for benchmarking and diagnostics where token usage and other - response metadata are needed alongside the parsed output. - - Returns: - tuple: (parsed_content, raw_response) - """ - messages = self._build_messages(document_type, document_schema, document_data) - response = self.send_message(messages=messages, file_doc_name=file_doc_name) - return self.get_content(response), response + self.ai_response = self.send_message(messages=messages, file_doc_name=file_doc_name) + return self.get_content(self.ai_response) def _build_messages( self, document_type: str, document_schema: dict, document_data: str From 3beb5555a32a4cd5f8c74422cccfb6f97fac00f7 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 17 Mar 2026 17:54:32 +0530 Subject: [PATCH 16/88] refactor: enhance BenchmarkRunner for improved precision and clarity in cost calculations --- transaction_parser/parser_benchmark/runner.py | 108 +++++++++--------- 1 file changed, 55 insertions(+), 53 deletions(-) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index cf576b9..541c47d 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -2,6 +2,7 @@ from timeit import default_timer import frappe +from frappe.utils import cint, flt from transaction_parser.transaction_parser.ai_integration.parser import AIParser from transaction_parser.transaction_parser.controllers import get_controller @@ -23,26 +24,24 @@ class BenchmarkRunner: def __init__(self, log_name: str): self.log = frappe.get_doc("Parser Benchmark Log", log_name) self.dataset = frappe.get_doc("Parser Benchmark Dataset", self.log.dataset) - - # intermediate state shared between steps - self._file_content = None - self._ai_content = None - self._controller = None + self.precision = cint(frappe.db.get_default("float_precision")) or 3 def run(self): self.log.status = "Running" self.log.ai_model = self.dataset.ai_model self.log.pdf_processor = self.dataset.pdf_processor self.log.save(ignore_permissions=True) - frappe.db.commit() + frappe.db.commit() # persist "Running" status before the long background job starts total_start = default_timer() try: file_doc = self._get_file_doc() - self._run_file_parsing(file_doc) - self._run_ai_parsing(file_doc) - self._run_document_generation(file_doc) + controller = self._get_controller(file_doc) + + file_content = self._run_file_parsing(file_doc) + ai_content = self._run_ai_parsing(controller, file_content, file_doc) + self._run_document_generation(controller, ai_content) self._calculate_cost() self.log.status = "Completed" @@ -52,9 +51,9 @@ def run(self): self.log.error = frappe.get_traceback() finally: - self.log.total_time = round(default_timer() - total_start, 4) + self.log.total_time = flt(default_timer() - total_start, self.precision) self.log.save(ignore_permissions=True) - frappe.db.commit() + frappe.db.commit() # background jobs don't auto-commit; persist final results return self.log.name @@ -63,12 +62,28 @@ def run(self): def _get_file_doc(self): return frappe.get_last_doc("File", filters={"file_url": self.dataset.file}) - def _get_controller(self): - cls = get_controller(self.dataset.country, self.dataset.transaction_type) - controller = cls(company=self.dataset.company) + def _get_controller(self, file_doc): + ds = self.dataset + cls = get_controller(ds.country, ds.transaction_type) + + controller = cls(company=ds.company) controller.initialize() + controller.file = file_doc + return controller + def _get_cost_row(self): + try: + settings = frappe.get_cached_doc("Parser Benchmark Settings") + except Exception: + return None + + for row in settings.token_costs: + if row.ai_model == self.dataset.ai_model: + return row + + return None + # ── step 1: file parsing ──────────────────────────────── def _run_file_parsing(self, file_doc): @@ -85,67 +100,52 @@ def _run_file_parsing(self, file_doc): pdf_processor, ) finally: - self.log.file_parse_time = round(default_timer() - start, 4) + self.log.file_parse_time = flt(default_timer() - start, self.precision) _, peak = tracemalloc.get_traced_memory() tracemalloc.stop() - self.log.file_parse_memory = round(peak / 1024 / 1024, 2) # bytes → MB + self.log.file_parse_memory = flt( + peak / 1024 / 1024, self.precision + ) # bytes → MB self.log.file_content = content - self._file_content = content + return content # ── step 2: AI parsing ────────────────────────────────── - def _run_ai_parsing(self, file_doc): - self._controller = self._get_controller() - self._controller.file = file_doc - - schema = self._controller.get_schema() + def _run_ai_parsing(self, controller, file_content, file_doc): parser = AIParser(self.dataset.ai_model) start = default_timer() ai_content = parser.parse( - document_type=self._controller.DOCTYPE, - document_schema=schema, - document_data=self._file_content, + document_type=controller.DOCTYPE, + document_schema=controller.get_schema(), + document_data=file_content, file_doc_name=file_doc.name, ) - self.log.ai_parse_time = round(default_timer() - start, 4) + self.log.ai_parse_time = flt(default_timer() - start, self.precision) - # token usage usage = parser.ai_response.get("usage", {}) self.log.prompt_tokens = usage.get("prompt_tokens", 0) self.log.completion_tokens = usage.get("completion_tokens", 0) self.log.total_tokens = usage.get("total_tokens", 0) - - # parsed content self.log.ai_response = frappe.as_json(ai_content, indent=2) - self._ai_content = ai_content + + return ai_content # ── step 3: document generation ───────────────────────── - def _run_document_generation(self, file_doc): - c = self._controller - c.data = self._ai_content - c.create_document() - c.doc.db_set("is_created_by_benchmark", 1) + def _run_document_generation(self, controller, ai_content): + controller.data = ai_content + controller.create_document() + controller.doc.db_set("is_created_by_benchmark", 1) - self.log.document_type = c.DOCTYPE - self.log.document_name = c.doc.name + self.log.document_type = controller.DOCTYPE + self.log.document_name = controller.doc.name # ── step 4: cost calculation ──────────────────────────── def _calculate_cost(self): - try: - settings = frappe.get_cached_doc("Parser Benchmark Settings") - except Exception: - return - - cost_row = None - for row in settings.token_costs: - if row.ai_model == self.dataset.ai_model: - cost_row = row - break - + cost_row = self._get_cost_row() if not cost_row: return @@ -156,10 +156,12 @@ def _calculate_cost(self): prompt = self.log.prompt_tokens or 0 completion = self.log.completion_tokens or 0 - self.log.input_cost = round( - prompt * cost_row.input_cost_per_million / 1_000_000, 6 + self.log.input_cost = flt( + prompt * cost_row.input_cost_per_million / 1_000_000, self.precision + ) + self.log.output_cost = flt( + completion * cost_row.output_cost_per_million / 1_000_000, self.precision ) - self.log.output_cost = round( - completion * cost_row.output_cost_per_million / 1_000_000, 6 + self.log.total_cost = flt( + self.log.input_cost + self.log.output_cost, self.precision ) - self.log.total_cost = round(self.log.input_cost + self.log.output_cost, 6) From 32d3d1bd9a9b96792fb4db595309fbaff5ade250 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 11:54:41 +0530 Subject: [PATCH 17/88] fix: update modified timestamps and correct field types in benchmark documents --- .../parser_benchmark_dataset.json | 2 +- .../parser_benchmark_log.json | 4 +- .../parser_benchmark_log.py | 2 +- transaction_parser/parser_benchmark/runner.py | 58 ++++++++++++------- 4 files changed, 42 insertions(+), 24 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index aad1886..b4c122c 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -120,7 +120,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-17 12:17:35.880185", + "modified": "2026-03-18 06:59:00.336896", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index 98d0c83..4da8c05 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -199,7 +199,7 @@ }, { "fieldname": "ai_response", - "fieldtype": "JSON", + "fieldtype": "Code", "label": "AI Response", "read_only": 1 }, @@ -276,7 +276,7 @@ "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-17 12:19:10.850540", + "modified": "2026-03-18 06:50:06.637142", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Log", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py index 0c6e306..3edb898 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -15,7 +15,7 @@ class ParserBenchmarkLog(Document): ai_model: DF.Data | None ai_parse_time: DF.Float - ai_response: DF.JSON | None + ai_response: DF.Code | None completion_tokens: DF.Int currency: DF.Link | None dataset: DF.Link diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index 541c47d..797672a 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -2,10 +2,18 @@ from timeit import default_timer import frappe +from frappe.core.doctype.file.file import File from frappe.utils import cint, flt +from transaction_parser.parser_benchmark.doctype.parser_benchmark_dataset.parser_benchmark_dataset import ( + ParserBenchmarkDataset, +) +from transaction_parser.parser_benchmark.doctype.parser_benchmark_log.parser_benchmark_log import ( + ParserBenchmarkLog, +) from transaction_parser.transaction_parser.ai_integration.parser import AIParser from transaction_parser.transaction_parser.controllers import get_controller +from transaction_parser.transaction_parser.controllers.transaction import Transaction from transaction_parser.transaction_parser.utils.file_processor import FileProcessor from transaction_parser.transaction_parser.utils.pdf_processor import get_pdf_processor @@ -22,8 +30,10 @@ class BenchmarkRunner: """ def __init__(self, log_name: str): - self.log = frappe.get_doc("Parser Benchmark Log", log_name) - self.dataset = frappe.get_doc("Parser Benchmark Dataset", self.log.dataset) + self.log: ParserBenchmarkLog = frappe.get_doc("Parser Benchmark Log", log_name) + self.dataset: ParserBenchmarkDataset = frappe.get_doc( + "Parser Benchmark Dataset", self.log.dataset + ) self.precision = cint(frappe.db.get_default("float_precision")) or 3 def run(self): @@ -36,13 +46,13 @@ def run(self): total_start = default_timer() try: - file_doc = self._get_file_doc() - controller = self._get_controller(file_doc) + file_doc: File = self._get_file_doc() + self.controller: Transaction = self._get_controller(file_doc) file_content = self._run_file_parsing(file_doc) - ai_content = self._run_ai_parsing(controller, file_content, file_doc) - self._run_document_generation(controller, ai_content) + ai_content = self._run_ai_parsing(file_content, file_doc.name) self._calculate_cost() + self._run_document_generation(ai_content) self.log.status = "Completed" @@ -62,19 +72,26 @@ def run(self): def _get_file_doc(self): return frappe.get_last_doc("File", filters={"file_url": self.dataset.file}) - def _get_controller(self, file_doc): + def _get_controller(self, file_doc: File) -> Transaction: ds = self.dataset cls = get_controller(ds.country, ds.transaction_type) controller = cls(company=ds.company) - controller.initialize() + controller.initialize() # manual initialization to set up doctype/schema without needing document data controller.file = file_doc + controller.ai_model = self.dataset.ai_model return controller def _get_cost_row(self): + from transaction_parser.parser_benchmark.doctype.parser_benchmark_settings.parser_benchmark_settings import ( + ParserBenchmarkSettings, + ) + try: - settings = frappe.get_cached_doc("Parser Benchmark Settings") + settings: ParserBenchmarkSettings = frappe.get_cached_doc( + "Parser Benchmark Settings" + ) except Exception: return None @@ -86,7 +103,7 @@ def _get_cost_row(self): # ── step 1: file parsing ──────────────────────────────── - def _run_file_parsing(self, file_doc): + def _run_file_parsing(self, file_doc: File) -> str: pdf_processor = None if file_doc.file_type == "PDF" and self.dataset.pdf_processor: pdf_processor = get_pdf_processor(self.dataset.pdf_processor) @@ -112,15 +129,15 @@ def _run_file_parsing(self, file_doc): # ── step 2: AI parsing ────────────────────────────────── - def _run_ai_parsing(self, controller, file_content, file_doc): + def _run_ai_parsing(self, file_content: str, file_name: str) -> dict: parser = AIParser(self.dataset.ai_model) start = default_timer() ai_content = parser.parse( - document_type=controller.DOCTYPE, - document_schema=controller.get_schema(), + document_type=self.controller.DOCTYPE, + document_schema=self.controller.get_schema(), document_data=file_content, - file_doc_name=file_doc.name, + file_doc_name=file_name, ) self.log.ai_parse_time = flt(default_timer() - start, self.precision) @@ -134,16 +151,17 @@ def _run_ai_parsing(self, controller, file_content, file_doc): # ── step 3: document generation ───────────────────────── - def _run_document_generation(self, controller, ai_content): - controller.data = ai_content - controller.create_document() - controller.doc.db_set("is_created_by_benchmark", 1) + def _run_document_generation(self, ai_content: dict): + self.controller.data = ai_content + self.controller.create_document() + self.controller.doc.db_set("is_created_by_benchmark", 1) - self.log.document_type = controller.DOCTYPE - self.log.document_name = controller.doc.name + self.log.document_type = self.controller.DOCTYPE + self.log.document_name = self.controller.doc.name # ── step 4: cost calculation ──────────────────────────── + # Code is not reaching here, need to fix the issue first def _calculate_cost(self): cost_row = self._get_cost_row() if not cost_row: From 0f7eb1460ee59b5edd076b131ac810dde2a596ba Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 11:57:00 +0530 Subject: [PATCH 18/88] revert: remove redundant benchmark fields from CUSTOM_FIELDS and update patch execution order --- transaction_parser/install.py | 16 +--------------- transaction_parser/patches.txt | 2 +- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/transaction_parser/install.py b/transaction_parser/install.py index f8221e6..8e6f35c 100644 --- a/transaction_parser/install.py +++ b/transaction_parser/install.py @@ -11,14 +11,7 @@ "label": "Is Created By Transaction Parser", "read_only": 1, "insert_after": "is_internal_customer", - }, - { - "fieldname": "is_created_by_benchmark", - "fieldtype": "Check", - "label": "Is Created By Benchmark", - "read_only": 1, - "insert_after": "is_created_by_transaction_parser", - }, + } ], "Purchase Invoice": [ { @@ -28,13 +21,6 @@ "read_only": 1, "insert_after": "is_internal_supplier", }, - { - "fieldname": "is_created_by_benchmark", - "fieldtype": "Check", - "label": "Is Created By Benchmark", - "read_only": 1, - "insert_after": "is_created_by_transaction_parser", - }, ], "Communication": [ { diff --git a/transaction_parser/patches.txt b/transaction_parser/patches.txt index ea34737..27002f4 100644 --- a/transaction_parser/patches.txt +++ b/transaction_parser/patches.txt @@ -4,5 +4,5 @@ [post_model_sync] # Patches added in this section will be executed after doctypes are migrated -execute:from transaction_parser.install import after_install; after_install() #3 +execute:from transaction_parser.install import after_install; after_install() #2 transaction_parser.patches.set_default_pdf_processor #1 From 22a12ed157f7494d55ef36316eb03cf6b57a2b5d Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 12:01:59 +0530 Subject: [PATCH 19/88] refactor: remove unused fields from ParserBenchmarkLog and update field order --- .../parser_benchmark_log.json | 38 ++++--------------- .../parser_benchmark_log.py | 2 - 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index 4da8c05..95aaa4a 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -36,10 +36,7 @@ "total_cost", "section_break_ai_content", "ai_response", - "result_tab", - "document_type", - "column_break_wdyy", - "document_name", + "error_tab", "section_break_error", "error" ], @@ -203,30 +200,10 @@ "label": "AI Response", "read_only": 1 }, - { - "fieldname": "result_tab", - "fieldtype": "Tab Break", - "label": "Result" - }, - { - "fieldname": "document_type", - "fieldtype": "Link", - "label": "Document Type", - "options": "DocType", - "read_only": 1 - }, - { - "fieldname": "document_name", - "fieldtype": "Dynamic Link", - "label": "Document", - "options": "document_type", - "read_only": 1 - }, { "depends_on": "eval: doc.status === 'Failed'", "fieldname": "section_break_error", - "fieldtype": "Section Break", - "label": "Error" + "fieldtype": "Section Break" }, { "fieldname": "error", @@ -234,10 +211,6 @@ "label": "Error Traceback", "read_only": 1 }, - { - "fieldname": "column_break_wdyy", - "fieldtype": "Column Break" - }, { "fieldname": "section_break_nmcv", "fieldtype": "Section Break" @@ -271,12 +244,17 @@ { "fieldname": "section_break_yqhs", "fieldtype": "Section Break" + }, + { + "fieldname": "error_tab", + "fieldtype": "Tab Break", + "label": "Error" } ], "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-18 06:50:06.637142", + "modified": "2026-03-18 07:31:12.048039", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Log", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py index 3edb898..b168824 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -19,8 +19,6 @@ class ParserBenchmarkLog(Document): completion_tokens: DF.Int currency: DF.Link | None dataset: DF.Link - document_name: DF.DynamicLink | None - document_type: DF.Link | None error: DF.Code | None file_content: DF.Code | None file_parse_memory: DF.Float From e234932145795f0ede244efe47bf16aedd065bf5 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 12:31:32 +0530 Subject: [PATCH 20/88] refactor: update field types and modify structure in benchmark dataset and log --- .../parser_benchmark_dataset.json | 5 +- .../parser_benchmark_dataset.py | 2 +- .../parser_benchmark_log.json | 98 +++++++++++++++++-- .../parser_benchmark_log.py | 4 + 4 files changed, 95 insertions(+), 14 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index b4c122c..f0aba8a 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -97,8 +97,7 @@ "fieldname": "company", "fieldtype": "Link", "label": "Company", - "options": "Company", - "reqd": 1 + "options": "Company" }, { "fieldname": "page_limit", @@ -120,7 +119,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-18 06:59:00.336896", + "modified": "2026-03-18 07:55:54.431360", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 6af1df6..4b49b8f 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -24,7 +24,7 @@ class ParserBenchmarkDataset(Document): "Google Gemini Pro", "Google Gemini Flash", ] - company: DF.Link + company: DF.Link | None country: DF.Literal["India", "Other"] enabled: DF.Check file: DF.Attach diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index 95aaa4a..05b86af 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -5,22 +5,35 @@ "doctype": "DocType", "engine": "InnoDB", "field_order": [ + "config_section", "dataset", - "status", - "total_time", + "transaction_type", "column_break_summary", - "ai_model", - "pdf_processor", + "country", + "company", + "section_break_peyo", + "total_time", + "column_break_fmal", + "status", "section_break_nmcv", "naming_series", "column_break_ubhs", "file_parsing_tab", - "file_parse_time", + "pdf_processor", "column_break_file_metrics", + "page_limit", + "section_break_umzr", + "file_parse_time", + "column_break_kgph", "file_parse_memory", "section_break_file_content", "file_content", "ai_response_tab", + "other_details_section", + "ai_model", + "column_break_fsxb", + "ai_parse_time", + "api_cost_section", "input_token_cost", "currency", "column_break_ncdl", @@ -29,7 +42,6 @@ "prompt_tokens", "completion_tokens", "total_tokens", - "ai_parse_time", "column_break_ai", "input_cost", "output_cost", @@ -191,10 +203,10 @@ }, { "fieldname": "section_break_ai_content", - "fieldtype": "Section Break", - "label": "AI Output" + "fieldtype": "Section Break" }, { + "description": "Only document content", "fieldname": "ai_response", "fieldtype": "Code", "label": "AI Response", @@ -212,6 +224,7 @@ "read_only": 1 }, { + "collapsible": 1, "fieldname": "section_break_nmcv", "fieldtype": "Section Break" }, @@ -243,18 +256,83 @@ }, { "fieldname": "section_break_yqhs", - "fieldtype": "Section Break" + "fieldtype": "Section Break", + "label": "Token Details" }, { "fieldname": "error_tab", "fieldtype": "Tab Break", "label": "Error" + }, + { + "fieldname": "api_cost_section", + "fieldtype": "Section Break", + "label": "API Cost" + }, + { + "fieldname": "other_details_section", + "fieldtype": "Section Break", + "label": "Details" + }, + { + "fieldname": "column_break_fsxb", + "fieldtype": "Column Break" + }, + { + "fieldname": "transaction_type", + "fieldtype": "Select", + "label": "Transaction Type", + "options": "Sales Order\nExpense", + "read_only": 1 + }, + { + "fieldname": "country", + "fieldtype": "Select", + "label": "Country", + "options": "India\nOther", + "read_only": 1 + }, + { + "fieldname": "company", + "fieldtype": "Link", + "label": "Company", + "options": "Company", + "read_only": 1 + }, + { + "fieldname": "section_break_peyo", + "fieldtype": "Section Break" + }, + { + "fieldname": "column_break_fmal", + "fieldtype": "Column Break" + }, + { + "fieldname": "config_section", + "fieldtype": "Section Break", + "label": "Config" + }, + { + "default": "0", + "fieldname": "page_limit", + "fieldtype": "Int", + "label": "Page Limit", + "non_negative": 1, + "read_only": 1 + }, + { + "fieldname": "section_break_umzr", + "fieldtype": "Section Break" + }, + { + "fieldname": "column_break_kgph", + "fieldtype": "Column Break" } ], "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-18 07:31:12.048039", + "modified": "2026-03-18 07:58:21.307417", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Log", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py index b168824..3f03825 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -16,7 +16,9 @@ class ParserBenchmarkLog(Document): ai_model: DF.Data | None ai_parse_time: DF.Float ai_response: DF.Code | None + company: DF.Link | None completion_tokens: DF.Int + country: DF.Literal["India", "Other"] currency: DF.Link | None dataset: DF.Link error: DF.Code | None @@ -28,12 +30,14 @@ class ParserBenchmarkLog(Document): naming_series: DF.Literal["PAR-BM-LOG-"] output_cost: DF.Currency output_token_cost: DF.Currency + page_limit: DF.Int pdf_processor: DF.Data | None prompt_tokens: DF.Int status: DF.Literal["Queued", "Running", "Completed", "Failed"] total_cost: DF.Currency total_time: DF.Float total_tokens: DF.Int + transaction_type: DF.Literal["Sales Order", "Expense"] # end: auto-generated types pass From ca02fd7a009debd3518da12c534705c1be068aac Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 12:42:25 +0530 Subject: [PATCH 21/88] refactor: reorder fields in parser benchmark dataset and add column break for improved layout --- .../parser_benchmark_dataset.json | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index f0aba8a..afc1a89 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -9,15 +9,16 @@ "column_break_title", "enabled", "section_break_file", - "file", "column_break_file", + "file", "transaction_type", + "column_break_gbap", "country", + "company", "processing_section", "ai_model", "pdf_processor", "column_break_processing", - "company", "page_limit", "section_break_bhsg", "naming_series" @@ -115,11 +116,15 @@ "fieldtype": "Select", "label": "Naming Series", "options": "PAR-BM-DTS-" + }, + { + "fieldname": "column_break_gbap", + "fieldtype": "Column Break" } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-18 07:55:54.431360", + "modified": "2026-03-18 08:03:14.089586", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", From a0e9f832cbdba95117399d9b490b55e0328b5b0a Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 12:46:27 +0530 Subject: [PATCH 22/88] refactor: update AI model options and modify timestamps in benchmark settings --- .../parser_benchmark_dataset.json | 4 +- .../parser_benchmark_dataset.py | 4 +- .../transaction_parser_settings.json | 6 +-- .../transaction_parser_settings.py | 43 +++++++++++++++++++ 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index afc1a89..f2da857 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -80,7 +80,7 @@ "fieldtype": "Select", "in_list_view": 1, "label": "AI Model", - "options": "DeepSeek Chat\nDeepSeek Reasoner\nOpenAI gpt-4o\nOpenAI gpt-4o-mini\nOpenAI gpt-5\nOpenAI gpt-5-mini\nGoogle Gemini Pro\nGoogle Gemini Flash", + "options": "DeepSeek Chat\nDeepSeek Reasoner\nOpenAI gpt-4o\nOpenAI gpt-4o-mini\nOpenAI gpt-5\nOpenAI gpt-5-mini\nGoogle Gemini Pro-2.5\nGoogle Gemini Flash-2.5", "reqd": 1 }, { @@ -124,7 +124,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-18 08:03:14.089586", + "modified": "2026-03-18 08:15:07.566307", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 4b49b8f..0640dce 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -21,8 +21,8 @@ class ParserBenchmarkDataset(Document): "OpenAI gpt-4o-mini", "OpenAI gpt-5", "OpenAI gpt-5-mini", - "Google Gemini Pro", - "Google Gemini Flash", + "Google Gemini Pro-2.5", + "Google Gemini Flash-2.5", ] company: DF.Link | None country: DF.Literal["India", "Other"] diff --git a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json index 53dc83c..2bc8dcd 100644 --- a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json +++ b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json @@ -91,7 +91,7 @@ "fieldtype": "Select", "label": "Default AI Model", "mandatory_depends_on": "eval: doc.enabled", - "options": "DeepSeek Chat\nDeepSeek Reasoner\nOpenAI gpt-4o\nOpenAI gpt-4o-mini\nOpenAI gpt-5\nOpenAI gpt-5-mini\nGoogle Gemini Pro\nGoogle Gemini Flash" + "options": "DeepSeek Chat\nDeepSeek Reasoner\nOpenAI gpt-4o\nOpenAI gpt-4o-mini\nOpenAI gpt-5\nOpenAI gpt-5-mini\nGoogle Gemini Pro-2.5\nGoogle Gemini Flash-2.5" }, { "default": "OCRMyPDF", @@ -166,7 +166,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2026-03-14 13:35:17.150533", + "modified": "2026-03-18 08:14:24.869817", "modified_by": "Administrator", "module": "Transaction Parser", "name": "Transaction Parser Settings", @@ -187,4 +187,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} +} \ No newline at end of file diff --git a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py index 95b4c43..44d0f51 100644 --- a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py +++ b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py @@ -12,6 +12,49 @@ class TransactionParserSettings(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + from transaction_parser.transaction_parser.doctype.transaction_parser_api_key_item.transaction_parser_api_key_item import ( + TransactionParserAPIKeyItem, + ) + from transaction_parser.transaction_parser.doctype.transaction_parser_email_account.transaction_parser_email_account import ( + TransactionParserEmailAccount, + ) + from transaction_parser.transaction_parser.doctype.transaction_parser_party_email.transaction_parser_party_email import ( + TransactionParserPartyEmail, + ) + + address_schema: DF.JSON | None + api_keys: DF.Table[TransactionParserAPIKeyItem] + base_schema: DF.JSON | None + default_ai_model: DF.Literal[ + "DeepSeek Chat", + "DeepSeek Reasoner", + "OpenAI gpt-4o", + "OpenAI gpt-4o-mini", + "OpenAI gpt-5", + "OpenAI gpt-5-mini", + "Google Gemini Pro-2.5", + "Google Gemini Flash-2.5", + ] + enabled: DF.Check + incoming_email_accounts: DF.Table[TransactionParserEmailAccount] + invoice_lookback_count: DF.Int + item_schema: DF.JSON | None + parse_incoming_emails: DF.Check + parse_party_emails: DF.Check + party_emails: DF.Table[TransactionParserPartyEmail] + party_schema: DF.JSON | None + pdf_processor: DF.Literal["OCRMyPDF", "Docling"] + tax_schema: DF.JSON | None + + # end: auto-generated types # TODO: can we check API creds? def validate(self): self.validate_lookback_count() From 5344529b2f9f6c1fe4b5d887d3e87bd3b5e70e9a Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 14:29:03 +0530 Subject: [PATCH 23/88] refactor: enhance parser benchmark dataset structure with additional AI model options and scheduling fields --- .../parser_benchmark_dataset.json | 194 +++++++++++++++--- .../parser_benchmark_dataset.py | 28 ++- 2 files changed, 184 insertions(+), 38 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index f2da857..3a35d6c 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -9,17 +9,39 @@ "column_break_title", "enabled", "section_break_file", - "column_break_file", "file", "transaction_type", "column_break_gbap", "country", "company", "processing_section", - "ai_model", - "pdf_processor", - "column_break_processing", + "column_break_ai_models", + "openai_gpt_4o_mini", + "openai_gpt_4o", + "openai_gpt_5_mini", + "openai_gpt_5", + "column_break_leid", + "deepseek_chat", + "deepseek_reasoner", + "column_break_nqke", + "google_gemini_flash_25", + "google_gemini_pro_25", + "pdf_processor_section", + "ocrmypdf", + "column_break_pdf_processor", + "docling", + "other_config_section", "page_limit", + "schedule_section", + "monday", + "thursday", + "sunday", + "column_break_schedule", + "tuesday", + "friday", + "column_break_szid", + "wednesday", + "saturday", "section_break_bhsg", "naming_series" ], @@ -50,10 +72,6 @@ "label": "File", "reqd": 1 }, - { - "fieldname": "column_break_file", - "fieldtype": "Column Break" - }, { "fieldname": "transaction_type", "fieldtype": "Select", @@ -70,35 +88,98 @@ "options": "India\nOther", "reqd": 1 }, + { + "fieldname": "column_break_gbap", + "fieldtype": "Column Break" + }, + { + "fieldname": "company", + "fieldtype": "Link", + "label": "Company", + "options": "Company" + }, { "fieldname": "processing_section", "fieldtype": "Section Break", - "label": "Processing Configuration" + "label": "AI Models" }, { - "fieldname": "ai_model", - "fieldtype": "Select", - "in_list_view": 1, - "label": "AI Model", - "options": "DeepSeek Chat\nDeepSeek Reasoner\nOpenAI gpt-4o\nOpenAI gpt-4o-mini\nOpenAI gpt-5\nOpenAI gpt-5-mini\nGoogle Gemini Pro-2.5\nGoogle Gemini Flash-2.5", - "reqd": 1 + "default": "0", + "fieldname": "deepseek_chat", + "fieldtype": "Check", + "label": "DeepSeek Chat" }, { - "fieldname": "pdf_processor", - "fieldtype": "Select", - "label": "PDF Processor", - "options": "OCRMyPDF\nDocling", - "reqd": 1 + "default": "0", + "fieldname": "deepseek_reasoner", + "fieldtype": "Check", + "label": "DeepSeek Reasoner" + }, + { + "default": "0", + "fieldname": "openai_gpt_4o", + "fieldtype": "Check", + "label": "OpenAI gpt-4o" }, { - "fieldname": "column_break_processing", + "default": "0", + "fieldname": "openai_gpt_4o_mini", + "fieldtype": "Check", + "label": "OpenAI gpt-4o-mini" + }, + { + "fieldname": "column_break_ai_models", "fieldtype": "Column Break" }, { - "fieldname": "company", - "fieldtype": "Link", - "label": "Company", - "options": "Company" + "default": "0", + "fieldname": "openai_gpt_5", + "fieldtype": "Check", + "label": "OpenAI gpt-5" + }, + { + "default": "0", + "fieldname": "openai_gpt_5_mini", + "fieldtype": "Check", + "label": "OpenAI gpt-5-mini" + }, + { + "default": "0", + "fieldname": "google_gemini_pro_25", + "fieldtype": "Check", + "label": "Google Gemini Pro-2.5" + }, + { + "default": "0", + "fieldname": "google_gemini_flash_25", + "fieldtype": "Check", + "label": "Google Gemini Flash-2.5" + }, + { + "fieldname": "pdf_processor_section", + "fieldtype": "Section Break", + "label": "PDF Processors" + }, + { + "default": "0", + "fieldname": "ocrmypdf", + "fieldtype": "Check", + "label": "OCRMyPDF" + }, + { + "fieldname": "column_break_pdf_processor", + "fieldtype": "Column Break" + }, + { + "default": "0", + "fieldname": "docling", + "fieldtype": "Check", + "label": "Docling" + }, + { + "fieldname": "other_config_section", + "fieldtype": "Section Break", + "label": "Other Configuration" }, { "fieldname": "page_limit", @@ -106,6 +187,57 @@ "label": "Page Limit", "non_negative": 1 }, + { + "fieldname": "schedule_section", + "fieldtype": "Section Break", + "label": "Schedule" + }, + { + "default": "0", + "fieldname": "monday", + "fieldtype": "Check", + "label": "Monday" + }, + { + "default": "0", + "fieldname": "tuesday", + "fieldtype": "Check", + "label": "Tuesday" + }, + { + "default": "0", + "fieldname": "wednesday", + "fieldtype": "Check", + "label": "Wednesday" + }, + { + "default": "0", + "fieldname": "thursday", + "fieldtype": "Check", + "label": "Thursday" + }, + { + "fieldname": "column_break_schedule", + "fieldtype": "Column Break" + }, + { + "default": "0", + "fieldname": "friday", + "fieldtype": "Check", + "label": "Friday" + }, + { + "default": "0", + "fieldname": "saturday", + "fieldtype": "Check", + "label": "Saturday" + }, + { + "default": "0", + "fieldname": "sunday", + "fieldtype": "Check", + "label": "Sunday" + }, { "fieldname": "section_break_bhsg", "fieldtype": "Section Break" @@ -118,13 +250,21 @@ "options": "PAR-BM-DTS-" }, { - "fieldname": "column_break_gbap", + "fieldname": "column_break_leid", + "fieldtype": "Column Break" + }, + { + "fieldname": "column_break_nqke", + "fieldtype": "Column Break" + }, + { + "fieldname": "column_break_szid", "fieldtype": "Column Break" } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-18 08:15:07.566307", + "modified": "2026-03-18 09:58:01.274730", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 0640dce..0b35f3e 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -14,25 +14,31 @@ class ParserBenchmarkDataset(Document): if TYPE_CHECKING: from frappe.types import DF - ai_model: DF.Literal[ - "DeepSeek Chat", - "DeepSeek Reasoner", - "OpenAI gpt-4o", - "OpenAI gpt-4o-mini", - "OpenAI gpt-5", - "OpenAI gpt-5-mini", - "Google Gemini Pro-2.5", - "Google Gemini Flash-2.5", - ] company: DF.Link | None country: DF.Literal["India", "Other"] + deepseek_chat: DF.Check + deepseek_reasoner: DF.Check + docling: DF.Check enabled: DF.Check file: DF.Attach + friday: DF.Check + google_gemini_flash_25: DF.Check + google_gemini_pro_25: DF.Check + monday: DF.Check naming_series: DF.Literal["PAR-BM-DTS-"] + ocrmypdf: DF.Check + openai_gpt_4o: DF.Check + openai_gpt_4o_mini: DF.Check + openai_gpt_5: DF.Check + openai_gpt_5_mini: DF.Check page_limit: DF.Int - pdf_processor: DF.Literal["OCRMyPDF", "Docling"] + saturday: DF.Check + sunday: DF.Check + thursday: DF.Check title: DF.Data transaction_type: DF.Literal["Sales Order", "Expense"] + tuesday: DF.Check + wednesday: DF.Check # end: auto-generated types pass From c5a601d25fcc69ffb5bda633164373a6977be0ef Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 14:30:28 +0530 Subject: [PATCH 24/88] refactor: remove unnecessary label from "Run Benchmark" button in Parser Benchmark Dataset --- .../parser_benchmark_dataset/parser_benchmark_dataset.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js index 80522dc..0472ce4 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js @@ -4,7 +4,7 @@ frappe.ui.form.on("Parser Benchmark Dataset", { refresh(frm) { if (!frm.is_new() && frm.doc.enabled) { - frm.add_custom_button(__("Run Benchmark"), () => run_benchmark(frm), __("Actions")); + frm.add_custom_button(__("Run Benchmark"), () => run_benchmark(frm)); } set_pdf_processor_options(frm); From a05f8d53affcf8116d798bd1da6967bdf1f6857f Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 14:32:28 +0530 Subject: [PATCH 25/88] refactor: add standard filter options for status, transaction type, and country fields in parser benchmark log --- .../doctype/parser_benchmark_log/parser_benchmark_log.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index 05b86af..bbdb83d 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -75,6 +75,7 @@ "fieldname": "status", "fieldtype": "Select", "in_list_view": 1, + "in_standard_filter": 1, "label": "Status", "options": "Queued\nRunning\nCompleted\nFailed", "read_only": 1 @@ -281,6 +282,8 @@ { "fieldname": "transaction_type", "fieldtype": "Select", + "in_list_view": 1, + "in_standard_filter": 1, "label": "Transaction Type", "options": "Sales Order\nExpense", "read_only": 1 @@ -288,6 +291,8 @@ { "fieldname": "country", "fieldtype": "Select", + "in_list_view": 1, + "in_standard_filter": 1, "label": "Country", "options": "India\nOther", "read_only": 1 @@ -332,7 +337,7 @@ "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-18 07:58:21.307417", + "modified": "2026-03-18 10:01:56.943584", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Log", From 316180a696cb361f8fa26340f9061d744dd01c7d Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 14:36:09 +0530 Subject: [PATCH 26/88] refactor: update AI model and PDF processor fields to use select options in benchmark log and token cost --- .../parser_benchmark_log/parser_benchmark_log.json | 8 +++++--- .../parser_benchmark_log/parser_benchmark_log.py | 13 +++++++++++-- .../parser_benchmark_token_cost.json | 4 ++-- .../parser_benchmark_token_cost.py | 4 ++-- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index bbdb83d..5b22bb9 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -86,15 +86,17 @@ }, { "fieldname": "ai_model", - "fieldtype": "Data", + "fieldtype": "Select", "in_list_view": 1, "label": "AI Model", + "options": "DeepSeek Chat\nDeepSeek Reasoner\nOpenAI gpt-4o\nOpenAI gpt-4o-mini\nOpenAI gpt-5\nOpenAI gpt-5-mini\nGoogle Gemini Pro-2.5\nGoogle Gemini Flash-2.5", "read_only": 1 }, { "fieldname": "pdf_processor", - "fieldtype": "Data", + "fieldtype": "Select", "label": "PDF Processor", + "options": "OCRMyPDF\nDocling", "read_only": 1 }, { @@ -337,7 +339,7 @@ "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-18 10:01:56.943584", + "modified": "2026-03-18 10:04:36.919978", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Log", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py index 3f03825..e673d0a 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -13,7 +13,16 @@ class ParserBenchmarkLog(Document): if TYPE_CHECKING: from frappe.types import DF - ai_model: DF.Data | None + ai_model: DF.Literal[ + "DeepSeek Chat", + "DeepSeek Reasoner", + "OpenAI gpt-4o", + "OpenAI gpt-4o-mini", + "OpenAI gpt-5", + "OpenAI gpt-5-mini", + "Google Gemini Pro-2.5", + "Google Gemini Flash-2.5", + ] ai_parse_time: DF.Float ai_response: DF.Code | None company: DF.Link | None @@ -31,7 +40,7 @@ class ParserBenchmarkLog(Document): output_cost: DF.Currency output_token_cost: DF.Currency page_limit: DF.Int - pdf_processor: DF.Data | None + pdf_processor: DF.Literal["OCRMyPDF", "Docling"] prompt_tokens: DF.Int status: DF.Literal["Queued", "Running", "Completed", "Failed"] total_cost: DF.Currency diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json index b34524c..030ebd6 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json @@ -16,7 +16,7 @@ "fieldtype": "Select", "in_list_view": 1, "label": "AI Model", - "options": "DeepSeek Chat\nDeepSeek Reasoner\nOpenAI gpt-4o\nOpenAI gpt-4o-mini\nOpenAI gpt-5\nOpenAI gpt-5-mini\nGoogle Gemini Pro\nGoogle Gemini Flash", + "options": "DeepSeek Chat\nDeepSeek Reasoner\nOpenAI gpt-4o\nOpenAI gpt-4o-mini\nOpenAI gpt-5\nOpenAI gpt-5-mini\nGoogle Gemini Pro-2.5\nGoogle Gemini Flash-2.5", "reqd": 1 }, { @@ -56,7 +56,7 @@ ], "istable": 1, "links": [], - "modified": "2026-03-17 12:33:28.293555", + "modified": "2026-03-18 10:05:11.453785", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Token Cost", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py index 7c2f236..fa33751 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py @@ -20,8 +20,8 @@ class ParserBenchmarkTokenCost(Document): "OpenAI gpt-4o-mini", "OpenAI gpt-5", "OpenAI gpt-5-mini", - "Google Gemini Pro", - "Google Gemini Flash", + "Google Gemini Pro-2.5", + "Google Gemini Flash-2.5", ] currency: DF.Link input_cost_per_million: DF.Currency From 4dcc6f2f3fb127c4e2356c60f1b6e33d66f2af0c Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 14:48:21 +0530 Subject: [PATCH 27/88] refactor: update parser benchmark settings and dataset structure by removing schedule fields and adding enabled option --- .../parser_benchmark_dataset.json | 72 ++-------------- .../parser_benchmark_dataset.py | 7 -- .../parser_benchmark_settings.json | 86 ++++++++++++++++++- .../parser_benchmark_settings.py | 8 ++ 4 files changed, 99 insertions(+), 74 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index 3a35d6c..2cb4c37 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -23,25 +23,16 @@ "column_break_leid", "deepseek_chat", "deepseek_reasoner", - "column_break_nqke", + "column_break_zgpf", "google_gemini_flash_25", "google_gemini_pro_25", "pdf_processor_section", "ocrmypdf", - "column_break_pdf_processor", "docling", + "column_break_pdf_processor", "other_config_section", "page_limit", - "schedule_section", - "monday", - "thursday", - "sunday", - "column_break_schedule", - "tuesday", - "friday", - "column_break_szid", - "wednesday", - "saturday", + "column_break_jjht", "section_break_bhsg", "naming_series" ], @@ -187,57 +178,6 @@ "label": "Page Limit", "non_negative": 1 }, - { - "fieldname": "schedule_section", - "fieldtype": "Section Break", - "label": "Schedule" - }, - { - "default": "0", - "fieldname": "monday", - "fieldtype": "Check", - "label": "Monday" - }, - { - "default": "0", - "fieldname": "tuesday", - "fieldtype": "Check", - "label": "Tuesday" - }, - { - "default": "0", - "fieldname": "wednesday", - "fieldtype": "Check", - "label": "Wednesday" - }, - { - "default": "0", - "fieldname": "thursday", - "fieldtype": "Check", - "label": "Thursday" - }, - { - "fieldname": "column_break_schedule", - "fieldtype": "Column Break" - }, - { - "default": "0", - "fieldname": "friday", - "fieldtype": "Check", - "label": "Friday" - }, - { - "default": "0", - "fieldname": "saturday", - "fieldtype": "Check", - "label": "Saturday" - }, - { - "default": "0", - "fieldname": "sunday", - "fieldtype": "Check", - "label": "Sunday" - }, { "fieldname": "section_break_bhsg", "fieldtype": "Section Break" @@ -254,17 +194,17 @@ "fieldtype": "Column Break" }, { - "fieldname": "column_break_nqke", + "fieldname": "column_break_jjht", "fieldtype": "Column Break" }, { - "fieldname": "column_break_szid", + "fieldname": "column_break_zgpf", "fieldtype": "Column Break" } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-18 09:58:01.274730", + "modified": "2026-03-18 10:17:58.099090", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 0b35f3e..2e60983 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -21,10 +21,8 @@ class ParserBenchmarkDataset(Document): docling: DF.Check enabled: DF.Check file: DF.Attach - friday: DF.Check google_gemini_flash_25: DF.Check google_gemini_pro_25: DF.Check - monday: DF.Check naming_series: DF.Literal["PAR-BM-DTS-"] ocrmypdf: DF.Check openai_gpt_4o: DF.Check @@ -32,13 +30,8 @@ class ParserBenchmarkDataset(Document): openai_gpt_5: DF.Check openai_gpt_5_mini: DF.Check page_limit: DF.Int - saturday: DF.Check - sunday: DF.Check - thursday: DF.Check title: DF.Data transaction_type: DF.Literal["Sales Order", "Expense"] - tuesday: DF.Check - wednesday: DF.Check # end: auto-generated types pass diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json index ea894cf..6875c8e 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json @@ -4,6 +4,19 @@ "doctype": "DocType", "engine": "InnoDB", "field_order": [ + "enabled", + "column_break_gpyw", + "schedule_section", + "monday", + "thursday", + "sunday", + "column_break_schedule", + "tuesday", + "friday", + "column_break_kaxp", + "wednesday", + "saturday", + "section_break_kvgf", "token_costs" ], "fields": [ @@ -13,11 +26,82 @@ "label": "Token Costs", "options": "Parser Benchmark Token Cost", "reqd": 1 + }, + { + "depends_on": "eval: doc.enabled", + "fieldname": "schedule_section", + "fieldtype": "Section Break", + "label": "Schedule" + }, + { + "default": "0", + "fieldname": "monday", + "fieldtype": "Check", + "label": "Monday" + }, + { + "default": "0", + "fieldname": "tuesday", + "fieldtype": "Check", + "label": "Tuesday" + }, + { + "default": "0", + "fieldname": "wednesday", + "fieldtype": "Check", + "label": "Wednesday" + }, + { + "default": "0", + "fieldname": "thursday", + "fieldtype": "Check", + "label": "Thursday" + }, + { + "fieldname": "column_break_schedule", + "fieldtype": "Column Break" + }, + { + "default": "0", + "fieldname": "friday", + "fieldtype": "Check", + "label": "Friday" + }, + { + "default": "0", + "fieldname": "saturday", + "fieldtype": "Check", + "label": "Saturday" + }, + { + "default": "0", + "fieldname": "sunday", + "fieldtype": "Check", + "label": "Sunday" + }, + { + "fieldname": "column_break_kaxp", + "fieldtype": "Column Break" + }, + { + "depends_on": "eval: doc.enabled", + "fieldname": "section_break_kvgf", + "fieldtype": "Section Break" + }, + { + "fieldname": "column_break_gpyw", + "fieldtype": "Column Break" + }, + { + "default": "1", + "fieldname": "enabled", + "fieldtype": "Check", + "label": "Enabled" } ], "issingle": 1, "links": [], - "modified": "2026-03-17 12:28:36.279956", + "modified": "2026-03-18 10:15:09.444532", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Settings", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py index 9ee4fae..c131618 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py @@ -17,7 +17,15 @@ class ParserBenchmarkSettings(Document): ParserBenchmarkTokenCost, ) + enabled: DF.Check + friday: DF.Check + monday: DF.Check + saturday: DF.Check + sunday: DF.Check + thursday: DF.Check token_costs: DF.Table[ParserBenchmarkTokenCost] + tuesday: DF.Check + wednesday: DF.Check # end: auto-generated types pass From 20943df86ce290e9d80df333ddbd4cd0ae98852c Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 15:34:54 +0530 Subject: [PATCH 28/88] refactor: enhance parser benchmark functionality with scheduling and validation for AI models and PDF processors --- transaction_parser/hooks.py | 6 + .../parser_benchmark_dataset.js | 12 +- .../parser_benchmark_dataset.py | 104 ++++++++++++++---- .../parser_benchmark_settings.py | 42 ++++++- transaction_parser/parser_benchmark/runner.py | 46 ++------ 5 files changed, 151 insertions(+), 59 deletions(-) diff --git a/transaction_parser/hooks.py b/transaction_parser/hooks.py index c7f1d6e..255ed1d 100644 --- a/transaction_parser/hooks.py +++ b/transaction_parser/hooks.py @@ -34,3 +34,9 @@ } export_python_type_annotations = True + +scheduler_events = { + "daily": [ + "transaction_parser.parser_benchmark.doctype.parser_benchmark_settings.parser_benchmark_settings.run_scheduled_benchmarks", + ], +} diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js index 0472ce4..8155da7 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js @@ -16,17 +16,19 @@ function run_benchmark(frm) { method: "transaction_parser.parser_benchmark.doctype.parser_benchmark_dataset.parser_benchmark_dataset.run_benchmark", args: { dataset_name: frm.doc.name }, freeze: true, - freeze_message: __("Queuing benchmark..."), + freeze_message: __("Queuing benchmarks..."), callback(r) { - if (r.message) { + if (r.message && r.message.length) { frappe.show_alert({ - message: __("Benchmark queued. Redirecting to log..."), + message: __("{0} benchmark(s) queued.", [r.message.length]), indicator: "green", }); - frappe.set_route("Form", "Parser Benchmark Log", r.message); + frappe.set_route("List", "Parser Benchmark Log", { + dataset: frm.doc.name, + }); } else { frappe.show_alert({ - message: __("Failed to queue benchmark. Please try again."), + message: __("No benchmarks queued. Check model/processor selections."), indicator: "red", }); } diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 2e60983..cd2ae7c 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -2,8 +2,26 @@ # For license information, please see license.txt import frappe +from frappe import _ from frappe.model.document import Document +# Maps dataset checkbox fieldnames → model/processor display names +AI_MODEL_FIELD_MAP = { + "deepseek_chat": "DeepSeek Chat", + "deepseek_reasoner": "DeepSeek Reasoner", + "openai_gpt_4o": "OpenAI gpt-4o", + "openai_gpt_4o_mini": "OpenAI gpt-4o-mini", + "openai_gpt_5": "OpenAI gpt-5", + "openai_gpt_5_mini": "OpenAI gpt-5-mini", + "google_gemini_pro_25": "Google Gemini Pro-2.5", + "google_gemini_flash_25": "Google Gemini Flash-2.5", +} + +PDF_PROCESSOR_FIELD_MAP = { + "ocrmypdf": "OCRMyPDF", + "docling": "Docling", +} + class ParserBenchmarkDataset(Document): # begin: auto-generated types @@ -34,32 +52,80 @@ class ParserBenchmarkDataset(Document): transaction_type: DF.Literal["Sales Order", "Expense"] # end: auto-generated types - pass + def validate(self): + self.validate_selections() + + def validate_selections(self): + if not self.get_selected_models(): + frappe.throw(_("Please select at least one AI Model.")) + + if not self.get_selected_processors(): + frappe.throw(_("Please select at least one PDF Processor.")) + + def get_selected_models(self) -> list[str]: + """Return list of selected AI model names.""" + return [ + label + for field, label in AI_MODEL_FIELD_MAP.items() + if self.get(field) + ] + + def get_selected_processors(self) -> list[str]: + """Return list of selected PDF processor names.""" + return [ + label + for field, label in PDF_PROCESSOR_FIELD_MAP.items() + if self.get(field) + ] @frappe.whitelist() def run_benchmark(dataset_name: str): - """Create a Benchmark Log and enqueue the benchmark run.""" + """Create Benchmark Logs for each model x processor combo and enqueue runs.""" frappe.has_permission("Parser Benchmark Dataset", "write", throw=True) - log = frappe.get_doc( - { - "doctype": "Parser Benchmark Log", - "dataset": dataset_name, - "status": "Queued", - } - ).insert(ignore_permissions=True) - - frappe.db.commit() # Ensure the log is saved before the background job picks it up - - frappe.enqueue( - _run_benchmark, - log_name=log.name, - queue="long", - now=frappe.conf.developer_mode, - ) + dataset = frappe.get_doc("Parser Benchmark Dataset", dataset_name) + log_names = _create_and_enqueue_logs(dataset) + + if not log_names: + frappe.throw(_("No model/processor combinations selected.")) + + return log_names + + +def _create_and_enqueue_logs(dataset) -> list[str]: + """Create one log per model x processor combo and enqueue each for background execution.""" + log_names = [] + + for ai_model in dataset.get_selected_models(): + for pdf_processor in dataset.get_selected_processors(): + log = frappe.get_doc( + { + "doctype": "Parser Benchmark Log", + "dataset": dataset.name, + "status": "Queued", + "ai_model": ai_model, + "pdf_processor": pdf_processor, + "transaction_type": dataset.transaction_type, + "country": dataset.country, + "company": dataset.company, + "page_limit": dataset.page_limit, + } + ).insert(ignore_permissions=True) + + log_names.append(log.name) + + frappe.db.commit() + + for log_name in log_names: + frappe.enqueue( + _run_benchmark, + log_name=log_name, + queue="long", + now=frappe.conf.developer_mode, + ) - return log.name + return log_names def _run_benchmark(log_name: str): diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py index c131618..37dd592 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py @@ -1,7 +1,19 @@ # Copyright (c) 2026, Resilient Tech and contributors # For license information, please see license.txt +import frappe from frappe.model.document import Document +from frappe.utils import getdate + +WEEKDAY_FIELDS = [ + "monday", + "tuesday", + "wednesday", + "thursday", + "friday", + "saturday", + "sunday", +] class ParserBenchmarkSettings(Document): @@ -28,4 +40,32 @@ class ParserBenchmarkSettings(Document): wednesday: DF.Check # end: auto-generated types - pass + def is_scheduled_today(self) -> bool: + """Check if today's weekday is enabled in the schedule.""" + today_index = getdate().weekday() # 0 = Monday + return bool(self.get(WEEKDAY_FIELDS[today_index])) + + +def run_scheduled_benchmarks(): + """Scheduled job: runs all enabled datasets if today is a scheduled day.""" + from transaction_parser.parser_benchmark.doctype.parser_benchmark_dataset.parser_benchmark_dataset import ( + _create_and_enqueue_logs, + ) + + try: + settings: ParserBenchmarkSettings = frappe.get_cached_doc("Parser Benchmark Settings") + except Exception: + return + + if not settings.enabled or not settings.is_scheduled_today(): + return + + datasets = frappe.get_all( + "Parser Benchmark Dataset", + filters={"enabled": 1}, + pluck="name", + ) + + for dataset_name in datasets: + dataset = frappe.get_doc("Parser Benchmark Dataset", dataset_name) + _create_and_enqueue_logs(dataset) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index 797672a..d5ac343 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -20,13 +20,11 @@ class BenchmarkRunner: """ - Wraps the existing Transaction Parser flow to capture - performance metrics (time, memory, tokens, cost) for benchmarking. + Runs a single benchmark for one AI model + PDF processor combination. Flow: 1. File parsing → time, memory, extracted content 2. AI parsing → time, tokens, cost, AI response - 3. Doc generation → linked document (Sales Order / Purchase Invoice) """ def __init__(self, log_name: str): @@ -38,10 +36,8 @@ def __init__(self, log_name: str): def run(self): self.log.status = "Running" - self.log.ai_model = self.dataset.ai_model - self.log.pdf_processor = self.dataset.pdf_processor self.log.save(ignore_permissions=True) - frappe.db.commit() # persist "Running" status before the long background job starts + frappe.db.commit() # persist "Running" status total_start = default_timer() @@ -50,9 +46,8 @@ def run(self): self.controller: Transaction = self._get_controller(file_doc) file_content = self._run_file_parsing(file_doc) - ai_content = self._run_ai_parsing(file_content, file_doc.name) + self._run_ai_parsing(file_content, file_doc.name) self._calculate_cost() - self._run_document_generation(ai_content) self.log.status = "Completed" @@ -63,7 +58,7 @@ def run(self): finally: self.log.total_time = flt(default_timer() - total_start, self.precision) self.log.save(ignore_permissions=True) - frappe.db.commit() # background jobs don't auto-commit; persist final results + frappe.db.commit() return self.log.name @@ -77,26 +72,20 @@ def _get_controller(self, file_doc: File) -> Transaction: cls = get_controller(ds.country, ds.transaction_type) controller = cls(company=ds.company) - controller.initialize() # manual initialization to set up doctype/schema without needing document data + controller.initialize() controller.file = file_doc - controller.ai_model = self.dataset.ai_model + controller.ai_model = self.log.ai_model return controller def _get_cost_row(self): - from transaction_parser.parser_benchmark.doctype.parser_benchmark_settings.parser_benchmark_settings import ( - ParserBenchmarkSettings, - ) - try: - settings: ParserBenchmarkSettings = frappe.get_cached_doc( - "Parser Benchmark Settings" - ) + settings = frappe.get_cached_doc("Parser Benchmark Settings") except Exception: return None for row in settings.token_costs: - if row.ai_model == self.dataset.ai_model: + if row.ai_model == self.log.ai_model: return row return None @@ -105,8 +94,8 @@ def _get_cost_row(self): def _run_file_parsing(self, file_doc: File) -> str: pdf_processor = None - if file_doc.file_type == "PDF" and self.dataset.pdf_processor: - pdf_processor = get_pdf_processor(self.dataset.pdf_processor) + if file_doc.file_type == "PDF" and self.log.pdf_processor: + pdf_processor = get_pdf_processor(self.log.pdf_processor) tracemalloc.start() start = default_timer() @@ -130,7 +119,7 @@ def _run_file_parsing(self, file_doc: File) -> str: # ── step 2: AI parsing ────────────────────────────────── def _run_ai_parsing(self, file_content: str, file_name: str) -> dict: - parser = AIParser(self.dataset.ai_model) + parser = AIParser(self.log.ai_model) start = default_timer() ai_content = parser.parse( @@ -149,19 +138,8 @@ def _run_ai_parsing(self, file_content: str, file_name: str) -> dict: return ai_content - # ── step 3: document generation ───────────────────────── - - def _run_document_generation(self, ai_content: dict): - self.controller.data = ai_content - self.controller.create_document() - self.controller.doc.db_set("is_created_by_benchmark", 1) - - self.log.document_type = self.controller.DOCTYPE - self.log.document_name = self.controller.doc.name - - # ── step 4: cost calculation ──────────────────────────── + # ── step 3: cost calculation ──────────────────────────── - # Code is not reaching here, need to fix the issue first def _calculate_cost(self): cost_row = self._get_cost_row() if not cost_row: From 02b2744cc8cec30d7bfc07b73cb4b58934bdb1cb Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 15:51:57 +0530 Subject: [PATCH 29/88] refactor: add party type and party fields to parser benchmark dataset with dynamic linking --- .../parser_benchmark_dataset.js | 10 ++++++ .../parser_benchmark_dataset.json | 33 +++++++++++++++++-- .../parser_benchmark_dataset.py | 12 +++---- transaction_parser/parser_benchmark/runner.py | 2 +- 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js index 8155da7..ba2c51c 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js @@ -2,6 +2,16 @@ // For license information, please see license.txt frappe.ui.form.on("Parser Benchmark Dataset", { + setup(frm) { + frm.set_query("party_type", function () { + return { + filters: { + name: ["in", Object.keys(frappe.boot.party_account_types)], + }, + }; + }); + }, + refresh(frm) { if (!frm.is_new() && frm.doc.enabled) { frm.add_custom_button(__("Run Benchmark"), () => run_benchmark(frm)); diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index 2cb4c37..5dbc736 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -13,7 +13,11 @@ "transaction_type", "column_break_gbap", "country", + "section_break_sobg", "company", + "column_break_qccq", + "party_type", + "party", "processing_section", "column_break_ai_models", "openai_gpt_4o_mini", @@ -61,7 +65,8 @@ "fieldname": "file", "fieldtype": "Attach", "label": "File", - "reqd": 1 + "reqd": 1, + "set_only_once": 1 }, { "fieldname": "transaction_type", @@ -89,6 +94,22 @@ "label": "Company", "options": "Company" }, + { + "fieldname": "party_type", + "fieldtype": "Link", + "label": "Party Type", + "options": "DocType", + "set_only_once": 1 + }, + { + "depends_on": "eval: doc.party_type", + "fieldname": "party", + "fieldtype": "Dynamic Link", + "label": "Party", + "mandatory_depends_on": "eval: doc.party_type", + "options": "party_type", + "set_only_once": 1 + }, { "fieldname": "processing_section", "fieldtype": "Section Break", @@ -200,11 +221,19 @@ { "fieldname": "column_break_zgpf", "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_sobg", + "fieldtype": "Section Break" + }, + { + "fieldname": "column_break_qccq", + "fieldtype": "Column Break" } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-18 10:17:58.099090", + "modified": "2026-03-18 11:19:11.810544", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index cd2ae7c..5692143 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -48,6 +48,8 @@ class ParserBenchmarkDataset(Document): openai_gpt_5: DF.Check openai_gpt_5_mini: DF.Check page_limit: DF.Int + party: DF.DynamicLink | None + party_type: DF.Link | None title: DF.Data transaction_type: DF.Literal["Sales Order", "Expense"] # end: auto-generated types @@ -64,18 +66,12 @@ def validate_selections(self): def get_selected_models(self) -> list[str]: """Return list of selected AI model names.""" - return [ - label - for field, label in AI_MODEL_FIELD_MAP.items() - if self.get(field) - ] + return [label for field, label in AI_MODEL_FIELD_MAP.items() if self.get(field)] def get_selected_processors(self) -> list[str]: """Return list of selected PDF processor names.""" return [ - label - for field, label in PDF_PROCESSOR_FIELD_MAP.items() - if self.get(field) + label for field, label in PDF_PROCESSOR_FIELD_MAP.items() if self.get(field) ] diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index d5ac343..973ad67 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -32,7 +32,7 @@ def __init__(self, log_name: str): self.dataset: ParserBenchmarkDataset = frappe.get_doc( "Parser Benchmark Dataset", self.log.dataset ) - self.precision = cint(frappe.db.get_default("float_precision")) or 3 + self.precision = 6 # to get 1-millionth of a token cost def run(self): self.log.status = "Running" From 2dcfb6051811bcc4fb1567314d141f628c55a942 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 16:30:53 +0530 Subject: [PATCH 30/88] refactor: update parser benchmark dataset and log to include party type and party fields, enhance benchmark runner with currency handling --- .../parser_benchmark_dataset.js | 34 ++++++++----------- .../parser_benchmark_dataset.json | 18 ++++++---- .../parser_benchmark_dataset.py | 4 ++- .../parser_benchmark_log.json | 24 +++++++++++-- .../parser_benchmark_log.py | 2 ++ transaction_parser/parser_benchmark/runner.py | 3 +- .../ai_integration/models.py | 4 +-- 7 files changed, 56 insertions(+), 33 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js index ba2c51c..ccf9a8f 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js @@ -2,22 +2,14 @@ // For license information, please see license.txt frappe.ui.form.on("Parser Benchmark Dataset", { - setup(frm) { - frm.set_query("party_type", function () { - return { - filters: { - name: ["in", Object.keys(frappe.boot.party_account_types)], - }, - }; - }); - }, - refresh(frm) { if (!frm.is_new() && frm.doc.enabled) { frm.add_custom_button(__("Run Benchmark"), () => run_benchmark(frm)); } + }, - set_pdf_processor_options(frm); + transaction_type(frm) { + set_party_type(frm); }, }); @@ -35,6 +27,7 @@ function run_benchmark(frm) { }); frappe.set_route("List", "Parser Benchmark Log", { dataset: frm.doc.name, + status: "Queued", }); } else { frappe.show_alert({ @@ -46,13 +39,14 @@ function run_benchmark(frm) { }); } -function set_pdf_processor_options(frm) { - frappe.call({ - method: "transaction_parser.parser_benchmark.doctype.parser_benchmark_dataset.parser_benchmark_dataset.get_pdf_processors", - callback(r) { - if (r.message) { - frm.set_df_property("pdf_processor", "options", r.message); - } - }, - }); +const PARTY_TYPE_MAP = { + "Sales Order": "Customer", + Expense: "Supplier", +}; + +function set_party_type(frm) { + const party_type = PARTY_TYPE_MAP[frm.doc.transaction_type]; + if (party_type && frm.doc.party_type !== party_type) { + frm.set_value("party_type", party_type); + } } diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index 5dbc736..e7eda91 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -55,6 +55,8 @@ "default": "1", "fieldname": "enabled", "fieldtype": "Check", + "in_list_view": 1, + "in_standard_filter": 1, "label": "Enabled" }, { @@ -97,9 +99,9 @@ { "fieldname": "party_type", "fieldtype": "Link", + "hidden": 1, "label": "Party Type", - "options": "DocType", - "set_only_once": 1 + "options": "DocType" }, { "depends_on": "eval: doc.party_type", @@ -107,8 +109,7 @@ "fieldtype": "Dynamic Link", "label": "Party", "mandatory_depends_on": "eval: doc.party_type", - "options": "party_type", - "set_only_once": 1 + "options": "party_type" }, { "fieldname": "processing_section", @@ -232,8 +233,13 @@ } ], "index_web_pages_for_search": 1, - "links": [], - "modified": "2026-03-18 11:19:11.810544", + "links": [ + { + "link_doctype": "Parser Benchmark Log", + "link_fieldname": "dataset" + } + ], + "modified": "2026-03-18 11:52:07.185717", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 5692143..7b2809c 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -105,6 +105,8 @@ def _create_and_enqueue_logs(dataset) -> list[str]: "transaction_type": dataset.transaction_type, "country": dataset.country, "company": dataset.company, + "party_type": dataset.party_type, + "party": dataset.party, "page_limit": dataset.page_limit, } ).insert(ignore_permissions=True) @@ -118,7 +120,7 @@ def _create_and_enqueue_logs(dataset) -> list[str]: _run_benchmark, log_name=log_name, queue="long", - now=frappe.conf.developer_mode, + # now=frappe.conf.developer_mode, ) return log_names diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index 5b22bb9..384f593 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -11,6 +11,8 @@ "column_break_summary", "country", "company", + "party_type", + "party", "section_break_peyo", "total_time", "column_break_fmal", @@ -68,7 +70,8 @@ "label": "Dataset", "options": "Parser Benchmark Dataset", "read_only": 1, - "reqd": 1 + "reqd": 1, + "search_index": 1 }, { "default": "Queued", @@ -306,6 +309,21 @@ "options": "Company", "read_only": 1 }, + { + "fieldname": "party_type", + "fieldtype": "Link", + "hidden": 1, + "label": "Party Type", + "options": "DocType", + "read_only": 1 + }, + { + "fieldname": "party", + "fieldtype": "Dynamic Link", + "label": "Party", + "options": "party_type", + "read_only": 1 + }, { "fieldname": "section_break_peyo", "fieldtype": "Section Break" @@ -339,7 +357,7 @@ "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-18 10:04:36.919978", + "modified": "2026-03-18 11:57:16.173340", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Log", @@ -363,4 +381,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} \ No newline at end of file +} diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py index e673d0a..4bae22d 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -40,6 +40,8 @@ class ParserBenchmarkLog(Document): output_cost: DF.Currency output_token_cost: DF.Currency page_limit: DF.Int + party: DF.DynamicLink | None + party_type: DF.Link | None pdf_processor: DF.Literal["OCRMyPDF", "Docling"] prompt_tokens: DF.Int status: DF.Literal["Queued", "Running", "Completed", "Failed"] diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index 973ad67..d27de4f 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -36,6 +36,7 @@ def __init__(self, log_name: str): def run(self): self.log.status = "Running" + self.log.currency = "USD" self.log.save(ignore_permissions=True) frappe.db.commit() # persist "Running" status @@ -71,7 +72,7 @@ def _get_controller(self, file_doc: File) -> Transaction: ds = self.dataset cls = get_controller(ds.country, ds.transaction_type) - controller = cls(company=ds.company) + controller = cls(company=ds.company, party=ds.party) controller.initialize() controller.file = file_doc controller.ai_model = self.log.ai_model diff --git a/transaction_parser/transaction_parser/ai_integration/models.py b/transaction_parser/transaction_parser/ai_integration/models.py index 9c6e3be..65dfd58 100644 --- a/transaction_parser/transaction_parser/ai_integration/models.py +++ b/transaction_parser/transaction_parser/ai_integration/models.py @@ -120,6 +120,6 @@ class GeminiFlash(Model): "OpenAI gpt-4o-mini": OpenAIGPT4oMini(), "OpenAI gpt-5": OpenAIGPT5(), "OpenAI gpt-5-mini": OpenAIGPT5Mini(), - "Google Gemini Pro": GeminiPro(), - "Google Gemini Flash": GeminiFlash(), + "Google Gemini Pro-2.5": GeminiPro(), + "Google Gemini Flash-2.5": GeminiFlash(), } From cd4922afae89497200cd72df53e3c9a5f0a56880 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 17:02:41 +0530 Subject: [PATCH 31/88] chore: minor fixes --- .../parser_benchmark_dataset.py | 19 ++++++++++++++++--- transaction_parser/parser_benchmark/runner.py | 6 +++--- transaction_parser/uninstall.py | 4 ++-- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 7b2809c..ad77580 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -55,12 +55,25 @@ class ParserBenchmarkDataset(Document): # end: auto-generated types def validate(self): - self.validate_selections() - - def validate_selections(self): + self.file_doc = frappe.get_last_doc("File", filters={"file_url": self.file}) + self.validate_file_type() + self.validate_selected_models() + self.validate_selected_processors() + + def validate_file_type(self): + if self.file_doc.file_type not in ["PDF", "CSV", "XLSX", "XLS"]: + frappe.throw( + _("Unsupported file type: {0}").format(self.file_doc.file_type) + ) + + def validate_selected_models(self): if not self.get_selected_models(): frappe.throw(_("Please select at least one AI Model.")) + def validate_selected_processors(self): + if self.file_doc.file_type != "PDF": + return + if not self.get_selected_processors(): frappe.throw(_("Please select at least one PDF Processor.")) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index d27de4f..05819fa 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -110,10 +110,10 @@ def _run_file_parsing(self, file_doc: File) -> str: self.log.file_parse_time = flt(default_timer() - start, self.precision) _, peak = tracemalloc.get_traced_memory() tracemalloc.stop() + self.log.file_parse_memory = flt( + peak / 1024 / 1024, self.precision + ) # bytes → MB - self.log.file_parse_memory = flt( - peak / 1024 / 1024, self.precision - ) # bytes → MB self.log.file_content = content return content diff --git a/transaction_parser/uninstall.py b/transaction_parser/uninstall.py index d54f2b3..8196823 100644 --- a/transaction_parser/uninstall.py +++ b/transaction_parser/uninstall.py @@ -2,8 +2,8 @@ FIELDS_TO_DELETE = { "Transaction Parser Settings": ["in_auto_create_supplier"], - "Sales Order": ["is_created_by_transaction_parser", "is_created_by_benchmark"], - "Purchase Invoice": ["is_created_by_transaction_parser", "is_created_by_benchmark"], + "Sales Order": ["is_created_by_transaction_parser"], + "Purchase Invoice": ["is_created_by_transaction_parser"], } From db21a5157f35de10c22f59f2f3528463cc4e36d5 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 17:26:11 +0530 Subject: [PATCH 32/88] refactor: update benchmark log to handle dynamic processor selection and modify PDF processor options --- .../parser_benchmark_dataset.py | 3 ++- .../parser_benchmark_log.json | 6 +++--- .../parser_benchmark_log/parser_benchmark_log.py | 2 +- transaction_parser/patches.txt | 1 + .../patches/rename_gemini_models.py | 16 ++++++++++++++++ 5 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 transaction_parser/patches/rename_gemini_models.py diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index ad77580..14f2a6f 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -105,9 +105,10 @@ def run_benchmark(dataset_name: str): def _create_and_enqueue_logs(dataset) -> list[str]: """Create one log per model x processor combo and enqueue each for background execution.""" log_names = [] + processors = dataset.get_selected_processors() or [None] for ai_model in dataset.get_selected_models(): - for pdf_processor in dataset.get_selected_processors(): + for pdf_processor in processors: log = frappe.get_doc( { "doctype": "Parser Benchmark Log", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index 384f593..ee57b7c 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -99,7 +99,7 @@ "fieldname": "pdf_processor", "fieldtype": "Select", "label": "PDF Processor", - "options": "OCRMyPDF\nDocling", + "options": "\nOCRMyPDF\nDocling", "read_only": 1 }, { @@ -357,7 +357,7 @@ "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-18 11:57:16.173340", + "modified": "2026-03-18 12:50:34.387945", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Log", @@ -381,4 +381,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} +} \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py index 4bae22d..239b1db 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -42,7 +42,7 @@ class ParserBenchmarkLog(Document): page_limit: DF.Int party: DF.DynamicLink | None party_type: DF.Link | None - pdf_processor: DF.Literal["OCRMyPDF", "Docling"] + pdf_processor: DF.Literal["", "OCRMyPDF", "Docling"] prompt_tokens: DF.Int status: DF.Literal["Queued", "Running", "Completed", "Failed"] total_cost: DF.Currency diff --git a/transaction_parser/patches.txt b/transaction_parser/patches.txt index 27002f4..7c87e23 100644 --- a/transaction_parser/patches.txt +++ b/transaction_parser/patches.txt @@ -1,6 +1,7 @@ [pre_model_sync] # Patches added in this section will be executed before doctypes are migrated # Read docs to understand patches: https://frappeframework.com/docs/v14/user/en/database-migrations +transaction_parser.patches.rename_gemini_models [post_model_sync] # Patches added in this section will be executed after doctypes are migrated diff --git a/transaction_parser/patches/rename_gemini_models.py b/transaction_parser/patches/rename_gemini_models.py new file mode 100644 index 0000000..a168455 --- /dev/null +++ b/transaction_parser/patches/rename_gemini_models.py @@ -0,0 +1,16 @@ +import frappe + +RENAME_MAP = { + "Google Gemini Pro": "Google Gemini Pro-2.5", + "Google Gemini Flash": "Google Gemini Flash-2.5", +} + +DOCTYPE = "Transaction Parser Settings" +FIELD = "default_ai_model" + + +def execute(): + current = frappe.db.get_single_value(DOCTYPE, FIELD) + + if current in RENAME_MAP: + frappe.db.set_single_value(DOCTYPE, FIELD, RENAME_MAP[current]) From a6b54a8badd200a921cad5fa58673bf5fb802884 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 17:40:29 +0530 Subject: [PATCH 33/88] chore: minor fixes --- .../parser_benchmark_dataset.py | 12 ++++++++++++ .../transaction_parser/ai_integration/parser.py | 5 ++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 14f2a6f..0428c60 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -109,6 +109,18 @@ def _create_and_enqueue_logs(dataset) -> list[str]: for ai_model in dataset.get_selected_models(): for pdf_processor in processors: + existing = frappe.db.exists( + "Parser Benchmark Log", + { + "dataset": dataset.name, + "ai_model": ai_model, + "pdf_processor": pdf_processor or "", + "status": ("in", ("Queued", "Running")), + }, + ) + if existing: + continue + log = frappe.get_doc( { "doctype": "Parser Benchmark Log", diff --git a/transaction_parser/transaction_parser/ai_integration/parser.py b/transaction_parser/transaction_parser/ai_integration/parser.py index 169fcb9..a9925e1 100644 --- a/transaction_parser/transaction_parser/ai_integration/parser.py +++ b/transaction_parser/transaction_parser/ai_integration/parser.py @@ -23,6 +23,7 @@ def __init__(self, model: str | None = None, settings=None): is_enabled(self.settings) self.model = self._get_model(model) + self.ai_response = {} if not self.model: frappe.throw(_(f"AI Model: {model} not found")) @@ -37,7 +38,9 @@ def parse( file_doc_name: str | None = None, ) -> dict: messages = self._build_messages(document_type, document_schema, document_data) - self.ai_response = self.send_message(messages=messages, file_doc_name=file_doc_name) + self.ai_response = self.send_message( + messages=messages, file_doc_name=file_doc_name + ) return self.get_content(self.ai_response) def _build_messages( From b0a2632097b92bff44f31181d8c18bfdfd527358 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 17:55:16 +0530 Subject: [PATCH 34/88] refactor: add file_type field to parser benchmark dataset and log, update validation logic --- .../parser_benchmark_dataset.json | 9 ++++++++- .../parser_benchmark_dataset.py | 17 +++++++++++------ .../parser_benchmark_log.json | 9 ++++++++- transaction_parser/parser_benchmark/runner.py | 2 +- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index e7eda91..d2b05c5 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -10,6 +10,7 @@ "enabled", "section_break_file", "file", + "file_type", "transaction_type", "column_break_gbap", "country", @@ -70,6 +71,12 @@ "reqd": 1, "set_only_once": 1 }, + { + "fieldname": "file_type", + "fieldtype": "Data", + "label": "File Type", + "read_only": 1 + }, { "fieldname": "transaction_type", "fieldtype": "Select", @@ -265,4 +272,4 @@ "sort_order": "DESC", "states": [], "title_field": "title" -} \ No newline at end of file +} diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 0428c60..fd1b7f9 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -54,24 +54,28 @@ class ParserBenchmarkDataset(Document): transaction_type: DF.Literal["Sales Order", "Expense"] # end: auto-generated types + SUPPORTED_FILE_TYPES = ("PDF", "CSV", "XLSX", "XLS") + def validate(self): - self.file_doc = frappe.get_last_doc("File", filters={"file_url": self.file}) + self.set_file_type() self.validate_file_type() self.validate_selected_models() self.validate_selected_processors() + def set_file_type(self): + file_doc = frappe.get_last_doc("File", filters={"file_url": self.file}) + self.file_type = file_doc.file_type + def validate_file_type(self): - if self.file_doc.file_type not in ["PDF", "CSV", "XLSX", "XLS"]: - frappe.throw( - _("Unsupported file type: {0}").format(self.file_doc.file_type) - ) + if self.file_type not in self.SUPPORTED_FILE_TYPES: + frappe.throw(_("Unsupported file type: {0}").format(self.file_type)) def validate_selected_models(self): if not self.get_selected_models(): frappe.throw(_("Please select at least one AI Model.")) def validate_selected_processors(self): - if self.file_doc.file_type != "PDF": + if self.file_type != "PDF": return if not self.get_selected_processors(): @@ -134,6 +138,7 @@ def _create_and_enqueue_logs(dataset) -> list[str]: "party_type": dataset.party_type, "party": dataset.party, "page_limit": dataset.page_limit, + "file_type": dataset.file_type, } ).insert(ignore_permissions=True) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index ee57b7c..162becb 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -22,6 +22,7 @@ "column_break_ubhs", "file_parsing_tab", "pdf_processor", + "file_type", "column_break_file_metrics", "page_limit", "section_break_umzr", @@ -109,6 +110,12 @@ "label": "Total Time (s)", "read_only": 1 }, + { + "fieldname": "file_type", + "fieldtype": "Data", + "label": "File Type", + "read_only": 1 + }, { "fieldname": "file_parsing_tab", "fieldtype": "Tab Break", @@ -381,4 +388,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} \ No newline at end of file +} diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index 05819fa..923507a 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -95,7 +95,7 @@ def _get_cost_row(self): def _run_file_parsing(self, file_doc: File) -> str: pdf_processor = None - if file_doc.file_type == "PDF" and self.log.pdf_processor: + if self.log.file_type == "PDF" and self.log.pdf_processor: pdf_processor = get_pdf_processor(self.log.pdf_processor) tracemalloc.start() From e6a749b3856e1e9ba9403fcd4701ed09339843c1 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 18 Mar 2026 18:10:29 +0530 Subject: [PATCH 35/88] refactor: update parser benchmark dataset and log to include file_type field, modify output_cost_per_million fieldtype to Currency, and enhance error messaging --- .../parser_benchmark_dataset/parser_benchmark_dataset.json | 4 ++-- .../parser_benchmark_dataset/parser_benchmark_dataset.py | 7 ++++++- .../doctype/parser_benchmark_log/parser_benchmark_log.json | 4 ++-- .../doctype/parser_benchmark_log/parser_benchmark_log.py | 1 + .../parser_benchmark_settings.json | 2 +- .../parser_benchmark_settings/parser_benchmark_settings.py | 7 +++---- .../parser_benchmark_token_cost.json | 4 ++-- .../parser_benchmark_token_cost.py | 2 +- 8 files changed, 18 insertions(+), 13 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index d2b05c5..9093be9 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -246,7 +246,7 @@ "link_fieldname": "dataset" } ], - "modified": "2026-03-18 11:52:07.185717", + "modified": "2026-03-18 13:39:37.507823", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", @@ -272,4 +272,4 @@ "sort_order": "DESC", "states": [], "title_field": "title" -} +} \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index fd1b7f9..e03b033 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -39,6 +39,7 @@ class ParserBenchmarkDataset(Document): docling: DF.Check enabled: DF.Check file: DF.Attach + file_type: DF.Data | None google_gemini_flash_25: DF.Check google_gemini_pro_25: DF.Check naming_series: DF.Literal["PAR-BM-DTS-"] @@ -101,7 +102,11 @@ def run_benchmark(dataset_name: str): log_names = _create_and_enqueue_logs(dataset) if not log_names: - frappe.throw(_("No model/processor combinations selected.")) + frappe.throw( + _( + "No new benchmarks to queue. All selected combinations are already queued or running." + ) + ) return log_names diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index 162becb..a635562 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -364,7 +364,7 @@ "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-18 12:50:34.387945", + "modified": "2026-03-18 13:39:29.730560", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Log", @@ -388,4 +388,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} +} \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py index 239b1db..1a11994 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -34,6 +34,7 @@ class ParserBenchmarkLog(Document): file_content: DF.Code | None file_parse_memory: DF.Float file_parse_time: DF.Float + file_type: DF.Data | None input_cost: DF.Currency input_token_cost: DF.Currency naming_series: DF.Literal["PAR-BM-LOG-"] diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json index 6875c8e..712b16e 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json @@ -101,7 +101,7 @@ ], "issingle": 1, "links": [], - "modified": "2026-03-18 10:15:09.444532", + "modified": "2026-03-18 13:40:00.122632", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Settings", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py index 37dd592..0643c38 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py @@ -52,10 +52,9 @@ def run_scheduled_benchmarks(): _create_and_enqueue_logs, ) - try: - settings: ParserBenchmarkSettings = frappe.get_cached_doc("Parser Benchmark Settings") - except Exception: - return + settings: ParserBenchmarkSettings = frappe.get_cached_doc( + "Parser Benchmark Settings" + ) if not settings.enabled or not settings.is_scheduled_today(): return diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json index 030ebd6..5321a7c 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json @@ -42,7 +42,7 @@ { "description": "Per 1M Tokens", "fieldname": "output_cost_per_million", - "fieldtype": "Float", + "fieldtype": "Currency", "in_list_view": 1, "label": "Output Cost", "non_negative": 1, @@ -56,7 +56,7 @@ ], "istable": 1, "links": [], - "modified": "2026-03-18 10:05:11.453785", + "modified": "2026-03-18 13:39:47.449246", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Token Cost", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py index fa33751..68ac63b 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.py @@ -25,7 +25,7 @@ class ParserBenchmarkTokenCost(Document): ] currency: DF.Link input_cost_per_million: DF.Currency - output_cost_per_million: DF.Float + output_cost_per_million: DF.Currency parent: DF.Data parentfield: DF.Data parenttype: DF.Data From 595267a3ff2f8141b30e97bdd026ec82d02f8670 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Thu, 19 Mar 2026 00:21:27 +0530 Subject: [PATCH 36/88] refactor: enhance processor validation and error handling in benchmark log creation --- .../parser_benchmark_dataset.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index e03b033..76814ee 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -77,6 +77,8 @@ def validate_selected_models(self): def validate_selected_processors(self): if self.file_type != "PDF": + self.ocrmypdf = None + self.docling = None return if not self.get_selected_processors(): @@ -114,7 +116,11 @@ def run_benchmark(dataset_name: str): def _create_and_enqueue_logs(dataset) -> list[str]: """Create one log per model x processor combo and enqueue each for background execution.""" log_names = [] - processors = dataset.get_selected_processors() or [None] + + if dataset.file_type == "PDF": + processors = dataset.get_selected_processors() or [None] + else: + processors = [None] for ai_model in dataset.get_selected_models(): for pdf_processor in processors: @@ -152,12 +158,15 @@ def _create_and_enqueue_logs(dataset) -> list[str]: frappe.db.commit() for log_name in log_names: - frappe.enqueue( - _run_benchmark, - log_name=log_name, - queue="long", - # now=frappe.conf.developer_mode, - ) + try: + frappe.enqueue( + _run_benchmark, + log_name=log_name, + queue="long", + ) + except Exception: + frappe.db.set_value("Parser Benchmark Log", log_name, "status", "Failed") + frappe.db.commit() return log_names From 5c3abb8880fa9a2171ed8419291c394ebaef3b17 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Thu, 19 Mar 2026 08:07:23 +0530 Subject: [PATCH 37/88] refactor: remove title field from ParserBenchmarkDataset class and adjust field order in JSON --- .../parser_benchmark_dataset.json | 16 +++------------- .../parser_benchmark_dataset.py | 1 - 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index 9093be9..dc217b6 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -5,9 +5,8 @@ "doctype": "DocType", "engine": "InnoDB", "field_order": [ - "title", - "column_break_title", "enabled", + "column_break_title", "section_break_file", "file", "file_type", @@ -42,12 +41,6 @@ "naming_series" ], "fields": [ - { - "fieldname": "title", - "fieldtype": "Data", - "label": "Title", - "reqd": 1 - }, { "fieldname": "column_break_title", "fieldtype": "Column Break" @@ -115,7 +108,6 @@ "fieldname": "party", "fieldtype": "Dynamic Link", "label": "Party", - "mandatory_depends_on": "eval: doc.party_type", "options": "party_type" }, { @@ -246,7 +238,7 @@ "link_fieldname": "dataset" } ], - "modified": "2026-03-18 13:39:37.507823", + "modified": "2026-03-19 03:36:39.325714", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", @@ -267,9 +259,7 @@ } ], "row_format": "Dynamic", - "show_title_field_in_link": 1, "sort_field": "modified", "sort_order": "DESC", - "states": [], - "title_field": "title" + "states": [] } \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 76814ee..442ce54 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -51,7 +51,6 @@ class ParserBenchmarkDataset(Document): page_limit: DF.Int party: DF.DynamicLink | None party_type: DF.Link | None - title: DF.Data transaction_type: DF.Literal["Sales Order", "Expense"] # end: auto-generated types From 1a5f1d328be6aff32e7e0bcb7188282f1d59164e Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Thu, 19 Mar 2026 09:07:08 +0530 Subject: [PATCH 38/88] refactor: update scheduler_events to use cron format for daily benchmarks --- transaction_parser/hooks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/transaction_parser/hooks.py b/transaction_parser/hooks.py index 255ed1d..15e7678 100644 --- a/transaction_parser/hooks.py +++ b/transaction_parser/hooks.py @@ -36,7 +36,10 @@ export_python_type_annotations = True scheduler_events = { - "daily": [ - "transaction_parser.parser_benchmark.doctype.parser_benchmark_settings.parser_benchmark_settings.run_scheduled_benchmarks", - ], + "cron": { + # at 2:00 am every day + "0 2 * * *": [ + "transaction_parser.parser_benchmark.doctype.parser_benchmark_settings.parser_benchmark_settings.run_scheduled_benchmarks", + ], + } } From 1877ef73c907da98d5a865cc5505c266af0804bb Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Fri, 20 Mar 2026 10:09:40 +0530 Subject: [PATCH 39/88] chore: temp party patch --- .../patches/set_party_in_dataset.py | 1006 +++++++++++++++++ 1 file changed, 1006 insertions(+) create mode 100644 transaction_parser/patches/set_party_in_dataset.py diff --git a/transaction_parser/patches/set_party_in_dataset.py b/transaction_parser/patches/set_party_in_dataset.py new file mode 100644 index 0000000..3ed4bdb --- /dev/null +++ b/transaction_parser/patches/set_party_in_dataset.py @@ -0,0 +1,1006 @@ +import frappe + +# ! NOTE: Temporary patch + +PARTY = [ + { + "name": "SO527790", + "file_name": "Bestellung 3600097229 Lieferant Nr 20203035.PDF", + "file_url": "/private/files/Bestellung 3600097229 Lieferant Nr 20203035.PDF", + "is_private": 1, + "customer": "C100122", + "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", + }, + { + "name": "SO527789", + "file_name": "Bestellung 3600097186 Lieferant Nr 20203035.PDF", + "file_url": "/private/files/Bestellung 3600097186 Lieferant Nr 20203035.PDF", + "is_private": 1, + "customer": "C100122", + "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", + }, + { + "name": "SO527783", + "file_name": "Lieferantenbestellung 70127077.xlsx", + "file_url": "/private/files/Lieferantenbestellung 70127077.xlsx", + "is_private": 1, + "customer": "100069", + "customer_name": "BRACK.CH AG", + }, + { + "name": "SO527782", + "file_name": "111944_Bestellung_4502190835_20260319.PDF", + "file_url": "/private/files/111944_Bestellung_4502190835_20260319.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527762", + "file_name": "order_a152091d-ad91-4fb3-9158-5afe3b43f813.csv", + "file_url": "/private/files/order_a152091d-ad91-4fb3-9158-5afe3b43f813.csv", + "is_private": 1, + "customer": "C102424", + "customer_name": "Haar-shop.ch AG", + }, + { + "name": "SO527727", + "file_name": "111944_Bestellung_4502187869_20260319.PDF", + "file_url": "/private/files/111944_Bestellung_4502187869_20260319.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527726", + "file_name": "Lieferantenbestellung 70126993.xlsx", + "file_url": "/private/files/Lieferantenbestellung 70126993.xlsx", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO527686", + "file_name": "112000_Bestellung_4502186288_20260317.PDF", + "file_url": "/private/files/112000_Bestellung_4502186288_20260317.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527679", + "file_name": "Bestellung 3600096912 Lieferant Nr 20203035.PDF", + "file_url": "/private/files/Bestellung 3600096912 Lieferant Nr 20203035.PDF", + "is_private": 1, + "customer": "C100122", + "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", + }, + { + "name": "SO527620", + "file_name": "Lieferantenbestellung 70126937.xlsx", + "file_url": "/private/files/Lieferantenbestellung 70126937.xlsx", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO527619", + "file_name": "112000_Bestellung_4502180368_20260309.PDF", + "file_url": "/private/files/112000_Bestellung_4502180368_20260309.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527595", + "file_name": "Einkauf_Bestellung99ffe0.csv", + "file_url": "/private/files/Einkauf_Bestellung99ffe0.csv", + "is_private": 1, + "customer": "100069", + "customer_name": "BRACK.CH AG", + }, + { + "name": "SO527594", + "file_name": "111944_Bestellung_4502185151_20260312.PDF", + "file_url": "/private/files/111944_Bestellung_4502185151_20260312.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527539", + "file_name": "111944_Bestellung_4502181439_20260311.PDF", + "file_url": "/private/files/111944_Bestellung_4502181439_20260311.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527538", + "file_name": "order_a14251cb-ef2a-4306-b7a3-9bac4d8c236e.csv", + "file_url": "/private/files/order_a14251cb-ef2a-4306-b7a3-9bac4d8c236e.csv", + "is_private": 1, + "customer": "C102424", + "customer_name": "Haar-shop.ch AG", + }, + { + "name": "SO527505", + "file_name": "111944_Bestellung_4502180283_20260307.PDF", + "file_url": "/private/files/111944_Bestellung_4502180283_20260307.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527504", + "file_name": "order_a13bf300-b510-4ea0-b07d-d3d005bf603c.csv", + "file_url": "/private/files/order_a13bf300-b510-4ea0-b07d-d3d005bf603c.csv", + "is_private": 1, + "customer": "C102424", + "customer_name": "Haar-shop.ch AG", + }, + { + "name": "SO527452", + "file_name": "Bestellung 3600096482 Lieferant Nr 20203035.PDF", + "file_url": "/private/files/Bestellung 3600096482 Lieferant Nr 20203035.PDF", + "is_private": 1, + "customer": "C100122", + "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", + }, + { + "name": "SO527344", + "file_name": "order_a133b6d9-f3a7-4629-877b-f72f4c751363.pdf", + "file_url": "/private/files/order_a133b6d9-f3a7-4629-877b-f72f4c751363.pdf", + "is_private": 1, + "customer": "C102424", + "customer_name": "Haar-shop.ch AG", + }, + { + "name": "SO527343", + "file_name": "Lieferantenbestellung 70126756.xlsx", + "file_url": "/private/files/Lieferantenbestellung 70126756.xlsx", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO527341", + "file_name": "111944_Bestellung_4502176704_20260303.PDF", + "file_url": "/private/files/111944_Bestellung_4502176704_20260303.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527285", + "file_name": "order_a12df8f0-ae26-43a0-b48e-09374819b081.csv", + "file_url": "/private/files/order_a12df8f0-ae26-43a0-b48e-09374819b081.csv", + "is_private": 1, + "customer": "C102424", + "customer_name": "Haar-shop.ch AG", + }, + { + "name": "SO527274", + "file_name": "111944_Bestellung_4502175021_20260227.PDF", + "file_url": "/private/files/111944_Bestellung_4502175021_20260227.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527273", + "file_name": "111944_Bestellung_4502175022_20260302.PDF", + "file_url": "/private/files/111944_Bestellung_4502175022_20260302.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527241", + "file_name": "Einkauf_Bestellung482a75.csv", + "file_url": "/private/files/Einkauf_Bestellung482a75.csv", + "is_private": 1, + "customer": "100069", + "customer_name": "BRACK.CH AG", + }, + { + "name": "SO527217", + "file_name": "Einkauf_Bestellung81ecda.csv", + "file_url": "/private/files/Einkauf_Bestellung81ecda.csv", + "is_private": 1, + "customer": "100069", + "customer_name": "BRACK.CH AG", + }, + { + "name": "SO527209", + "file_name": "order_a129be25-f590-47a8-9c17-6664d16c16d3.csv", + "file_url": "/private/files/order_a129be25-f590-47a8-9c17-6664d16c16d3.csv", + "is_private": 1, + "customer": "C102424", + "customer_name": "Haar-shop.ch AG", + }, + { + "name": "SO527206", + "file_name": "Bestellung 3600095890 Lieferant Nr 20203035.PDF", + "file_url": "/private/files/Bestellung 3600095890 Lieferant Nr 20203035.PDF", + "is_private": 1, + "customer": "C100122", + "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", + }, + { + "name": "SO527205", + "file_name": "Lieferantenbestellung 70126653.xlsx", + "file_url": "/private/files/Lieferantenbestellung 70126653.xlsx", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO527202", + "file_name": "111944_Bestellung_4502172320_20260226.PDF", + "file_url": "/private/files/111944_Bestellung_4502172320_20260226.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527201", + "file_name": "Bestellung 3600095878 Lieferant Nr 20203035.PDF", + "file_url": "/private/files/Bestellung 3600095878 Lieferant Nr 20203035.PDF", + "is_private": 1, + "customer": "C100122", + "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", + }, + { + "name": "SO527200", + "file_name": "111944_Bestellung_4502172321_20260228.PDF", + "file_url": "/private/files/111944_Bestellung_4502172321_20260228.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527167", + "file_name": "order_a12652f3-c8db-4c02-b44e-d1516ce83bfc.csv", + "file_url": "/private/files/order_a12652f3-c8db-4c02-b44e-d1516ce83bfc.csv", + "is_private": 1, + "customer": "C102424", + "customer_name": "Haar-shop.ch AG", + }, + { + "name": "SO527138", + "file_name": "order_a11fbe80-2651-4449-812d-6ea45fb30c43.csv", + "file_url": "/private/files/order_a11fbe80-2651-4449-812d-6ea45fb30c43.csv", + "is_private": 1, + "customer": "C102424", + "customer_name": "Haar-shop.ch AG", + }, + { + "name": "SO527137", + "file_name": "111944_Bestellung_4502170681_20260221.PDF", + "file_url": "/private/files/111944_Bestellung_4502170681_20260221.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527136", + "file_name": "112000_Bestellung_4502168855_20260223.PDF", + "file_url": "/private/files/112000_Bestellung_4502168855_20260223.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527135", + "file_name": "Lieferantenbestellung 70126567.xlsx", + "file_url": "/private/files/Lieferantenbestellung 70126567.xlsx", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO527067", + "file_name": "111944_Bestellung_4502168657_20260225.PDF", + "file_url": "/private/files/111944_Bestellung_4502168657_20260225.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527066", + "file_name": "Einkauf_Bestellung2e5890.csv", + "file_url": "/private/files/Einkauf_Bestellung2e5890.csv", + "is_private": 1, + "customer": "100069", + "customer_name": "BRACK.CH AG", + }, + { + "name": "SO527065", + "file_name": "111944_Bestellung_4502168447_20260221.PDF", + "file_url": "/private/files/111944_Bestellung_4502168447_20260221.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527064", + "file_name": "111944_Bestellung_4502168448_20260223.PDF", + "file_url": "/private/files/111944_Bestellung_4502168448_20260223.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527053", + "file_name": "BaQfc8R.png", + "file_url": "/private/files/BaQfc8R.png", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527047", + "file_name": "Bestellung 3600095574 Lieferant Nr 20203035.PDF", + "file_url": "/private/files/Bestellung 3600095574 Lieferant Nr 20203035.PDF", + "is_private": 1, + "customer": "C100122", + "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", + }, + { + "name": "SO527041", + "file_name": "order_a11bbdf0-5966-4148-aa23-c58e07f4f6a3.pdf", + "file_url": "/private/files/order_a11bbdf0-5966-4148-aa23-c58e07f4f6a3.pdf", + "is_private": 1, + "customer": "C102424", + "customer_name": "Haar-shop.ch AG", + }, + { + "name": "SO527040", + "file_name": "112000_Bestellung_4502168206_20260224.PDF", + "file_url": "/private/files/112000_Bestellung_4502168206_20260224.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527026", + "file_name": "Bestellung 3600095447 Lieferant Nr 20203035.PDF", + "file_url": "/private/files/Bestellung 3600095447 Lieferant Nr 20203035.PDF", + "is_private": 1, + "customer": "C100122", + "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", + }, + { + "name": "SO527018", + "file_name": "Lieferantenbestellung 70126470.xlsx", + "file_url": "/private/files/Lieferantenbestellung 70126470.xlsx", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO527017", + "file_name": "111944_Bestellung_4502167189_20260219.PDF", + "file_url": "/private/files/111944_Bestellung_4502167189_20260219.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527016", + "file_name": "111944_Bestellung_4502167190_20260227.PDF", + "file_url": "/private/files/111944_Bestellung_4502167190_20260227.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO527006", + "file_name": "order_a119a590-a3e7-410e-b0c0-1874bac7e304.pdf", + "file_url": "/private/files/order_a119a590-a3e7-410e-b0c0-1874bac7e304.pdf", + "is_private": 1, + "customer": "C102424", + "customer_name": "Haar-shop.ch AG", + }, + { + "name": "SO527000", + "file_name": "112000_Bestellung_4502166016_20260218.PDF", + "file_url": "/private/files/112000_Bestellung_4502166016_20260218.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO526971", + "file_name": "Lieferantenbestellung 70184985.xlsx", + "file_url": "/private/files/Lieferantenbestellung 70184985.xlsx", + "is_private": 1, + "customer": "C101705", + "customer_name": "PerfectHair AG", + }, + { + "name": "SO526967", + "file_name": "111944_Bestellung_4502164800_20260216.PDF", + "file_url": "/private/files/111944_Bestellung_4502164800_20260216.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO526966", + "file_name": "Lieferantenbestellung 70126395.xlsx", + "file_url": "/private/files/Lieferantenbestellung 70126395.xlsx", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO526964", + "file_name": "112000_Bestellung_4502161478_20260217.PDF", + "file_url": "/private/files/112000_Bestellung_4502161478_20260217.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO526904", + "file_name": "111944_Bestellung_4502162370_20260214.PDF", + "file_url": "/private/files/111944_Bestellung_4502162370_20260214.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO526896", + "file_name": "order_a1102207-7cd9-45f4-887d-f261a72ee2a3.pdf", + "file_url": "/private/files/order_a1102207-7cd9-45f4-887d-f261a72ee2a3.pdf", + "is_private": 1, + "customer": "C102424", + "customer_name": "Haar-shop.ch AG", + }, + { + "name": "SO526869", + "file_name": "Lieferantenbestellung 70126292.pdf", + "file_url": "/private/files/Lieferantenbestellung 70126292.pdf", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO526865", + "file_name": "112000_Bestellung_4502162633_20260213.PDF", + "file_url": "/private/files/112000_Bestellung_4502162633_20260213.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO526864", + "file_name": "Bestellung 3600095115 Lieferant Nr 20203035.PDF", + "file_url": "/private/files/Bestellung 3600095115 Lieferant Nr 20203035.PDF", + "is_private": 1, + "customer": "C100122", + "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", + }, + { + "name": "SO526827", + "file_name": "111944_Bestellung_4502161706_20260213.PDF", + "file_url": "/private/files/111944_Bestellung_4502161706_20260213.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO526826", + "file_name": "Einkauf_Bestellung4b7d54.csv", + "file_url": "/private/files/Einkauf_Bestellung4b7d54.csv", + "is_private": 1, + "customer": "100069", + "customer_name": "BRACK.CH AG", + }, + { + "name": "SO526802", + "file_name": "order_a0fbc923-8eac-4a14-bbed-441e1e666e77.pdf", + "file_url": "/private/files/order_a0fbc923-8eac-4a14-bbed-441e1e666e77.pdf", + "is_private": 1, + "customer": "C102424", + "customer_name": "Haar-shop.ch AG", + }, + { + "name": "SO526787", + "file_name": "Bestellung 3600094980 Lieferant Nr 20203035.PDF", + "file_url": "/private/files/Bestellung 3600094980 Lieferant Nr 20203035.PDF", + "is_private": 1, + "customer": "C100122", + "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", + }, + { + "name": "SO526786", + "file_name": "order_a10a03a6-4954-4e5f-99fd-bf752148e59f.pdf", + "file_url": "/private/files/order_a10a03a6-4954-4e5f-99fd-bf752148e59f.pdf", + "is_private": 1, + "customer": "C102424", + "customer_name": "Haar-shop.ch AG", + }, + { + "name": "SO526746", + "file_name": "Lieferantenbestellung 70126180.xlsx", + "file_url": "/private/files/Lieferantenbestellung 70126180.xlsx", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO526743", + "file_name": "111944_Bestellung_4502159532_20260211.PDF", + "file_url": "/private/files/111944_Bestellung_4502159532_20260211.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO526742", + "file_name": "111944_Bestellung_4502159533_20260211.PDF", + "file_url": "/private/files/111944_Bestellung_4502159533_20260211.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO526664", + "file_name": "111944_Bestellung_4502158143_20260207.PDF", + "file_url": "/private/files/111944_Bestellung_4502158143_20260207.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO526663", + "file_name": "111944_Bestellung_4502158144_20260209.PDF", + "file_url": "/private/files/111944_Bestellung_4502158144_20260209.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO526643", + "file_name": "Lieferantenbestellung 70125984.xlsx", + "file_url": "/private/files/Lieferantenbestellung 70125984.xlsx", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO526393", + "file_name": "order_a0ef91c7-8d5d-4156-a11a-79cec304d62b.pdf", + "file_url": "/private/files/order_a0ef91c7-8d5d-4156-a11a-79cec304d62b.pdf", + "is_private": 1, + "customer": "C102424", + "customer_name": "Haar-shop.ch AG", + }, + { + "name": "SO526387", + "file_name": "Lieferantenbestellung 70125781.xlsx", + "file_url": "/private/files/Lieferantenbestellung 70125781.xlsx", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO526385", + "file_name": "111944_Bestellung_4502150033_20260130.PDF", + "file_url": "/private/files/111944_Bestellung_4502150033_20260130.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO526353", + "file_name": "Lieferantenbestellung 70125781.xlsx", + "file_url": "/private/files/Lieferantenbestellung 70125781.xlsx", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO526101", + "file_name": "Lieferantenbestellung 70125507.xlsx", + "file_url": "/private/files/Lieferantenbestellung 70125507.xlsx", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO526019", + "file_name": "Lieferantenbestellung 70123859.csv", + "file_url": "/private/files/Lieferantenbestellung 70123859.csv", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO525925", + "file_name": "8h5UaEK.png", + "file_url": "/private/files/8h5UaEK.png", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO525817", + "file_name": "Einkauf_Bestellung685676.pdf", + "file_url": "/private/files/Einkauf_Bestellung685676.pdf", + "is_private": 1, + "customer": "100069", + "customer_name": "BRACK.CH AG", + }, + { + "name": "SO524706", + "file_name": "bLVrmGn.png", + "file_url": "/private/files/bLVrmGn.png", + "is_private": 1, + "customer": "100069", + "customer_name": "BRACK.CH AG", + }, + { + "name": "SO524474", + "file_name": "Lieferantenbestellung 70124156.csv", + "file_url": "/private/files/Lieferantenbestellung 70124156.csv", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO524212", + "file_name": "Einkauf_Bestellung.pdf", + "file_url": "/private/files/Einkauf_Bestellungca8331.pdf", + "is_private": 1, + "customer": "100069", + "customer_name": "BRACK.CH AG", + }, + { + "name": "SO524211", + "file_name": "Einkauf_Bestellungca8331.pdf", + "file_url": "/private/files/Einkauf_Bestellungca8331.pdf", + "is_private": 1, + "customer": "100069", + "customer_name": "BRACK.CH AG", + }, + { + "name": "SO524147", + "file_name": "111944_Bestellung_4502096157_20251126.PDF", + "file_url": "/private/files/111944_Bestellung_4502096157_20251126.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO524088", + "file_name": "111944_Bestellung_4502094116_20251125.PDF", + "file_url": "/private/files/111944_Bestellung_4502094116_20251125.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO524048", + "file_name": "111944_Bestellung_4502092807_20251122.PDF", + "file_url": "/private/files/111944_Bestellung_4502092807_20251122.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO524047", + "file_name": "111944_Bestellung_4502092808_20251127.PDF", + "file_url": "/private/files/111944_Bestellung_4502092808_20251127.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523904", + "file_name": "111944_Bestellung_4502090124_20251120.PDF", + "file_url": "/private/files/111944_Bestellung_4502090124_20251120.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523855", + "file_name": "112000_Bestellung_4502088756_20251119.PDF", + "file_url": "/private/files/112000_Bestellung_4502088756_20251119.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523845", + "file_name": "111944_Bestellung_4502086883_20251119.PDF", + "file_url": "/private/files/111944_Bestellung_4502086883_20251119.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523794", + "file_name": "111944_Bestellung_4502085790_20251114.PDF", + "file_url": "/private/files/111944_Bestellung_4502085790_20251114.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523712", + "file_name": "Lieferantenbestellung 70123207.csv", + "file_url": "/private/files/Lieferantenbestellung 70123207.csv", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO523711", + "file_name": "111944_Bestellung_4502081173_20251108.PDF", + "file_url": "/private/files/111944_Bestellung_4502081173_20251108.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523643", + "file_name": "111944_Bestellung_4502079140_20251107.PDF", + "file_url": "/private/files/111944_Bestellung_4502079140_20251107.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523642", + "file_name": "111944_Bestellung_4502079141_20251110.PDF", + "file_url": "/private/files/111944_Bestellung_4502079141_20251110.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523612", + "file_name": "111944_Bestellung_4502077753_20251107.PDF", + "file_url": "/private/files/111944_Bestellung_4502077753_20251107.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523611", + "file_name": "112000_Bestellung_4502077755_20251107.PDF", + "file_url": "/private/files/112000_Bestellung_4502077755_20251107.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523605", + "file_name": "111944_Bestellung_4502078150_20251104.PDF", + "file_url": "/private/files/111944_Bestellung_4502078150_20251104.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523566", + "file_name": "111944_Bestellung_4502075281_20251101.PDF", + "file_url": "/private/files/111944_Bestellung_4502075281_20251101.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523527", + "file_name": "111944_Bestellung_4502072369_20251101.PDF", + "file_url": "/private/files/111944_Bestellung_4502072369_20251101.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523497", + "file_name": "111944_Bestellung_4502069907_20251030.PDF", + "file_url": "/private/files/111944_Bestellung_4502069907_20251030.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523478", + "file_name": "111944_Bestellung_4502068446_20251029.PDF", + "file_url": "/private/files/111944_Bestellung_4502068446_20251029.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523453", + "file_name": "111944_Bestellung_4502069550_20251029.PDF", + "file_url": "/private/files/111944_Bestellung_4502069550_20251029.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523421", + "file_name": "111944_Bestellung_4502067202_20251025.PDF", + "file_url": "/private/files/111944_Bestellung_4502067202_20251025.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523371", + "file_name": "111944_Bestellung_4502062830_20251022.PDF", + "file_url": "/private/files/111944_Bestellung_4502062830_20251022.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523366", + "file_name": "111944_Bestellung_4502062727_20251022.PDF", + "file_url": "/private/files/111944_Bestellung_4502062727_20251022.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523359", + "file_name": "111944_Bestellung_4502063673_20251022.PDF", + "file_url": "/private/files/111944_Bestellung_4502063673_20251022.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523328", + "file_name": "111944_Bestellung_4502061809_20251017.PDF", + "file_url": "/private/files/111944_Bestellung_4502061809_20251017.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523298", + "file_name": "112000_Bestellung_4502060171_20251016.PDF", + "file_url": "/private/files/112000_Bestellung_4502060171_20251016.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523297", + "file_name": "111944_Bestellung_4502060170_20251016.PDF", + "file_url": "/private/files/111944_Bestellung_4502060170_20251016.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523288", + "file_name": "111944_Bestellung_4502061126_20251016.PDF", + "file_url": "/private/files/111944_Bestellung_4502061126_20251016.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO523185", + "file_name": "Einkauf_Bestellung (5)278b55.pdf", + "file_url": "/private/files/Einkauf_Bestellung (5)278b55.pdf", + "is_private": 1, + "customer": "100069", + "customer_name": "BRACK.CH AG", + }, + { + "name": "SO523047", + "file_name": "Bestellung 3600085201 Lieferant Nr 20203035.PDF", + "file_url": "/private/files/Bestellung 3600085201 Lieferant Nr 20203035.PDF", + "is_private": 1, + "customer": "C100122", + "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", + }, + { + "name": "SO522915", + "file_name": "Lieferantenbestellung 70121844.pdf", + "file_url": "/private/files/Lieferantenbestellung 70121844.pdf", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO522831", + "file_name": "111944_Bestellung_4502038760_20250922.PDF", + "file_url": "/private/files/111944_Bestellung_4502038760_20250922.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO522830", + "file_name": "111944_Bestellung_4502038759_20250920.PDF", + "file_url": "/private/files/111944_Bestellung_4502038759_20250920.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO522756", + "file_name": "Lieferantenbestellung 70121452.pdf", + "file_url": "/private/files/Lieferantenbestellung 70121452.pdf", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, + { + "name": "SO522425", + "file_name": "111944_Bestellung_4502023162_20250830.PDF", + "file_url": "/private/files/111944_Bestellung_4502023162_20250830.PDF", + "is_private": 1, + "customer": "C100931", + "customer_name": "Dipl. Ing. Fust", + }, + { + "name": "SO522362", + "file_name": "Bestellung 3600082331 Lieferant Nr 20203035.PDF", + "file_url": "/private/files/Bestellung 3600082331 Lieferant Nr 20203035.PDF", + "is_private": 1, + "customer": "C100122", + "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", + }, + { + "name": "SO522330", + "file_name": "Einkauf_Bestellunga81933.pdf", + "file_url": "/private/files/Einkauf_Bestellunga81933.pdf", + "is_private": 1, + "customer": "100069", + "customer_name": "BRACK.CH AG", + }, + { + "name": "SO522308", + "file_name": "Bestellung 3600082064 Lieferant Nr 20203035.PDF", + "file_url": "/private/files/Bestellung 3600082064 Lieferant Nr 20203035.PDF", + "is_private": 1, + "customer": "C100122", + "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", + }, + { + "name": "SO522293", + "file_name": "Lieferantenbestellung 70120310.pdf", + "file_url": "/private/files/Lieferantenbestellung 70120310.pdf", + "is_private": 1, + "customer": "C102340", + "customer_name": "4mybaby AG", + }, +] + + +def execute(): + file_url_to_customer = {entry["file_url"]: entry["customer"] for entry in PARTY} + + datasets = frappe.get_all( + "Parser Benchmark Dataset", + filters={ + "file": ["in", list(file_url_to_customer.keys())], + "party": ["is", "not set"], + }, + fields=["name", "file"], + ) + + updates = {} + for dataset in datasets: + updates[dataset.name] = { + "party_type": "Customer", + "party": file_url_to_customer[dataset.file], + } + + if updates: + frappe.db.bulk_update("Parser Benchmark Dataset", updates) + print(f"\n Updated {len(updates)} datasets with party information.") From 6989ec1e3fd58fcefbfbcabc4660633d37c3eaba Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Fri, 20 Mar 2026 17:42:31 +0530 Subject: [PATCH 40/88] refactor: enhance parser benchmark dataset and log with accuracy scoring and response hashing --- pyproject.toml | 1 + .../parser_benchmark_dataset.json | 23 +++- .../parser_benchmark_dataset.py | 1 + .../parser_benchmark_log.json | 51 ++++++++- .../parser_benchmark_log.py | 3 + transaction_parser/parser_benchmark/runner.py | 24 +++- transaction_parser/parser_benchmark/scorer.py | 107 ++++++++++++++++++ 7 files changed, 203 insertions(+), 7 deletions(-) create mode 100644 transaction_parser/parser_benchmark/scorer.py diff --git a/pyproject.toml b/pyproject.toml index a458259..ad24096 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "pymupdf~=1.26.3", "openai", "docling>=2.75.0", + "deepdiff>=8.0", ] [build-system] diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index dc217b6..e3602b2 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -10,8 +10,8 @@ "section_break_file", "file", "file_type", - "transaction_type", "column_break_gbap", + "transaction_type", "country", "section_break_sobg", "company", @@ -37,6 +37,8 @@ "other_config_section", "page_limit", "column_break_jjht", + "expected_result_section", + "expected_result", "section_break_bhsg", "naming_series" ], @@ -144,7 +146,7 @@ "fieldtype": "Column Break" }, { - "default": "0", + "default": "1", "fieldname": "openai_gpt_5", "fieldtype": "Check", "label": "OpenAI gpt-5" @@ -168,12 +170,13 @@ "label": "Google Gemini Flash-2.5" }, { + "depends_on": "eval: doc.file_type === \"PDF\"", "fieldname": "pdf_processor_section", "fieldtype": "Section Break", "label": "PDF Processors" }, { - "default": "0", + "default": "1", "fieldname": "ocrmypdf", "fieldtype": "Check", "label": "OCRMyPDF" @@ -199,6 +202,18 @@ "label": "Page Limit", "non_negative": 1 }, + { + "fieldname": "expected_result_section", + "fieldtype": "Section Break", + "label": "Expected Result" + }, + { + "description": "The correct/expected parsed output JSON for this dataset. Used to score AI accuracy.", + "fieldname": "expected_result", + "fieldtype": "Code", + "label": "Expected Result", + "options": "JSON" + }, { "fieldname": "section_break_bhsg", "fieldtype": "Section Break" @@ -238,7 +253,7 @@ "link_fieldname": "dataset" } ], - "modified": "2026-03-19 03:36:39.325714", + "modified": "2026-03-20 08:12:11.328907", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 442ce54..6f08fd2 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -38,6 +38,7 @@ class ParserBenchmarkDataset(Document): deepseek_reasoner: DF.Check docling: DF.Check enabled: DF.Check + expected_result: DF.Code | None file: DF.Attach file_type: DF.Data | None google_gemini_flash_25: DF.Check diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index a635562..bc2c1c5 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -8,8 +8,8 @@ "config_section", "dataset", "transaction_type", - "column_break_summary", "country", + "column_break_summary", "company", "party_type", "party", @@ -51,6 +51,13 @@ "total_cost", "section_break_ai_content", "ai_response", + "accuracy_tab", + "accuracy_section", + "accuracy_score", + "column_break_accuracy", + "response_hash", + "section_break_mismatches", + "field_mismatches", "error_tab", "section_break_error", "error" @@ -272,6 +279,46 @@ "fieldtype": "Section Break", "label": "Token Details" }, + { + "fieldname": "accuracy_tab", + "fieldtype": "Tab Break", + "label": "Accuracy" + }, + { + "fieldname": "accuracy_section", + "fieldtype": "Section Break" + }, + { + "description": "Percentage of fields matching the expected result (0-100)", + "fieldname": "accuracy_score", + "fieldtype": "Percent", + "in_list_view": 1, + "label": "Accuracy Score (%)", + "read_only": 1 + }, + { + "fieldname": "column_break_accuracy", + "fieldtype": "Column Break" + }, + { + "description": "Hash of AI response for consistency tracking across runs", + "fieldname": "response_hash", + "fieldtype": "Data", + "label": "Response Hash", + "read_only": 1 + }, + { + "fieldname": "section_break_mismatches", + "fieldtype": "Section Break", + "label": "Field Mismatches" + }, + { + "fieldname": "field_mismatches", + "fieldtype": "Code", + "label": "Field Mismatches", + "options": "JSON", + "read_only": 1 + }, { "fieldname": "error_tab", "fieldtype": "Tab Break", @@ -364,7 +411,7 @@ "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-18 13:39:29.730560", + "modified": "2026-03-20 08:14:27.215617", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Log", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py index 1a11994..ca8e9a4 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -13,6 +13,7 @@ class ParserBenchmarkLog(Document): if TYPE_CHECKING: from frappe.types import DF + accuracy_score: DF.Percent ai_model: DF.Literal[ "DeepSeek Chat", "DeepSeek Reasoner", @@ -31,6 +32,7 @@ class ParserBenchmarkLog(Document): currency: DF.Link | None dataset: DF.Link error: DF.Code | None + field_mismatches: DF.Code | None file_content: DF.Code | None file_parse_memory: DF.Float file_parse_time: DF.Float @@ -45,6 +47,7 @@ class ParserBenchmarkLog(Document): party_type: DF.Link | None pdf_processor: DF.Literal["", "OCRMyPDF", "Docling"] prompt_tokens: DF.Int + response_hash: DF.Data | None status: DF.Literal["Queued", "Running", "Completed", "Failed"] total_cost: DF.Currency total_time: DF.Float diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index 923507a..ba7eb63 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -47,8 +47,9 @@ def run(self): self.controller: Transaction = self._get_controller(file_doc) file_content = self._run_file_parsing(file_doc) - self._run_ai_parsing(file_content, file_doc.name) + ai_content = self._run_ai_parsing(file_content, file_doc.name) self._calculate_cost() + self._score_response(ai_content) self.log.status = "Completed" @@ -162,3 +163,24 @@ def _calculate_cost(self): self.log.total_cost = flt( self.log.input_cost + self.log.output_cost, self.precision ) + + # ── step 4: accuracy scoring ───────────────────────────── + + def _score_response(self, ai_content: dict): + from transaction_parser.parser_benchmark.scorer import ( + compute_response_hash, + score_response, + ) + + self.log.response_hash = compute_response_hash(ai_content) + + expected = self.dataset.expected_result + if not expected: + return + + if isinstance(expected, str): + expected = frappe.parse_json(expected) + + result = score_response(expected, ai_content) + self.log.accuracy_score = result["accuracy_score"] + self.log.field_mismatches = frappe.as_json(result["mismatches"], indent=2) diff --git a/transaction_parser/parser_benchmark/scorer.py b/transaction_parser/parser_benchmark/scorer.py new file mode 100644 index 0000000..17b8cd9 --- /dev/null +++ b/transaction_parser/parser_benchmark/scorer.py @@ -0,0 +1,107 @@ +import frappe +from deepdiff import DeepDiff, DeepHash +from frappe.utils import flt + + +def score_response(expected: dict, actual: dict, max_diffs: int = 500) -> dict: + """Compare AI response against expected result using DeepDiff.""" + diff = DeepDiff( + expected, + actual, + ignore_string_case=True, + ignore_order=True, + ignore_type_in_groups=[(dict, frappe._dict)], + significant_digits=2, + verbose_level=2, + max_diffs=max_diffs, + log_frequency_in_sec=0, + get_deep_distance=True, + ) + + distance = diff.get("deep_distance", 1) or 1 + accuracy = flt((1 - distance) * 100, 2) + + return { + "accuracy_score": accuracy, + "mismatches": _format_mismatches(diff), + } + + +def compute_response_hash(ai_response: dict | str) -> str: + """Deterministic hash of AI response for consistency tracking across runs.""" + if isinstance(ai_response, str): + ai_response = frappe.parse_json(ai_response) + + return DeepHash(ai_response)[ai_response][:16] + + +def _format_mismatches(diff: DeepDiff) -> list[dict]: + """Flatten DeepDiff into a simple list of {field, expected, actual}.""" + mismatches = [] + + for path, change in diff.get("values_changed", {}).items(): + mismatches.append({"field": path, "expected": change["old_value"], "actual": change["new_value"]}) + + for path, change in diff.get("type_changes", {}).items(): + mismatches.append({"field": path, "expected": change["old_value"], "actual": change["new_value"]}) + + for path, val in diff.get("dictionary_item_removed", {}).items(): + mismatches.append({"field": path, "expected": val, "actual": None}) + + for path, val in diff.get("dictionary_item_added", {}).items(): + mismatches.append({"field": path, "expected": None, "actual": val}) + + for path, val in diff.get("iterable_item_removed", {}).items(): + mismatches.append({"field": path, "expected": val, "actual": None}) + + for path, val in diff.get("iterable_item_added", {}).items(): + mismatches.append({"field": path, "expected": None, "actual": val}) + + return mismatches + + +# TODO: need some changes here +def get_consistency(dataset: str, ai_model: str, pdf_processor: str = "") -> dict: + """ + Check how consistent a model's response is for a given dataset + processor combo. + + Returns: + { + "total_runs": 5, + "unique_hashes": 2, + "consistency": 80.0, # % of runs with the most common hash + "hashes": {"abc123": 4, "def456": 1}, + } + """ + Log = frappe.qb.DocType("Parser Benchmark Log") + + logs = ( + frappe.qb.from_(Log) + .select(Log.response_hash) + .where(Log.dataset == dataset) + .where(Log.ai_model == ai_model) + .where(Log.pdf_processor == (pdf_processor or "")) + .where(Log.status == "Completed") + .where(Log.response_hash.isnotnull()) + .run(as_dict=True) + ) + + if not logs: + return {"total_runs": 0, "unique_hashes": 0, "consistency": 0.0, "hashes": {}} + + # count occurrences of each hash + hashes = {} + for log in logs: + h = log.response_hash + hashes[h] = hashes.get(h, 0) + 1 + + total_runs = len(logs) + most_common_count = max(hashes.values()) + consistency = flt((most_common_count / total_runs) * 100, 2) + + return { + "total_runs": total_runs, + "unique_hashes": len(hashes), + "consistency": consistency, + "hashes": hashes, + } From 651c3a031125577b899535215141eba6fb92953e Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Fri, 20 Mar 2026 17:58:24 +0530 Subject: [PATCH 41/88] chore: minor changes --- .../patches/set_party_in_dataset.py | 1006 ----------------- 1 file changed, 1006 deletions(-) delete mode 100644 transaction_parser/patches/set_party_in_dataset.py diff --git a/transaction_parser/patches/set_party_in_dataset.py b/transaction_parser/patches/set_party_in_dataset.py deleted file mode 100644 index 3ed4bdb..0000000 --- a/transaction_parser/patches/set_party_in_dataset.py +++ /dev/null @@ -1,1006 +0,0 @@ -import frappe - -# ! NOTE: Temporary patch - -PARTY = [ - { - "name": "SO527790", - "file_name": "Bestellung 3600097229 Lieferant Nr 20203035.PDF", - "file_url": "/private/files/Bestellung 3600097229 Lieferant Nr 20203035.PDF", - "is_private": 1, - "customer": "C100122", - "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", - }, - { - "name": "SO527789", - "file_name": "Bestellung 3600097186 Lieferant Nr 20203035.PDF", - "file_url": "/private/files/Bestellung 3600097186 Lieferant Nr 20203035.PDF", - "is_private": 1, - "customer": "C100122", - "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", - }, - { - "name": "SO527783", - "file_name": "Lieferantenbestellung 70127077.xlsx", - "file_url": "/private/files/Lieferantenbestellung 70127077.xlsx", - "is_private": 1, - "customer": "100069", - "customer_name": "BRACK.CH AG", - }, - { - "name": "SO527782", - "file_name": "111944_Bestellung_4502190835_20260319.PDF", - "file_url": "/private/files/111944_Bestellung_4502190835_20260319.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527762", - "file_name": "order_a152091d-ad91-4fb3-9158-5afe3b43f813.csv", - "file_url": "/private/files/order_a152091d-ad91-4fb3-9158-5afe3b43f813.csv", - "is_private": 1, - "customer": "C102424", - "customer_name": "Haar-shop.ch AG", - }, - { - "name": "SO527727", - "file_name": "111944_Bestellung_4502187869_20260319.PDF", - "file_url": "/private/files/111944_Bestellung_4502187869_20260319.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527726", - "file_name": "Lieferantenbestellung 70126993.xlsx", - "file_url": "/private/files/Lieferantenbestellung 70126993.xlsx", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO527686", - "file_name": "112000_Bestellung_4502186288_20260317.PDF", - "file_url": "/private/files/112000_Bestellung_4502186288_20260317.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527679", - "file_name": "Bestellung 3600096912 Lieferant Nr 20203035.PDF", - "file_url": "/private/files/Bestellung 3600096912 Lieferant Nr 20203035.PDF", - "is_private": 1, - "customer": "C100122", - "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", - }, - { - "name": "SO527620", - "file_name": "Lieferantenbestellung 70126937.xlsx", - "file_url": "/private/files/Lieferantenbestellung 70126937.xlsx", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO527619", - "file_name": "112000_Bestellung_4502180368_20260309.PDF", - "file_url": "/private/files/112000_Bestellung_4502180368_20260309.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527595", - "file_name": "Einkauf_Bestellung99ffe0.csv", - "file_url": "/private/files/Einkauf_Bestellung99ffe0.csv", - "is_private": 1, - "customer": "100069", - "customer_name": "BRACK.CH AG", - }, - { - "name": "SO527594", - "file_name": "111944_Bestellung_4502185151_20260312.PDF", - "file_url": "/private/files/111944_Bestellung_4502185151_20260312.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527539", - "file_name": "111944_Bestellung_4502181439_20260311.PDF", - "file_url": "/private/files/111944_Bestellung_4502181439_20260311.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527538", - "file_name": "order_a14251cb-ef2a-4306-b7a3-9bac4d8c236e.csv", - "file_url": "/private/files/order_a14251cb-ef2a-4306-b7a3-9bac4d8c236e.csv", - "is_private": 1, - "customer": "C102424", - "customer_name": "Haar-shop.ch AG", - }, - { - "name": "SO527505", - "file_name": "111944_Bestellung_4502180283_20260307.PDF", - "file_url": "/private/files/111944_Bestellung_4502180283_20260307.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527504", - "file_name": "order_a13bf300-b510-4ea0-b07d-d3d005bf603c.csv", - "file_url": "/private/files/order_a13bf300-b510-4ea0-b07d-d3d005bf603c.csv", - "is_private": 1, - "customer": "C102424", - "customer_name": "Haar-shop.ch AG", - }, - { - "name": "SO527452", - "file_name": "Bestellung 3600096482 Lieferant Nr 20203035.PDF", - "file_url": "/private/files/Bestellung 3600096482 Lieferant Nr 20203035.PDF", - "is_private": 1, - "customer": "C100122", - "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", - }, - { - "name": "SO527344", - "file_name": "order_a133b6d9-f3a7-4629-877b-f72f4c751363.pdf", - "file_url": "/private/files/order_a133b6d9-f3a7-4629-877b-f72f4c751363.pdf", - "is_private": 1, - "customer": "C102424", - "customer_name": "Haar-shop.ch AG", - }, - { - "name": "SO527343", - "file_name": "Lieferantenbestellung 70126756.xlsx", - "file_url": "/private/files/Lieferantenbestellung 70126756.xlsx", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO527341", - "file_name": "111944_Bestellung_4502176704_20260303.PDF", - "file_url": "/private/files/111944_Bestellung_4502176704_20260303.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527285", - "file_name": "order_a12df8f0-ae26-43a0-b48e-09374819b081.csv", - "file_url": "/private/files/order_a12df8f0-ae26-43a0-b48e-09374819b081.csv", - "is_private": 1, - "customer": "C102424", - "customer_name": "Haar-shop.ch AG", - }, - { - "name": "SO527274", - "file_name": "111944_Bestellung_4502175021_20260227.PDF", - "file_url": "/private/files/111944_Bestellung_4502175021_20260227.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527273", - "file_name": "111944_Bestellung_4502175022_20260302.PDF", - "file_url": "/private/files/111944_Bestellung_4502175022_20260302.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527241", - "file_name": "Einkauf_Bestellung482a75.csv", - "file_url": "/private/files/Einkauf_Bestellung482a75.csv", - "is_private": 1, - "customer": "100069", - "customer_name": "BRACK.CH AG", - }, - { - "name": "SO527217", - "file_name": "Einkauf_Bestellung81ecda.csv", - "file_url": "/private/files/Einkauf_Bestellung81ecda.csv", - "is_private": 1, - "customer": "100069", - "customer_name": "BRACK.CH AG", - }, - { - "name": "SO527209", - "file_name": "order_a129be25-f590-47a8-9c17-6664d16c16d3.csv", - "file_url": "/private/files/order_a129be25-f590-47a8-9c17-6664d16c16d3.csv", - "is_private": 1, - "customer": "C102424", - "customer_name": "Haar-shop.ch AG", - }, - { - "name": "SO527206", - "file_name": "Bestellung 3600095890 Lieferant Nr 20203035.PDF", - "file_url": "/private/files/Bestellung 3600095890 Lieferant Nr 20203035.PDF", - "is_private": 1, - "customer": "C100122", - "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", - }, - { - "name": "SO527205", - "file_name": "Lieferantenbestellung 70126653.xlsx", - "file_url": "/private/files/Lieferantenbestellung 70126653.xlsx", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO527202", - "file_name": "111944_Bestellung_4502172320_20260226.PDF", - "file_url": "/private/files/111944_Bestellung_4502172320_20260226.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527201", - "file_name": "Bestellung 3600095878 Lieferant Nr 20203035.PDF", - "file_url": "/private/files/Bestellung 3600095878 Lieferant Nr 20203035.PDF", - "is_private": 1, - "customer": "C100122", - "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", - }, - { - "name": "SO527200", - "file_name": "111944_Bestellung_4502172321_20260228.PDF", - "file_url": "/private/files/111944_Bestellung_4502172321_20260228.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527167", - "file_name": "order_a12652f3-c8db-4c02-b44e-d1516ce83bfc.csv", - "file_url": "/private/files/order_a12652f3-c8db-4c02-b44e-d1516ce83bfc.csv", - "is_private": 1, - "customer": "C102424", - "customer_name": "Haar-shop.ch AG", - }, - { - "name": "SO527138", - "file_name": "order_a11fbe80-2651-4449-812d-6ea45fb30c43.csv", - "file_url": "/private/files/order_a11fbe80-2651-4449-812d-6ea45fb30c43.csv", - "is_private": 1, - "customer": "C102424", - "customer_name": "Haar-shop.ch AG", - }, - { - "name": "SO527137", - "file_name": "111944_Bestellung_4502170681_20260221.PDF", - "file_url": "/private/files/111944_Bestellung_4502170681_20260221.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527136", - "file_name": "112000_Bestellung_4502168855_20260223.PDF", - "file_url": "/private/files/112000_Bestellung_4502168855_20260223.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527135", - "file_name": "Lieferantenbestellung 70126567.xlsx", - "file_url": "/private/files/Lieferantenbestellung 70126567.xlsx", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO527067", - "file_name": "111944_Bestellung_4502168657_20260225.PDF", - "file_url": "/private/files/111944_Bestellung_4502168657_20260225.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527066", - "file_name": "Einkauf_Bestellung2e5890.csv", - "file_url": "/private/files/Einkauf_Bestellung2e5890.csv", - "is_private": 1, - "customer": "100069", - "customer_name": "BRACK.CH AG", - }, - { - "name": "SO527065", - "file_name": "111944_Bestellung_4502168447_20260221.PDF", - "file_url": "/private/files/111944_Bestellung_4502168447_20260221.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527064", - "file_name": "111944_Bestellung_4502168448_20260223.PDF", - "file_url": "/private/files/111944_Bestellung_4502168448_20260223.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527053", - "file_name": "BaQfc8R.png", - "file_url": "/private/files/BaQfc8R.png", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527047", - "file_name": "Bestellung 3600095574 Lieferant Nr 20203035.PDF", - "file_url": "/private/files/Bestellung 3600095574 Lieferant Nr 20203035.PDF", - "is_private": 1, - "customer": "C100122", - "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", - }, - { - "name": "SO527041", - "file_name": "order_a11bbdf0-5966-4148-aa23-c58e07f4f6a3.pdf", - "file_url": "/private/files/order_a11bbdf0-5966-4148-aa23-c58e07f4f6a3.pdf", - "is_private": 1, - "customer": "C102424", - "customer_name": "Haar-shop.ch AG", - }, - { - "name": "SO527040", - "file_name": "112000_Bestellung_4502168206_20260224.PDF", - "file_url": "/private/files/112000_Bestellung_4502168206_20260224.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527026", - "file_name": "Bestellung 3600095447 Lieferant Nr 20203035.PDF", - "file_url": "/private/files/Bestellung 3600095447 Lieferant Nr 20203035.PDF", - "is_private": 1, - "customer": "C100122", - "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", - }, - { - "name": "SO527018", - "file_name": "Lieferantenbestellung 70126470.xlsx", - "file_url": "/private/files/Lieferantenbestellung 70126470.xlsx", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO527017", - "file_name": "111944_Bestellung_4502167189_20260219.PDF", - "file_url": "/private/files/111944_Bestellung_4502167189_20260219.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527016", - "file_name": "111944_Bestellung_4502167190_20260227.PDF", - "file_url": "/private/files/111944_Bestellung_4502167190_20260227.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO527006", - "file_name": "order_a119a590-a3e7-410e-b0c0-1874bac7e304.pdf", - "file_url": "/private/files/order_a119a590-a3e7-410e-b0c0-1874bac7e304.pdf", - "is_private": 1, - "customer": "C102424", - "customer_name": "Haar-shop.ch AG", - }, - { - "name": "SO527000", - "file_name": "112000_Bestellung_4502166016_20260218.PDF", - "file_url": "/private/files/112000_Bestellung_4502166016_20260218.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO526971", - "file_name": "Lieferantenbestellung 70184985.xlsx", - "file_url": "/private/files/Lieferantenbestellung 70184985.xlsx", - "is_private": 1, - "customer": "C101705", - "customer_name": "PerfectHair AG", - }, - { - "name": "SO526967", - "file_name": "111944_Bestellung_4502164800_20260216.PDF", - "file_url": "/private/files/111944_Bestellung_4502164800_20260216.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO526966", - "file_name": "Lieferantenbestellung 70126395.xlsx", - "file_url": "/private/files/Lieferantenbestellung 70126395.xlsx", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO526964", - "file_name": "112000_Bestellung_4502161478_20260217.PDF", - "file_url": "/private/files/112000_Bestellung_4502161478_20260217.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO526904", - "file_name": "111944_Bestellung_4502162370_20260214.PDF", - "file_url": "/private/files/111944_Bestellung_4502162370_20260214.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO526896", - "file_name": "order_a1102207-7cd9-45f4-887d-f261a72ee2a3.pdf", - "file_url": "/private/files/order_a1102207-7cd9-45f4-887d-f261a72ee2a3.pdf", - "is_private": 1, - "customer": "C102424", - "customer_name": "Haar-shop.ch AG", - }, - { - "name": "SO526869", - "file_name": "Lieferantenbestellung 70126292.pdf", - "file_url": "/private/files/Lieferantenbestellung 70126292.pdf", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO526865", - "file_name": "112000_Bestellung_4502162633_20260213.PDF", - "file_url": "/private/files/112000_Bestellung_4502162633_20260213.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO526864", - "file_name": "Bestellung 3600095115 Lieferant Nr 20203035.PDF", - "file_url": "/private/files/Bestellung 3600095115 Lieferant Nr 20203035.PDF", - "is_private": 1, - "customer": "C100122", - "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", - }, - { - "name": "SO526827", - "file_name": "111944_Bestellung_4502161706_20260213.PDF", - "file_url": "/private/files/111944_Bestellung_4502161706_20260213.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO526826", - "file_name": "Einkauf_Bestellung4b7d54.csv", - "file_url": "/private/files/Einkauf_Bestellung4b7d54.csv", - "is_private": 1, - "customer": "100069", - "customer_name": "BRACK.CH AG", - }, - { - "name": "SO526802", - "file_name": "order_a0fbc923-8eac-4a14-bbed-441e1e666e77.pdf", - "file_url": "/private/files/order_a0fbc923-8eac-4a14-bbed-441e1e666e77.pdf", - "is_private": 1, - "customer": "C102424", - "customer_name": "Haar-shop.ch AG", - }, - { - "name": "SO526787", - "file_name": "Bestellung 3600094980 Lieferant Nr 20203035.PDF", - "file_url": "/private/files/Bestellung 3600094980 Lieferant Nr 20203035.PDF", - "is_private": 1, - "customer": "C100122", - "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", - }, - { - "name": "SO526786", - "file_name": "order_a10a03a6-4954-4e5f-99fd-bf752148e59f.pdf", - "file_url": "/private/files/order_a10a03a6-4954-4e5f-99fd-bf752148e59f.pdf", - "is_private": 1, - "customer": "C102424", - "customer_name": "Haar-shop.ch AG", - }, - { - "name": "SO526746", - "file_name": "Lieferantenbestellung 70126180.xlsx", - "file_url": "/private/files/Lieferantenbestellung 70126180.xlsx", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO526743", - "file_name": "111944_Bestellung_4502159532_20260211.PDF", - "file_url": "/private/files/111944_Bestellung_4502159532_20260211.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO526742", - "file_name": "111944_Bestellung_4502159533_20260211.PDF", - "file_url": "/private/files/111944_Bestellung_4502159533_20260211.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO526664", - "file_name": "111944_Bestellung_4502158143_20260207.PDF", - "file_url": "/private/files/111944_Bestellung_4502158143_20260207.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO526663", - "file_name": "111944_Bestellung_4502158144_20260209.PDF", - "file_url": "/private/files/111944_Bestellung_4502158144_20260209.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO526643", - "file_name": "Lieferantenbestellung 70125984.xlsx", - "file_url": "/private/files/Lieferantenbestellung 70125984.xlsx", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO526393", - "file_name": "order_a0ef91c7-8d5d-4156-a11a-79cec304d62b.pdf", - "file_url": "/private/files/order_a0ef91c7-8d5d-4156-a11a-79cec304d62b.pdf", - "is_private": 1, - "customer": "C102424", - "customer_name": "Haar-shop.ch AG", - }, - { - "name": "SO526387", - "file_name": "Lieferantenbestellung 70125781.xlsx", - "file_url": "/private/files/Lieferantenbestellung 70125781.xlsx", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO526385", - "file_name": "111944_Bestellung_4502150033_20260130.PDF", - "file_url": "/private/files/111944_Bestellung_4502150033_20260130.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO526353", - "file_name": "Lieferantenbestellung 70125781.xlsx", - "file_url": "/private/files/Lieferantenbestellung 70125781.xlsx", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO526101", - "file_name": "Lieferantenbestellung 70125507.xlsx", - "file_url": "/private/files/Lieferantenbestellung 70125507.xlsx", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO526019", - "file_name": "Lieferantenbestellung 70123859.csv", - "file_url": "/private/files/Lieferantenbestellung 70123859.csv", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO525925", - "file_name": "8h5UaEK.png", - "file_url": "/private/files/8h5UaEK.png", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO525817", - "file_name": "Einkauf_Bestellung685676.pdf", - "file_url": "/private/files/Einkauf_Bestellung685676.pdf", - "is_private": 1, - "customer": "100069", - "customer_name": "BRACK.CH AG", - }, - { - "name": "SO524706", - "file_name": "bLVrmGn.png", - "file_url": "/private/files/bLVrmGn.png", - "is_private": 1, - "customer": "100069", - "customer_name": "BRACK.CH AG", - }, - { - "name": "SO524474", - "file_name": "Lieferantenbestellung 70124156.csv", - "file_url": "/private/files/Lieferantenbestellung 70124156.csv", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO524212", - "file_name": "Einkauf_Bestellung.pdf", - "file_url": "/private/files/Einkauf_Bestellungca8331.pdf", - "is_private": 1, - "customer": "100069", - "customer_name": "BRACK.CH AG", - }, - { - "name": "SO524211", - "file_name": "Einkauf_Bestellungca8331.pdf", - "file_url": "/private/files/Einkauf_Bestellungca8331.pdf", - "is_private": 1, - "customer": "100069", - "customer_name": "BRACK.CH AG", - }, - { - "name": "SO524147", - "file_name": "111944_Bestellung_4502096157_20251126.PDF", - "file_url": "/private/files/111944_Bestellung_4502096157_20251126.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO524088", - "file_name": "111944_Bestellung_4502094116_20251125.PDF", - "file_url": "/private/files/111944_Bestellung_4502094116_20251125.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO524048", - "file_name": "111944_Bestellung_4502092807_20251122.PDF", - "file_url": "/private/files/111944_Bestellung_4502092807_20251122.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO524047", - "file_name": "111944_Bestellung_4502092808_20251127.PDF", - "file_url": "/private/files/111944_Bestellung_4502092808_20251127.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523904", - "file_name": "111944_Bestellung_4502090124_20251120.PDF", - "file_url": "/private/files/111944_Bestellung_4502090124_20251120.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523855", - "file_name": "112000_Bestellung_4502088756_20251119.PDF", - "file_url": "/private/files/112000_Bestellung_4502088756_20251119.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523845", - "file_name": "111944_Bestellung_4502086883_20251119.PDF", - "file_url": "/private/files/111944_Bestellung_4502086883_20251119.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523794", - "file_name": "111944_Bestellung_4502085790_20251114.PDF", - "file_url": "/private/files/111944_Bestellung_4502085790_20251114.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523712", - "file_name": "Lieferantenbestellung 70123207.csv", - "file_url": "/private/files/Lieferantenbestellung 70123207.csv", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO523711", - "file_name": "111944_Bestellung_4502081173_20251108.PDF", - "file_url": "/private/files/111944_Bestellung_4502081173_20251108.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523643", - "file_name": "111944_Bestellung_4502079140_20251107.PDF", - "file_url": "/private/files/111944_Bestellung_4502079140_20251107.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523642", - "file_name": "111944_Bestellung_4502079141_20251110.PDF", - "file_url": "/private/files/111944_Bestellung_4502079141_20251110.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523612", - "file_name": "111944_Bestellung_4502077753_20251107.PDF", - "file_url": "/private/files/111944_Bestellung_4502077753_20251107.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523611", - "file_name": "112000_Bestellung_4502077755_20251107.PDF", - "file_url": "/private/files/112000_Bestellung_4502077755_20251107.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523605", - "file_name": "111944_Bestellung_4502078150_20251104.PDF", - "file_url": "/private/files/111944_Bestellung_4502078150_20251104.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523566", - "file_name": "111944_Bestellung_4502075281_20251101.PDF", - "file_url": "/private/files/111944_Bestellung_4502075281_20251101.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523527", - "file_name": "111944_Bestellung_4502072369_20251101.PDF", - "file_url": "/private/files/111944_Bestellung_4502072369_20251101.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523497", - "file_name": "111944_Bestellung_4502069907_20251030.PDF", - "file_url": "/private/files/111944_Bestellung_4502069907_20251030.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523478", - "file_name": "111944_Bestellung_4502068446_20251029.PDF", - "file_url": "/private/files/111944_Bestellung_4502068446_20251029.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523453", - "file_name": "111944_Bestellung_4502069550_20251029.PDF", - "file_url": "/private/files/111944_Bestellung_4502069550_20251029.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523421", - "file_name": "111944_Bestellung_4502067202_20251025.PDF", - "file_url": "/private/files/111944_Bestellung_4502067202_20251025.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523371", - "file_name": "111944_Bestellung_4502062830_20251022.PDF", - "file_url": "/private/files/111944_Bestellung_4502062830_20251022.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523366", - "file_name": "111944_Bestellung_4502062727_20251022.PDF", - "file_url": "/private/files/111944_Bestellung_4502062727_20251022.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523359", - "file_name": "111944_Bestellung_4502063673_20251022.PDF", - "file_url": "/private/files/111944_Bestellung_4502063673_20251022.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523328", - "file_name": "111944_Bestellung_4502061809_20251017.PDF", - "file_url": "/private/files/111944_Bestellung_4502061809_20251017.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523298", - "file_name": "112000_Bestellung_4502060171_20251016.PDF", - "file_url": "/private/files/112000_Bestellung_4502060171_20251016.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523297", - "file_name": "111944_Bestellung_4502060170_20251016.PDF", - "file_url": "/private/files/111944_Bestellung_4502060170_20251016.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523288", - "file_name": "111944_Bestellung_4502061126_20251016.PDF", - "file_url": "/private/files/111944_Bestellung_4502061126_20251016.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO523185", - "file_name": "Einkauf_Bestellung (5)278b55.pdf", - "file_url": "/private/files/Einkauf_Bestellung (5)278b55.pdf", - "is_private": 1, - "customer": "100069", - "customer_name": "BRACK.CH AG", - }, - { - "name": "SO523047", - "file_name": "Bestellung 3600085201 Lieferant Nr 20203035.PDF", - "file_url": "/private/files/Bestellung 3600085201 Lieferant Nr 20203035.PDF", - "is_private": 1, - "customer": "C100122", - "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", - }, - { - "name": "SO522915", - "file_name": "Lieferantenbestellung 70121844.pdf", - "file_url": "/private/files/Lieferantenbestellung 70121844.pdf", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO522831", - "file_name": "111944_Bestellung_4502038760_20250922.PDF", - "file_url": "/private/files/111944_Bestellung_4502038760_20250922.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO522830", - "file_name": "111944_Bestellung_4502038759_20250920.PDF", - "file_url": "/private/files/111944_Bestellung_4502038759_20250920.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO522756", - "file_name": "Lieferantenbestellung 70121452.pdf", - "file_url": "/private/files/Lieferantenbestellung 70121452.pdf", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, - { - "name": "SO522425", - "file_name": "111944_Bestellung_4502023162_20250830.PDF", - "file_url": "/private/files/111944_Bestellung_4502023162_20250830.PDF", - "is_private": 1, - "customer": "C100931", - "customer_name": "Dipl. Ing. Fust", - }, - { - "name": "SO522362", - "file_name": "Bestellung 3600082331 Lieferant Nr 20203035.PDF", - "file_url": "/private/files/Bestellung 3600082331 Lieferant Nr 20203035.PDF", - "is_private": 1, - "customer": "C100122", - "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", - }, - { - "name": "SO522330", - "file_name": "Einkauf_Bestellunga81933.pdf", - "file_url": "/private/files/Einkauf_Bestellunga81933.pdf", - "is_private": 1, - "customer": "100069", - "customer_name": "BRACK.CH AG", - }, - { - "name": "SO522308", - "file_name": "Bestellung 3600082064 Lieferant Nr 20203035.PDF", - "file_url": "/private/files/Bestellung 3600082064 Lieferant Nr 20203035.PDF", - "is_private": 1, - "customer": "C100122", - "customer_name": "INTERDISCOUNT Division der Coop Genossenschaft", - }, - { - "name": "SO522293", - "file_name": "Lieferantenbestellung 70120310.pdf", - "file_url": "/private/files/Lieferantenbestellung 70120310.pdf", - "is_private": 1, - "customer": "C102340", - "customer_name": "4mybaby AG", - }, -] - - -def execute(): - file_url_to_customer = {entry["file_url"]: entry["customer"] for entry in PARTY} - - datasets = frappe.get_all( - "Parser Benchmark Dataset", - filters={ - "file": ["in", list(file_url_to_customer.keys())], - "party": ["is", "not set"], - }, - fields=["name", "file"], - ) - - updates = {} - for dataset in datasets: - updates[dataset.name] = { - "party_type": "Customer", - "party": file_url_to_customer[dataset.file], - } - - if updates: - frappe.db.bulk_update("Parser Benchmark Dataset", updates) - print(f"\n Updated {len(updates)} datasets with party information.") From 4b91704990916d8b7ee2e56194e32be60b04288a Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Fri, 20 Mar 2026 17:59:06 +0530 Subject: [PATCH 42/88] chore: minor changes --- transaction_parser/parser_benchmark/scorer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/transaction_parser/parser_benchmark/scorer.py b/transaction_parser/parser_benchmark/scorer.py index 17b8cd9..5366392 100644 --- a/transaction_parser/parser_benchmark/scorer.py +++ b/transaction_parser/parser_benchmark/scorer.py @@ -60,7 +60,6 @@ def _format_mismatches(diff: DeepDiff) -> list[dict]: return mismatches -# TODO: need some changes here def get_consistency(dataset: str, ai_model: str, pdf_processor: str = "") -> dict: """ Check how consistent a model's response is for a given dataset + processor combo. From 381820548678162e5a88355aa79320310c62aa78 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Sat, 21 Mar 2026 15:16:35 +0530 Subject: [PATCH 43/88] refactor: remove response_hash from ParserBenchmarkLog and related scoring logic --- .../parser_benchmark_log.json | 10 +-- .../parser_benchmark_log.py | 1 - transaction_parser/parser_benchmark/runner.py | 7 +- transaction_parser/parser_benchmark/scorer.py | 72 ++++--------------- 4 files changed, 17 insertions(+), 73 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index bc2c1c5..52bb3a9 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -55,7 +55,6 @@ "accuracy_section", "accuracy_score", "column_break_accuracy", - "response_hash", "section_break_mismatches", "field_mismatches", "error_tab", @@ -300,13 +299,6 @@ "fieldname": "column_break_accuracy", "fieldtype": "Column Break" }, - { - "description": "Hash of AI response for consistency tracking across runs", - "fieldname": "response_hash", - "fieldtype": "Data", - "label": "Response Hash", - "read_only": 1 - }, { "fieldname": "section_break_mismatches", "fieldtype": "Section Break", @@ -411,7 +403,7 @@ "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-20 08:14:27.215617", + "modified": "2026-03-21 10:46:06.203098", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Log", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py index ca8e9a4..9cb664f 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -47,7 +47,6 @@ class ParserBenchmarkLog(Document): party_type: DF.Link | None pdf_processor: DF.Literal["", "OCRMyPDF", "Docling"] prompt_tokens: DF.Int - response_hash: DF.Data | None status: DF.Literal["Queued", "Running", "Completed", "Failed"] total_cost: DF.Currency total_time: DF.Float diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index ba7eb63..7d920af 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -167,12 +167,7 @@ def _calculate_cost(self): # ── step 4: accuracy scoring ───────────────────────────── def _score_response(self, ai_content: dict): - from transaction_parser.parser_benchmark.scorer import ( - compute_response_hash, - score_response, - ) - - self.log.response_hash = compute_response_hash(ai_content) + from transaction_parser.parser_benchmark.scorer import score_response expected = self.dataset.expected_result if not expected: diff --git a/transaction_parser/parser_benchmark/scorer.py b/transaction_parser/parser_benchmark/scorer.py index 5366392..3347abc 100644 --- a/transaction_parser/parser_benchmark/scorer.py +++ b/transaction_parser/parser_benchmark/scorer.py @@ -1,5 +1,5 @@ import frappe -from deepdiff import DeepDiff, DeepHash +from deepdiff import DeepDiff from frappe.utils import flt @@ -27,23 +27,27 @@ def score_response(expected: dict, actual: dict, max_diffs: int = 500) -> dict: } -def compute_response_hash(ai_response: dict | str) -> str: - """Deterministic hash of AI response for consistency tracking across runs.""" - if isinstance(ai_response, str): - ai_response = frappe.parse_json(ai_response) - - return DeepHash(ai_response)[ai_response][:16] - - def _format_mismatches(diff: DeepDiff) -> list[dict]: """Flatten DeepDiff into a simple list of {field, expected, actual}.""" mismatches = [] for path, change in diff.get("values_changed", {}).items(): - mismatches.append({"field": path, "expected": change["old_value"], "actual": change["new_value"]}) + mismatches.append( + { + "field": path, + "expected": change["old_value"], + "actual": change["new_value"], + } + ) for path, change in diff.get("type_changes", {}).items(): - mismatches.append({"field": path, "expected": change["old_value"], "actual": change["new_value"]}) + mismatches.append( + { + "field": path, + "expected": change["old_value"], + "actual": change["new_value"], + } + ) for path, val in diff.get("dictionary_item_removed", {}).items(): mismatches.append({"field": path, "expected": val, "actual": None}) @@ -58,49 +62,3 @@ def _format_mismatches(diff: DeepDiff) -> list[dict]: mismatches.append({"field": path, "expected": None, "actual": val}) return mismatches - - -def get_consistency(dataset: str, ai_model: str, pdf_processor: str = "") -> dict: - """ - Check how consistent a model's response is for a given dataset + processor combo. - - Returns: - { - "total_runs": 5, - "unique_hashes": 2, - "consistency": 80.0, # % of runs with the most common hash - "hashes": {"abc123": 4, "def456": 1}, - } - """ - Log = frappe.qb.DocType("Parser Benchmark Log") - - logs = ( - frappe.qb.from_(Log) - .select(Log.response_hash) - .where(Log.dataset == dataset) - .where(Log.ai_model == ai_model) - .where(Log.pdf_processor == (pdf_processor or "")) - .where(Log.status == "Completed") - .where(Log.response_hash.isnotnull()) - .run(as_dict=True) - ) - - if not logs: - return {"total_runs": 0, "unique_hashes": 0, "consistency": 0.0, "hashes": {}} - - # count occurrences of each hash - hashes = {} - for log in logs: - h = log.response_hash - hashes[h] = hashes.get(h, 0) + 1 - - total_runs = len(logs) - most_common_count = max(hashes.values()) - consistency = flt((most_common_count / total_runs) * 100, 2) - - return { - "total_runs": total_runs, - "unique_hashes": len(hashes), - "consistency": consistency, - "hashes": hashes, - } From 8c77061b034c3d1a314ab4a0b86e22e559339e5a Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Sat, 21 Mar 2026 15:30:12 +0530 Subject: [PATCH 44/88] fix: correct default distance value in score_response function --- transaction_parser/parser_benchmark/scorer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transaction_parser/parser_benchmark/scorer.py b/transaction_parser/parser_benchmark/scorer.py index 3347abc..b5ab257 100644 --- a/transaction_parser/parser_benchmark/scorer.py +++ b/transaction_parser/parser_benchmark/scorer.py @@ -18,7 +18,7 @@ def score_response(expected: dict, actual: dict, max_diffs: int = 500) -> dict: get_deep_distance=True, ) - distance = diff.get("deep_distance", 1) or 1 + distance = diff.get("deep_distance", 0) accuracy = flt((1 - distance) * 100, 2) return { From c2e7bb37e97701b10ab55637f8cfc7035e2eda22 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Sat, 21 Mar 2026 15:37:36 +0530 Subject: [PATCH 45/88] refactor: enhance mismatch formatting to handle unhandled DeepDiff change types --- transaction_parser/parser_benchmark/scorer.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/transaction_parser/parser_benchmark/scorer.py b/transaction_parser/parser_benchmark/scorer.py index b5ab257..2267eed 100644 --- a/transaction_parser/parser_benchmark/scorer.py +++ b/transaction_parser/parser_benchmark/scorer.py @@ -27,6 +27,12 @@ def score_response(expected: dict, actual: dict, max_diffs: int = 500) -> dict: } +_CHANGED_TYPES = {"values_changed", "type_changes"} +_REMOVED_TYPES = {"dictionary_item_removed", "iterable_item_removed"} +_ADDED_TYPES = {"dictionary_item_added", "iterable_item_added"} +_HANDLED_TYPES = _CHANGED_TYPES | _REMOVED_TYPES | _ADDED_TYPES | {"deep_distance"} + + def _format_mismatches(diff: DeepDiff) -> list[dict]: """Flatten DeepDiff into a simple list of {field, expected, actual}.""" mismatches = [] @@ -61,4 +67,19 @@ def _format_mismatches(diff: DeepDiff) -> list[dict]: for path, val in diff.get("iterable_item_added", {}).items(): mismatches.append({"field": path, "expected": None, "actual": val}) + # catch-all for unhandled DeepDiff change types + for change_type, changes in diff.items(): + if change_type in _HANDLED_TYPES: + continue + + if isinstance(changes, dict): + for path, val in changes.items(): + mismatches.append( + {"field": f"{change_type}: {path}", "expected": None, "actual": val} + ) + else: + mismatches.append( + {"field": change_type, "expected": None, "actual": str(changes)} + ) + return mismatches From e0e8558a1d74776d4e4590ff98a5e85ba19c41db Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Sat, 21 Mar 2026 17:05:02 +0530 Subject: [PATCH 46/88] refactor: add validation for expected result to ensure valid JSON format --- .../parser_benchmark_dataset.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 6f08fd2..dca3379 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -60,10 +60,25 @@ class ParserBenchmarkDataset(Document): def validate(self): self.set_file_type() self.validate_file_type() + self.validate_expected_result() self.validate_selected_models() self.validate_selected_processors() + def validate_expected_result(self): + if not self.expected_result: + return + + try: + frappe.parse_json(self.expected_result) + except Exception: + frappe.throw( + title=_("Invalid JSON"), msg=_("Expected Result must be valid JSON.") + ) + def set_file_type(self): + if not self.has_value_changed("file"): + return + file_doc = frappe.get_last_doc("File", filters={"file_url": self.file}) self.file_type = file_doc.file_type From 8e90ef73f2bdd9237dbf5ee08413bd438f61900c Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Sat, 21 Mar 2026 18:22:25 +0530 Subject: [PATCH 47/88] refactor: update conditions and add fields for Parser Benchmark Dataset and Log --- .../parser_benchmark_dataset.js | 2 +- .../parser_benchmark_dataset.json | 37 ++++++++++++++-- .../parser_benchmark_dataset.py | 15 +++---- .../parser_benchmark_log.json | 16 ++++--- .../parser_benchmark_log.py | 43 +++++++++++++++++-- .../parser_benchmark_settings.py | 2 +- transaction_parser/parser_benchmark/runner.py | 2 +- 7 files changed, 91 insertions(+), 26 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js index ccf9a8f..90f7718 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js @@ -3,7 +3,7 @@ frappe.ui.form.on("Parser Benchmark Dataset", { refresh(frm) { - if (!frm.is_new() && frm.doc.enabled) { + if (frm.doc.docstatus === 1 && frm.doc.enabled) { frm.add_custom_button(__("Run Benchmark"), () => run_benchmark(frm)); } }, diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index e3602b2..cfa1da8 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -40,7 +40,9 @@ "expected_result_section", "expected_result", "section_break_bhsg", - "naming_series" + "naming_series", + "column_break_aoce", + "amended_from" ], "fields": [ { @@ -48,6 +50,7 @@ "fieldtype": "Column Break" }, { + "allow_on_submit": 1, "default": "1", "fieldname": "enabled", "fieldtype": "Check", @@ -63,8 +66,8 @@ "fieldname": "file", "fieldtype": "Attach", "label": "File", - "reqd": 1, - "set_only_once": 1 + "print_hide": 1, + "reqd": 1 }, { "fieldname": "file_type", @@ -118,24 +121,28 @@ "label": "AI Models" }, { + "allow_on_submit": 1, "default": "0", "fieldname": "deepseek_chat", "fieldtype": "Check", "label": "DeepSeek Chat" }, { + "allow_on_submit": 1, "default": "0", "fieldname": "deepseek_reasoner", "fieldtype": "Check", "label": "DeepSeek Reasoner" }, { + "allow_on_submit": 1, "default": "0", "fieldname": "openai_gpt_4o", "fieldtype": "Check", "label": "OpenAI gpt-4o" }, { + "allow_on_submit": 1, "default": "0", "fieldname": "openai_gpt_4o_mini", "fieldtype": "Check", @@ -146,24 +153,28 @@ "fieldtype": "Column Break" }, { + "allow_on_submit": 1, "default": "1", "fieldname": "openai_gpt_5", "fieldtype": "Check", "label": "OpenAI gpt-5" }, { + "allow_on_submit": 1, "default": "0", "fieldname": "openai_gpt_5_mini", "fieldtype": "Check", "label": "OpenAI gpt-5-mini" }, { + "allow_on_submit": 1, "default": "0", "fieldname": "google_gemini_pro_25", "fieldtype": "Check", "label": "Google Gemini Pro-2.5" }, { + "allow_on_submit": 1, "default": "0", "fieldname": "google_gemini_flash_25", "fieldtype": "Check", @@ -176,6 +187,7 @@ "label": "PDF Processors" }, { + "allow_on_submit": 1, "default": "1", "fieldname": "ocrmypdf", "fieldtype": "Check", @@ -186,6 +198,7 @@ "fieldtype": "Column Break" }, { + "allow_on_submit": 1, "default": "0", "fieldname": "docling", "fieldtype": "Check", @@ -208,6 +221,7 @@ "label": "Expected Result" }, { + "allow_on_submit": 1, "description": "The correct/expected parsed output JSON for this dataset. Used to score AI accuracy.", "fieldname": "expected_result", "fieldtype": "Code", @@ -215,6 +229,7 @@ "options": "JSON" }, { + "collapsible": 1, "fieldname": "section_break_bhsg", "fieldtype": "Section Break" }, @@ -244,16 +259,30 @@ { "fieldname": "column_break_qccq", "fieldtype": "Column Break" + }, + { + "fieldname": "amended_from", + "fieldtype": "Link", + "label": "Amended From", + "no_copy": 1, + "options": "Parser Benchmark Dataset", + "print_hide": 1, + "read_only": 1 + }, + { + "fieldname": "column_break_aoce", + "fieldtype": "Column Break" } ], "index_web_pages_for_search": 1, + "is_submittable": 1, "links": [ { "link_doctype": "Parser Benchmark Log", "link_fieldname": "dataset" } ], - "modified": "2026-03-20 08:12:11.328907", + "modified": "2026-03-21 13:51:36.810990", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index dca3379..db171aa 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -32,6 +32,7 @@ class ParserBenchmarkDataset(Document): if TYPE_CHECKING: from frappe.types import DF + amended_from: DF.Link | None company: DF.Link | None country: DF.Literal["India", "Other"] deepseek_chat: DF.Check @@ -76,7 +77,7 @@ def validate_expected_result(self): ) def set_file_type(self): - if not self.has_value_changed("file"): + if self.file_type and not self.has_value_changed("file"): return file_doc = frappe.get_last_doc("File", filters={"file_url": self.file}) @@ -116,6 +117,9 @@ def run_benchmark(dataset_name: str): frappe.has_permission("Parser Benchmark Dataset", "write", throw=True) dataset = frappe.get_doc("Parser Benchmark Dataset", dataset_name) + if dataset.docstatus != 1: + frappe.throw(_("Dataset must be submitted before running benchmarks.")) + log_names = _create_and_enqueue_logs(dataset) if not log_names: @@ -154,17 +158,10 @@ def _create_and_enqueue_logs(dataset) -> list[str]: log = frappe.get_doc( { "doctype": "Parser Benchmark Log", - "dataset": dataset.name, "status": "Queued", + "dataset": dataset.name, "ai_model": ai_model, "pdf_processor": pdf_processor, - "transaction_type": dataset.transaction_type, - "country": dataset.country, - "company": dataset.company, - "party_type": dataset.party_type, - "party": dataset.party, - "page_limit": dataset.page_limit, - "file_type": dataset.file_type, } ).insert(ignore_permissions=True) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index 52bb3a9..53b0849 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -103,6 +103,7 @@ "read_only": 1 }, { + "depends_on": "eval: doc.file_type === \"PDF\"", "fieldname": "pdf_processor", "fieldtype": "Select", "label": "PDF Processor", @@ -119,6 +120,7 @@ { "fieldname": "file_type", "fieldtype": "Data", + "is_virtual": 1, "label": "File Type", "read_only": 1 }, @@ -333,8 +335,7 @@ { "fieldname": "transaction_type", "fieldtype": "Select", - "in_list_view": 1, - "in_standard_filter": 1, + "is_virtual": 1, "label": "Transaction Type", "options": "Sales Order\nExpense", "read_only": 1 @@ -342,8 +343,7 @@ { "fieldname": "country", "fieldtype": "Select", - "in_list_view": 1, - "in_standard_filter": 1, + "is_virtual": 1, "label": "Country", "options": "India\nOther", "read_only": 1 @@ -351,6 +351,7 @@ { "fieldname": "company", "fieldtype": "Link", + "is_virtual": 1, "label": "Company", "options": "Company", "read_only": 1 @@ -359,6 +360,7 @@ "fieldname": "party_type", "fieldtype": "Link", "hidden": 1, + "is_virtual": 1, "label": "Party Type", "options": "DocType", "read_only": 1 @@ -366,6 +368,7 @@ { "fieldname": "party", "fieldtype": "Dynamic Link", + "is_virtual": 1, "label": "Party", "options": "party_type", "read_only": 1 @@ -384,11 +387,10 @@ "label": "Config" }, { - "default": "0", "fieldname": "page_limit", "fieldtype": "Int", + "is_virtual": 1, "label": "Page Limit", - "non_negative": 1, "read_only": 1 }, { @@ -403,7 +405,7 @@ "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-21 10:46:06.203098", + "modified": "2026-03-21 13:50:57.098387", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Log", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py index 9cb664f..5c489f6 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -1,6 +1,7 @@ # Copyright (c) 2026, Resilient Tech and contributors # For license information, please see license.txt +import frappe from frappe.model.document import Document @@ -36,13 +37,11 @@ class ParserBenchmarkLog(Document): file_content: DF.Code | None file_parse_memory: DF.Float file_parse_time: DF.Float - file_type: DF.Data | None input_cost: DF.Currency input_token_cost: DF.Currency naming_series: DF.Literal["PAR-BM-LOG-"] output_cost: DF.Currency output_token_cost: DF.Currency - page_limit: DF.Int party: DF.DynamicLink | None party_type: DF.Link | None pdf_processor: DF.Literal["", "OCRMyPDF", "Docling"] @@ -54,4 +53,42 @@ class ParserBenchmarkLog(Document): transaction_type: DF.Literal["Sales Order", "Expense"] # end: auto-generated types - pass + def _get_dataset(self): + if not hasattr(self, "_dataset_doc"): + self._dataset_doc = frappe.get_cached_doc( + "Parser Benchmark Dataset", self.dataset + ) + + return self._dataset_doc + + def get_from_dataset(self, fieldname: str): + dataset = self._get_dataset() + return dataset.get(fieldname) if dataset else None + + @property + def transaction_type(self): + return self.get_from_dataset("transaction_type") + + @property + def country(self): + return self.get_from_dataset("country") + + @property + def company(self): + return self.get_from_dataset("company") + + @property + def party_type(self): + return self.get_from_dataset("party_type") + + @property + def party(self): + return self.get_from_dataset("party") + + @property + def page_limit(self): + return self.get_from_dataset("page_limit") or 0 + + @property + def file_type(self): + return self.get_from_dataset("file_type") diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py index 0643c38..5398206 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py @@ -61,7 +61,7 @@ def run_scheduled_benchmarks(): datasets = frappe.get_all( "Parser Benchmark Dataset", - filters={"enabled": 1}, + filters={"enabled": 1, "docstatus": 1}, pluck="name", ) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index 7d920af..cf475f4 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -3,7 +3,7 @@ import frappe from frappe.core.doctype.file.file import File -from frappe.utils import cint, flt +from frappe.utils import flt from transaction_parser.parser_benchmark.doctype.parser_benchmark_dataset.parser_benchmark_dataset import ( ParserBenchmarkDataset, From 019f946a2b1ba39cc669e49c60cf46c7ed2b2c4e Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Sat, 21 Mar 2026 18:41:16 +0530 Subject: [PATCH 48/88] refactor: update validate_selected_processors to reset processor fields for non-PDF file types --- .../parser_benchmark_dataset/parser_benchmark_dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index db171aa..0ab5f41 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -93,8 +93,8 @@ def validate_selected_models(self): def validate_selected_processors(self): if self.file_type != "PDF": - self.ocrmypdf = None - self.docling = None + for field in PDF_PROCESSOR_FIELD_MAP: + self.set(field, 0) return if not self.get_selected_processors(): From ef37aaf6eba47c0f9277b755799aeef93ac6bf4b Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 24 Mar 2026 10:40:10 +0530 Subject: [PATCH 49/88] refactor: streamline validation logic by consolidating expected result checks --- .../parser_benchmark_dataset.js | 2 ++ .../parser_benchmark_dataset.py | 26 +++++++++---------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js index 90f7718..39b8605 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.js @@ -6,6 +6,8 @@ frappe.ui.form.on("Parser Benchmark Dataset", { if (frm.doc.docstatus === 1 && frm.doc.enabled) { frm.add_custom_button(__("Run Benchmark"), () => run_benchmark(frm)); } + + if (frm.doc.docstatus === 0) set_party_type(frm); }, transaction_type(frm) { diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 0ab5f41..c042037 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -61,20 +61,9 @@ class ParserBenchmarkDataset(Document): def validate(self): self.set_file_type() self.validate_file_type() - self.validate_expected_result() self.validate_selected_models() self.validate_selected_processors() - - def validate_expected_result(self): - if not self.expected_result: - return - - try: - frappe.parse_json(self.expected_result) - except Exception: - frappe.throw( - title=_("Invalid JSON"), msg=_("Expected Result must be valid JSON.") - ) + self.validate_expected_result() def set_file_type(self): if self.file_type and not self.has_value_changed("file"): @@ -94,12 +83,23 @@ def validate_selected_models(self): def validate_selected_processors(self): if self.file_type != "PDF": for field in PDF_PROCESSOR_FIELD_MAP: - self.set(field, 0) + self.set(field, 0) return if not self.get_selected_processors(): frappe.throw(_("Please select at least one PDF Processor.")) + def validate_expected_result(self): + if not self.expected_result: + return + + try: + frappe.parse_json(self.expected_result) + except Exception: + frappe.throw( + title=_("Invalid JSON"), msg=_("Expected Result must be valid JSON.") + ) + def get_selected_models(self) -> list[str]: """Return list of selected AI model names.""" return [label for field, label in AI_MODEL_FIELD_MAP.items() if self.get(field)] From 1198a3b98a8eae02aafc5cec05edb158c8a803df Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 24 Mar 2026 11:17:33 +0530 Subject: [PATCH 50/88] refactor: add dataset check in _get_dataset and enhance _normalize_empty function for better handling of empty values --- .../parser_benchmark_log.py | 3 +++ transaction_parser/parser_benchmark/scorer.py | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py index 5c489f6..feec737 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -54,6 +54,9 @@ class ParserBenchmarkLog(Document): # end: auto-generated types def _get_dataset(self): + if not self.dataset: + return None + if not hasattr(self, "_dataset_doc"): self._dataset_doc = frappe.get_cached_doc( "Parser Benchmark Dataset", self.dataset diff --git a/transaction_parser/parser_benchmark/scorer.py b/transaction_parser/parser_benchmark/scorer.py index 2267eed..8e53609 100644 --- a/transaction_parser/parser_benchmark/scorer.py +++ b/transaction_parser/parser_benchmark/scorer.py @@ -3,8 +3,28 @@ from frappe.utils import flt +def _normalize_empty(obj): + """Recursively convert empty strings to None so `""` vs `None` is not a mismatch. + + Leaves `0`, `False`, and other falsy values untouched. + """ + if isinstance(obj, dict): + return {k: _normalize_empty(v) for k, v in obj.items()} + + if isinstance(obj, list): + return [_normalize_empty(v) for v in obj] + + if obj == "": + return None + + return obj + + def score_response(expected: dict, actual: dict, max_diffs: int = 500) -> dict: """Compare AI response against expected result using DeepDiff.""" + expected = _normalize_empty(expected) + actual = _normalize_empty(actual) + diff = DeepDiff( expected, actual, From f32b98553d1195cf3c56f20c7d9fb656aa230955 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 25 Mar 2026 00:52:08 +0530 Subject: [PATCH 51/88] refactor: add currency field to log creation for better financial tracking --- .../parser_benchmark_dataset/parser_benchmark_dataset.json | 5 +++-- .../parser_benchmark_dataset/parser_benchmark_dataset.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index cfa1da8..e97dd69 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -210,6 +210,7 @@ "label": "Other Configuration" }, { + "allow_on_submit": 1, "fieldname": "page_limit", "fieldtype": "Int", "label": "Page Limit", @@ -282,7 +283,7 @@ "link_fieldname": "dataset" } ], - "modified": "2026-03-21 13:51:36.810990", + "modified": "2026-03-24 14:57:58.689734", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", @@ -306,4 +307,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} \ No newline at end of file +} diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index c042037..361ca53 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -162,6 +162,7 @@ def _create_and_enqueue_logs(dataset) -> list[str]: "dataset": dataset.name, "ai_model": ai_model, "pdf_processor": pdf_processor, + "currency": "USD", } ).insert(ignore_permissions=True) From 2da7ff24292f0c414eed091fa62a2e06840ed751 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 25 Mar 2026 01:00:49 +0530 Subject: [PATCH 52/88] feat: add Transaction Parser Accuracy Analysis report files and implementation --- .../parser_benchmark/report/__init__.py | 0 .../__init__.py | 0 .../transaction_parser_accuracy_analysis.js | 6 ++++ .../transaction_parser_accuracy_analysis.json | 28 +++++++++++++++++++ .../transaction_parser_accuracy_analysis.py | 15 ++++++++++ 5 files changed, 49 insertions(+) create mode 100644 transaction_parser/parser_benchmark/report/__init__.py create mode 100644 transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/__init__.py create mode 100644 transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js create mode 100644 transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.json create mode 100644 transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py diff --git a/transaction_parser/parser_benchmark/report/__init__.py b/transaction_parser/parser_benchmark/report/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/__init__.py b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js new file mode 100644 index 0000000..de1977a --- /dev/null +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js @@ -0,0 +1,6 @@ +// Copyright (c) 2026, Resilient Tech and contributors +// For license information, please see license.txt + +frappe.query_reports["Transaction Parser Accuracy Analysis"] = { + filters: [], +}; diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.json b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.json new file mode 100644 index 0000000..c96356c --- /dev/null +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.json @@ -0,0 +1,28 @@ +{ + "add_total_row": 0, + "add_translate_data": 0, + "columns": [], + "creation": "2026-03-24 20:26:18.136226", + "disabled": 0, + "docstatus": 0, + "doctype": "Report", + "filters": [], + "idx": 0, + "is_standard": "Yes", + "letter_head": null, + "modified": "2026-03-24 20:26:18.136226", + "modified_by": "Administrator", + "module": "Parser Benchmark", + "name": "Transaction Parser Accuracy Analysis", + "owner": "Administrator", + "prepared_report": 0, + "ref_doctype": "Parser Benchmark Log", + "report_name": "Transaction Parser Accuracy Analysis", + "report_type": "Script Report", + "roles": [ + { + "role": "System Manager" + } + ], + "timeout": 0 +} \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py new file mode 100644 index 0000000..cbcf540 --- /dev/null +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py @@ -0,0 +1,15 @@ +# Copyright (c) 2026, Resilient Tech and contributors +# For license information, please see license.txt + +import frappe + + +class TransactionParserAccuracyAnalysis: + def execute(self, filters: frappe._dict = None): + columns, data = [], [] + return columns, data + + +def execute(filters=None): + filters = frappe._dict(filters or {}) + return TransactionParserAccuracyAnalysis().execute(filters) From 1cd7f659f331f6b25336321ce44df7a2b7cdf2e2 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 25 Mar 2026 11:53:00 +0530 Subject: [PATCH 53/88] feat: enhance Transaction Parser Accuracy Analysis with new filters and data handling --- .../transaction_parser_accuracy_analysis.js | 92 +++- .../transaction_parser_accuracy_analysis.json | 2 +- .../transaction_parser_accuracy_analysis.py | 401 +++++++++++++++++- 3 files changed, 488 insertions(+), 7 deletions(-) diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js index de1977a..132e87e 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js @@ -1,6 +1,96 @@ // Copyright (c) 2026, Resilient Tech and contributors // For license information, please see license.txt +const AI_MODELS = [ + "DeepSeek Chat", + "DeepSeek Reasoner", + "OpenAI gpt-4o", + "OpenAI gpt-4o-mini", + "OpenAI gpt-5", + "OpenAI gpt-5-mini", + "Google Gemini Pro-2.5", + "Google Gemini Flash-2.5", +]; + +const FILE_TYPES = ["PDF", "CSV", "XLSX", "XLS"]; + +const PDF_PROCESSORS = ["OCRMyPDF", "Docling"]; + +const PARTY_TYPE_MAP = { + "Sales Order": "Customer", + Expense: "Supplier", +}; + +function make_options(items, txt) { + return items + .filter((v) => !txt || v.toLowerCase().includes(txt.toLowerCase())) + .map((v) => ({ value: v, description: "" })); +} + frappe.query_reports["Transaction Parser Accuracy Analysis"] = { - filters: [], + tree: true, + initial_depth: 1, + + onload(report) { + set_party_type(report); + }, + + filters: [ + { + fieldname: "company", + label: __("Company"), + fieldtype: "Link", + options: "Company", + reqd: 1, + default: frappe.defaults.get_user_default("Company"), + }, + { + fieldname: "transaction_type", + label: __("Transaction Type"), + fieldtype: "Select", + options: "\nSales Order\nExpense", + reqd: 1, + default: "Sales Order", + on_change() { + set_party_type(frappe.query_report); + }, + }, + { + fieldname: "party_type", + label: __("Party Type"), + fieldtype: "Link", + options: "DocType", + hidden: 1, + }, + { + fieldname: "party", + label: __("Party"), + fieldtype: "Dynamic Link", + options: "party_type", + }, + { + fieldname: "file_type", + label: __("File Type"), + fieldtype: "MultiSelectList", + get_data: (txt) => make_options(FILE_TYPES, txt), + }, + { + fieldname: "ai_model", + label: __("AI Model"), + fieldtype: "MultiSelectList", + get_data: (txt) => make_options(AI_MODELS, txt), + }, + { + fieldname: "pdf_processor", + label: __("PDF Processor"), + fieldtype: "MultiSelectList", + get_data: (txt) => make_options(PDF_PROCESSORS, txt), + }, + ], }; + +function set_party_type(report) { + const transaction_type = report.get_filter_value("transaction_type"); + const party_type = PARTY_TYPE_MAP[transaction_type] || ""; + report.set_filter_value("party_type", party_type); +} diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.json b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.json index c96356c..75401e4 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.json +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.json @@ -10,7 +10,7 @@ "idx": 0, "is_standard": "Yes", "letter_head": null, - "modified": "2026-03-24 20:26:18.136226", + "modified": "2026-03-25 06:20:26.780337", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Transaction Parser Accuracy Analysis", diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py index cbcf540..7c3cb89 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py @@ -1,15 +1,406 @@ # Copyright (c) 2026, Resilient Tech and contributors # For license information, please see license.txt +import json +from collections import Counter, defaultdict +from enum import StrEnum + import frappe +from frappe import _ + +PARTY_TYPE_MAP = { + "Sales Order": "Customer", + "Expense": "Supplier", +} + +# Sorting order for child rows within each party group +_AI_MODEL_ORDER = { + "OpenAI gpt-5": 0, + "OpenAI gpt-5-mini": 1, + "OpenAI gpt-4o": 2, + "OpenAI gpt-4o-mini": 3, + "Google Gemini Pro-2.5": 4, + "Google Gemini Flash-2.5": 5, + "DeepSeek Reasoner": 6, + "DeepSeek Chat": 7, +} + +_PDF_PROCESSOR_ORDER = { + "OCRMyPDF": 0, + "Docling": 1, +} + +_FILE_TYPE_ORDER = { + "PDF": 0, + "CSV": 1, + "XLSX": 2, + "XLS": 3, +} + +class Col(StrEnum): + """Column fieldnames — single source of truth for the report.""" -class TransactionParserAccuracyAnalysis: - def execute(self, filters: frappe._dict = None): - columns, data = [], [] - return columns, data + PARTY = "party" + PARTY_NAME = "party_name" + ACCURACY_SCORE = "accuracy_score" + AI_MODEL = "ai_model" + PDF_PROCESSOR = "pdf_processor" + FILE_TYPE = "file_type" + FILE_PARSE_TIME = "file_parse_time" + FILE_PARSE_MEMORY = "file_parse_memory" + AI_PARSE_TIME = "ai_parse_time" + TOTAL_TIME = "total_time" + TOTAL_COST = "total_cost" + PROMPT_TOKENS = "prompt_tokens" + COMPLETION_TOKENS = "completion_tokens" + TOTAL_TOKENS = "total_tokens" + CURRENCY = "currency" + DATASET = "dataset" + MISMATCH_COUNT = "mismatch_count" + TOP_MISMATCHES = "top_mismatches" + + +# Fields averaged in party-group summary rows +_AVG_FIELDS = ( + Col.ACCURACY_SCORE, + Col.FILE_PARSE_TIME, + Col.FILE_PARSE_MEMORY, + Col.AI_PARSE_TIME, + Col.TOTAL_TIME, + Col.PROMPT_TOKENS, + Col.COMPLETION_TOKENS, + Col.TOTAL_TOKENS, +) + +# Fields summed in party-group summary rows +_SUM_FIELDS = (Col.TOTAL_COST,) def execute(filters=None): filters = frappe._dict(filters or {}) - return TransactionParserAccuracyAnalysis().execute(filters) + return AccuracyAnalysisReport(filters).run() + + +class AccuracyAnalysisReport: + def __init__(self, filters: frappe._dict): + self.filters = filters + self._set_party_type() + self.group_by_party = True # Always group — even for a single party + + def run(self): + self.data = [self._build_row(r) for r in self._fetch_logs()] + + if self.group_by_party: + self._group_by_party() + + return self._get_columns(), self.data + + # ── Columns ────────────────────────────────────────────────────── + + def _get_columns(self): + return [ + { + "fieldname": Col.PARTY, + "label": _("Party"), + "fieldtype": "Data", + "width": 200, + }, + { + "fieldname": Col.PARTY_NAME, + "label": _("Party Name"), + "fieldtype": "Data", + "width": 200, + }, + { + "fieldname": Col.DATASET, + "label": _("Dataset"), + "fieldtype": "Link", + "options": "Parser Benchmark Dataset", + "width": 160, + }, + { + "fieldname": Col.ACCURACY_SCORE, + "label": _("Accuracy (%)"), + "fieldtype": "Percent", + "width": 120, + }, + { + "fieldname": Col.AI_MODEL, + "label": _("AI Model"), + "fieldtype": "Data", + "width": 180, + }, + { + "fieldname": Col.PDF_PROCESSOR, + "label": _("Processor"), + "fieldtype": "Data", + "width": 110, + }, + { + "fieldname": Col.FILE_TYPE, + "label": _("File Type"), + "fieldtype": "Data", + "width": 90, + }, + { + "fieldname": Col.FILE_PARSE_TIME, + "label": _("File Parse (s)"), + "fieldtype": "Float", + "width": 120, + "precision": 2, + }, + { + "fieldname": Col.FILE_PARSE_MEMORY, + "label": _("Memory (MB)"), + "fieldtype": "Float", + "width": 110, + "precision": 2, + }, + { + "fieldname": Col.AI_PARSE_TIME, + "label": _("AI Parse (s)"), + "fieldtype": "Float", + "width": 110, + "precision": 2, + }, + { + "fieldname": Col.TOTAL_TIME, + "label": _("Total (s)"), + "fieldtype": "Float", + "width": 100, + "precision": 2, + }, + { + "fieldname": Col.TOTAL_COST, + "label": _("Total Cost"), + "fieldtype": "Currency", + "width": 110, + "options": Col.CURRENCY, + }, + { + "fieldname": Col.PROMPT_TOKENS, + "label": _("Prompt Tokens"), + "fieldtype": "Int", + "width": 120, + }, + { + "fieldname": Col.COMPLETION_TOKENS, + "label": _("Compl. Tokens"), + "fieldtype": "Int", + "width": 120, + }, + { + "fieldname": Col.TOTAL_TOKENS, + "label": _("Total Tokens"), + "fieldtype": "Int", + "width": 110, + }, + { + "fieldname": Col.MISMATCH_COUNT, + "label": _("Mismatches"), + "fieldtype": "Int", + "width": 100, + }, + { + "fieldname": Col.TOP_MISMATCHES, + "label": _("Top Mismatched Fields"), + "fieldtype": "Data", + "width": 300, + }, + ] + + # ── SQL query ──────────────────────────────────────────────────── + + def _fetch_logs(self): + conditions, values = self._build_conditions() + + return frappe.db.sql( + f""" + SELECT + log.ai_model, + log.pdf_processor, + log.accuracy_score, + log.file_parse_time, + log.file_parse_memory, + log.ai_parse_time, + log.total_time, + log.total_cost, + log.prompt_tokens, + log.completion_tokens, + log.total_tokens, + log.currency, + log.field_mismatches, + log.dataset, + ds.party, + ds.file_type, + COALESCE(cust.customer_name, supp.supplier_name, ds.party) AS party_name + FROM `tabParser Benchmark Log` log + JOIN `tabParser Benchmark Dataset` ds ON log.dataset = ds.name + LEFT JOIN `tabCustomer` cust + ON ds.party_type = 'Customer' AND ds.party = cust.name + LEFT JOIN `tabSupplier` supp + ON ds.party_type = 'Supplier' AND ds.party = supp.name + WHERE log.status = 'Completed' + {conditions} + ORDER BY ds.party, log.ai_model, ds.file_type + """, + values=values, + as_dict=True, + ) + + def _build_conditions(self): + conditions: list[str] = [] + values: dict = {} + + for column, key in ( + ("ds.company", "company"), + ("ds.transaction_type", "transaction_type"), + ("ds.party_type", "party_type"), + ("ds.party", "party"), + ): + if self.filters.get(key): + conditions.append(f"AND {column} = %({key})s") + values[key] = self.filters[key] + + for column, key in ( + ("ds.file_type", "file_type"), + ("log.ai_model", "ai_model"), + ("log.pdf_processor", "pdf_processor"), + ): + self._add_in_condition(conditions, values, column, key) + + return "\n ".join(conditions), values + + def _add_in_condition(self, conditions, values, column, key): + """Append an ``IN (...)`` clause for a multi-select filter.""" + raw = self.filters.get(key) + if not raw: + return + + items = raw if isinstance(raw, list) else [raw] + placeholders = [] + for i, val in enumerate(items): + param = f"{key}_{i}" + placeholders.append(f"%({param})s") + values[param] = val + + conditions.append(f"AND {column} IN ({', '.join(placeholders)})") + + # ── Helpers ────────────────────────────────────────────────────── + + def _set_party_type(self): + """Derive party_type from transaction_type when not explicitly set.""" + transaction_type = self.filters.get("transaction_type") + if transaction_type and not self.filters.get("party_type"): + self.filters["party_type"] = PARTY_TYPE_MAP.get(transaction_type) + + def _build_row(self, r): + """Build a single detail row from a log record.""" + mismatches = self._parse_mismatches(r.field_mismatches) + mismatch_fields = [self._short_field_name(m["field"]) for m in mismatches] + + return { + Col.PARTY: r.party or _("No Party"), + Col.PARTY_NAME: r.party_name or "", + Col.ACCURACY_SCORE: r.accuracy_score, + Col.AI_MODEL: r.ai_model, + Col.PDF_PROCESSOR: r.pdf_processor, + Col.FILE_TYPE: r.file_type, + Col.DATASET: r.dataset, + Col.FILE_PARSE_TIME: r.file_parse_time, + Col.FILE_PARSE_MEMORY: r.file_parse_memory, + Col.AI_PARSE_TIME: r.ai_parse_time, + Col.TOTAL_TIME: r.total_time, + Col.TOTAL_COST: r.total_cost, + Col.PROMPT_TOKENS: r.prompt_tokens, + Col.COMPLETION_TOKENS: r.completion_tokens, + Col.TOTAL_TOKENS: r.total_tokens, + Col.CURRENCY: r.currency, + Col.MISMATCH_COUNT: len(mismatches), + Col.TOP_MISMATCHES: ", ".join(mismatch_fields[:5]) + if mismatch_fields + else "", + "_mismatch_fields": mismatch_fields, + } + + @staticmethod + def _parse_mismatches(raw) -> list[dict]: + """Parse field_mismatches JSON string into a list.""" + if not raw: + return [] + try: + data = json.loads(raw) if isinstance(raw, str) else raw + return data if isinstance(data, list) else [] + except (json.JSONDecodeError, TypeError): + return [] + + @staticmethod + def _short_field_name(field: str) -> str: + """Extract a readable field name from DeepDiff path like root['items'][0]['qty'].""" + import re + + keys = re.findall(r"\['(.*?)'\]", field) + return ".".join(keys) if keys else field + + # ── Grouping ───────────────────────────────────────────────────── + + def _group_by_party(self): + """Group data by party, creating tree view with indent levels.""" + if not self.data: + return + + grouped = defaultdict(list) + for row in self.data: + party = row.get(Col.PARTY) or _("No Party") + row["indent"] = 1 + grouped[party].append(row) + + tree_data = [] + for party, rows in grouped.items(): + rows.sort(key=self._sort_key) + tree_data.append(self._group_row(party, rows)) + tree_data.extend(rows) + + self.data = tree_data + + @staticmethod + def _sort_key(row): + """Sort key for child rows: AI Model → PDF Processor → File Type.""" + return ( + _AI_MODEL_ORDER.get(row.get(Col.AI_MODEL), 99), + _PDF_PROCESSOR_ORDER.get(row.get(Col.PDF_PROCESSOR), 99), + _FILE_TYPE_ORDER.get(row.get(Col.FILE_TYPE), 99), + ) + + def _group_row(self, party, rows): + """Aggregated summary row for a party group (indent 0).""" + count = len(rows) + row = { + Col.PARTY: party, + Col.PARTY_NAME: rows[0].get(Col.PARTY_NAME), + Col.CURRENCY: rows[0].get(Col.CURRENCY), + "indent": 0, + } + + for field in _AVG_FIELDS: + vals = [r.get(field) or 0 for r in rows] + row[field] = round(sum(vals) / count, 2) if count else 0 + + for field in _SUM_FIELDS: + row[field] = sum(r.get(field) or 0 for r in rows) + + # aggregate mismatch info across all child rows + all_fields = [] + for r in rows: + all_fields.extend(r.get("_mismatch_fields", [])) + + row[Col.MISMATCH_COUNT] = sum(r.get(Col.MISMATCH_COUNT, 0) for r in rows) + + if all_fields: + top = Counter(all_fields).most_common(5) + row[Col.TOP_MISMATCHES] = ", ".join(f"{name} ({cnt})" for name, cnt in top) + else: + row[Col.TOP_MISMATCHES] = "" + + return row From f5b048b569bcdebd02e05fe5c505af590f77a507 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 25 Mar 2026 12:49:20 +0530 Subject: [PATCH 54/88] feat: enhance Accuracy Analysis Report with new metrics and aggregation logic --- .../transaction_parser_accuracy_analysis.py | 118 ++++++++++++++++-- 1 file changed, 107 insertions(+), 11 deletions(-) diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py index 7c3cb89..f5099b7 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py @@ -57,7 +57,9 @@ class Col(StrEnum): TOTAL_TOKENS = "total_tokens" CURRENCY = "currency" DATASET = "dataset" - MISMATCH_COUNT = "mismatch_count" + RUN_COUNT = "run_count" + MISMATCH_RATE = "mismatch_rate" + UNIQUE_MISMATCHES = "unique_mismatches" TOP_MISMATCHES = "top_mismatches" @@ -90,10 +92,15 @@ def __init__(self, filters: frappe._dict): def run(self): self.data = [self._build_row(r) for r in self._fetch_logs()] + self._aggregate_by_config() if self.group_by_party: self._group_by_party() + # strip internal keys before sending to client + for row in self.data: + row.pop("_mismatch_fields", None) + return self._get_columns(), self.data # ── Columns ────────────────────────────────────────────────────── @@ -119,6 +126,12 @@ def _get_columns(self): "options": "Parser Benchmark Dataset", "width": 160, }, + { + "fieldname": Col.RUN_COUNT, + "label": _("Runs"), + "fieldtype": "Int", + "width": 70, + }, { "fieldname": Col.ACCURACY_SCORE, "label": _("Accuracy (%)"), @@ -197,10 +210,16 @@ def _get_columns(self): "width": 110, }, { - "fieldname": Col.MISMATCH_COUNT, - "label": _("Mismatches"), + "fieldname": Col.MISMATCH_RATE, + "label": _("Mismatch Rate (%)"), + "fieldtype": "Percent", + "width": 140, + }, + { + "fieldname": Col.UNIQUE_MISMATCHES, + "label": _("Unique Fields"), "fieldtype": "Int", - "width": 100, + "width": 110, }, { "fieldname": Col.TOP_MISMATCHES, @@ -300,6 +319,12 @@ def _build_row(self, r): mismatches = self._parse_mismatches(r.field_mismatches) mismatch_fields = [self._short_field_name(m["field"]) for m in mismatches] + if mismatch_fields: + top = Counter(mismatch_fields).most_common(5) + top_str = ", ".join(f"{name} x{cnt}" for name, cnt in top) + else: + top_str = "" + return { Col.PARTY: r.party or _("No Party"), Col.PARTY_NAME: r.party_name or "", @@ -317,10 +342,9 @@ def _build_row(self, r): Col.COMPLETION_TOKENS: r.completion_tokens, Col.TOTAL_TOKENS: r.total_tokens, Col.CURRENCY: r.currency, - Col.MISMATCH_COUNT: len(mismatches), - Col.TOP_MISMATCHES: ", ".join(mismatch_fields[:5]) - if mismatch_fields - else "", + Col.MISMATCH_RATE: round(100 - (r.accuracy_score or 0), 2), + Col.UNIQUE_MISMATCHES: len(set(mismatch_fields)), + Col.TOP_MISMATCHES: top_str, "_mismatch_fields": mismatch_fields, } @@ -343,6 +367,74 @@ def _short_field_name(field: str) -> str: keys = re.findall(r"\['(.*?)'\]", field) return ".".join(keys) if keys else field + # ── Aggregation ────────────────────────────────────────────────── + + def _aggregate_by_config(self): + """Collapse multiple runs of the same config into one averaged row. + + Groups by (party, ai_model, pdf_processor, file_type) and averages + numeric fields. The Dataset column shows the latest dataset if all + runs share one, otherwise left blank. + """ + if not self.data: + return + + groups: dict[tuple, list[dict]] = defaultdict(list) + for row in self.data: + key = ( + row.get(Col.DATASET), + row.get(Col.AI_MODEL), + row.get(Col.PDF_PROCESSOR) or "", + row.get(Col.FILE_TYPE), + ) + groups[key].append(row) + + aggregated = [] + for _key, rows in groups.items(): + count = len(rows) + agg = { + Col.PARTY: rows[0].get(Col.PARTY), + Col.PARTY_NAME: rows[0].get(Col.PARTY_NAME), + Col.AI_MODEL: rows[0].get(Col.AI_MODEL), + Col.PDF_PROCESSOR: rows[0].get(Col.PDF_PROCESSOR), + Col.FILE_TYPE: rows[0].get(Col.FILE_TYPE), + Col.CURRENCY: rows[0].get(Col.CURRENCY), + Col.RUN_COUNT: count, + } + + # unique datasets — show if only one, else blank + agg[Col.DATASET] = rows[0].get(Col.DATASET, "") + + for field in _AVG_FIELDS: + vals = [r.get(field) or 0 for r in rows] + agg[field] = round(sum(vals) / count, 2) if count else 0 + + for field in _SUM_FIELDS: + agg[field] = round(sum(r.get(field) or 0 for r in rows) / count, 6) + + # aggregate mismatches + all_fields = [] + for r in rows: + all_fields.extend(r.get("_mismatch_fields", [])) + + agg[Col.MISMATCH_RATE] = round(100 - (agg.get(Col.ACCURACY_SCORE) or 0), 2) + agg[Col.UNIQUE_MISMATCHES] = round( + sum(r.get(Col.UNIQUE_MISMATCHES, 0) for r in rows) / count + ) + + if all_fields: + top = Counter(all_fields).most_common(5) + agg[Col.TOP_MISMATCHES] = ", ".join( + f"{name} x{cnt}" for name, cnt in top + ) + else: + agg[Col.TOP_MISMATCHES] = "" + + agg["_mismatch_fields"] = all_fields + aggregated.append(agg) + + self.data = aggregated + # ── Grouping ───────────────────────────────────────────────────── def _group_by_party(self): @@ -380,6 +472,7 @@ def _group_row(self, party, rows): Col.PARTY: party, Col.PARTY_NAME: rows[0].get(Col.PARTY_NAME), Col.CURRENCY: rows[0].get(Col.CURRENCY), + Col.RUN_COUNT: sum(r.get(Col.RUN_COUNT, 1) for r in rows), "indent": 0, } @@ -388,18 +481,21 @@ def _group_row(self, party, rows): row[field] = round(sum(vals) / count, 2) if count else 0 for field in _SUM_FIELDS: - row[field] = sum(r.get(field) or 0 for r in rows) + row[field] = round(sum(r.get(field) or 0 for r in rows), 6) # aggregate mismatch info across all child rows all_fields = [] for r in rows: all_fields.extend(r.get("_mismatch_fields", [])) - row[Col.MISMATCH_COUNT] = sum(r.get(Col.MISMATCH_COUNT, 0) for r in rows) + row[Col.MISMATCH_RATE] = round(100 - (row.get(Col.ACCURACY_SCORE) or 0), 2) + row[Col.UNIQUE_MISMATCHES] = round( + sum(r.get(Col.UNIQUE_MISMATCHES, 0) for r in rows) / count + ) if all_fields: top = Counter(all_fields).most_common(5) - row[Col.TOP_MISMATCHES] = ", ".join(f"{name} ({cnt})" for name, cnt in top) + row[Col.TOP_MISMATCHES] = ", ".join(f"{name} x{cnt}" for name, cnt in top) else: row[Col.TOP_MISMATCHES] = "" From 299e95e13411c31b9855b18b8602f164e3a70010 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 25 Mar 2026 13:51:12 +0530 Subject: [PATCH 55/88] fix: remove ignore_order parameter from DeepDiff in score_response function --- transaction_parser/parser_benchmark/scorer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/transaction_parser/parser_benchmark/scorer.py b/transaction_parser/parser_benchmark/scorer.py index 8e53609..a06f053 100644 --- a/transaction_parser/parser_benchmark/scorer.py +++ b/transaction_parser/parser_benchmark/scorer.py @@ -29,7 +29,6 @@ def score_response(expected: dict, actual: dict, max_diffs: int = 500) -> dict: expected, actual, ignore_string_case=True, - ignore_order=True, ignore_type_in_groups=[(dict, frappe._dict)], significant_digits=2, verbose_level=2, From e870e00dab3e3965fd51eff1fb9af52dae6d477c Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 25 Mar 2026 16:57:36 +0530 Subject: [PATCH 56/88] refactor: optimize SQL query construction in AccuracyAnalysisReport --- .../transaction_parser_accuracy_analysis.py | 86 ++++++++----------- 1 file changed, 37 insertions(+), 49 deletions(-) diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py index f5099b7..fd57500 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py @@ -7,6 +7,7 @@ import frappe from frappe import _ +from frappe.query_builder.functions import Coalesce PARTY_TYPE_MAP = { "Sales Order": "Customer", @@ -229,14 +230,23 @@ def _get_columns(self): }, ] - # ── SQL query ──────────────────────────────────────────────────── + # ── Query ───────────────────────────────────────────────────── def _fetch_logs(self): - conditions, values = self._build_conditions() - - return frappe.db.sql( - f""" - SELECT + log = frappe.qb.DocType("Parser Benchmark Log") + ds = frappe.qb.DocType("Parser Benchmark Dataset") + cust = frappe.qb.DocType("Customer") + supp = frappe.qb.DocType("Supplier") + + query = ( + frappe.qb.from_(log) + .join(ds) + .on(log.dataset == ds.name) + .left_join(cust) + .on((ds.party_type == "Customer") & (ds.party == cust.name)) + .left_join(supp) + .on((ds.party_type == "Supplier") & (ds.party == supp.name)) + .select( log.ai_model, log.pdf_processor, log.accuracy_score, @@ -253,58 +263,36 @@ def _fetch_logs(self): log.dataset, ds.party, ds.file_type, - COALESCE(cust.customer_name, supp.supplier_name, ds.party) AS party_name - FROM `tabParser Benchmark Log` log - JOIN `tabParser Benchmark Dataset` ds ON log.dataset = ds.name - LEFT JOIN `tabCustomer` cust - ON ds.party_type = 'Customer' AND ds.party = cust.name - LEFT JOIN `tabSupplier` supp - ON ds.party_type = 'Supplier' AND ds.party = supp.name - WHERE log.status = 'Completed' - {conditions} - ORDER BY ds.party, log.ai_model, ds.file_type - """, - values=values, - as_dict=True, + Coalesce(cust.customer_name, supp.supplier_name, ds.party).as_( + "party_name" + ), + ) + .where(log.status == "Completed") + .orderby(ds.party, log.ai_model, ds.file_type) ) - def _build_conditions(self): - conditions: list[str] = [] - values: dict = {} - + # exact-match filters for column, key in ( - ("ds.company", "company"), - ("ds.transaction_type", "transaction_type"), - ("ds.party_type", "party_type"), - ("ds.party", "party"), + (ds.company, "company"), + (ds.transaction_type, "transaction_type"), + (ds.party_type, "party_type"), + (ds.party, "party"), ): if self.filters.get(key): - conditions.append(f"AND {column} = %({key})s") - values[key] = self.filters[key] + query = query.where(column == self.filters[key]) + # multi-select IN filters for column, key in ( - ("ds.file_type", "file_type"), - ("log.ai_model", "ai_model"), - ("log.pdf_processor", "pdf_processor"), + (ds.file_type, "file_type"), + (log.ai_model, "ai_model"), + (log.pdf_processor, "pdf_processor"), ): - self._add_in_condition(conditions, values, column, key) - - return "\n ".join(conditions), values - - def _add_in_condition(self, conditions, values, column, key): - """Append an ``IN (...)`` clause for a multi-select filter.""" - raw = self.filters.get(key) - if not raw: - return - - items = raw if isinstance(raw, list) else [raw] - placeholders = [] - for i, val in enumerate(items): - param = f"{key}_{i}" - placeholders.append(f"%({param})s") - values[param] = val + values = self.filters.get(key) + if values: + items = values if isinstance(values, list) else [values] + query = query.where(column.isin(items)) - conditions.append(f"AND {column} IN ({', '.join(placeholders)})") + return query.run(as_dict=True) # ── Helpers ────────────────────────────────────────────────────── From 0f1fe55d1024bd7dfc1e5c25ab6e1c38f5b2a8c9 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 25 Mar 2026 17:20:58 +0530 Subject: [PATCH 57/88] fix: update default value for enabled field in Parser Benchmark Settings --- .../parser_benchmark_settings/parser_benchmark_settings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json index 712b16e..d786e18 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json @@ -93,7 +93,7 @@ "fieldtype": "Column Break" }, { - "default": "1", + "default": "0", "fieldname": "enabled", "fieldtype": "Check", "label": "Enabled" @@ -101,7 +101,7 @@ ], "issingle": 1, "links": [], - "modified": "2026-03-18 13:40:00.122632", + "modified": "2026-03-25 12:49:58.838421", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Settings", From a842310041c6d00e70c3921705e909a7bcc5c7d4 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 25 Mar 2026 17:47:56 +0530 Subject: [PATCH 58/88] feat: add commit information fields to Parser Benchmark Log and Settings --- .../parser_benchmark_dataset.py | 30 +++++++++++++++++++ .../parser_benchmark_log.json | 29 +++++++++++++++++- .../parser_benchmark_log.py | 2 ++ .../parser_benchmark_settings.json | 5 ++-- 4 files changed, 62 insertions(+), 4 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 361ca53..59769ca 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -141,6 +141,8 @@ def _create_and_enqueue_logs(dataset) -> list[str]: else: processors = [None] + commit_info = get_commit_info() + for ai_model in dataset.get_selected_models(): for pdf_processor in processors: existing = frappe.db.exists( @@ -163,6 +165,7 @@ def _create_and_enqueue_logs(dataset) -> list[str]: "ai_model": ai_model, "pdf_processor": pdf_processor, "currency": "USD", + **commit_info, } ).insert(ignore_permissions=True) @@ -190,6 +193,33 @@ def _run_benchmark(log_name: str): BenchmarkRunner(log_name).run() +def get_commit_info() -> dict: + """Return the current git commit hash and message for the transaction_parser app.""" + import subprocess + + app_path = frappe.get_app_path("transaction_parser") + + try: + result = subprocess.run( + ["git", "log", "-1", "--format=%H%n%s"], + cwd=app_path, + capture_output=True, + text=True, + timeout=5, + ) + + if result.returncode == 0: + lines = result.stdout.strip().split("\n", 1) + return { + "commit_hash": lines[0] if lines else "", + "commit_message": lines[1] if len(lines) > 1 else "", + } + except Exception: + pass + + return {} + + @frappe.whitelist() def get_pdf_processors(): frappe.has_permission("Parser Benchmark Dataset", "write", throw=True) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index 53b0849..107c914 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -17,6 +17,10 @@ "total_time", "column_break_fmal", "status", + "version_section", + "commit_hash", + "column_break_version", + "commit_message", "section_break_nmcv", "naming_series", "column_break_ubhs", @@ -244,6 +248,29 @@ "label": "Error Traceback", "read_only": 1 }, + { + "collapsible": 1, + "fieldname": "version_section", + "fieldtype": "Section Break", + "label": "Version" + }, + { + "fieldname": "commit_hash", + "fieldtype": "Data", + "label": "Commit Hash", + "length": 40, + "read_only": 1 + }, + { + "fieldname": "column_break_version", + "fieldtype": "Column Break" + }, + { + "fieldname": "commit_message", + "fieldtype": "Small Text", + "label": "Commit Message", + "read_only": 1 + }, { "collapsible": 1, "fieldname": "section_break_nmcv", @@ -405,7 +432,7 @@ "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-21 13:50:57.098387", + "modified": "2026-03-25 13:08:38.340789", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Log", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py index feec737..754fa34 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -27,6 +27,8 @@ class ParserBenchmarkLog(Document): ] ai_parse_time: DF.Float ai_response: DF.Code | None + commit_hash: DF.Data | None + commit_message: DF.SmallText | None company: DF.Link | None completion_tokens: DF.Int country: DF.Literal["India", "Other"] diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json index d786e18..d588b26 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json @@ -24,8 +24,7 @@ "fieldname": "token_costs", "fieldtype": "Table", "label": "Token Costs", - "options": "Parser Benchmark Token Cost", - "reqd": 1 + "options": "Parser Benchmark Token Cost" }, { "depends_on": "eval: doc.enabled", @@ -101,7 +100,7 @@ ], "issingle": 1, "links": [], - "modified": "2026-03-25 12:49:58.838421", + "modified": "2026-03-25 12:53:24.601045", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Settings", From 87403dc0fb116954df70fea4555cccd5d6875493 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 25 Mar 2026 18:23:33 +0530 Subject: [PATCH 59/88] refactor: rename and update log creation functions for benchmark processing --- .../parser_benchmark_dataset.py | 64 ++++++------------- .../parser_benchmark_settings.py | 5 +- transaction_parser/parser_benchmark/runner.py | 9 ++- 3 files changed, 29 insertions(+), 49 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 59769ca..edd786f 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -116,50 +116,41 @@ def run_benchmark(dataset_name: str): """Create Benchmark Logs for each model x processor combo and enqueue runs.""" frappe.has_permission("Parser Benchmark Dataset", "write", throw=True) - dataset = frappe.get_doc("Parser Benchmark Dataset", dataset_name) - if dataset.docstatus != 1: + if frappe.db.get_value("Parser Benchmark Dataset", dataset_name, "docstatus") != 1: frappe.throw(_("Dataset must be submitted before running benchmarks.")) - log_names = _create_and_enqueue_logs(dataset) + log_names = create_and_enqueue_benchmark_logs(dataset_name) if not log_names: frappe.throw( _( - "No new benchmarks to queue. All selected combinations are already queued or running." + "No new benchmarks to queue. Please check if the dataset is properly configured" ) ) return log_names -def _create_and_enqueue_logs(dataset) -> list[str]: +def create_and_enqueue_benchmark_logs(dataset_name: str) -> list[str]: """Create one log per model x processor combo and enqueue each for background execution.""" - log_names = [] - - if dataset.file_type == "PDF": - processors = dataset.get_selected_processors() or [None] - else: - processors = [None] + dataset: ParserBenchmarkDataset = frappe.get_cached_doc( + "Parser Benchmark Dataset", dataset_name + ) + models = dataset.get_selected_models() + processors = ( + dataset.get_selected_processors() or [None] + if dataset.file_type == "PDF" + else [None] + ) commit_info = get_commit_info() + log_names = [] - for ai_model in dataset.get_selected_models(): + for ai_model in models: for pdf_processor in processors: - existing = frappe.db.exists( - "Parser Benchmark Log", - { - "dataset": dataset.name, - "ai_model": ai_model, - "pdf_processor": pdf_processor or "", - "status": ("in", ("Queued", "Running")), - }, - ) - if existing: - continue - - log = frappe.get_doc( + log = frappe.new_doc("Parser Benchmark Log") + log.update( { - "doctype": "Parser Benchmark Log", "status": "Queued", "dataset": dataset.name, "ai_model": ai_model, @@ -167,19 +158,15 @@ def _create_and_enqueue_logs(dataset) -> list[str]: "currency": "USD", **commit_info, } - ).insert(ignore_permissions=True) - + ) + log.insert(ignore_permissions=True) log_names.append(log.name) frappe.db.commit() for log_name in log_names: try: - frappe.enqueue( - _run_benchmark, - log_name=log_name, - queue="long", - ) + frappe.enqueue(_run_benchmark, log_name=log_name, queue="long") except Exception: frappe.db.set_value("Parser Benchmark Log", log_name, "status", "Failed") frappe.db.commit() @@ -218,14 +205,3 @@ def get_commit_info() -> dict: pass return {} - - -@frappe.whitelist() -def get_pdf_processors(): - frappe.has_permission("Parser Benchmark Dataset", "write", throw=True) - - from transaction_parser.transaction_parser.utils.pdf_processor import ( - get_available_pdf_processors, - ) - - return get_available_pdf_processors() diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py index 5398206..6057f85 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py @@ -49,7 +49,7 @@ def is_scheduled_today(self) -> bool: def run_scheduled_benchmarks(): """Scheduled job: runs all enabled datasets if today is a scheduled day.""" from transaction_parser.parser_benchmark.doctype.parser_benchmark_dataset.parser_benchmark_dataset import ( - _create_and_enqueue_logs, + create_and_enqueue_benchmark_logs, ) settings: ParserBenchmarkSettings = frappe.get_cached_doc( @@ -66,5 +66,4 @@ def run_scheduled_benchmarks(): ) for dataset_name in datasets: - dataset = frappe.get_doc("Parser Benchmark Dataset", dataset_name) - _create_and_enqueue_logs(dataset) + create_and_enqueue_benchmark_logs(dataset_name) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index cf475f4..e44cf89 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -11,6 +11,9 @@ from transaction_parser.parser_benchmark.doctype.parser_benchmark_log.parser_benchmark_log import ( ParserBenchmarkLog, ) +from transaction_parser.parser_benchmark.doctype.parser_benchmark_settings.parser_benchmark_settings import ( + ParserBenchmarkSettings, +) from transaction_parser.transaction_parser.ai_integration.parser import AIParser from transaction_parser.transaction_parser.controllers import get_controller from transaction_parser.transaction_parser.controllers.transaction import Transaction @@ -73,7 +76,7 @@ def _get_controller(self, file_doc: File) -> Transaction: ds = self.dataset cls = get_controller(ds.country, ds.transaction_type) - controller = cls(company=ds.company, party=ds.party) + controller: Transaction = cls(company=ds.company, party=ds.party) controller.initialize() controller.file = file_doc controller.ai_model = self.log.ai_model @@ -82,7 +85,9 @@ def _get_controller(self, file_doc: File) -> Transaction: def _get_cost_row(self): try: - settings = frappe.get_cached_doc("Parser Benchmark Settings") + settings: ParserBenchmarkSettings = frappe.get_cached_doc( + "Parser Benchmark Settings" + ) except Exception: return None From f7cf22ebabfd13e90efb8c88f370cc3225575a09 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 25 Mar 2026 18:46:01 +0530 Subject: [PATCH 60/88] feat: enhance score_response function to accept significant_digits parameter for improved accuracy --- transaction_parser/parser_benchmark/runner.py | 11 +++++++++-- transaction_parser/parser_benchmark/scorer.py | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index e44cf89..ce1c94a 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -3,7 +3,7 @@ import frappe from frappe.core.doctype.file.file import File -from frappe.utils import flt +from frappe.utils import cint, flt from transaction_parser.parser_benchmark.doctype.parser_benchmark_dataset.parser_benchmark_dataset import ( ParserBenchmarkDataset, @@ -37,6 +37,9 @@ def __init__(self, log_name: str): ) self.precision = 6 # to get 1-millionth of a token cost + # for accuracy score + self.significant_digits = cint(frappe.db.get_default("float_precision")) or 2 + def run(self): self.log.status = "Running" self.log.currency = "USD" @@ -181,6 +184,10 @@ def _score_response(self, ai_content: dict): if isinstance(expected, str): expected = frappe.parse_json(expected) - result = score_response(expected, ai_content) + result = score_response( + expected, + ai_content, + significant_digits=self.significant_digits, + ) self.log.accuracy_score = result["accuracy_score"] self.log.field_mismatches = frappe.as_json(result["mismatches"], indent=2) diff --git a/transaction_parser/parser_benchmark/scorer.py b/transaction_parser/parser_benchmark/scorer.py index a06f053..2e3d9dd 100644 --- a/transaction_parser/parser_benchmark/scorer.py +++ b/transaction_parser/parser_benchmark/scorer.py @@ -20,7 +20,13 @@ def _normalize_empty(obj): return obj -def score_response(expected: dict, actual: dict, max_diffs: int = 500) -> dict: +def score_response( + expected: dict, + actual: dict, + *, + max_diffs: int = 500, + significant_digits: int = 2, +) -> dict: """Compare AI response against expected result using DeepDiff.""" expected = _normalize_empty(expected) actual = _normalize_empty(actual) @@ -29,8 +35,9 @@ def score_response(expected: dict, actual: dict, max_diffs: int = 500) -> dict: expected, actual, ignore_string_case=True, + ignore_numeric_type_changes=True, ignore_type_in_groups=[(dict, frappe._dict)], - significant_digits=2, + significant_digits=significant_digits, verbose_level=2, max_diffs=max_diffs, log_frequency_in_sec=0, From e7781d97f51dd5d652a5bb8f8b691ce3e0285c98 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Thu, 26 Mar 2026 15:40:34 +0530 Subject: [PATCH 61/88] chore: linter fix with db-commit msg --- .../parser_benchmark_dataset/parser_benchmark_dataset.py | 5 +++-- transaction_parser/parser_benchmark/runner.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index edd786f..fd3e9a4 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -162,14 +162,15 @@ def create_and_enqueue_benchmark_logs(dataset_name: str) -> list[str]: log.insert(ignore_permissions=True) log_names.append(log.name) - frappe.db.commit() + # commit before enqueuing so background jobs can read the inserted logs + frappe.db.commit() # nosemgrep for log_name in log_names: try: frappe.enqueue(_run_benchmark, log_name=log_name, queue="long") except Exception: frappe.db.set_value("Parser Benchmark Log", log_name, "status", "Failed") - frappe.db.commit() + frappe.db.commit() # nosemgrep -- persist Failed status when enqueue fails return log_names diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index ce1c94a..1cc3fb4 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -44,7 +44,7 @@ def run(self): self.log.status = "Running" self.log.currency = "USD" self.log.save(ignore_permissions=True) - frappe.db.commit() # persist "Running" status + frappe.db.commit() # nosemgrep -- persist "Running" status before long-running benchmark total_start = default_timer() @@ -66,7 +66,7 @@ def run(self): finally: self.log.total_time = flt(default_timer() - total_start, self.precision) self.log.save(ignore_permissions=True) - frappe.db.commit() + frappe.db.commit() # nosemgrep -- persist final results inside background job return self.log.name From bd7b3a364bc24b9b511f15c9d3e7ba904f17a9c1 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Thu, 26 Mar 2026 16:00:41 +0530 Subject: [PATCH 62/88] fix: add error handling for benchmark log enqueueing and improve tracemalloc management --- .../parser_benchmark_settings.py | 8 +++++++- transaction_parser/parser_benchmark/runner.py | 9 +++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py index 6057f85..4ca20d1 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py @@ -66,4 +66,10 @@ def run_scheduled_benchmarks(): ) for dataset_name in datasets: - create_and_enqueue_benchmark_logs(dataset_name) + try: + create_and_enqueue_benchmark_logs(dataset_name) + except Exception: + frappe.log_error( + title=f"Failed to enqueue benchmark for dataset {dataset_name}", + message=frappe.get_traceback(), + ) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index 1cc3fb4..886c29f 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -107,7 +107,11 @@ def _run_file_parsing(self, file_doc: File) -> str: if self.log.file_type == "PDF" and self.log.pdf_processor: pdf_processor = get_pdf_processor(self.log.pdf_processor) - tracemalloc.start() + # to prevent stopping an already running tracemalloc instance + was_tracing = tracemalloc.is_tracing() + if not was_tracing: + tracemalloc.start() + start = default_timer() try: content = FileProcessor().get_content( @@ -118,7 +122,8 @@ def _run_file_parsing(self, file_doc: File) -> str: finally: self.log.file_parse_time = flt(default_timer() - start, self.precision) _, peak = tracemalloc.get_traced_memory() - tracemalloc.stop() + if not was_tracing: + tracemalloc.stop() self.log.file_parse_memory = flt( peak / 1024 / 1024, self.precision ) # bytes → MB From 41f0d23cb9931feb0601de9a14cd2c30b7cea4a0 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Fri, 27 Mar 2026 17:11:58 +0530 Subject: [PATCH 63/88] feat: proper accuracy measuring --- pyproject.toml | 1 - .../parser_benchmark_dataset.json | 22 +- .../parser_benchmark_dataset.py | 38 ++- .../__init__.py | 0 .../parser_benchmark_expected_field.json | 41 +++ .../parser_benchmark_expected_field.py | 36 +++ .../parser_benchmark_key_weight/__init__.py | 0 .../parser_benchmark_key_weight.json | 46 ++++ .../parser_benchmark_key_weight.py | 36 +++ .../parser_benchmark_log.json | 20 +- .../parser_benchmark_log.py | 6 +- .../parser_benchmark_score_detail/__init__.py | 0 .../parser_benchmark_score_detail.json | 76 ++++++ .../parser_benchmark_score_detail.py | 26 ++ .../parser_benchmark_settings.json | 19 +- .../parser_benchmark_settings.py | 4 + .../parser_benchmark_token_cost.json | 2 +- .../transaction_parser_accuracy_analysis.py | 156 +++++------ transaction_parser/parser_benchmark/runner.py | 47 +++- transaction_parser/parser_benchmark/scorer.py | 242 +++++++++++------- 20 files changed, 595 insertions(+), 223 deletions(-) create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_expected_field/__init__.py create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_expected_field/parser_benchmark_expected_field.json create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_expected_field/parser_benchmark_expected_field.py create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_key_weight/__init__.py create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_key_weight/parser_benchmark_key_weight.json create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_key_weight/parser_benchmark_key_weight.py create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_score_detail/__init__.py create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_score_detail/parser_benchmark_score_detail.json create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_score_detail/parser_benchmark_score_detail.py diff --git a/pyproject.toml b/pyproject.toml index ad24096..a458259 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,6 @@ dependencies = [ "pymupdf~=1.26.3", "openai", "docling>=2.75.0", - "deepdiff>=8.0", ] [build-system] diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index e97dd69..fb07441 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -37,8 +37,8 @@ "other_config_section", "page_limit", "column_break_jjht", - "expected_result_section", - "expected_result", + "expected_fields_section", + "expected_fields", "section_break_bhsg", "naming_series", "column_break_aoce", @@ -217,17 +217,17 @@ "non_negative": 1 }, { - "fieldname": "expected_result_section", + "fieldname": "expected_fields_section", "fieldtype": "Section Break", - "label": "Expected Result" + "label": "Expected Fields" }, { "allow_on_submit": 1, - "description": "The correct/expected parsed output JSON for this dataset. Used to score AI accuracy.", - "fieldname": "expected_result", - "fieldtype": "Code", - "label": "Expected Result", - "options": "JSON" + "description": "Add one row per response key you want to score. Pick the key and paste only its expected JSON value.", + "fieldname": "expected_fields", + "fieldtype": "Table", + "label": "Expected Fields", + "options": "Parser Benchmark Expected Field" }, { "collapsible": 1, @@ -283,7 +283,7 @@ "link_fieldname": "dataset" } ], - "modified": "2026-03-24 14:57:58.689734", + "modified": "2026-03-27 10:32:46.620190", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", @@ -307,4 +307,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} +} \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index fd3e9a4..6d0ca6e 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -32,6 +32,10 @@ class ParserBenchmarkDataset(Document): if TYPE_CHECKING: from frappe.types import DF + from transaction_parser.parser_benchmark.doctype.parser_benchmark_expected_field.parser_benchmark_expected_field import ( + ParserBenchmarkExpectedField, + ) + amended_from: DF.Link | None company: DF.Link | None country: DF.Literal["India", "Other"] @@ -39,7 +43,7 @@ class ParserBenchmarkDataset(Document): deepseek_reasoner: DF.Check docling: DF.Check enabled: DF.Check - expected_result: DF.Code | None + expected_fields: DF.Table[ParserBenchmarkExpectedField] file: DF.Attach file_type: DF.Data | None google_gemini_flash_25: DF.Check @@ -63,7 +67,7 @@ def validate(self): self.validate_file_type() self.validate_selected_models() self.validate_selected_processors() - self.validate_expected_result() + self.validate_expected_fields() def set_file_type(self): if self.file_type and not self.has_value_changed("file"): @@ -89,16 +93,29 @@ def validate_selected_processors(self): if not self.get_selected_processors(): frappe.throw(_("Please select at least one PDF Processor.")) - def validate_expected_result(self): - if not self.expected_result: + def validate_expected_fields(self): + if not self.expected_fields: return - try: - frappe.parse_json(self.expected_result) - except Exception: - frappe.throw( - title=_("Invalid JSON"), msg=_("Expected Result must be valid JSON.") - ) + seen_keys = set() + for row in self.expected_fields: + if row.key in seen_keys: + frappe.throw( + _("Duplicate key '{0}' in Expected Fields row {1}").format( + row.key, row.idx + ) + ) + seen_keys.add(row.key) + + try: + frappe.parse_json(row.expected_json) + except Exception: + frappe.throw( + title=_("Invalid JSON"), + msg=_( + "Expected JSON in row {0} (key: {1}) must be valid JSON." + ).format(row.idx, row.key), + ) def get_selected_models(self) -> list[str]: """Return list of selected AI model names.""" @@ -165,6 +182,7 @@ def create_and_enqueue_benchmark_logs(dataset_name: str) -> list[str]: # commit before enqueuing so background jobs can read the inserted logs frappe.db.commit() # nosemgrep + # TODO: pass a dataset doc and the settings... for log_name in log_names: try: frappe.enqueue(_run_benchmark, log_name=log_name, queue="long") diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_expected_field/__init__.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_expected_field/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_expected_field/parser_benchmark_expected_field.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_expected_field/parser_benchmark_expected_field.json new file mode 100644 index 0000000..1d69fdf --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_expected_field/parser_benchmark_expected_field.json @@ -0,0 +1,41 @@ +{ + "actions": [], + "creation": "2026-03-27 00:00:00", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "key", + "expected_json" + ], + "fields": [ + { + "fieldname": "key", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Key", + "options": "document_number\ndocument_date\ncurrency\nitem_list\ntotals\npayment_terms\nlocal_terms\ndelivery_date\nbuyer\nvendor\ncompany\nsupplier", + "reqd": 1 + }, + { + "allow_on_submit": 1, + "fieldname": "expected_json", + "fieldtype": "Code", + "in_list_view": 1, + "label": "Expected JSON", + "options": "JSON", + "reqd": 1 + } + ], + "istable": 1, + "links": [], + "modified": "2026-03-27 10:56:30.834125", + "modified_by": "Administrator", + "module": "Parser Benchmark", + "name": "Parser Benchmark Expected Field", + "owner": "Administrator", + "permissions": [], + "row_format": "Dynamic", + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_expected_field/parser_benchmark_expected_field.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_expected_field/parser_benchmark_expected_field.py new file mode 100644 index 0000000..670f9da --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_expected_field/parser_benchmark_expected_field.py @@ -0,0 +1,36 @@ +# Copyright (c) 2026, Resilient Tech and contributors +# For license information, please see license.txt + +from frappe.model.document import Document + + +class ParserBenchmarkExpectedField(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + expected_json: DF.Code + key: DF.Literal[ + "document_number", + "document_date", + "currency", + "item_list", + "totals", + "payment_terms", + "local_terms", + "delivery_date", + "buyer", + "vendor", + "company", + "supplier", + ] + parent: DF.Data + parentfield: DF.Data + parenttype: DF.Data + # end: auto-generated types + + pass diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_key_weight/__init__.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_key_weight/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_key_weight/parser_benchmark_key_weight.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_key_weight/parser_benchmark_key_weight.json new file mode 100644 index 0000000..4c0c322 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_key_weight/parser_benchmark_key_weight.json @@ -0,0 +1,46 @@ +{ + "actions": [], + "creation": "2026-03-27 00:00:00", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "key", + "column_break_qaca", + "weight" + ], + "fields": [ + { + "fieldname": "key", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Key", + "options": "document_number\ndocument_date\ncurrency\nitem_list\ntotals\npayment_terms\nlocal_terms\ndelivery_date\nbuyer\nvendor\ncompany\nsupplier", + "reqd": 1 + }, + { + "default": "1", + "fieldname": "weight", + "fieldtype": "Float", + "in_list_view": 1, + "label": "Weight", + "non_negative": 1, + "reqd": 1 + }, + { + "fieldname": "column_break_qaca", + "fieldtype": "Column Break" + } + ], + "istable": 1, + "links": [], + "modified": "2026-03-27 11:00:43.378124", + "modified_by": "Administrator", + "module": "Parser Benchmark", + "name": "Parser Benchmark Key Weight", + "owner": "Administrator", + "permissions": [], + "row_format": "Dynamic", + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_key_weight/parser_benchmark_key_weight.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_key_weight/parser_benchmark_key_weight.py new file mode 100644 index 0000000..117ba33 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_key_weight/parser_benchmark_key_weight.py @@ -0,0 +1,36 @@ +# Copyright (c) 2026, Resilient Tech and contributors +# For license information, please see license.txt + +from frappe.model.document import Document + + +class ParserBenchmarkKeyWeight(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + key: DF.Literal[ + "document_number", + "document_date", + "currency", + "item_list", + "totals", + "payment_terms", + "local_terms", + "delivery_date", + "buyer", + "vendor", + "company", + "supplier", + ] + parent: DF.Data + parentfield: DF.Data + parenttype: DF.Data + weight: DF.Float + # end: auto-generated types + + pass diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index 107c914..9ba294b 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -59,8 +59,8 @@ "accuracy_section", "accuracy_score", "column_break_accuracy", - "section_break_mismatches", - "field_mismatches", + "score_details_section", + "score_details", "error_tab", "section_break_error", "error" @@ -329,15 +329,15 @@ "fieldtype": "Column Break" }, { - "fieldname": "section_break_mismatches", + "fieldname": "score_details_section", "fieldtype": "Section Break", - "label": "Field Mismatches" + "label": "Score Details" }, { - "fieldname": "field_mismatches", - "fieldtype": "Code", - "label": "Field Mismatches", - "options": "JSON", + "fieldname": "score_details", + "fieldtype": "Table", + "label": "Score Details", + "options": "Parser Benchmark Score Detail", "read_only": 1 }, { @@ -432,7 +432,7 @@ "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2026-03-25 13:08:38.340789", + "modified": "2026-03-27 10:32:31.037034", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Log", @@ -456,4 +456,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} \ No newline at end of file +} diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py index 754fa34..0b9e29a 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -14,6 +14,10 @@ class ParserBenchmarkLog(Document): if TYPE_CHECKING: from frappe.types import DF + from transaction_parser.parser_benchmark.doctype.parser_benchmark_score_detail.parser_benchmark_score_detail import ( + ParserBenchmarkScoreDetail, + ) + accuracy_score: DF.Percent ai_model: DF.Literal[ "DeepSeek Chat", @@ -35,7 +39,6 @@ class ParserBenchmarkLog(Document): currency: DF.Link | None dataset: DF.Link error: DF.Code | None - field_mismatches: DF.Code | None file_content: DF.Code | None file_parse_memory: DF.Float file_parse_time: DF.Float @@ -48,6 +51,7 @@ class ParserBenchmarkLog(Document): party_type: DF.Link | None pdf_processor: DF.Literal["", "OCRMyPDF", "Docling"] prompt_tokens: DF.Int + score_details: DF.Table[ParserBenchmarkScoreDetail] status: DF.Literal["Queued", "Running", "Completed", "Failed"] total_cost: DF.Currency total_time: DF.Float diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_score_detail/__init__.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_score_detail/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_score_detail/parser_benchmark_score_detail.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_score_detail/parser_benchmark_score_detail.json new file mode 100644 index 0000000..b8fd5d6 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_score_detail/parser_benchmark_score_detail.json @@ -0,0 +1,76 @@ +{ + "actions": [], + "creation": "2026-03-27 00:00:00", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "key", + "total", + "column_break_jhbx", + "matched", + "accuracy", + "section_break_zimm", + "mismatches" + ], + "fields": [ + { + "fieldname": "key", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Key", + "read_only": 1, + "reqd": 1 + }, + { + "description": "Number of leaf fields whose value matched the expected result", + "fieldname": "matched", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Matched", + "read_only": 1 + }, + { + "description": "Total number of leaf fields compared for this key", + "fieldname": "total", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Total", + "read_only": 1 + }, + { + "fieldname": "accuracy", + "fieldtype": "Percent", + "in_list_view": 1, + "label": "Accuracy (%)", + "read_only": 1 + }, + { + "description": "List of fields that did not match, with expected and actual values", + "fieldname": "mismatches", + "fieldtype": "Code", + "label": "Mismatches", + "options": "JSON", + "read_only": 1 + }, + { + "fieldname": "column_break_jhbx", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_zimm", + "fieldtype": "Section Break" + } + ], + "istable": 1, + "links": [], + "modified": "2026-03-27 12:30:17.720466", + "modified_by": "Administrator", + "module": "Parser Benchmark", + "name": "Parser Benchmark Score Detail", + "owner": "Administrator", + "permissions": [], + "row_format": "Dynamic", + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_score_detail/parser_benchmark_score_detail.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_score_detail/parser_benchmark_score_detail.py new file mode 100644 index 0000000..d70cb77 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_score_detail/parser_benchmark_score_detail.py @@ -0,0 +1,26 @@ +# Copyright (c) 2026, Resilient Tech and contributors +# For license information, please see license.txt + +from frappe.model.document import Document + + +class ParserBenchmarkScoreDetail(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + accuracy: DF.Percent + key: DF.Data + matched: DF.Int + mismatches: DF.Code | None + parent: DF.Data + parentfield: DF.Data + parenttype: DF.Data + total: DF.Int + # end: auto-generated types + + pass diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json index d588b26..6eda028 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.json @@ -17,7 +17,9 @@ "wednesday", "saturday", "section_break_kvgf", - "token_costs" + "token_costs", + "key_weights_section", + "key_weights" ], "fields": [ { @@ -83,7 +85,6 @@ "fieldtype": "Column Break" }, { - "depends_on": "eval: doc.enabled", "fieldname": "section_break_kvgf", "fieldtype": "Section Break" }, @@ -95,12 +96,22 @@ "default": "0", "fieldname": "enabled", "fieldtype": "Check", - "label": "Enabled" + "label": "Scheduling Enabled" + }, + { + "fieldname": "key_weights_section", + "fieldtype": "Section Break" + }, + { + "fieldname": "key_weights", + "fieldtype": "Table", + "label": "Key Weights", + "options": "Parser Benchmark Key Weight" } ], "issingle": 1, "links": [], - "modified": "2026-03-25 12:53:24.601045", + "modified": "2026-03-27 12:19:19.982234", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Settings", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py index 4ca20d1..0db8c50 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py @@ -25,12 +25,16 @@ class ParserBenchmarkSettings(Document): if TYPE_CHECKING: from frappe.types import DF + from transaction_parser.parser_benchmark.doctype.parser_benchmark_key_weight.parser_benchmark_key_weight import ( + ParserBenchmarkKeyWeight, + ) from transaction_parser.parser_benchmark.doctype.parser_benchmark_token_cost.parser_benchmark_token_cost import ( ParserBenchmarkTokenCost, ) enabled: DF.Check friday: DF.Check + key_weights: DF.Table[ParserBenchmarkKeyWeight] monday: DF.Check saturday: DF.Check sunday: DF.Check diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json index 5321a7c..ce0da61 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_token_cost/parser_benchmark_token_cost.json @@ -56,7 +56,7 @@ ], "istable": 1, "links": [], - "modified": "2026-03-18 13:39:47.449246", + "modified": "2026-03-27 10:36:00.699499", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Token Cost", diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py index fd57500..307abfc 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py @@ -1,8 +1,11 @@ # Copyright (c) 2026, Resilient Tech and contributors # For license information, please see license.txt -import json -from collections import Counter, defaultdict +# TODO: Need to refactor and Test +# TODO: Need to add, what does getting mismatched!!! +# TODO: Need to create a other report as well like comparing with commits + +from collections import defaultdict from enum import StrEnum import frappe @@ -59,9 +62,7 @@ class Col(StrEnum): CURRENCY = "currency" DATASET = "dataset" RUN_COUNT = "run_count" - MISMATCH_RATE = "mismatch_rate" - UNIQUE_MISMATCHES = "unique_mismatches" - TOP_MISMATCHES = "top_mismatches" + KEY_SCORES = "key_scores" # Fields averaged in party-group summary rows @@ -89,10 +90,13 @@ class AccuracyAnalysisReport: def __init__(self, filters: frappe._dict): self.filters = filters self._set_party_type() - self.group_by_party = True # Always group — even for a single party + self.group_by_party = True def run(self): - self.data = [self._build_row(r) for r in self._fetch_logs()] + logs = self._fetch_logs() + score_details_map = self._fetch_score_details([r.log_name for r in logs]) + + self.data = [self._build_row(r, score_details_map) for r in logs] self._aggregate_by_config() if self.group_by_party: @@ -100,7 +104,7 @@ def run(self): # strip internal keys before sending to client for row in self.data: - row.pop("_mismatch_fields", None) + row.pop("_key_accuracies", None) return self._get_columns(), self.data @@ -211,22 +215,10 @@ def _get_columns(self): "width": 110, }, { - "fieldname": Col.MISMATCH_RATE, - "label": _("Mismatch Rate (%)"), - "fieldtype": "Percent", - "width": 140, - }, - { - "fieldname": Col.UNIQUE_MISMATCHES, - "label": _("Unique Fields"), - "fieldtype": "Int", - "width": 110, - }, - { - "fieldname": Col.TOP_MISMATCHES, - "label": _("Top Mismatched Fields"), + "fieldname": Col.KEY_SCORES, + "label": _("Key Scores"), "fieldtype": "Data", - "width": 300, + "width": 350, }, ] @@ -247,6 +239,7 @@ def _fetch_logs(self): .left_join(supp) .on((ds.party_type == "Supplier") & (ds.party == supp.name)) .select( + log.name.as_("log_name"), log.ai_model, log.pdf_processor, log.accuracy_score, @@ -259,7 +252,6 @@ def _fetch_logs(self): log.completion_tokens, log.total_tokens, log.currency, - log.field_mismatches, log.dataset, ds.party, ds.file_type, @@ -294,6 +286,26 @@ def _fetch_logs(self): return query.run(as_dict=True) + def _fetch_score_details(self, log_names: list[str]) -> dict[str, list[dict]]: + """Fetch score_details child rows for all logs at once.""" + if not log_names: + return {} + + sd = frappe.qb.DocType("Parser Benchmark Score Detail") + rows = ( + frappe.qb.from_(sd) + .select(sd.parent, sd.key, sd.matched, sd.total, sd.accuracy) + .where(sd.parent.isin(log_names)) + .orderby(sd.idx) + .run(as_dict=True) + ) + + details_map: dict[str, list[dict]] = defaultdict(list) + for row in rows: + details_map[row.parent].append(row) + + return details_map + # ── Helpers ────────────────────────────────────────────────────── def _set_party_type(self): @@ -302,16 +314,15 @@ def _set_party_type(self): if transaction_type and not self.filters.get("party_type"): self.filters["party_type"] = PARTY_TYPE_MAP.get(transaction_type) - def _build_row(self, r): + def _build_row(self, r, score_details_map): """Build a single detail row from a log record.""" - mismatches = self._parse_mismatches(r.field_mismatches) - mismatch_fields = [self._short_field_name(m["field"]) for m in mismatches] + details = score_details_map.get(r.log_name, []) + key_accuracies = {d["key"]: d["accuracy"] for d in details} - if mismatch_fields: - top = Counter(mismatch_fields).most_common(5) - top_str = ", ".join(f"{name} x{cnt}" for name, cnt in top) + if key_accuracies: + key_str = ", ".join(f"{k}: {v:.0f}%" for k, v in key_accuracies.items()) else: - top_str = "" + key_str = "" return { Col.PARTY: r.party or _("No Party"), @@ -330,40 +341,14 @@ def _build_row(self, r): Col.COMPLETION_TOKENS: r.completion_tokens, Col.TOTAL_TOKENS: r.total_tokens, Col.CURRENCY: r.currency, - Col.MISMATCH_RATE: round(100 - (r.accuracy_score or 0), 2), - Col.UNIQUE_MISMATCHES: len(set(mismatch_fields)), - Col.TOP_MISMATCHES: top_str, - "_mismatch_fields": mismatch_fields, + Col.KEY_SCORES: key_str, + "_key_accuracies": key_accuracies, } - @staticmethod - def _parse_mismatches(raw) -> list[dict]: - """Parse field_mismatches JSON string into a list.""" - if not raw: - return [] - try: - data = json.loads(raw) if isinstance(raw, str) else raw - return data if isinstance(data, list) else [] - except (json.JSONDecodeError, TypeError): - return [] - - @staticmethod - def _short_field_name(field: str) -> str: - """Extract a readable field name from DeepDiff path like root['items'][0]['qty'].""" - import re - - keys = re.findall(r"\['(.*?)'\]", field) - return ".".join(keys) if keys else field - # ── Aggregation ────────────────────────────────────────────────── def _aggregate_by_config(self): - """Collapse multiple runs of the same config into one averaged row. - - Groups by (party, ai_model, pdf_processor, file_type) and averages - numeric fields. The Dataset column shows the latest dataset if all - runs share one, otherwise left blank. - """ + """Collapse multiple runs of the same config into one averaged row.""" if not self.data: return @@ -390,7 +375,6 @@ def _aggregate_by_config(self): Col.RUN_COUNT: count, } - # unique datasets — show if only one, else blank agg[Col.DATASET] = rows[0].get(Col.DATASET, "") for field in _AVG_FIELDS: @@ -400,25 +384,22 @@ def _aggregate_by_config(self): for field in _SUM_FIELDS: agg[field] = round(sum(r.get(field) or 0 for r in rows) / count, 6) - # aggregate mismatches - all_fields = [] + # aggregate per-key accuracies + all_key_accs: dict[str, list[float]] = defaultdict(list) for r in rows: - all_fields.extend(r.get("_mismatch_fields", [])) + for k, v in r.get("_key_accuracies", {}).items(): + all_key_accs[k].append(v or 0) - agg[Col.MISMATCH_RATE] = round(100 - (agg.get(Col.ACCURACY_SCORE) or 0), 2) - agg[Col.UNIQUE_MISMATCHES] = round( - sum(r.get(Col.UNIQUE_MISMATCHES, 0) for r in rows) / count + avg_key_accs = { + k: round(sum(v) / len(v), 1) for k, v in all_key_accs.items() + } + agg[Col.KEY_SCORES] = ( + ", ".join(f"{k}: {v:.0f}%" for k, v in avg_key_accs.items()) + if avg_key_accs + else "" ) + agg["_key_accuracies"] = avg_key_accs - if all_fields: - top = Counter(all_fields).most_common(5) - agg[Col.TOP_MISMATCHES] = ", ".join( - f"{name} x{cnt}" for name, cnt in top - ) - else: - agg[Col.TOP_MISMATCHES] = "" - - agg["_mismatch_fields"] = all_fields aggregated.append(agg) self.data = aggregated @@ -471,20 +452,17 @@ def _group_row(self, party, rows): for field in _SUM_FIELDS: row[field] = round(sum(r.get(field) or 0 for r in rows), 6) - # aggregate mismatch info across all child rows - all_fields = [] + # aggregate per-key accuracies across all child rows + all_key_accs: dict[str, list[float]] = defaultdict(list) for r in rows: - all_fields.extend(r.get("_mismatch_fields", [])) - - row[Col.MISMATCH_RATE] = round(100 - (row.get(Col.ACCURACY_SCORE) or 0), 2) - row[Col.UNIQUE_MISMATCHES] = round( - sum(r.get(Col.UNIQUE_MISMATCHES, 0) for r in rows) / count + for k, v in r.get("_key_accuracies", {}).items(): + all_key_accs[k].append(v or 0) + + avg_key_accs = {k: round(sum(v) / len(v), 1) for k, v in all_key_accs.items()} + row[Col.KEY_SCORES] = ( + ", ".join(f"{k}: {v:.0f}%" for k, v in avg_key_accs.items()) + if avg_key_accs + else "" ) - if all_fields: - top = Counter(all_fields).most_common(5) - row[Col.TOP_MISMATCHES] = ", ".join(f"{name} x{cnt}" for name, cnt in top) - else: - row[Col.TOP_MISMATCHES] = "" - return row diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index 886c29f..96e6e72 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -30,8 +30,10 @@ class BenchmarkRunner: 2. AI parsing → time, tokens, cost, AI response """ + # TODO: can pass settings / dataset here? def __init__(self, log_name: str): self.log: ParserBenchmarkLog = frappe.get_doc("Parser Benchmark Log", log_name) + # TODO: can use cached doc... self.dataset: ParserBenchmarkDataset = frappe.get_doc( "Parser Benchmark Dataset", self.log.dataset ) @@ -182,17 +184,44 @@ def _calculate_cost(self): def _score_response(self, ai_content: dict): from transaction_parser.parser_benchmark.scorer import score_response - expected = self.dataset.expected_result - if not expected: + expected_fields = self.dataset.expected_fields + if not expected_fields: return - if isinstance(expected, str): - expected = frappe.parse_json(expected) + weights = self._get_key_weights() result = score_response( - expected, - ai_content, - significant_digits=self.significant_digits, + expected_fields=[ + {"key": row.key, "expected_json": row.expected_json} + for row in expected_fields + ], + actual=ai_content, + weights=weights, + precision=self.significant_digits, ) - self.log.accuracy_score = result["accuracy_score"] - self.log.field_mismatches = frappe.as_json(result["mismatches"], indent=2) + + self.log.accuracy_score = result["overall_accuracy"] + + for detail in result["details"]: + self.log.append( + "score_details", + { + "key": detail["key"], + "matched": detail["matched"], + "total": detail["total"], + "accuracy": detail["accuracy"], + "mismatches": frappe.as_json(detail["mismatches"], indent=2) + if detail["mismatches"] + else None, + }, + ) + + # TODO: settings can be in init... + def _get_key_weights(self) -> dict[str, float]: + """Load key weights from Parser Benchmark Settings.""" + try: + settings = frappe.get_cached_doc("Parser Benchmark Settings") + except Exception: + return {} + + return {row.key: row.weight for row in (settings.key_weights or [])} diff --git a/transaction_parser/parser_benchmark/scorer.py b/transaction_parser/parser_benchmark/scorer.py index 2e3d9dd..3ee081c 100644 --- a/transaction_parser/parser_benchmark/scorer.py +++ b/transaction_parser/parser_benchmark/scorer.py @@ -1,111 +1,179 @@ import frappe -from deepdiff import DeepDiff from frappe.utils import flt -def _normalize_empty(obj): - """Recursively convert empty strings to None so `""` vs `None` is not a mismatch. +def _normalize(obj): + """Recursively normalize values for comparison. - Leaves `0`, `False`, and other falsy values untouched. + - Empty strings → None + - Strings → stripped and lowercased + - frappe._dict → plain dict """ if isinstance(obj, dict): - return {k: _normalize_empty(v) for k, v in obj.items()} + return {k: _normalize(v) for k, v in obj.items()} if isinstance(obj, list): - return [_normalize_empty(v) for v in obj] + return [_normalize(v) for v in obj] if obj == "": return None + if isinstance(obj, str): + return obj.strip().lower() + return obj +def _compare_scalar( + expected, actual, path: str, precision: int +) -> tuple[int, int, list]: + """Compare two scalar (non-dict, non-list) values. + + Returns (matched, total, mismatches). + """ + if expected is None and actual is None: + return 1, 1, [] + + if expected is None or actual is None: + return 0, 1, [{"field": path, "expected": expected, "actual": actual}] + + # numeric comparison with tolerance + if isinstance(expected, int | float) and isinstance(actual, int | float): + if flt(expected, precision) == flt(actual, precision): + return 1, 1, [] + return 0, 1, [{"field": path, "expected": expected, "actual": actual}] + + # string comparison (already lowered by _normalize) + if str(expected) == str(actual): + return 1, 1, [] + + return 0, 1, [{"field": path, "expected": expected, "actual": actual}] + + +def _compare(expected, actual, path: str, precision: int) -> tuple[int, int, list]: + """Recursively compare expected vs actual, counting leaf field matches. + + Only keys/indices present in `expected` are scored — extra keys in + `actual` are ignored. Lists are compared index-by-index (order matters). + + Returns (matched, total, mismatches). + """ + if isinstance(expected, dict): + matched = total = 0 + mismatches = [] + + for key, exp_val in expected.items(): + child_path = f"{path}.{key}" if path else key + act_val = actual.get(key) if isinstance(actual, dict) else None + m, t, mm = _compare(exp_val, act_val, child_path, precision) + matched += m + total += t + mismatches.extend(mm) + + return matched, total, mismatches + + if isinstance(expected, list): + matched = total = 0 + mismatches = [] + actual_list = actual if isinstance(actual, list) else [] + + for idx, exp_item in enumerate(expected): + child_path = f"{path}[{idx}]" + act_item = actual_list[idx] if idx < len(actual_list) else None + + if act_item is None: + # missing actual item — count all leaves in expected as mismatched + m, t, mm = _compare(exp_item, None, child_path, precision) + mismatches.extend(mm) + else: + m, t, mm = _compare(exp_item, act_item, child_path, precision) + mismatches.extend(mm) + + matched += m + total += t + + return matched, total, mismatches + + # scalar + return _compare_scalar(expected, actual, path, precision) + + +def score_key(expected, actual, key: str, precision: int = 2) -> dict: + """Score a single top-level key. + + Args: + expected: The expected value (parsed JSON) for this key. + actual: The actual AI response value for this key. + key: The key name (used as path prefix in mismatch reports). + precision: Decimal precision for numeric comparisons. + + Returns: + {"key": str, "matched": int, "total": int, "accuracy": float, "mismatches": list} + """ + exp_normalized = _normalize(expected) + act_normalized = _normalize(actual) + + matched, total, mismatches = _compare( + exp_normalized, act_normalized, key, precision + ) + + accuracy = flt((matched / total) * 100, 2) if total else 100.0 + + return { + "key": key, + "matched": matched, + "total": total, + "accuracy": accuracy, + "mismatches": mismatches, + } + + def score_response( - expected: dict, + expected_fields: list[dict], actual: dict, *, - max_diffs: int = 500, - significant_digits: int = 2, + weights: dict[str, float] | None = None, + precision: int = 2, ) -> dict: - """Compare AI response against expected result using DeepDiff.""" - expected = _normalize_empty(expected) - actual = _normalize_empty(actual) - - diff = DeepDiff( - expected, - actual, - ignore_string_case=True, - ignore_numeric_type_changes=True, - ignore_type_in_groups=[(dict, frappe._dict)], - significant_digits=significant_digits, - verbose_level=2, - max_diffs=max_diffs, - log_frequency_in_sec=0, - get_deep_distance=True, - ) + """Score AI response against expected fields with per-key breakdown. + + Args: + expected_fields: List of {"key": str, "expected_json": str|dict} rows + from the Dataset child table. + actual: The full AI response dict. + weights: {key_name: weight} from Settings. Defaults to 1 for all keys. + precision: Decimal precision for numeric comparisons. + + Returns: + {"overall_accuracy": float, "details": list[dict]} + where each detail is the output of score_key(). + """ + weights = weights or {} + details = [] - distance = diff.get("deep_distance", 0) - accuracy = flt((1 - distance) * 100, 2) + weighted_matched = 0.0 + weighted_total = 0.0 - return { - "accuracy_score": accuracy, - "mismatches": _format_mismatches(diff), - } + for row in expected_fields: + key = row["key"] + expected = row["expected_json"] + + if isinstance(expected, str): + expected = frappe.parse_json(expected) + actual_value = actual.get(key) if isinstance(actual, dict) else None + result = score_key(expected, actual_value, key, precision) + details.append(result) -_CHANGED_TYPES = {"values_changed", "type_changes"} -_REMOVED_TYPES = {"dictionary_item_removed", "iterable_item_removed"} -_ADDED_TYPES = {"dictionary_item_added", "iterable_item_added"} -_HANDLED_TYPES = _CHANGED_TYPES | _REMOVED_TYPES | _ADDED_TYPES | {"deep_distance"} - - -def _format_mismatches(diff: DeepDiff) -> list[dict]: - """Flatten DeepDiff into a simple list of {field, expected, actual}.""" - mismatches = [] - - for path, change in diff.get("values_changed", {}).items(): - mismatches.append( - { - "field": path, - "expected": change["old_value"], - "actual": change["new_value"], - } - ) - - for path, change in diff.get("type_changes", {}).items(): - mismatches.append( - { - "field": path, - "expected": change["old_value"], - "actual": change["new_value"], - } - ) - - for path, val in diff.get("dictionary_item_removed", {}).items(): - mismatches.append({"field": path, "expected": val, "actual": None}) - - for path, val in diff.get("dictionary_item_added", {}).items(): - mismatches.append({"field": path, "expected": None, "actual": val}) - - for path, val in diff.get("iterable_item_removed", {}).items(): - mismatches.append({"field": path, "expected": val, "actual": None}) - - for path, val in diff.get("iterable_item_added", {}).items(): - mismatches.append({"field": path, "expected": None, "actual": val}) - - # catch-all for unhandled DeepDiff change types - for change_type, changes in diff.items(): - if change_type in _HANDLED_TYPES: - continue - - if isinstance(changes, dict): - for path, val in changes.items(): - mismatches.append( - {"field": f"{change_type}: {path}", "expected": None, "actual": val} - ) - else: - mismatches.append( - {"field": change_type, "expected": None, "actual": str(changes)} - ) - - return mismatches + w = weights.get(key, 1.0) + weighted_matched += result["matched"] * w + weighted_total += result["total"] * w + + overall_accuracy = ( + flt((weighted_matched / weighted_total) * 100, 2) if weighted_total else 0.0 + ) + + return { + "overall_accuracy": overall_accuracy, + "details": details, + } From 32810fa8e3cf6ca0749a10e9d6bccf37186af428 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Mon, 30 Mar 2026 12:21:02 +0530 Subject: [PATCH 64/88] refactor: add validation for key weights to prevent duplicates --- .../parser_benchmark_settings.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py index 0db8c50..103e2e8 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_settings/parser_benchmark_settings.py @@ -2,6 +2,7 @@ # For license information, please see license.txt import frappe +from frappe import _ from frappe.model.document import Document from frappe.utils import getdate @@ -44,6 +45,23 @@ class ParserBenchmarkSettings(Document): wednesday: DF.Check # end: auto-generated types + def validate(self): + self.validate_wights() + + def validate_wights(self): + if not self.key_weights: + return + + seen_keys = set() + for row in self.key_weights: + if row.key in seen_keys: + frappe.throw( + _("Duplicate key '{0}' in Key Weights row {1}").format( + row.key, row.idx + ) + ) + seen_keys.add(row.key) + def is_scheduled_today(self) -> bool: """Check if today's weekday is enabled in the schedule.""" today_index = getdate().weekday() # 0 = Monday From 61964e7bd6963a1d2f764d119e7e65a4b47e02a0 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Mon, 30 Mar 2026 12:50:38 +0530 Subject: [PATCH 65/88] refactor: replace hardcoded DOCTYPE with constant and improve BenchmarkRunner initialization --- .../parser_benchmark_dataset.py | 19 +++++++----- transaction_parser/parser_benchmark/runner.py | 31 +++++++------------ 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 6d0ca6e..861a245 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -22,6 +22,8 @@ "docling": "Docling", } +DOCTYPE = "Parser Benchmark Dataset" + class ParserBenchmarkDataset(Document): # begin: auto-generated types @@ -131,9 +133,9 @@ def get_selected_processors(self) -> list[str]: @frappe.whitelist() def run_benchmark(dataset_name: str): """Create Benchmark Logs for each model x processor combo and enqueue runs.""" - frappe.has_permission("Parser Benchmark Dataset", "write", throw=True) + frappe.has_permission(DOCTYPE, "write", throw=True) - if frappe.db.get_value("Parser Benchmark Dataset", dataset_name, "docstatus") != 1: + if frappe.db.get_value(DOCTYPE, dataset_name, "docstatus") != 1: frappe.throw(_("Dataset must be submitted before running benchmarks.")) log_names = create_and_enqueue_benchmark_logs(dataset_name) @@ -150,12 +152,10 @@ def run_benchmark(dataset_name: str): def create_and_enqueue_benchmark_logs(dataset_name: str) -> list[str]: """Create one log per model x processor combo and enqueue each for background execution.""" - dataset: ParserBenchmarkDataset = frappe.get_cached_doc( - "Parser Benchmark Dataset", dataset_name - ) + dataset: ParserBenchmarkDataset = frappe.get_cached_doc(DOCTYPE, dataset_name) models = dataset.get_selected_models() processors = ( - dataset.get_selected_processors() or [None] + (dataset.get_selected_processors() or [None]) if dataset.file_type == "PDF" else [None] ) @@ -182,10 +182,13 @@ def create_and_enqueue_benchmark_logs(dataset_name: str) -> list[str]: # commit before enqueuing so background jobs can read the inserted logs frappe.db.commit() # nosemgrep - # TODO: pass a dataset doc and the settings... for log_name in log_names: try: - frappe.enqueue(_run_benchmark, log_name=log_name, queue="long") + frappe.enqueue( + _run_benchmark, + log_name=log_name, + queue="long", + ) except Exception: frappe.db.set_value("Parser Benchmark Log", log_name, "status", "Failed") frappe.db.commit() # nosemgrep -- persist Failed status when enqueue fails diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index 96e6e72..716fffe 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -30,13 +30,19 @@ class BenchmarkRunner: 2. AI parsing → time, tokens, cost, AI response """ - # TODO: can pass settings / dataset here? - def __init__(self, log_name: str): + def __init__( + self, + log_name: str, + dataset: ParserBenchmarkDataset | None = None, + settings: ParserBenchmarkSettings | None = None, + ): self.log: ParserBenchmarkLog = frappe.get_doc("Parser Benchmark Log", log_name) - # TODO: can use cached doc... - self.dataset: ParserBenchmarkDataset = frappe.get_doc( + self.dataset: ParserBenchmarkDataset = dataset or frappe.get_cached_doc( "Parser Benchmark Dataset", self.log.dataset ) + self.settings: ParserBenchmarkSettings = settings or frappe.get_cached_doc( + "Parser Benchmark Settings" + ) self.precision = 6 # to get 1-millionth of a token cost # for accuracy score @@ -89,14 +95,7 @@ def _get_controller(self, file_doc: File) -> Transaction: return controller def _get_cost_row(self): - try: - settings: ParserBenchmarkSettings = frappe.get_cached_doc( - "Parser Benchmark Settings" - ) - except Exception: - return None - - for row in settings.token_costs: + for row in self.settings.token_costs: if row.ai_model == self.log.ai_model: return row @@ -216,12 +215,6 @@ def _score_response(self, ai_content: dict): }, ) - # TODO: settings can be in init... def _get_key_weights(self) -> dict[str, float]: """Load key weights from Parser Benchmark Settings.""" - try: - settings = frappe.get_cached_doc("Parser Benchmark Settings") - except Exception: - return {} - - return {row.key: row.weight for row in (settings.key_weights or [])} + return {row.key: row.weight for row in (self.settings.key_weights or [])} From 70a54ff5c8c59df5238243c18a9826b2c308a04d Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Mon, 30 Mar 2026 13:28:39 +0530 Subject: [PATCH 66/88] feat: add Transaction Parser Version Comparison report --- .../__init__.py | 0 .../transaction_parser_version_comparison.js | 96 +++++ ...transaction_parser_version_comparison.json | 28 ++ .../transaction_parser_version_comparison.py | 387 ++++++++++++++++++ 4 files changed, 511 insertions(+) create mode 100644 transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/__init__.py create mode 100644 transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js create mode 100644 transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.json create mode 100644 transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/__init__.py b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js new file mode 100644 index 0000000..3b199bc --- /dev/null +++ b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js @@ -0,0 +1,96 @@ +// Copyright (c) 2026, Resilient Tech and contributors +// For license information, please see license.txt + +const AI_MODELS = [ + "DeepSeek Chat", + "DeepSeek Reasoner", + "OpenAI gpt-4o", + "OpenAI gpt-4o-mini", + "OpenAI gpt-5", + "OpenAI gpt-5-mini", + "Google Gemini Pro-2.5", + "Google Gemini Flash-2.5", +]; + +const FILE_TYPES = ["PDF", "CSV", "XLSX", "XLS"]; + +const PDF_PROCESSORS = ["OCRMyPDF", "Docling"]; + +const PARTY_TYPE_MAP = { + "Sales Order": "Customer", + Expense: "Supplier", +}; + +function make_options(items, txt) { + return items + .filter((v) => !txt || v.toLowerCase().includes(txt.toLowerCase())) + .map((v) => ({ value: v, description: "" })); +} + +frappe.query_reports["Transaction Parser Version Comparison"] = { + tree: true, + initial_depth: 1, + + onload(report) { + set_party_type(report); + }, + + filters: [ + { + fieldname: "company", + label: __("Company"), + fieldtype: "Link", + options: "Company", + reqd: 1, + default: frappe.defaults.get_user_default("Company"), + }, + { + fieldname: "transaction_type", + label: __("Transaction Type"), + fieldtype: "Select", + options: "\nSales Order\nExpense", + reqd: 1, + default: "Sales Order", + on_change() { + set_party_type(frappe.query_report); + }, + }, + { + fieldname: "party_type", + label: __("Party Type"), + fieldtype: "Link", + options: "DocType", + hidden: 1, + }, + { + fieldname: "party", + label: __("Party"), + fieldtype: "Dynamic Link", + options: "party_type", + }, + { + fieldname: "file_type", + label: __("File Type"), + fieldtype: "MultiSelectList", + get_data: (txt) => make_options(FILE_TYPES, txt), + }, + { + fieldname: "ai_model", + label: __("AI Model"), + fieldtype: "MultiSelectList", + get_data: (txt) => make_options(AI_MODELS, txt), + }, + { + fieldname: "pdf_processor", + label: __("PDF Processor"), + fieldtype: "MultiSelectList", + get_data: (txt) => make_options(PDF_PROCESSORS, txt), + }, + ], +}; + +function set_party_type(report) { + const transaction_type = report.get_filter_value("transaction_type"); + const party_type = PARTY_TYPE_MAP[transaction_type] || ""; + report.set_filter_value("party_type", party_type); +} diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.json b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.json new file mode 100644 index 0000000..a0fbf68 --- /dev/null +++ b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.json @@ -0,0 +1,28 @@ +{ + "add_total_row": 0, + "add_translate_data": 0, + "columns": [], + "creation": "2026-03-30 00:00:00", + "disabled": 0, + "docstatus": 0, + "doctype": "Report", + "filters": [], + "idx": 0, + "is_standard": "Yes", + "letter_head": null, + "modified": "2026-03-30 09:38:02.128996", + "modified_by": "Administrator", + "module": "Parser Benchmark", + "name": "Transaction Parser Version Comparison", + "owner": "Administrator", + "prepared_report": 0, + "ref_doctype": "Parser Benchmark Log", + "report_name": "Transaction Parser Version Comparison", + "report_type": "Script Report", + "roles": [ + { + "role": "System Manager" + } + ], + "timeout": 0 +} \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py new file mode 100644 index 0000000..56f0792 --- /dev/null +++ b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py @@ -0,0 +1,387 @@ +# Copyright (c) 2026, Resilient Tech and contributors +# For license information, please see license.txt + +from collections import defaultdict +from enum import StrEnum + +import frappe +from frappe import _ +from frappe.query_builder.functions import Coalesce + +PARTY_TYPE_MAP = { + "Sales Order": "Customer", + "Expense": "Supplier", +} + +# Sorting order for child rows within each party group +_AI_MODEL_ORDER = { + "OpenAI gpt-5": 0, + "OpenAI gpt-5-mini": 1, + "OpenAI gpt-4o": 2, + "OpenAI gpt-4o-mini": 3, + "Google Gemini Pro-2.5": 4, + "Google Gemini Flash-2.5": 5, + "DeepSeek Reasoner": 6, + "DeepSeek Chat": 7, +} + +_PDF_PROCESSOR_ORDER = { + "OCRMyPDF": 0, + "Docling": 1, +} + +_FILE_TYPE_ORDER = { + "PDF": 0, + "CSV": 1, + "XLSX": 2, + "XLS": 3, +} + + +class Col(StrEnum): + """Column fieldnames — single source of truth for the report.""" + + PARTY = "party" + PARTY_NAME = "party_name" + DATASET = "dataset" + AI_MODEL = "ai_model" + PDF_PROCESSOR = "pdf_processor" + FILE_TYPE = "file_type" + COMMIT_HASH = "commit_hash" + COMMIT_MESSAGE = "commit_message" + ACCURACY_SCORE = "accuracy_score" + KEY_SCORES = "key_scores" + RUN_COUNT = "run_count" + + +def execute(filters=None): + filters = frappe._dict(filters or {}) + return VersionComparisonReport(filters).run() + + +class VersionComparisonReport: + def __init__(self, filters: frappe._dict): + self.filters = filters + self._set_party_type() + + def run(self): + logs = self._fetch_logs() + if not logs: + return self._get_columns(), [] + + score_details_map = self._fetch_score_details([r.log_name for r in logs]) + + self.data = [self._build_row(r, score_details_map) for r in logs] + self._aggregate_by_config() + self._group_by_party() + + # strip internal keys + for row in self.data: + row.pop("_key_accuracies", None) + + return self._get_columns(), self.data + + # ── Columns ────────────────────────────────────────────────────── + + def _get_columns(self): + return [ + { + "fieldname": Col.PARTY, + "label": _("Party"), + "fieldtype": "Data", + "width": 200, + }, + { + "fieldname": Col.PARTY_NAME, + "label": _("Party Name"), + "fieldtype": "Data", + "width": 200, + }, + { + "fieldname": Col.DATASET, + "label": _("Dataset"), + "fieldtype": "Link", + "options": "Parser Benchmark Dataset", + "width": 160, + }, + { + "fieldname": Col.AI_MODEL, + "label": _("AI Model"), + "fieldtype": "Data", + "width": 180, + }, + { + "fieldname": Col.PDF_PROCESSOR, + "label": _("Processor"), + "fieldtype": "Data", + "width": 110, + }, + { + "fieldname": Col.FILE_TYPE, + "label": _("File Type"), + "fieldtype": "Data", + "width": 90, + }, + { + "fieldname": Col.COMMIT_HASH, + "label": _("Commit"), + "fieldtype": "Data", + "width": 100, + }, + { + "fieldname": Col.COMMIT_MESSAGE, + "label": _("Commit Message"), + "fieldtype": "Data", + "width": 250, + }, + { + "fieldname": Col.RUN_COUNT, + "label": _("Runs"), + "fieldtype": "Int", + "width": 60, + }, + { + "fieldname": Col.ACCURACY_SCORE, + "label": _("Accuracy (%)"), + "fieldtype": "Percent", + "width": 120, + }, + { + "fieldname": Col.KEY_SCORES, + "label": _("Key Scores"), + "fieldtype": "Data", + "width": 350, + }, + ] + + # ── Query ──────────────────────────────────────────────────────── + + def _fetch_logs(self): + log = frappe.qb.DocType("Parser Benchmark Log") + ds = frappe.qb.DocType("Parser Benchmark Dataset") + cust = frappe.qb.DocType("Customer") + supp = frappe.qb.DocType("Supplier") + + query = ( + frappe.qb.from_(log) + .join(ds) + .on(log.dataset == ds.name) + .left_join(cust) + .on((ds.party_type == "Customer") & (ds.party == cust.name)) + .left_join(supp) + .on((ds.party_type == "Supplier") & (ds.party == supp.name)) + .select( + log.name.as_("log_name"), + log.ai_model, + log.pdf_processor, + log.accuracy_score, + log.dataset, + log.commit_hash, + log.commit_message, + ds.party, + ds.file_type, + Coalesce(cust.customer_name, supp.supplier_name, ds.party).as_( + "party_name" + ), + ) + .where(log.status == "Completed") + .where(Coalesce(log.commit_hash, "") != "") + .orderby(ds.party, log.ai_model, ds.file_type) + ) + + # exact-match filters + for column, key in ( + (ds.company, "company"), + (ds.transaction_type, "transaction_type"), + (ds.party_type, "party_type"), + (ds.party, "party"), + ): + if self.filters.get(key): + query = query.where(column == self.filters[key]) + + # multi-select IN filters + for column, key in ( + (ds.file_type, "file_type"), + (log.ai_model, "ai_model"), + (log.pdf_processor, "pdf_processor"), + ): + values = self.filters.get(key) + if values: + items = values if isinstance(values, list) else [values] + query = query.where(column.isin(items)) + + return query.run(as_dict=True) + + def _fetch_score_details(self, log_names: list[str]) -> dict[str, list[dict]]: + """Fetch score_details child rows for all logs at once.""" + if not log_names: + return {} + + sd = frappe.qb.DocType("Parser Benchmark Score Detail") + rows = ( + frappe.qb.from_(sd) + .select(sd.parent, sd.key, sd.matched, sd.total, sd.accuracy) + .where(sd.parent.isin(log_names)) + .orderby(sd.idx) + .run(as_dict=True) + ) + + details_map: dict[str, list[dict]] = defaultdict(list) + for row in rows: + details_map[row.parent].append(row) + + return details_map + + # ── Helpers ────────────────────────────────────────────────────── + + def _set_party_type(self): + """Derive party_type from transaction_type when not explicitly set.""" + transaction_type = self.filters.get("transaction_type") + if transaction_type and not self.filters.get("party_type"): + self.filters["party_type"] = PARTY_TYPE_MAP.get(transaction_type) + + def _build_row(self, r, score_details_map): + """Build a single detail row from a log record.""" + details = score_details_map.get(r.log_name, []) + key_accuracies = {d["key"]: d["accuracy"] for d in details} + + key_str = ( + ", ".join(f"{k}: {v:.0f}%" for k, v in key_accuracies.items()) + if key_accuracies + else "" + ) + + short_hash = (r.commit_hash or "")[:7] + commit_msg = (r.commit_message or "").split("\n")[0][:80] + + return { + Col.PARTY: r.party or _("No Party"), + Col.PARTY_NAME: r.party_name or "", + Col.DATASET: r.dataset, + Col.AI_MODEL: r.ai_model, + Col.PDF_PROCESSOR: r.pdf_processor, + Col.FILE_TYPE: r.file_type, + Col.COMMIT_HASH: short_hash, + Col.COMMIT_MESSAGE: commit_msg, + Col.ACCURACY_SCORE: r.accuracy_score, + Col.KEY_SCORES: key_str, + Col.RUN_COUNT: 1, + "_key_accuracies": key_accuracies, + } + + # ── Aggregation ────────────────────────────────────────────────── + + def _aggregate_by_config(self): + """Collapse multiple runs of same config + commit into one averaged row.""" + if not self.data: + return + + groups: dict[tuple, list[dict]] = defaultdict(list) + for row in self.data: + key = ( + row.get(Col.DATASET), + row.get(Col.AI_MODEL), + row.get(Col.PDF_PROCESSOR) or "", + row.get(Col.FILE_TYPE), + row.get(Col.COMMIT_HASH), + ) + groups[key].append(row) + + aggregated = [] + for _key, rows in groups.items(): + count = len(rows) + agg = { + Col.PARTY: rows[0].get(Col.PARTY), + Col.PARTY_NAME: rows[0].get(Col.PARTY_NAME), + Col.DATASET: rows[0].get(Col.DATASET), + Col.AI_MODEL: rows[0].get(Col.AI_MODEL), + Col.PDF_PROCESSOR: rows[0].get(Col.PDF_PROCESSOR), + Col.FILE_TYPE: rows[0].get(Col.FILE_TYPE), + Col.COMMIT_HASH: rows[0].get(Col.COMMIT_HASH), + Col.COMMIT_MESSAGE: rows[0].get(Col.COMMIT_MESSAGE), + Col.RUN_COUNT: count, + } + + # average accuracy + vals = [r.get(Col.ACCURACY_SCORE) or 0 for r in rows] + agg[Col.ACCURACY_SCORE] = round(sum(vals) / count, 2) if count else 0 + + # aggregate per-key accuracies + all_key_accs: dict[str, list[float]] = defaultdict(list) + for r in rows: + for k, v in r.get("_key_accuracies", {}).items(): + all_key_accs[k].append(v or 0) + + avg_key_accs = { + k: round(sum(v) / len(v), 1) for k, v in all_key_accs.items() + } + agg[Col.KEY_SCORES] = ( + ", ".join(f"{k}: {v:.0f}%" for k, v in avg_key_accs.items()) + if avg_key_accs + else "" + ) + agg["_key_accuracies"] = avg_key_accs + + aggregated.append(agg) + + self.data = aggregated + + # ── Grouping ───────────────────────────────────────────────────── + + def _group_by_party(self): + """Group data by party, creating tree view with indent levels.""" + if not self.data: + return + + grouped = defaultdict(list) + for row in self.data: + party = row.get(Col.PARTY) or _("No Party") + row["indent"] = 1 + grouped[party].append(row) + + tree_data = [] + for party, rows in grouped.items(): + rows.sort(key=self._sort_key) + tree_data.append(self._group_row(party, rows)) + tree_data.extend(rows) + + self.data = tree_data + + @staticmethod + def _sort_key(row): + """Sort: AI Model → PDF Processor → File Type → Commit Hash.""" + return ( + _AI_MODEL_ORDER.get(row.get(Col.AI_MODEL), 99), + _PDF_PROCESSOR_ORDER.get(row.get(Col.PDF_PROCESSOR), 99), + _FILE_TYPE_ORDER.get(row.get(Col.FILE_TYPE), 99), + row.get(Col.COMMIT_HASH) or "", + ) + + def _group_row(self, party, rows): + """Aggregated summary row for a party group (indent 0).""" + count = len(rows) + row = { + Col.PARTY: party, + Col.PARTY_NAME: rows[0].get(Col.PARTY_NAME), + Col.RUN_COUNT: sum(r.get(Col.RUN_COUNT, 1) for r in rows), + "indent": 0, + } + + # average accuracy across all child rows + vals = [r.get(Col.ACCURACY_SCORE) or 0 for r in rows] + row[Col.ACCURACY_SCORE] = round(sum(vals) / count, 2) if count else 0 + + # aggregate per-key accuracies + all_key_accs: dict[str, list[float]] = defaultdict(list) + for r in rows: + for k, v in r.get("_key_accuracies", {}).items(): + all_key_accs[k].append(v or 0) + + avg_key_accs = {k: round(sum(v) / len(v), 1) for k, v in all_key_accs.items()} + row[Col.KEY_SCORES] = ( + ", ".join(f"{k}: {v:.0f}%" for k, v in avg_key_accs.items()) + if avg_key_accs + else "" + ) + + return row From cd3affb7ae6b58abc176f144cd140c350d752878 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Mon, 30 Mar 2026 13:34:20 +0530 Subject: [PATCH 67/88] feat: add option to include disabled datasets in accuracy analysis and version comparison reports --- .../transaction_parser_accuracy_analysis.js | 6 ++++++ .../transaction_parser_accuracy_analysis.py | 3 +++ .../transaction_parser_version_comparison.js | 6 ++++++ .../transaction_parser_version_comparison.py | 3 +++ 4 files changed, 18 insertions(+) diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js index 132e87e..c22fab8 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js @@ -86,6 +86,12 @@ frappe.query_reports["Transaction Parser Accuracy Analysis"] = { fieldtype: "MultiSelectList", get_data: (txt) => make_options(PDF_PROCESSORS, txt), }, + { + fieldname: "include_disabled_datasets", + label: __("Include Disabled Datasets"), + fieldtype: "Check", + default: 0, + }, ], }; diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py index 307abfc..e6a063b 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py @@ -263,6 +263,9 @@ def _fetch_logs(self): .orderby(ds.party, log.ai_model, ds.file_type) ) + if not self.filters.get("include_disabled_datasets"): + query = query.where(ds.enabled == 1) + # exact-match filters for column, key in ( (ds.company, "company"), diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js index 3b199bc..6061239 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js +++ b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js @@ -86,6 +86,12 @@ frappe.query_reports["Transaction Parser Version Comparison"] = { fieldtype: "MultiSelectList", get_data: (txt) => make_options(PDF_PROCESSORS, txt), }, + { + fieldname: "include_disabled_datasets", + label: __("Include Disabled Datasets"), + fieldtype: "Check", + default: 0, + }, ], }; diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py index 56f0792..0f3f644 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py +++ b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py @@ -189,6 +189,9 @@ def _fetch_logs(self): .orderby(ds.party, log.ai_model, ds.file_type) ) + if not self.filters.get("include_disabled_datasets"): + query = query.where(ds.enabled == 1) + # exact-match filters for column, key in ( (ds.company, "company"), From afd74c7f1453c07f205d926b1c4db434b3180eae Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Mon, 30 Mar 2026 17:25:29 +0530 Subject: [PATCH 68/88] feat: implement multiple file support in Parser Benchmark Dataset and related reports --- .../parser_benchmark_dataset.json | 48 ++++++------ .../parser_benchmark_dataset.py | 52 ++++++------- .../parser_benchmark_dataset_file/__init__.py | 2 + .../parser_benchmark_dataset_file.json | 38 ++++++++++ .../parser_benchmark_dataset_file.py | 23 ++++++ .../parser_benchmark_log.json | 10 +-- .../parser_benchmark_log.py | 4 - .../transaction_parser_accuracy_analysis.js | 14 ++-- .../transaction_parser_accuracy_analysis.py | 27 ++----- .../transaction_parser_version_comparison.js | 14 ++-- .../transaction_parser_version_comparison.py | 27 ++----- transaction_parser/parser_benchmark/runner.py | 47 +++++++----- transaction_parser/patches.txt | 2 + .../patches/populate_dataset_files_table.py | 52 +++++++++++++ .../patches/remove_dataset_file_field.py | 73 +++++++++++++++++++ 15 files changed, 293 insertions(+), 140 deletions(-) create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/__init__.py create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/parser_benchmark_dataset_file.json create mode 100644 transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/parser_benchmark_dataset_file.py create mode 100644 transaction_parser/patches/populate_dataset_files_table.py create mode 100644 transaction_parser/patches/remove_dataset_file_field.py diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index fb07441..22b7ea8 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -6,11 +6,8 @@ "engine": "InnoDB", "field_order": [ "enabled", + "is_multiple_files", "column_break_title", - "section_break_file", - "file", - "file_type", - "column_break_gbap", "transaction_type", "country", "section_break_sobg", @@ -18,6 +15,8 @@ "column_break_qccq", "party_type", "party", + "files_section", + "files", "processing_section", "column_break_ai_models", "openai_gpt_4o_mini", @@ -59,20 +58,12 @@ "label": "Enabled" }, { - "fieldname": "section_break_file", - "fieldtype": "Section Break" - }, - { - "fieldname": "file", - "fieldtype": "Attach", - "label": "File", - "print_hide": 1, - "reqd": 1 - }, - { - "fieldname": "file_type", - "fieldtype": "Data", - "label": "File Type", + "allow_on_submit": 1, + "default": "0", + "fieldname": "is_multiple_files", + "fieldtype": "Check", + "in_standard_filter": 1, + "label": "Multiple Files", "read_only": 1 }, { @@ -91,10 +82,7 @@ "options": "India\nOther", "reqd": 1 }, - { - "fieldname": "column_break_gbap", - "fieldtype": "Column Break" - }, + { "fieldname": "company", "fieldtype": "Link", @@ -115,6 +103,19 @@ "label": "Party", "options": "party_type" }, + { + "fieldname": "files_section", + "fieldtype": "Section Break", + "label": "Files" + }, + { + "allow_on_submit": 1, + "fieldname": "files", + "fieldtype": "Table", + "label": "Files", + "options": "Parser Benchmark Dataset File", + "reqd": 1 + }, { "fieldname": "processing_section", "fieldtype": "Section Break", @@ -181,7 +182,6 @@ "label": "Google Gemini Flash-2.5" }, { - "depends_on": "eval: doc.file_type === \"PDF\"", "fieldname": "pdf_processor_section", "fieldtype": "Section Break", "label": "PDF Processors" @@ -307,4 +307,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} \ No newline at end of file +} diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 861a245..23c6d95 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -34,6 +34,9 @@ class ParserBenchmarkDataset(Document): if TYPE_CHECKING: from frappe.types import DF + from transaction_parser.parser_benchmark.doctype.parser_benchmark_dataset_file.parser_benchmark_dataset_file import ( + ParserBenchmarkDatasetFile, + ) from transaction_parser.parser_benchmark.doctype.parser_benchmark_expected_field.parser_benchmark_expected_field import ( ParserBenchmarkExpectedField, ) @@ -46,10 +49,10 @@ class ParserBenchmarkDataset(Document): docling: DF.Check enabled: DF.Check expected_fields: DF.Table[ParserBenchmarkExpectedField] - file: DF.Attach - file_type: DF.Data | None + files: DF.Table[ParserBenchmarkDatasetFile] google_gemini_flash_25: DF.Check google_gemini_pro_25: DF.Check + is_multiple_files: DF.Check naming_series: DF.Literal["PAR-BM-DTS-"] ocrmypdf: DF.Check openai_gpt_4o: DF.Check @@ -62,39 +65,24 @@ class ParserBenchmarkDataset(Document): transaction_type: DF.Literal["Sales Order", "Expense"] # end: auto-generated types - SUPPORTED_FILE_TYPES = ("PDF", "CSV", "XLSX", "XLS") - def validate(self): - self.set_file_type() - self.validate_file_type() + self.validate_files() self.validate_selected_models() - self.validate_selected_processors() self.validate_expected_fields() - def set_file_type(self): - if self.file_type and not self.has_value_changed("file"): - return + def validate_files(self): + """Set file_type for each row and auto-set is_multiple_files.""" + for row in self.files: + if row.file and (not row.file_type or row.has_value_changed("file")): + file_doc = frappe.get_last_doc("File", filters={"file_url": row.file}) + row.file_type = file_doc.file_type - file_doc = frappe.get_last_doc("File", filters={"file_url": self.file}) - self.file_type = file_doc.file_type - - def validate_file_type(self): - if self.file_type not in self.SUPPORTED_FILE_TYPES: - frappe.throw(_("Unsupported file type: {0}").format(self.file_type)) + self.is_multiple_files = len(self.files) > 1 def validate_selected_models(self): if not self.get_selected_models(): frappe.throw(_("Please select at least one AI Model.")) - def validate_selected_processors(self): - if self.file_type != "PDF": - for field in PDF_PROCESSOR_FIELD_MAP: - self.set(field, 0) - return - - if not self.get_selected_processors(): - frappe.throw(_("Please select at least one PDF Processor.")) - def validate_expected_fields(self): if not self.expected_fields: return @@ -129,6 +117,18 @@ def get_selected_processors(self) -> list[str]: label for field, label in PDF_PROCESSOR_FIELD_MAP.items() if self.get(field) ] + def has_pdf_file(self) -> bool: + """Check if any file in the child table is a PDF.""" + return any(row.file_type == "PDF" for row in self.files) + + def get_file_docs(self) -> list: + """Return File documents for each row in the files child table.""" + file_docs = [] + for row in self.files: + file_doc = frappe.get_last_doc("File", filters={"file_url": row.file}) + file_docs.append(file_doc) + return file_docs + @frappe.whitelist() def run_benchmark(dataset_name: str): @@ -156,7 +156,7 @@ def create_and_enqueue_benchmark_logs(dataset_name: str) -> list[str]: models = dataset.get_selected_models() processors = ( (dataset.get_selected_processors() or [None]) - if dataset.file_type == "PDF" + if dataset.has_pdf_file() else [None] ) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/__init__.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/__init__.py new file mode 100644 index 0000000..c4fea77 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026, Resilient Tech and contributors +# For license information, please see license.txt diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/parser_benchmark_dataset_file.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/parser_benchmark_dataset_file.json new file mode 100644 index 0000000..e3683f7 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/parser_benchmark_dataset_file.json @@ -0,0 +1,38 @@ +{ + "actions": [], + "creation": "2026-03-30 00:00:00", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "file", + "file_type" + ], + "fields": [ + { + "fieldname": "file", + "fieldtype": "Attach", + "in_list_view": 1, + "label": "File", + "reqd": 1 + }, + { + "fieldname": "file_type", + "fieldtype": "Data", + "in_list_view": 1, + "label": "File Type", + "read_only": 1 + } + ], + "istable": 1, + "links": [], + "modified": "2026-03-30 00:00:00", + "modified_by": "Administrator", + "module": "Parser Benchmark", + "name": "Parser Benchmark Dataset File", + "owner": "Administrator", + "permissions": [], + "row_format": "Dynamic", + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/parser_benchmark_dataset_file.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/parser_benchmark_dataset_file.py new file mode 100644 index 0000000..78b5e42 --- /dev/null +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/parser_benchmark_dataset_file.py @@ -0,0 +1,23 @@ +# Copyright (c) 2026, Resilient Tech and contributors +# For license information, please see license.txt + +from frappe.model.document import Document + + +class ParserBenchmarkDatasetFile(Document): + # begin: auto-generated types + # This code is auto-generated. Do not modify anything in this block. + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from frappe.types import DF + + file: DF.Attach + file_type: DF.Data | None + parent: DF.Data + parentfield: DF.Data + parenttype: DF.Data + # end: auto-generated types + + pass diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json index 9ba294b..0bbe3c6 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.json @@ -26,7 +26,6 @@ "column_break_ubhs", "file_parsing_tab", "pdf_processor", - "file_type", "column_break_file_metrics", "page_limit", "section_break_umzr", @@ -107,7 +106,6 @@ "read_only": 1 }, { - "depends_on": "eval: doc.file_type === \"PDF\"", "fieldname": "pdf_processor", "fieldtype": "Select", "label": "PDF Processor", @@ -121,13 +119,7 @@ "label": "Total Time (s)", "read_only": 1 }, - { - "fieldname": "file_type", - "fieldtype": "Data", - "is_virtual": 1, - "label": "File Type", - "read_only": 1 - }, + { "fieldname": "file_parsing_tab", "fieldtype": "Tab Break", diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py index 0b9e29a..5cd5b7f 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_log/parser_benchmark_log.py @@ -97,7 +97,3 @@ def party(self): @property def page_limit(self): return self.get_from_dataset("page_limit") or 0 - - @property - def file_type(self): - return self.get_from_dataset("file_type") diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js index c22fab8..5c84d22 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js @@ -12,8 +12,6 @@ const AI_MODELS = [ "Google Gemini Flash-2.5", ]; -const FILE_TYPES = ["PDF", "CSV", "XLSX", "XLS"]; - const PDF_PROCESSORS = ["OCRMyPDF", "Docling"]; const PARTY_TYPE_MAP = { @@ -68,12 +66,6 @@ frappe.query_reports["Transaction Parser Accuracy Analysis"] = { fieldtype: "Dynamic Link", options: "party_type", }, - { - fieldname: "file_type", - label: __("File Type"), - fieldtype: "MultiSelectList", - get_data: (txt) => make_options(FILE_TYPES, txt), - }, { fieldname: "ai_model", label: __("AI Model"), @@ -92,6 +84,12 @@ frappe.query_reports["Transaction Parser Accuracy Analysis"] = { fieldtype: "Check", default: 0, }, + { + fieldname: "is_multiple_files", + label: __("Multiple Files Only"), + fieldtype: "Check", + default: 0, + }, ], }; diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py index e6a063b..45e11ba 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py @@ -34,13 +34,6 @@ "Docling": 1, } -_FILE_TYPE_ORDER = { - "PDF": 0, - "CSV": 1, - "XLSX": 2, - "XLS": 3, -} - class Col(StrEnum): """Column fieldnames — single source of truth for the report.""" @@ -50,7 +43,6 @@ class Col(StrEnum): ACCURACY_SCORE = "accuracy_score" AI_MODEL = "ai_model" PDF_PROCESSOR = "pdf_processor" - FILE_TYPE = "file_type" FILE_PARSE_TIME = "file_parse_time" FILE_PARSE_MEMORY = "file_parse_memory" AI_PARSE_TIME = "ai_parse_time" @@ -155,12 +147,6 @@ def _get_columns(self): "fieldtype": "Data", "width": 110, }, - { - "fieldname": Col.FILE_TYPE, - "label": _("File Type"), - "fieldtype": "Data", - "width": 90, - }, { "fieldname": Col.FILE_PARSE_TIME, "label": _("File Parse (s)"), @@ -254,18 +240,20 @@ def _fetch_logs(self): log.currency, log.dataset, ds.party, - ds.file_type, Coalesce(cust.customer_name, supp.supplier_name, ds.party).as_( "party_name" ), ) .where(log.status == "Completed") - .orderby(ds.party, log.ai_model, ds.file_type) + .orderby(ds.party, log.ai_model) ) if not self.filters.get("include_disabled_datasets"): query = query.where(ds.enabled == 1) + if self.filters.get("is_multiple_files"): + query = query.where(ds.is_multiple_files == 1) + # exact-match filters for column, key in ( (ds.company, "company"), @@ -278,7 +266,6 @@ def _fetch_logs(self): # multi-select IN filters for column, key in ( - (ds.file_type, "file_type"), (log.ai_model, "ai_model"), (log.pdf_processor, "pdf_processor"), ): @@ -333,7 +320,6 @@ def _build_row(self, r, score_details_map): Col.ACCURACY_SCORE: r.accuracy_score, Col.AI_MODEL: r.ai_model, Col.PDF_PROCESSOR: r.pdf_processor, - Col.FILE_TYPE: r.file_type, Col.DATASET: r.dataset, Col.FILE_PARSE_TIME: r.file_parse_time, Col.FILE_PARSE_MEMORY: r.file_parse_memory, @@ -361,7 +347,6 @@ def _aggregate_by_config(self): row.get(Col.DATASET), row.get(Col.AI_MODEL), row.get(Col.PDF_PROCESSOR) or "", - row.get(Col.FILE_TYPE), ) groups[key].append(row) @@ -373,7 +358,6 @@ def _aggregate_by_config(self): Col.PARTY_NAME: rows[0].get(Col.PARTY_NAME), Col.AI_MODEL: rows[0].get(Col.AI_MODEL), Col.PDF_PROCESSOR: rows[0].get(Col.PDF_PROCESSOR), - Col.FILE_TYPE: rows[0].get(Col.FILE_TYPE), Col.CURRENCY: rows[0].get(Col.CURRENCY), Col.RUN_COUNT: count, } @@ -430,11 +414,10 @@ def _group_by_party(self): @staticmethod def _sort_key(row): - """Sort key for child rows: AI Model → PDF Processor → File Type.""" + """Sort key for child rows: AI Model → PDF Processor.""" return ( _AI_MODEL_ORDER.get(row.get(Col.AI_MODEL), 99), _PDF_PROCESSOR_ORDER.get(row.get(Col.PDF_PROCESSOR), 99), - _FILE_TYPE_ORDER.get(row.get(Col.FILE_TYPE), 99), ) def _group_row(self, party, rows): diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js index 6061239..2c252e2 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js +++ b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js @@ -12,8 +12,6 @@ const AI_MODELS = [ "Google Gemini Flash-2.5", ]; -const FILE_TYPES = ["PDF", "CSV", "XLSX", "XLS"]; - const PDF_PROCESSORS = ["OCRMyPDF", "Docling"]; const PARTY_TYPE_MAP = { @@ -68,12 +66,6 @@ frappe.query_reports["Transaction Parser Version Comparison"] = { fieldtype: "Dynamic Link", options: "party_type", }, - { - fieldname: "file_type", - label: __("File Type"), - fieldtype: "MultiSelectList", - get_data: (txt) => make_options(FILE_TYPES, txt), - }, { fieldname: "ai_model", label: __("AI Model"), @@ -92,6 +84,12 @@ frappe.query_reports["Transaction Parser Version Comparison"] = { fieldtype: "Check", default: 0, }, + { + fieldname: "is_multiple_files", + label: __("Multiple Files Only"), + fieldtype: "Check", + default: 0, + }, ], }; diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py index 0f3f644..e0e7fb1 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py +++ b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py @@ -30,13 +30,6 @@ "Docling": 1, } -_FILE_TYPE_ORDER = { - "PDF": 0, - "CSV": 1, - "XLSX": 2, - "XLS": 3, -} - class Col(StrEnum): """Column fieldnames — single source of truth for the report.""" @@ -46,7 +39,6 @@ class Col(StrEnum): DATASET = "dataset" AI_MODEL = "ai_model" PDF_PROCESSOR = "pdf_processor" - FILE_TYPE = "file_type" COMMIT_HASH = "commit_hash" COMMIT_MESSAGE = "commit_message" ACCURACY_SCORE = "accuracy_score" @@ -116,12 +108,6 @@ def _get_columns(self): "fieldtype": "Data", "width": 110, }, - { - "fieldname": Col.FILE_TYPE, - "label": _("File Type"), - "fieldtype": "Data", - "width": 90, - }, { "fieldname": Col.COMMIT_HASH, "label": _("Commit"), @@ -179,19 +165,21 @@ def _fetch_logs(self): log.commit_hash, log.commit_message, ds.party, - ds.file_type, Coalesce(cust.customer_name, supp.supplier_name, ds.party).as_( "party_name" ), ) .where(log.status == "Completed") .where(Coalesce(log.commit_hash, "") != "") - .orderby(ds.party, log.ai_model, ds.file_type) + .orderby(ds.party, log.ai_model) ) if not self.filters.get("include_disabled_datasets"): query = query.where(ds.enabled == 1) + if self.filters.get("is_multiple_files"): + query = query.where(ds.is_multiple_files == 1) + # exact-match filters for column, key in ( (ds.company, "company"), @@ -204,7 +192,6 @@ def _fetch_logs(self): # multi-select IN filters for column, key in ( - (ds.file_type, "file_type"), (log.ai_model, "ai_model"), (log.pdf_processor, "pdf_processor"), ): @@ -263,7 +250,6 @@ def _build_row(self, r, score_details_map): Col.DATASET: r.dataset, Col.AI_MODEL: r.ai_model, Col.PDF_PROCESSOR: r.pdf_processor, - Col.FILE_TYPE: r.file_type, Col.COMMIT_HASH: short_hash, Col.COMMIT_MESSAGE: commit_msg, Col.ACCURACY_SCORE: r.accuracy_score, @@ -285,7 +271,6 @@ def _aggregate_by_config(self): row.get(Col.DATASET), row.get(Col.AI_MODEL), row.get(Col.PDF_PROCESSOR) or "", - row.get(Col.FILE_TYPE), row.get(Col.COMMIT_HASH), ) groups[key].append(row) @@ -299,7 +284,6 @@ def _aggregate_by_config(self): Col.DATASET: rows[0].get(Col.DATASET), Col.AI_MODEL: rows[0].get(Col.AI_MODEL), Col.PDF_PROCESSOR: rows[0].get(Col.PDF_PROCESSOR), - Col.FILE_TYPE: rows[0].get(Col.FILE_TYPE), Col.COMMIT_HASH: rows[0].get(Col.COMMIT_HASH), Col.COMMIT_MESSAGE: rows[0].get(Col.COMMIT_MESSAGE), Col.RUN_COUNT: count, @@ -352,11 +336,10 @@ def _group_by_party(self): @staticmethod def _sort_key(row): - """Sort: AI Model → PDF Processor → File Type → Commit Hash.""" + """Sort: AI Model → PDF Processor → Commit Hash.""" return ( _AI_MODEL_ORDER.get(row.get(Col.AI_MODEL), 99), _PDF_PROCESSOR_ORDER.get(row.get(Col.PDF_PROCESSOR), 99), - _FILE_TYPE_ORDER.get(row.get(Col.FILE_TYPE), 99), row.get(Col.COMMIT_HASH) or "", ) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index 716fffe..8f949d7 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -2,6 +2,7 @@ from timeit import default_timer import frappe +from frappe import _ from frappe.core.doctype.file.file import File from frappe.utils import cint, flt @@ -57,11 +58,11 @@ def run(self): total_start = default_timer() try: - file_doc: File = self._get_file_doc() - self.controller: Transaction = self._get_controller(file_doc) + file_docs: list[File] = self._get_file_docs() + self.controller: Transaction = self._get_controller(file_docs[0]) - file_content = self._run_file_parsing(file_doc) - ai_content = self._run_ai_parsing(file_content, file_doc.name) + file_content = self._run_file_parsing(file_docs) + ai_content = self._run_ai_parsing(file_content, file_docs[0].name) self._calculate_cost() self._score_response(ai_content) @@ -80,8 +81,11 @@ def run(self): # ── helpers ────────────────────────────────────────────── - def _get_file_doc(self): - return frappe.get_last_doc("File", filters={"file_url": self.dataset.file}) + def _get_file_docs(self) -> list[File]: + file_docs = self.dataset.get_file_docs() + if not file_docs: + frappe.throw(_("No files in dataset {0}").format(self.dataset.name)) + return file_docs def _get_controller(self, file_doc: File) -> Transaction: ds = self.dataset @@ -103,11 +107,7 @@ def _get_cost_row(self): # ── step 1: file parsing ──────────────────────────────── - def _run_file_parsing(self, file_doc: File) -> str: - pdf_processor = None - if self.log.file_type == "PDF" and self.log.pdf_processor: - pdf_processor = get_pdf_processor(self.log.pdf_processor) - + def _run_file_parsing(self, file_docs: list[File]) -> str: # to prevent stopping an already running tracemalloc instance was_tracing = tracemalloc.is_tracing() if not was_tracing: @@ -115,10 +115,23 @@ def _run_file_parsing(self, file_doc: File) -> str: start = default_timer() try: - content = FileProcessor().get_content( - file_doc, - self.dataset.page_limit or None, - pdf_processor, + contents = [] + for file_doc in file_docs: + pdf_processor = None + if file_doc.file_type == "PDF" and self.log.pdf_processor: + pdf_processor = get_pdf_processor(self.log.pdf_processor) + + content = FileProcessor().get_content( + file_doc, + self.dataset.page_limit or None, + pdf_processor, + ) + contents.append(content) + + combined = ( + "\n\n--- Document Separator ---\n\n".join(contents) + if len(contents) > 1 + else contents[0] ) finally: self.log.file_parse_time = flt(default_timer() - start, self.precision) @@ -129,8 +142,8 @@ def _run_file_parsing(self, file_doc: File) -> str: peak / 1024 / 1024, self.precision ) # bytes → MB - self.log.file_content = content - return content + self.log.file_content = combined + return combined # ── step 2: AI parsing ────────────────────────────────── diff --git a/transaction_parser/patches.txt b/transaction_parser/patches.txt index 7c87e23..d95cd72 100644 --- a/transaction_parser/patches.txt +++ b/transaction_parser/patches.txt @@ -2,8 +2,10 @@ # Patches added in this section will be executed before doctypes are migrated # Read docs to understand patches: https://frappeframework.com/docs/v14/user/en/database-migrations transaction_parser.patches.rename_gemini_models +transaction_parser.patches.remove_dataset_file_field [post_model_sync] # Patches added in this section will be executed after doctypes are migrated execute:from transaction_parser.install import after_install; after_install() #2 transaction_parser.patches.set_default_pdf_processor #1 +transaction_parser.patches.populate_dataset_files_table diff --git a/transaction_parser/patches/populate_dataset_files_table.py b/transaction_parser/patches/populate_dataset_files_table.py new file mode 100644 index 0000000..07a415f --- /dev/null +++ b/transaction_parser/patches/populate_dataset_files_table.py @@ -0,0 +1,52 @@ +""" +Populate the new ``files`` child table on Parser Benchmark Dataset. + +After model_sync creates the ``Parser Benchmark Dataset File`` child table, +this patch reads the File documents that were previously attached (by the +``remove_dataset_file_field`` pre_model_sync patch) and inserts them as child +rows so the new child-table based workflow works seamlessly. +""" + +import frappe + + +def execute(): + datasets = frappe.get_all("Parser Benchmark Dataset", fields=["name"]) + + for ds in datasets: + # Skip if already has files in child table + if frappe.db.count("Parser Benchmark Dataset File", {"parent": ds.name}): + continue + + # Find File docs attached to this dataset + files = frappe.get_all( + "File", + filters={ + "attached_to_doctype": "Parser Benchmark Dataset", + "attached_to_name": ds.name, + }, + fields=["file_url", "file_type"], + ) + + if not files: + continue + + for idx, f in enumerate(files, 1): + child = frappe.new_doc("Parser Benchmark Dataset File") + child.update( + { + "parent": ds.name, + "parenttype": "Parser Benchmark Dataset", + "parentfield": "files", + "idx": idx, + "file": f.file_url, + "file_type": f.file_type or "", + } + ) + child.db_insert() + + # Update is_multiple_files flag + is_multiple = 1 if len(files) > 1 else 0 + frappe.db.set_value( + "Parser Benchmark Dataset", ds.name, "is_multiple_files", is_multiple + ) diff --git a/transaction_parser/patches/remove_dataset_file_field.py b/transaction_parser/patches/remove_dataset_file_field.py new file mode 100644 index 0000000..87e2b48 --- /dev/null +++ b/transaction_parser/patches/remove_dataset_file_field.py @@ -0,0 +1,73 @@ +""" +Migrate `file` field data on Parser Benchmark Dataset to Frappe File attachments. + +Before the `file` column is dropped (pre_model_sync), ensure every Dataset that +had a file URL stored in the `file` field has a corresponding File doc properly +linked via `attached_to_doctype` / `attached_to_name`. +""" + +import frappe + + +def execute(): + if not frappe.db.has_column("Parser Benchmark Dataset", "file"): + return + + datasets = frappe.get_all( + "Parser Benchmark Dataset", + filters={"file": ("is", "set")}, + fields=["name", "file"], + ) + + for ds in datasets: + file_url = ds.file + if not file_url: + continue + + # Check if a properly-linked File doc already exists + existing = frappe.db.exists( + "File", + { + "file_url": file_url, + "attached_to_doctype": "Parser Benchmark Dataset", + "attached_to_name": ds.name, + }, + ) + + if existing: + continue + + # Try to find an unlinked File doc with the same URL and link it + unlinked = frappe.db.get_value( + "File", + {"file_url": file_url}, + ["name", "attached_to_doctype", "attached_to_name"], + as_dict=True, + ) + + if unlinked: + if not unlinked.attached_to_doctype: + # Link the orphan File doc to this dataset + frappe.db.set_value( + "File", + unlinked.name, + { + "attached_to_doctype": "Parser Benchmark Dataset", + "attached_to_name": ds.name, + }, + ) + else: + # File is attached to something else — create a copy + _create_attachment(ds.name, file_url) + else: + # No File doc exists at all — create one + _create_attachment(ds.name, file_url) + + +def _create_attachment(dataset_name: str, file_url: str): + """Create a new File doc attached to the given dataset.""" + f = frappe.new_doc("File") + f.file_url = file_url + f.attached_to_doctype = "Parser Benchmark Dataset" + f.attached_to_name = dataset_name + f.insert(ignore_permissions=True) From cbefeb916a207ed39bfb75fc01333b77f8878767 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Mon, 30 Mar 2026 18:05:05 +0530 Subject: [PATCH 69/88] fix: add recalculation of accuracy scores for completed Parser Benchmark Logs --- transaction_parser/patches.txt | 1 + .../patches/recalculate_accuracy.py | 96 +++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 transaction_parser/patches/recalculate_accuracy.py diff --git a/transaction_parser/patches.txt b/transaction_parser/patches.txt index 7c87e23..d88f879 100644 --- a/transaction_parser/patches.txt +++ b/transaction_parser/patches.txt @@ -7,3 +7,4 @@ transaction_parser.patches.rename_gemini_models # Patches added in this section will be executed after doctypes are migrated execute:from transaction_parser.install import after_install; after_install() #2 transaction_parser.patches.set_default_pdf_processor #1 +transaction_parser.patches.recalculate_accuracy diff --git a/transaction_parser/patches/recalculate_accuracy.py b/transaction_parser/patches/recalculate_accuracy.py new file mode 100644 index 0000000..f2985fb --- /dev/null +++ b/transaction_parser/patches/recalculate_accuracy.py @@ -0,0 +1,96 @@ +import frappe +from frappe.utils import cint + + +def execute(): + """Enqueue recalculation of accuracy scores for all completed Parser Benchmark Logs.""" + log_names = frappe.get_all( + "Parser Benchmark Log", + filters={"status": "Completed", "ai_response": ["is", "set"]}, + pluck="name", + ) + + if not log_names: + return + + frappe.enqueue( + recalculate_accuracy, + log_names=log_names, + queue="long", + timeout=3600, + ) + + +def recalculate_accuracy(log_names: list[str]): + """Recalculate accuracy scores for the given Parser Benchmark Logs. + + Uses cached docs for datasets to avoid repeated DB reads. + Commits in batches of 100 to avoid long-running transactions. + """ + from transaction_parser.parser_benchmark.scorer import score_response + + settings = frappe.get_cached_doc("Parser Benchmark Settings") + weights = {row.key: row.weight for row in (settings.key_weights or [])} + precision = cint(frappe.db.get_default("float_precision")) or 2 + + BATCH_SIZE = 100 + + for idx, log_name in enumerate(log_names, start=1): + try: + _recalculate_log(log_name, weights, precision, score_response) + except Exception: + frappe.log_error( + title=f"Recalculate Accuracy: {log_name}", + message=frappe.get_traceback(), + ) + + if idx % BATCH_SIZE == 0: + frappe.db.commit() # nosemgrep + + frappe.db.commit() # nosemgrep + + +def _recalculate_log(log_name, weights, precision, score_response): + """Rescore a single log and update its score details.""" + log = frappe.get_doc("Parser Benchmark Log", log_name) + + if not log.ai_response: + return + + dataset = frappe.get_cached_doc("Parser Benchmark Dataset", log.dataset) + + if not dataset.expected_fields: + return + + ai_content = frappe.parse_json(log.ai_response) + + result = score_response( + expected_fields=[ + {"key": row.key, "expected_json": row.expected_json} + for row in dataset.expected_fields + ], + actual=ai_content, + weights=weights, + precision=precision, + ) + + # clear old score details + log.score_details = [] + + for detail in result["details"]: + log.append( + "score_details", + { + "key": detail["key"], + "matched": detail["matched"], + "total": detail["total"], + "accuracy": detail["accuracy"], + "mismatches": frappe.as_json(detail["mismatches"], indent=2) + if detail["mismatches"] + else None, + }, + ) + + log.accuracy_score = result["overall_accuracy"] + log.flags.ignore_validate = True + log.save(ignore_permissions=True) From b8919826ab826e0521326e8b5c2072feb1a6885d Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Mon, 30 Mar 2026 18:21:25 +0530 Subject: [PATCH 70/88] fix: filter completed logs by document status in accuracy analysis and version comparison reports --- .../transaction_parser_accuracy_analysis.py | 1 + .../transaction_parser_version_comparison.py | 1 + 2 files changed, 2 insertions(+) diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py index e6a063b..3d1e7ac 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py @@ -260,6 +260,7 @@ def _fetch_logs(self): ), ) .where(log.status == "Completed") + .where(ds.docstatus == 1) .orderby(ds.party, log.ai_model, ds.file_type) ) diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py index 0f3f644..6773fb8 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py +++ b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py @@ -185,6 +185,7 @@ def _fetch_logs(self): ), ) .where(log.status == "Completed") + .where(ds.docstatus == 1) .where(Coalesce(log.commit_hash, "") != "") .orderby(ds.party, log.ai_model, ds.file_type) ) From 494aa65d41a8efa21304269189c7194c3a8e92c7 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Mon, 30 Mar 2026 18:45:28 +0530 Subject: [PATCH 71/88] refactor: enhance Parser Benchmark Dataset with multiple file support and validation --- .../parser_benchmark_dataset.json | 22 ++++++++++++++----- .../parser_benchmark_dataset.py | 3 +++ .../parser_benchmark_dataset_file.json | 9 ++++++-- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json index 22b7ea8..86fdcfb 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.json @@ -6,17 +6,19 @@ "engine": "InnoDB", "field_order": [ "enabled", - "is_multiple_files", "column_break_title", + "files_section", + "files", + "is_multiple_files", + "section_break_txfs", "transaction_type", + "column_break_xusg", "country", "section_break_sobg", "company", "column_break_qccq", "party_type", "party", - "files_section", - "files", "processing_section", "column_break_ai_models", "openai_gpt_4o_mini", @@ -82,7 +84,6 @@ "options": "India\nOther", "reqd": 1 }, - { "fieldname": "company", "fieldtype": "Link", @@ -273,6 +274,15 @@ { "fieldname": "column_break_aoce", "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_txfs", + "fieldtype": "Section Break", + "label": "Transaction Details" + }, + { + "fieldname": "column_break_xusg", + "fieldtype": "Column Break" } ], "index_web_pages_for_search": 1, @@ -283,7 +293,7 @@ "link_fieldname": "dataset" } ], - "modified": "2026-03-27 10:32:46.620190", + "modified": "2026-03-30 15:01:03.977940", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset", @@ -307,4 +317,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} +} \ No newline at end of file diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 23c6d95..26feffa 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -70,6 +70,9 @@ def validate(self): self.validate_selected_models() self.validate_expected_fields() + def before_update_after_submit(self): + self.validate_files() + def validate_files(self): """Set file_type for each row and auto-set is_multiple_files.""" for row in self.files: diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/parser_benchmark_dataset_file.json b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/parser_benchmark_dataset_file.json index e3683f7..5f54a45 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/parser_benchmark_dataset_file.json +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset_file/parser_benchmark_dataset_file.json @@ -5,6 +5,7 @@ "engine": "InnoDB", "field_order": [ "file", + "column_break_yahx", "file_type" ], "fields": [ @@ -21,11 +22,15 @@ "in_list_view": 1, "label": "File Type", "read_only": 1 + }, + { + "fieldname": "column_break_yahx", + "fieldtype": "Column Break" } ], "istable": 1, "links": [], - "modified": "2026-03-30 00:00:00", + "modified": "2026-03-30 15:01:27.752102", "modified_by": "Administrator", "module": "Parser Benchmark", "name": "Parser Benchmark Dataset File", @@ -35,4 +40,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} +} \ No newline at end of file From 7471bc5223fed251f2b8d53a3d28414c2512d719 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 31 Mar 2026 11:18:55 +0530 Subject: [PATCH 72/88] fix: add file type validation for Parser Benchmark Dataset --- .../parser_benchmark_dataset.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py index 26feffa..cc7c55c 100644 --- a/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py +++ b/transaction_parser/parser_benchmark/doctype/parser_benchmark_dataset/parser_benchmark_dataset.py @@ -65,6 +65,8 @@ class ParserBenchmarkDataset(Document): transaction_type: DF.Literal["Sales Order", "Expense"] # end: auto-generated types + SUPPORTED_FILE_TYPES = ("PDF", "CSV", "XLSX", "XLS") + def validate(self): self.validate_files() self.validate_selected_models() @@ -80,6 +82,17 @@ def validate_files(self): file_doc = frappe.get_last_doc("File", filters={"file_url": row.file}) row.file_type = file_doc.file_type + if row.file_type not in self.SUPPORTED_FILE_TYPES: + frappe.throw( + _( + "File '{0}' has unsupported type '{1}'. Supported types are:
{2}." + ).format( + file_doc.file_name, + row.file_type, + "
".join(self.SUPPORTED_FILE_TYPES), + ) + ) + self.is_multiple_files = len(self.files) > 1 def validate_selected_models(self): From ed3e5f5df1dabc9206e3b3685ac2c321e3cd6f43 Mon Sep 17 00:00:00 2001 From: Karm Soni Date: Tue, 31 Mar 2026 13:27:16 +0530 Subject: [PATCH 73/88] feat: enhance transaction parsing to support multiple file attachments --- .../transaction_parser/__init__.py | 55 ++++++++++++++----- .../ai_integration/parser.py | 52 +++++++++++++----- .../ai_integration/prompts.py | 10 +++- .../controllers/transaction.py | 41 +++++++++++--- .../transaction_parser_settings.json | 9 +++ .../overrides/communication.py | 33 ++++++++++- .../utils/file_processor.py | 19 +++++++ 7 files changed, 177 insertions(+), 42 deletions(-) diff --git a/transaction_parser/transaction_parser/__init__.py b/transaction_parser/transaction_parser/__init__.py index 83555ce..fcc2c56 100644 --- a/transaction_parser/transaction_parser/__init__.py +++ b/transaction_parser/transaction_parser/__init__.py @@ -34,30 +34,49 @@ def parse(transaction, country, file_url, ai_model=None, page_limit=None): def _parse( country, transaction, - file_url, + file_urls, ai_model=None, page_limit=None, user=None, party=None, company=None, + communication_name=None, ): try: - file = None - filename = file_url.split("/")[-1] + if ( + isinstance(file_urls, str) + and file_urls.startswith("[") + and file_urls.endswith("]") + ): + file_urls = frappe.parse_json(file_urls) - file = frappe.get_last_doc("File", filters={"file_url": file_url}) - filename = file.file_name + elif isinstance(file_urls, str): + file_urls = [file_urls] + + file_names = frappe.get_list( + "File", filters={"file_url": ("in", file_urls)}, pluck="name" + ) + + files = [] + for file_name in file_names: + file = frappe.get_doc("File", file_name) + files.append(file) controller = get_controller(country, transaction)(party=party, company=company) - doc = controller.generate(file, ai_model, page_limit) + doc = controller.generate(files, ai_model, page_limit, communication_name) + filenames = ( + ", ".join([f.file_name for f in files]) + if len(files) > 1 + else files[0].file_name + ) notification = { "document_type": TRANSACTION_MAP[transaction], "document_name": doc.name, "subject": _("{0} {1} generated from {2}").format( _(TRANSACTION_MAP[transaction]), doc.name, - filename, + filenames, ), } @@ -69,19 +88,27 @@ def _parse( and frappe.flags.skip_duplicate_error ): notification = { - "document_type": "File", - "document_name": file.name if file else filename, - "subject": _("Duplicate entry found for {0}").format(filename), + "document_type": "Communication" if communication_name else "File", + "document_name": ( + communication_name if communication_name else files[0].name + ), + "subject": _("Duplicate entry found for {0}").format(file_urls), "message": str(e), } return error_log = frappe.log_error( "Transaction Parser API Error", - reference_doctype="File", - reference_name=file.name if file else filename, + reference_doctype="Communication" if communication_name else "File", + reference_name=( + communication_name + if communication_name + else files[0].name + if files + else None + ), ) - message = _("Failed to generate {0} from {1}").format(_(transaction), filename) + message = _("Failed to generate {0} from {1}").format(_(transaction), file_urls) notification = { "document_type": error_log.doctype, @@ -90,7 +117,7 @@ def _parse( "message": str(e), } - email_failure(user, message, str(e), file_url) + email_failure(user, message, str(e), file_urls) finally: if notification: diff --git a/transaction_parser/transaction_parser/ai_integration/parser.py b/transaction_parser/transaction_parser/ai_integration/parser.py index f74eec2..d7cd2f1 100644 --- a/transaction_parser/transaction_parser/ai_integration/parser.py +++ b/transaction_parser/transaction_parser/ai_integration/parser.py @@ -34,18 +34,32 @@ def parse( document_type: str, document_schema: dict, document_data: str, - file_doc_name: str | None = None, + doc_name: str | None = None, + file_count: int = 1, + is_communication: bool = False, ) -> dict: - messages = self._build_messages(document_type, document_schema, document_data) - response = self.send_message(messages=messages, file_doc_name=file_doc_name) + messages = self._build_messages( + document_type, document_schema, document_data, file_count + ) + + response = self.send_message( + messages=messages, + doc_name=doc_name, + is_communication=is_communication, + ) + return self.get_content(response) def _build_messages( - self, document_type: str, document_schema: dict, document_data: str + self, + document_type: str, + document_schema: dict, + document_data: str, + file_count: int = 1, ) -> tuple: """Build the message structure for AI API call.""" system_prompt = get_system_prompt(document_schema) - user_prompt = get_user_prompt(document_type, document_data) + user_prompt = get_user_prompt(document_type, document_data, file_count) return ( { @@ -58,9 +72,14 @@ def _build_messages( }, ) - def send_message(self, messages: tuple, file_doc_name: str | None = None) -> dict: + def send_message( + self, + messages: tuple, + doc_name: str | None = None, + is_communication: bool = False, + ) -> dict: """Send messages to AI API and handle the response.""" - log = self._create_log_entry(file_doc_name) + log = self._create_log_entry(doc_name, is_communication) try: response = self._make_api_call(messages) @@ -81,16 +100,19 @@ def send_message(self, messages: tuple, file_doc_name: str | None = None) -> dic finally: enqueue_integration_request(**log) - def _create_log_entry(self, file_doc_name: str | None) -> frappe._dict: + def _create_log_entry( + self, doc_name: str | None, is_communication: bool = False + ) -> frappe._dict: """Create a log entry for the API call.""" log = frappe._dict(url=self.model.base_url) - if file_doc_name: - log.update( - { - "reference_doctype": "File", - "reference_name": file_doc_name, - } - ) + + log.update( + { + "reference_doctype": "Communication" if is_communication else "File", + "reference_name": doc_name, + } + ) + return log def _make_api_call(self, messages: tuple) -> Any: diff --git a/transaction_parser/transaction_parser/ai_integration/prompts.py b/transaction_parser/transaction_parser/ai_integration/prompts.py index 95cf2e0..9e01e5a 100644 --- a/transaction_parser/transaction_parser/ai_integration/prompts.py +++ b/transaction_parser/transaction_parser/ai_integration/prompts.py @@ -34,10 +34,16 @@ def get_system_prompt(document_schema: dict) -> str: {document_schema}""" -def get_user_prompt(document_type: str, document_data: str) -> str: +def get_user_prompt(document_type: str, document_data: str, file_count: int = 1) -> str: input_doc_type = INPUT_DOCUMENTS.get(document_type, "document") - return f"""Generate {document_type} for given {input_doc_type} according to above JSON schema. + base_instruction = f"Generate {document_type} for given {input_doc_type} according to above JSON schema." + + if file_count > 1: + multi_file_instruction = f"\n\nIMPORTANT: The data below contains information from {file_count} related documents. Consolidate and merge the information from all documents to create a single unified {document_type}. Combine item lists, sum totals appropriately, and merge party/address information." + base_instruction += multi_file_instruction + + return f"""{base_instruction} Document data is given below: {document_data}""" diff --git a/transaction_parser/transaction_parser/controllers/transaction.py b/transaction_parser/transaction_parser/controllers/transaction.py index 05a59c0..8ae5e7c 100644 --- a/transaction_parser/transaction_parser/controllers/transaction.py +++ b/transaction_parser/transaction_parser/controllers/transaction.py @@ -29,12 +29,20 @@ def __init__( self.company = company def generate( - self, file, ai_model: str | None = None, page_limit: int | None = None + self, + files, + ai_model: str | None = None, + page_limit: int | None = None, + communication_name: str | None = None, ): self.initialize() - self.file = file + if isinstance(files, str): + files = [files] + + self.files = files self.ai_model = ai_model + self.communication_name = communication_name self.data = self._parse_file_content(ai_model, page_limit) self.doc = frappe.get_doc({"doctype": self.DOCTYPE}) self.doc.is_created_by_transaction_parser = 1 @@ -49,7 +57,8 @@ def generate( def initialize(self) -> None: # file processing - self.file = None + self.files = None + self.communication_name = None # output schema self.schema = None @@ -72,14 +81,27 @@ def initialize(self) -> None: def _parse_file_content( self, ai_model: str | None = None, page_limit: int | None = None ) -> dict: - content = FileProcessor().get_content(self.file, page_limit) + file_processor = FileProcessor() + doc_name = None + file_count = len(self.files) + + if len(self.files) > 1: + content = file_processor.get_combined_content(self.files, page_limit) + doc_name = self.communication_name + + else: + content = file_processor.get_content(self.files, page_limit) + doc_name = self.files[0].name + schema = self.get_schema() return AIParser(ai_model, self.settings).parse( document_type=self.DOCTYPE, document_schema=schema, document_data=content, - file_doc_name=self.file.name, + doc_name=doc_name, + file_count=file_count, + is_communication=bool(self.communication_name), ) ################################### @@ -271,9 +293,12 @@ def _set_flags(self) -> None: self.doc.flags.ignore_links = True def _attach_file(self) -> None: - self.file.attached_to_doctype = self.DOCTYPE - self.file.attached_to_name = self.doc.name - self.file.save() + files_to_attach = self.files if isinstance(self.files, list) else [self.files] + + for file_doc in files_to_attach: + file_doc.attached_to_doctype = self.DOCTYPE + file_doc.attached_to_name = self.doc.name + file_doc.save() def set_exchange_rate(self, from_currency, date, args): company_currency = erpnext.get_company_currency(self.doc.company) diff --git a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json index 73a5a16..007ec5a 100644 --- a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json +++ b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json @@ -15,6 +15,7 @@ "email_configuration_section", "parse_incoming_emails", "parse_party_emails", + "process_one_document_per_communication", "incoming_email_accounts", "party_emails", "custom_fields_tab", @@ -113,6 +114,14 @@ "fieldtype": "Check", "label": "Parse Party Emails" }, + { + "default": "1", + "depends_on": "eval: doc.parse_incoming_emails", + "description": "When enabled, all attachments from a communication are combined to create a single document. When disabled, each attachment creates a separate document.", + "fieldname": "process_one_document_per_communication", + "fieldtype": "Check", + "label": "Process One Document Per Communication" + }, { "fieldname": "email_configuration_section", "fieldtype": "Section Break", diff --git a/transaction_parser/transaction_parser/overrides/communication.py b/transaction_parser/transaction_parser/overrides/communication.py index 679ae95..48e8e12 100644 --- a/transaction_parser/transaction_parser/overrides/communication.py +++ b/transaction_parser/transaction_parser/overrides/communication.py @@ -98,8 +98,18 @@ def _process_attachments( else: country = frappe.db.get_value("Company", company, "country") + supported_extensions = {"pdf", "xlsx", "xls", "csv"} + filtered_attachments = [ + attachment + for attachment in attachments + if attachment.file_url.split(".")[-1].lower() in supported_extensions + ] + + if not filtered_attachments: + return + sorted_attachments = sorted( - attachments, + filtered_attachments, key=lambda attachment: {"xlsx": 0, "csv": 1, "pdf": 2}.get( attachment.file_url.split(".")[-1].lower(), 3 ), @@ -122,16 +132,33 @@ def _process_attachments( def _parse_attachments( doc, country, transaction_type, attachments, ai_model, user, party, company ): - for attachment in attachments: + settings = frappe.get_cached_doc("Transaction Parser Settings") + + if settings.process_one_document_per_communication: + file_urls = [attachment.file_url for attachment in attachments] _parse( country=country, transaction=transaction_type, - file_url=attachment.file_url, + file_urls=file_urls, ai_model=ai_model, user=user, party=party, company=company, + communication_name=doc.name, ) frappe.db.commit() + else: + for attachment in attachments: + _parse( + country=country, + transaction=transaction_type, + file_urls=attachment.file_url, + ai_model=ai_model, + user=user, + party=party, + company=company, + communication_name=doc.name, + ) + frappe.db.commit() doc.db_set("is_processed_by_transaction_parser", 1) diff --git a/transaction_parser/transaction_parser/utils/file_processor.py b/transaction_parser/transaction_parser/utils/file_processor.py index 8a3ccd9..46a0eac 100644 --- a/transaction_parser/transaction_parser/utils/file_processor.py +++ b/transaction_parser/transaction_parser/utils/file_processor.py @@ -22,6 +22,25 @@ def get_content(self, doc, page_limit=None): else: frappe.throw(_("Only PDF, CSV, and Excel files are supported")) + def get_combined_content(self, docs, page_limit=None): + """Combine content from multiple files with clear separators.""" + if not docs: + frappe.throw(_("No files provided for processing")) + + if not isinstance(docs, list): + docs = [docs] + + if len(docs) == 1: + return self.get_content(docs[0], page_limit) + + combined_parts = [] + for index, doc in enumerate(docs, 1): + separator = f"{'=' * 60}\nDocument {index}: {doc.file_name} ({doc.file_type})\n{'=' * 60}" + content = self.get_content(doc, page_limit) + combined_parts.append(f"{separator}\n\n{content}") + + return "\n\n".join(combined_parts) + def _process_pdf(self, doc, page_limit=None): """Process PDF files with OCR and page limiting.""" self.file = io.BytesIO(doc.get_content()) From edb56d0f9a2d1c200a2cd635017cea310380f463 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Tue, 31 Mar 2026 18:59:04 +0530 Subject: [PATCH 74/88] feat: add log names column and formatter to accuracy analysis and version comparison reports --- .../transaction_parser_accuracy_analysis.js | 14 ++++ .../transaction_parser_accuracy_analysis.py | 80 ++++++++++++------ .../transaction_parser_version_comparison.js | 14 ++++ .../transaction_parser_version_comparison.py | 81 +++++++++++++------ 4 files changed, 138 insertions(+), 51 deletions(-) diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js index 5c84d22..5032a3c 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.js @@ -33,6 +33,20 @@ frappe.query_reports["Transaction Parser Accuracy Analysis"] = { set_party_type(report); }, + formatter(value, row, column, data, default_formatter) { + if (column.fieldname === "log_names" && value) { + const names = value.split(",").filter(Boolean); + if (!names.length) return value; + + const filters = frappe.utils.get_url_from_dict({ + name: JSON.stringify(["in", names]), + }); + const url = `/app/parser-benchmark-log?${filters}`; + return `See Logs (${names.length})`; + } + return default_formatter(value, row, column, data); + }, + filters: [ { fieldname: "company", diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py index 27283ad..50e34f2 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py +++ b/transaction_parser/parser_benchmark/report/transaction_parser_accuracy_analysis/transaction_parser_accuracy_analysis.py @@ -55,6 +55,7 @@ class Col(StrEnum): DATASET = "dataset" RUN_COUNT = "run_count" KEY_SCORES = "key_scores" + LOG_NAMES = "log_names" # Fields averaged in party-group summary rows @@ -94,16 +95,21 @@ def run(self): if self.group_by_party: self._group_by_party() + # discover all unique key names for dynamic columns + all_keys = dict.fromkeys( + k for row in self.data for k in (row.get("_key_accuracies") or {}) + ) + # strip internal keys before sending to client for row in self.data: row.pop("_key_accuracies", None) - return self._get_columns(), self.data + return self._get_columns(list(all_keys)), self.data # ── Columns ────────────────────────────────────────────────────── - def _get_columns(self): - return [ + def _get_columns(self, key_names=None): + columns = [ { "fieldname": Col.PARTY, "label": _("Party"), @@ -200,13 +206,29 @@ def _get_columns(self): "fieldtype": "Int", "width": 110, }, + ] + + # dynamic per-key accuracy columns + for key in key_names or []: + columns.append( + { + "fieldname": f"key_{key}", + "label": _(key.replace("_", " ").title() + " (%)"), + "fieldtype": "Percent", + "width": 120, + } + ) + + columns.append( { - "fieldname": Col.KEY_SCORES, - "label": _("Key Scores"), + "fieldname": Col.LOG_NAMES, + "label": _("Logs"), "fieldtype": "Data", - "width": 350, - }, - ] + "width": 100, + } + ) + + return columns # ── Query ───────────────────────────────────────────────────── @@ -310,12 +332,7 @@ def _build_row(self, r, score_details_map): details = score_details_map.get(r.log_name, []) key_accuracies = {d["key"]: d["accuracy"] for d in details} - if key_accuracies: - key_str = ", ".join(f"{k}: {v:.0f}%" for k, v in key_accuracies.items()) - else: - key_str = "" - - return { + row = { Col.PARTY: r.party or _("No Party"), Col.PARTY_NAME: r.party_name or "", Col.ACCURACY_SCORE: r.accuracy_score, @@ -331,10 +348,16 @@ def _build_row(self, r, score_details_map): Col.COMPLETION_TOKENS: r.completion_tokens, Col.TOTAL_TOKENS: r.total_tokens, Col.CURRENCY: r.currency, - Col.KEY_SCORES: key_str, + Col.LOG_NAMES: r.log_name, "_key_accuracies": key_accuracies, } + # per-key accuracy as separate fields + for k, v in key_accuracies.items(): + row[f"key_{k}"] = v + + return row + # ── Aggregation ────────────────────────────────────────────────── def _aggregate_by_config(self): @@ -365,6 +388,11 @@ def _aggregate_by_config(self): agg[Col.DATASET] = rows[0].get(Col.DATASET, "") + # collect all log names used + agg[Col.LOG_NAMES] = ",".join( + r.get(Col.LOG_NAMES) for r in rows if r.get(Col.LOG_NAMES) + ) + for field in _AVG_FIELDS: vals = [r.get(field) or 0 for r in rows] agg[field] = round(sum(vals) / count, 2) if count else 0 @@ -381,13 +409,11 @@ def _aggregate_by_config(self): avg_key_accs = { k: round(sum(v) / len(v), 1) for k, v in all_key_accs.items() } - agg[Col.KEY_SCORES] = ( - ", ".join(f"{k}: {v:.0f}%" for k, v in avg_key_accs.items()) - if avg_key_accs - else "" - ) agg["_key_accuracies"] = avg_key_accs + for k, v in avg_key_accs.items(): + agg[f"key_{k}"] = v + aggregated.append(agg) self.data = aggregated @@ -432,6 +458,11 @@ def _group_row(self, party, rows): "indent": 0, } + # collect all log names from children + row[Col.LOG_NAMES] = ",".join( + r.get(Col.LOG_NAMES) for r in rows if r.get(Col.LOG_NAMES) + ) + for field in _AVG_FIELDS: vals = [r.get(field) or 0 for r in rows] row[field] = round(sum(vals) / count, 2) if count else 0 @@ -446,10 +477,9 @@ def _group_row(self, party, rows): all_key_accs[k].append(v or 0) avg_key_accs = {k: round(sum(v) / len(v), 1) for k, v in all_key_accs.items()} - row[Col.KEY_SCORES] = ( - ", ".join(f"{k}: {v:.0f}%" for k, v in avg_key_accs.items()) - if avg_key_accs - else "" - ) + for k, v in avg_key_accs.items(): + row[f"key_{k}"] = v + + row["_key_accuracies"] = avg_key_accs return row diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js index 2c252e2..bd831ca 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js +++ b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.js @@ -33,6 +33,20 @@ frappe.query_reports["Transaction Parser Version Comparison"] = { set_party_type(report); }, + formatter(value, row, column, data, default_formatter) { + if (column.fieldname === "log_names" && value) { + const names = value.split(",").filter(Boolean); + if (!names.length) return value; + + const filters = frappe.utils.get_url_from_dict({ + name: JSON.stringify(["in", names]), + }); + const url = `/app/parser-benchmark-log?${filters}`; + return `See Logs (${names.length})`; + } + return default_formatter(value, row, column, data); + }, + filters: [ { fieldname: "company", diff --git a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py index e20aae9..5af7dda 100644 --- a/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py +++ b/transaction_parser/parser_benchmark/report/transaction_parser_version_comparison/transaction_parser_version_comparison.py @@ -44,6 +44,7 @@ class Col(StrEnum): ACCURACY_SCORE = "accuracy_score" KEY_SCORES = "key_scores" RUN_COUNT = "run_count" + LOG_NAMES = "log_names" def execute(filters=None): @@ -67,16 +68,21 @@ def run(self): self._aggregate_by_config() self._group_by_party() + # discover all unique key names for dynamic columns + all_keys = dict.fromkeys( + k for row in self.data for k in (row.get("_key_accuracies") or {}) + ) + # strip internal keys for row in self.data: row.pop("_key_accuracies", None) - return self._get_columns(), self.data + return self._get_columns(list(all_keys)), self.data # ── Columns ────────────────────────────────────────────────────── - def _get_columns(self): - return [ + def _get_columns(self, key_names=None): + columns = [ { "fieldname": Col.PARTY, "label": _("Party"), @@ -132,13 +138,29 @@ def _get_columns(self): "fieldtype": "Percent", "width": 120, }, + ] + + # dynamic per-key accuracy columns + for key in key_names or []: + columns.append( + { + "fieldname": f"key_{key}", + "label": _(key.replace("_", " ").title() + " (%)"), + "fieldtype": "Percent", + "width": 120, + } + ) + + columns.append( { - "fieldname": Col.KEY_SCORES, - "label": _("Key Scores"), + "fieldname": Col.LOG_NAMES, + "label": _("Logs"), "fieldtype": "Data", - "width": 350, - }, - ] + "width": 100, + } + ) + + return columns # ── Query ──────────────────────────────────────────────────────── @@ -236,16 +258,10 @@ def _build_row(self, r, score_details_map): details = score_details_map.get(r.log_name, []) key_accuracies = {d["key"]: d["accuracy"] for d in details} - key_str = ( - ", ".join(f"{k}: {v:.0f}%" for k, v in key_accuracies.items()) - if key_accuracies - else "" - ) - short_hash = (r.commit_hash or "")[:7] commit_msg = (r.commit_message or "").split("\n")[0][:80] - return { + row = { Col.PARTY: r.party or _("No Party"), Col.PARTY_NAME: r.party_name or "", Col.DATASET: r.dataset, @@ -254,11 +270,17 @@ def _build_row(self, r, score_details_map): Col.COMMIT_HASH: short_hash, Col.COMMIT_MESSAGE: commit_msg, Col.ACCURACY_SCORE: r.accuracy_score, - Col.KEY_SCORES: key_str, + Col.LOG_NAMES: r.log_name, Col.RUN_COUNT: 1, "_key_accuracies": key_accuracies, } + # per-key accuracy as separate fields + for k, v in key_accuracies.items(): + row[f"key_{k}"] = v + + return row + # ── Aggregation ────────────────────────────────────────────────── def _aggregate_by_config(self): @@ -290,6 +312,11 @@ def _aggregate_by_config(self): Col.RUN_COUNT: count, } + # collect all log names used + agg[Col.LOG_NAMES] = ",".join( + r.get(Col.LOG_NAMES) for r in rows if r.get(Col.LOG_NAMES) + ) + # average accuracy vals = [r.get(Col.ACCURACY_SCORE) or 0 for r in rows] agg[Col.ACCURACY_SCORE] = round(sum(vals) / count, 2) if count else 0 @@ -303,13 +330,11 @@ def _aggregate_by_config(self): avg_key_accs = { k: round(sum(v) / len(v), 1) for k, v in all_key_accs.items() } - agg[Col.KEY_SCORES] = ( - ", ".join(f"{k}: {v:.0f}%" for k, v in avg_key_accs.items()) - if avg_key_accs - else "" - ) agg["_key_accuracies"] = avg_key_accs + for k, v in avg_key_accs.items(): + agg[f"key_{k}"] = v + aggregated.append(agg) self.data = aggregated @@ -354,6 +379,11 @@ def _group_row(self, party, rows): "indent": 0, } + # collect all log names from children + row[Col.LOG_NAMES] = ",".join( + r.get(Col.LOG_NAMES) for r in rows if r.get(Col.LOG_NAMES) + ) + # average accuracy across all child rows vals = [r.get(Col.ACCURACY_SCORE) or 0 for r in rows] row[Col.ACCURACY_SCORE] = round(sum(vals) / count, 2) if count else 0 @@ -365,10 +395,9 @@ def _group_row(self, party, rows): all_key_accs[k].append(v or 0) avg_key_accs = {k: round(sum(v) / len(v), 1) for k, v in all_key_accs.items()} - row[Col.KEY_SCORES] = ( - ", ".join(f"{k}: {v:.0f}%" for k, v in avg_key_accs.items()) - if avg_key_accs - else "" - ) + for k, v in avg_key_accs.items(): + row[f"key_{k}"] = v + + row["_key_accuracies"] = avg_key_accs return row From 995852f4251e597de1ae616ae1028fd2714ff6c9 Mon Sep 17 00:00:00 2001 From: Karm Soni Date: Wed, 1 Apr 2026 00:18:02 +0530 Subject: [PATCH 75/88] feat: refactor file processing to support multiple attachments and improve response merging --- .../ai_integration/parser.py | 8 +- .../ai_integration/prompts.py | 10 +- .../controllers/transaction.py | 42 +- .../transaction_parser_settings.json | 374 ++++++++--------- .../overrides/communication.py | 1 + .../utils/file_processor.py | 19 - .../utils/response_merger.py | 386 ++++++++++++++++++ 7 files changed, 609 insertions(+), 231 deletions(-) create mode 100644 transaction_parser/transaction_parser/utils/response_merger.py diff --git a/transaction_parser/transaction_parser/ai_integration/parser.py b/transaction_parser/transaction_parser/ai_integration/parser.py index d7cd2f1..7cbe23e 100644 --- a/transaction_parser/transaction_parser/ai_integration/parser.py +++ b/transaction_parser/transaction_parser/ai_integration/parser.py @@ -35,12 +35,9 @@ def parse( document_schema: dict, document_data: str, doc_name: str | None = None, - file_count: int = 1, is_communication: bool = False, ) -> dict: - messages = self._build_messages( - document_type, document_schema, document_data, file_count - ) + messages = self._build_messages(document_type, document_schema, document_data) response = self.send_message( messages=messages, @@ -55,11 +52,10 @@ def _build_messages( document_type: str, document_schema: dict, document_data: str, - file_count: int = 1, ) -> tuple: """Build the message structure for AI API call.""" system_prompt = get_system_prompt(document_schema) - user_prompt = get_user_prompt(document_type, document_data, file_count) + user_prompt = get_user_prompt(document_type, document_data) return ( { diff --git a/transaction_parser/transaction_parser/ai_integration/prompts.py b/transaction_parser/transaction_parser/ai_integration/prompts.py index 9e01e5a..95cf2e0 100644 --- a/transaction_parser/transaction_parser/ai_integration/prompts.py +++ b/transaction_parser/transaction_parser/ai_integration/prompts.py @@ -34,16 +34,10 @@ def get_system_prompt(document_schema: dict) -> str: {document_schema}""" -def get_user_prompt(document_type: str, document_data: str, file_count: int = 1) -> str: +def get_user_prompt(document_type: str, document_data: str) -> str: input_doc_type = INPUT_DOCUMENTS.get(document_type, "document") - base_instruction = f"Generate {document_type} for given {input_doc_type} according to above JSON schema." - - if file_count > 1: - multi_file_instruction = f"\n\nIMPORTANT: The data below contains information from {file_count} related documents. Consolidate and merge the information from all documents to create a single unified {document_type}. Combine item lists, sum totals appropriately, and merge party/address information." - base_instruction += multi_file_instruction - - return f"""{base_instruction} + return f"""Generate {document_type} for given {input_doc_type} according to above JSON schema. Document data is given below: {document_data}""" diff --git a/transaction_parser/transaction_parser/controllers/transaction.py b/transaction_parser/transaction_parser/controllers/transaction.py index 8ae5e7c..69c5fd7 100644 --- a/transaction_parser/transaction_parser/controllers/transaction.py +++ b/transaction_parser/transaction_parser/controllers/transaction.py @@ -7,6 +7,9 @@ from transaction_parser.transaction_parser.ai_integration.parser import AIParser from transaction_parser.transaction_parser.utils import to_dict from transaction_parser.transaction_parser.utils.file_processor import FileProcessor +from transaction_parser.transaction_parser.utils.response_merger import ( + ResponseMerger, +) class Transaction: @@ -81,29 +84,46 @@ def initialize(self) -> None: def _parse_file_content( self, ai_model: str | None = None, page_limit: int | None = None ) -> dict: - file_processor = FileProcessor() - doc_name = None - file_count = len(self.files) - if len(self.files) > 1: - content = file_processor.get_combined_content(self.files, page_limit) - doc_name = self.communication_name + return self._parse_multiple_files(ai_model, page_limit) - else: - content = file_processor.get_content(self.files, page_limit) - doc_name = self.files[0].name + return self._parse_single_file(self.files[0], ai_model, page_limit) + def _parse_single_file( + self, + file, + ai_model: str | None = None, + page_limit: int | None = None, + ) -> dict: + content = FileProcessor().get_content(file, page_limit) schema = self.get_schema() return AIParser(ai_model, self.settings).parse( document_type=self.DOCTYPE, document_schema=schema, document_data=content, - doc_name=doc_name, - file_count=file_count, + doc_name=self.communication_name or file.name, is_communication=bool(self.communication_name), ) + def _parse_multiple_files( + self, ai_model: str | None = None, page_limit: int | None = None + ) -> dict: + response = self._parse_single_file(self.files[0], ai_model, page_limit) + merger = ResponseMerger( + response, + schema=self.get_schema(), + ) + + for file in self.files[1:]: + if merger.is_complete(): + break + + new_response = self._parse_single_file(file, ai_model, page_limit) + merger.merge(new_response) + + return merger.response + ################################### ########## Output Schema ########## ################################### diff --git a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json index 007ec5a..96706b2 100644 --- a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json +++ b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json @@ -1,189 +1,189 @@ { - "actions": [], - "allow_rename": 1, - "creation": "2025-03-11 16:54:51.622854", - "doctype": "DocType", - "engine": "InnoDB", - "field_order": [ - "api_tab", - "enabled", - "ai_model_section", - "default_ai_model", - "api_keys", - "transaction_configurations_section", - "invoice_lookback_count", - "email_configuration_section", - "parse_incoming_emails", - "parse_party_emails", - "process_one_document_per_communication", - "incoming_email_accounts", - "party_emails", - "custom_fields_tab", - "custom_fields_section", - "column_break_kdtw", - "base_schema", - "tax_schema", - "address_schema", - "party_schema", - "item_schema" - ], - "fields": [ - { - "fieldname": "api_tab", - "fieldtype": "Tab Break", - "label": "API" - }, - { - "default": "0", - "fieldname": "enabled", - "fieldtype": "Check", - "label": "Enabled" - }, - { - "depends_on": "eval: doc.enabled", - "fieldname": "custom_fields_tab", - "fieldtype": "Tab Break", - "label": "Customizations" - }, - { - "fieldname": "custom_fields_section", - "fieldtype": "Section Break", - "label": "Custom Fields" - }, - { - "depends_on": "eval: doc.enabled", - "fieldname": "column_break_kdtw", - "fieldtype": "Column Break" - }, - { - "depends_on": "eval: doc.enabled", - "fieldname": "base_schema", - "fieldtype": "JSON", - "label": "Base Schema" - }, - { - "depends_on": "eval: doc.enabled", - "fieldname": "tax_schema", - "fieldtype": "JSON", - "label": "Tax Schema" - }, - { - "depends_on": "eval: doc.enabled", - "fieldname": "address_schema", - "fieldtype": "JSON", - "label": "Address Schema" - }, - { - "depends_on": "eval: doc.enabled", - "fieldname": "party_schema", - "fieldtype": "JSON", - "label": "Party Schema" - }, - { - "depends_on": "eval: doc.enabled", - "fieldname": "item_schema", - "fieldtype": "JSON", - "label": "Item Schema" - }, - { - "depends_on": "eval: doc.enabled", - "fieldname": "default_ai_model", - "fieldtype": "Select", - "label": "Default AI Model", - "mandatory_depends_on": "eval: doc.enabled", - "options": "DeepSeek Chat\nDeepSeek Reasoner\nOpenAI gpt-4o\nOpenAI gpt-4o-mini\nOpenAI gpt-5\nOpenAI gpt-5-mini\nGoogle Gemini Pro\nGoogle Gemini Flash" - }, - { - "depends_on": "eval: doc.enabled", - "fieldname": "api_keys", - "fieldtype": "Table", - "label": "API Keys", - "options": "Transaction Parser API Key Item" - }, - { - "default": "0", - "fieldname": "parse_incoming_emails", - "fieldtype": "Check", - "label": "Parse Incoming Emails" - }, - { - "default": "0", - "depends_on": "eval: doc.parse_incoming_emails", - "description": "Directly Parse Emails from Party", - "fieldname": "parse_party_emails", - "fieldtype": "Check", - "label": "Parse Party Emails" - }, - { - "default": "1", - "depends_on": "eval: doc.parse_incoming_emails", - "description": "When enabled, all attachments from a communication are combined to create a single document. When disabled, each attachment creates a separate document.", - "fieldname": "process_one_document_per_communication", - "fieldtype": "Check", - "label": "Process One Document Per Communication" - }, - { - "fieldname": "email_configuration_section", - "fieldtype": "Section Break", - "label": "Incoming Email Configurations" - }, - { - "fieldname": "ai_model_section", - "fieldtype": "Section Break", - "label": "AI Model Configurations" - }, - { - "depends_on": "eval: doc.parse_incoming_emails", - "description": "If an email is received on any of these Email Accounts, an attempt will be made to generate the specified Transaction from its attachments.", - "fieldname": "incoming_email_accounts", - "fieldtype": "Table", - "label": "Incoming Email Accounts", - "mandatory_depends_on": "eval: doc.parse_incoming_emails && !doc.parse_party_emails", - "options": "Transaction Parser Email Account" - }, - { - "depends_on": "eval: doc.parse_incoming_emails", - "description": "Mapping Party based on Email Address", - "fieldname": "party_emails", - "fieldtype": "Table", - "label": "Party Emails", - "options": "Transaction Parser Party Email" - }, - { - "fieldname": "invoice_lookback_count", - "fieldtype": "Int", - "in_list_view": 1, - "label": "Number of Past Invoices to Consider for Item Code Selection", - "reqd": 1 - }, - { - "fieldname": "transaction_configurations_section", - "fieldtype": "Section Break", - "label": "Transaction Configurations" - } - ], - "index_web_pages_for_search": 1, - "issingle": 1, - "links": [], - "modified": "2025-09-08 08:48:58.870032", - "modified_by": "Administrator", - "module": "Transaction Parser", - "name": "Transaction Parser Settings", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "print": 1, - "read": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "row_format": "Dynamic", - "sort_field": "modified", - "sort_order": "DESC", - "states": [] + "actions": [], + "allow_rename": 1, + "creation": "2025-03-11 16:54:51.622854", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "api_tab", + "enabled", + "ai_model_section", + "default_ai_model", + "api_keys", + "transaction_configurations_section", + "invoice_lookback_count", + "email_configuration_section", + "parse_incoming_emails", + "parse_party_emails", + "process_one_document_per_communication", + "incoming_email_accounts", + "party_emails", + "custom_fields_tab", + "custom_fields_section", + "column_break_kdtw", + "base_schema", + "tax_schema", + "address_schema", + "party_schema", + "item_schema" + ], + "fields": [ + { + "fieldname": "api_tab", + "fieldtype": "Tab Break", + "label": "API" + }, + { + "default": "0", + "fieldname": "enabled", + "fieldtype": "Check", + "label": "Enabled" + }, + { + "depends_on": "eval: doc.enabled", + "fieldname": "custom_fields_tab", + "fieldtype": "Tab Break", + "label": "Customizations" + }, + { + "fieldname": "custom_fields_section", + "fieldtype": "Section Break", + "label": "Custom Fields" + }, + { + "depends_on": "eval: doc.enabled", + "fieldname": "column_break_kdtw", + "fieldtype": "Column Break" + }, + { + "depends_on": "eval: doc.enabled", + "fieldname": "base_schema", + "fieldtype": "JSON", + "label": "Base Schema" + }, + { + "depends_on": "eval: doc.enabled", + "fieldname": "tax_schema", + "fieldtype": "JSON", + "label": "Tax Schema" + }, + { + "depends_on": "eval: doc.enabled", + "fieldname": "address_schema", + "fieldtype": "JSON", + "label": "Address Schema" + }, + { + "depends_on": "eval: doc.enabled", + "fieldname": "party_schema", + "fieldtype": "JSON", + "label": "Party Schema" + }, + { + "depends_on": "eval: doc.enabled", + "fieldname": "item_schema", + "fieldtype": "JSON", + "label": "Item Schema" + }, + { + "depends_on": "eval: doc.enabled", + "fieldname": "default_ai_model", + "fieldtype": "Select", + "label": "Default AI Model", + "mandatory_depends_on": "eval: doc.enabled", + "options": "DeepSeek Chat\nDeepSeek Reasoner\nOpenAI gpt-4o\nOpenAI gpt-4o-mini\nOpenAI gpt-5\nOpenAI gpt-5-mini\nGoogle Gemini Pro\nGoogle Gemini Flash" + }, + { + "depends_on": "eval: doc.enabled", + "fieldname": "api_keys", + "fieldtype": "Table", + "label": "API Keys", + "options": "Transaction Parser API Key Item" + }, + { + "default": "0", + "fieldname": "parse_incoming_emails", + "fieldtype": "Check", + "label": "Parse Incoming Emails" + }, + { + "default": "0", + "depends_on": "eval: doc.parse_incoming_emails", + "description": "Directly Parse Emails from Party", + "fieldname": "parse_party_emails", + "fieldtype": "Check", + "label": "Parse Party Emails" + }, + { + "default": "1", + "depends_on": "eval: doc.parse_incoming_emails", + "description": "When enabled, all attachments from a communication are combined to create a single document. When disabled, each attachment creates a separate document.", + "fieldname": "process_one_document_per_communication", + "fieldtype": "Check", + "label": "Process One Document Per Communication" + }, + { + "fieldname": "email_configuration_section", + "fieldtype": "Section Break", + "label": "Incoming Email Configurations" + }, + { + "fieldname": "ai_model_section", + "fieldtype": "Section Break", + "label": "AI Model Configurations" + }, + { + "depends_on": "eval: doc.parse_incoming_emails", + "description": "If an email is received on any of these Email Accounts, an attempt will be made to generate the specified Transaction from its attachments.", + "fieldname": "incoming_email_accounts", + "fieldtype": "Table", + "label": "Incoming Email Accounts", + "mandatory_depends_on": "eval: doc.parse_incoming_emails && !doc.parse_party_emails", + "options": "Transaction Parser Email Account" + }, + { + "depends_on": "eval: doc.parse_incoming_emails", + "description": "Mapping Party based on Email Address", + "fieldname": "party_emails", + "fieldtype": "Table", + "label": "Party Emails", + "options": "Transaction Parser Party Email" + }, + { + "fieldname": "invoice_lookback_count", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Number of Past Invoices to Consider for Item Code Selection", + "reqd": 1 + }, + { + "fieldname": "transaction_configurations_section", + "fieldtype": "Section Break", + "label": "Transaction Configurations" + } + ], + "index_web_pages_for_search": 1, + "issingle": 1, + "links": [], + "modified": "2025-09-08 08:48:58.870032", + "modified_by": "Administrator", + "module": "Transaction Parser", + "name": "Transaction Parser Settings", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "row_format": "Dynamic", + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/transaction_parser/transaction_parser/overrides/communication.py b/transaction_parser/transaction_parser/overrides/communication.py index 48e8e12..99c1b7e 100644 --- a/transaction_parser/transaction_parser/overrides/communication.py +++ b/transaction_parser/transaction_parser/overrides/communication.py @@ -126,6 +126,7 @@ def _process_attachments( party=party, company=default_company if not company else company, queue="long", + now=True, ) diff --git a/transaction_parser/transaction_parser/utils/file_processor.py b/transaction_parser/transaction_parser/utils/file_processor.py index 46a0eac..8a3ccd9 100644 --- a/transaction_parser/transaction_parser/utils/file_processor.py +++ b/transaction_parser/transaction_parser/utils/file_processor.py @@ -22,25 +22,6 @@ def get_content(self, doc, page_limit=None): else: frappe.throw(_("Only PDF, CSV, and Excel files are supported")) - def get_combined_content(self, docs, page_limit=None): - """Combine content from multiple files with clear separators.""" - if not docs: - frappe.throw(_("No files provided for processing")) - - if not isinstance(docs, list): - docs = [docs] - - if len(docs) == 1: - return self.get_content(docs[0], page_limit) - - combined_parts = [] - for index, doc in enumerate(docs, 1): - separator = f"{'=' * 60}\nDocument {index}: {doc.file_name} ({doc.file_type})\n{'=' * 60}" - content = self.get_content(doc, page_limit) - combined_parts.append(f"{separator}\n\n{content}") - - return "\n\n".join(combined_parts) - def _process_pdf(self, doc, page_limit=None): """Process PDF files with OCR and page limiting.""" self.file = io.BytesIO(doc.get_content()) diff --git a/transaction_parser/transaction_parser/utils/response_merger.py b/transaction_parser/transaction_parser/utils/response_merger.py new file mode 100644 index 0000000..0398a41 --- /dev/null +++ b/transaction_parser/transaction_parser/utils/response_merger.py @@ -0,0 +1,386 @@ +from dataclasses import dataclass +from typing import Any + +from frappe import _dict + + +@dataclass +class FieldType: + """Base class for field types in schema.""" + + required: bool + + def is_empty(self, value: Any) -> bool: + """Check if a value is considered empty.""" + return ( + value is None + or value == "" + or (isinstance(value, list | dict) and len(value) == 0) + ) + + +@dataclass +class PrimitiveField(FieldType): + """Represents a primitive field (string, number, date, etc.).""" + + pass + + +@dataclass +class ObjectField(FieldType): + """Represents a nested object field with child fields.""" + + children: dict[str, FieldType] + + +@dataclass +class ListField(FieldType): + """Represents a list/array field.""" + + item_type: FieldType # Type of items in the list + + +class SchemaParser: + """Parses document schema and builds structured field type hierarchy.""" + + def parse(self, schema: dict) -> dict[str, FieldType]: + """ + Parse schema dictionary into structured field types. + + Args: + schema: Schema dictionary to parse + + Returns: + Dictionary mapping field names to FieldType instances + """ + fields = {} + + for key, value in schema.items(): + fields[key] = self._parse_field(value) + + return fields + + def _parse_field(self, schema_value: Any) -> FieldType: + """ + Parse a single field value into appropriate FieldType. + + Args: + schema_value: Value from schema (string, list, or dict) + + Returns: + Appropriate FieldType instance + """ + # List field: schema value is a list + if isinstance(schema_value, list): + required = self._is_required(schema_value) + + # Empty list or primitive list items + if not schema_value: + return ListField( + required=required, item_type=PrimitiveField(required=True) + ) + + item_schema = schema_value[0] + + # List of objects (e.g., item_list) + if isinstance(item_schema, dict): + item_fields = self.parse(item_schema) + return ListField( + required=required, + item_type=ObjectField(required=True, children=item_fields), + ) + + # List of primitives (e.g., emails, phones) + else: + return ListField( + required=required, + item_type=PrimitiveField(required=self._is_required(item_schema)), + ) + + # Object field: schema value is a dict + elif isinstance(schema_value, dict): + children = self.parse(schema_value) + return ObjectField(required=True, children=children) + + # Primitive field: schema value is a string + else: + return PrimitiveField(required=self._is_required(schema_value)) + + def _is_required(self, schema_value: Any) -> bool: + """ + Check if field is required based on schema notation. + + Args: + schema_value: Schema value (string or other type) + + Returns: + True if required, False if optional + """ + if isinstance(schema_value, str): + return "| null" not in schema_value and "| 0" not in schema_value + + return True + + +class ResponseMerger: + """Schema-driven merger for AI responses from multiple attachments.""" + + def __init__(self, response: dict, schema: dict): + """ + Initialize ResponseMerger with a base response and optional schema. + + Args: + response: Initial response dict from first file parsing + schema: Document schema dict for automatic field detection + """ + self.response = _dict(response) if isinstance(response, dict) else response + self.schema = schema + + # Parse schema into structured field types + parser = SchemaParser() + self.fields = parser.parse(schema) + + def is_complete(self) -> bool: + """ + Check if all required fields are filled. + + Returns: + True if complete, False otherwise + """ + return len(self.get_missing_fields()) == 0 + + def get_missing_fields(self) -> list[str]: + """ + Get list of missing required field paths. + + Returns: + List of dot-separated field paths that are missing + """ + missing = [] + self._check_missing_fields(self.fields, self.response, "", missing) + return missing + + def _check_missing_fields( + self, + fields: dict[str, FieldType], + data: dict, + path_prefix: str, + missing: list[str], + ) -> None: + """ + Recursively check for missing required fields. + + Args: + fields: Field type definitions + data: Current data dict to check + path_prefix: Current path prefix for nested fields + missing: List to append missing field paths to + """ + for key, field_type in fields.items(): + field_path = f"{path_prefix}.{key}" if path_prefix else key + value = data.get(key) if isinstance(data, dict) else None + + # Check primitive fields + if isinstance(field_type, PrimitiveField): + if field_type.required and field_type.is_empty(value): + missing.append(field_path) + + # Check object fields recursively + elif isinstance(field_type, ObjectField): + if field_type.required and field_type.is_empty(value): + missing.append(field_path) + + elif value: + self._check_missing_fields( + field_type.children, value, field_path, missing + ) + + # List fields are checked during merging, not at top level + # (we care about item contents, not just list existence) + + def merge(self, new_response: dict) -> None: + """ + Merge new response into existing response. + + Args: + new_response: New response dict to merge from + """ + self._merge_fields(self.fields, self.response, new_response) + + def _merge_fields( + self, + fields: dict[str, FieldType], + target: dict, + source: dict, + ) -> None: + """ + Merge source data into target based on field definitions. + + Args: + fields: Field type definitions + target: Target dict to merge into + source: Source dict to merge from + """ + for key, field_type in fields.items(): + source_value = source.get(key) + + # Skip if source doesn't have this field + if source_value is None: + continue + + # Handle primitive fields + if isinstance(field_type, PrimitiveField): + self._merge_primitive(target, key, source_value) + + # Handle object fields + elif isinstance(field_type, ObjectField): + self._merge_object(field_type, target, key, source_value) + + # Handle list fields + elif isinstance(field_type, ListField): + self._merge_list(field_type, target, key, source_value) + + def _merge_primitive(self, target: dict, key: str, source_value: Any) -> None: + """ + Merge primitive field value. + + Args: + target: Target dict + key: Field key + source_value: Value from source + """ + # Only fill if target is empty + if not target.get(key) and source_value: + target[key] = source_value + + def _merge_object( + self, + field_type: ObjectField, + target: dict, + key: str, + source_value: dict, + ) -> None: + """ + Merge object field recursively. + + Args: + field_type: ObjectField definition + target: Target dict + key: Field key + source_value: Object value from source + """ + # Ensure target has the object + if key not in target: + target[key] = {} + + # Recursively merge children + self._merge_fields(field_type.children, target[key], source_value) + + def _merge_list( + self, + field_type: ListField, + target: dict, + key: str, + source_value: list, + ) -> None: + """ + Merge list field by matching items. + + Args: + field_type: ListField definition + target: Target dict + key: Field key + source_value: List value from source + """ + target_list = target.get(key, []) + + # If target list is empty, use source list + if not target_list: + target[key] = source_value + return + + # If source list is empty, nothing to merge + if not source_value: + return + + # Only merge if items are objects (not primitive lists) + if not isinstance(field_type.item_type, ObjectField): + return + + # Match and merge items + for target_item in target_list: + matched_item = self._find_matching_item( + target_item, source_value, field_type.item_type + ) + if matched_item: + self._merge_fields( + field_type.item_type.children, target_item, matched_item + ) + + def _find_matching_item( + self, + target_item: dict, + source_items: list[dict], + item_type: ObjectField, + ) -> dict | None: + """ + Find matching item in source list using intelligent matching. + + Matching strategy: + - Extract key fields from item schema (party_item_code, quantity, rate, description) + - Require at least 2 matching fields for a match + + Args: + target_item: Item to find match for + source_items: List of candidate items + item_type: ObjectField describing item structure + + Returns: + Matching item or None + """ + # Define priority key fields for matching + key_field_names = ["party_item_code", "quantity", "rate", "description"] + + # Filter to only fields that exist in schema + available_keys = [ + name for name in key_field_names if name in item_type.children + ] + + # Try to find match + for source_item in source_items: + if self._items_match(target_item, source_item, available_keys): + return source_item + + return None + + def _items_match( + self, + item1: dict, + item2: dict, + key_fields: list[str], + ) -> bool: + """ + Check if two items match based on key fields. + + Requires at least 2 matching fields. + + Args: + item1: First item + item2: Second item + key_fields: List of key field names to check + + Returns: + True if items match, False otherwise + """ + matches = 0 + + for field in key_fields: + value1 = item1.get(field) + value2 = item2.get(field) + + # Both must exist and be equal + if value1 and value2 and value1 == value2: + matches += 1 + + # Require at least 2 matching fields + return matches >= 2 From 1906196fc898e458dd0960ed00777b69ae343a77 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 1 Apr 2026 11:00:21 +0530 Subject: [PATCH 76/88] feat: enhance AI parsing and prompts with company details integration --- transaction_parser/parser_benchmark/runner.py | 1 + .../public/js/transaction_parser_dialog.js | 11 +++++++ .../transaction_parser/__init__.py | 3 +- .../ai_integration/parser.py | 29 +++++++++++++++++-- .../ai_integration/prompts.py | 21 ++++++++++++-- .../controllers/transaction.py | 1 + 6 files changed, 59 insertions(+), 7 deletions(-) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index 8f949d7..a4f59a0 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -156,6 +156,7 @@ def _run_ai_parsing(self, file_content: str, file_name: str) -> dict: document_schema=self.controller.get_schema(), document_data=file_content, file_doc_name=file_name, + company=self.dataset.company, ) self.log.ai_parse_time = flt(default_timer() - start, self.precision) diff --git a/transaction_parser/public/js/transaction_parser_dialog.js b/transaction_parser/public/js/transaction_parser_dialog.js index 1382721..919052f 100644 --- a/transaction_parser/public/js/transaction_parser_dialog.js +++ b/transaction_parser/public/js/transaction_parser_dialog.js @@ -52,6 +52,17 @@ async function create_transaction_parser_dialog(transaction_type, list_view) { default: get_default_country(), reqd: 1, }, + { + fieldtype: "Section Break", + }, + { + fieldname: "company", + label: __("Company"), + fieldtype: "Link", + options: "Company", + default: frappe.defaults.get_user_default("Company"), + reqd: 1, + }, ], primary_action_label: __("Submit"), primary_action(values) { diff --git a/transaction_parser/transaction_parser/__init__.py b/transaction_parser/transaction_parser/__init__.py index abd27da..fb35787 100644 --- a/transaction_parser/transaction_parser/__init__.py +++ b/transaction_parser/transaction_parser/__init__.py @@ -15,7 +15,7 @@ @frappe.whitelist() -def parse(transaction, country, file_url, ai_model=None, page_limit=None): +def parse(transaction, country, file_url, ai_model=None, page_limit=None, company=None): is_enabled() frappe.has_permission(TRANSACTION_MAP[transaction], "create", throw=True) @@ -27,6 +27,7 @@ def parse(transaction, country, file_url, ai_model=None, page_limit=None): file_url=cstr(file_url), ai_model=cstr(ai_model), page_limit=cint(page_limit), + company=cstr(company) if company else None, queue="long", now=frappe.conf.developer_mode, ) diff --git a/transaction_parser/transaction_parser/ai_integration/parser.py b/transaction_parser/transaction_parser/ai_integration/parser.py index a9925e1..8349288 100644 --- a/transaction_parser/transaction_parser/ai_integration/parser.py +++ b/transaction_parser/transaction_parser/ai_integration/parser.py @@ -36,19 +36,27 @@ def parse( document_schema: dict, document_data: str, file_doc_name: str | None = None, + company: str | None = None, ) -> dict: - messages = self._build_messages(document_type, document_schema, document_data) + messages = self._build_messages( + document_type, document_schema, document_data, company + ) self.ai_response = self.send_message( messages=messages, file_doc_name=file_doc_name ) return self.get_content(self.ai_response) def _build_messages( - self, document_type: str, document_schema: dict, document_data: str + self, + document_type: str, + document_schema: dict, + document_data: str, + company: str | None = None, ) -> tuple: """Build the message structure for AI API call.""" + company_info = self._get_company_info(company) if company else "" system_prompt = get_system_prompt(document_schema) - user_prompt = get_user_prompt(document_type, document_data) + user_prompt = get_user_prompt(document_type, document_data, company_info) return ( { @@ -61,6 +69,21 @@ def _build_messages( }, ) + @staticmethod + def _get_company_info(company: str) -> str: + """Build a company context string with name and address if available.""" + from frappe.contacts.doctype.address.address import get_company_address + from frappe.utils import strip_html + + info = f"Company: {company}" + + address = get_company_address(company) + if address.company_address_display: + address_text = strip_html(address.company_address_display).strip() + info += f"\nLocated at: {address_text}" + + return info + def send_message(self, messages: tuple, file_doc_name: str | None = None) -> dict: """Send messages to AI API and handle the response.""" log = self._create_log_entry(file_doc_name) diff --git a/transaction_parser/transaction_parser/ai_integration/prompts.py b/transaction_parser/transaction_parser/ai_integration/prompts.py index 95cf2e0..8db715b 100644 --- a/transaction_parser/transaction_parser/ai_integration/prompts.py +++ b/transaction_parser/transaction_parser/ai_integration/prompts.py @@ -5,7 +5,8 @@ def get_system_prompt(document_schema: dict) -> str: - return f"""You are a JSON data extraction and validation expert for your company's ERP platform. + prompt = f"""You are a JSON data extraction and validation expert for an ERP platform. + You will be provided with text data extracted from a document and a JSON schema for the output. Your role is to: @@ -33,11 +34,25 @@ def get_system_prompt(document_schema: dict) -> str: JSON schema is given below: {document_schema}""" + return prompt + -def get_user_prompt(document_type: str, document_data: str) -> str: +def get_user_prompt( + document_type: str, document_data: str, company_info: str = "" +) -> str: input_doc_type = INPUT_DOCUMENTS.get(document_type, "document") - return f"""Generate {document_type} for given {input_doc_type} according to above JSON schema. + company_context = "" + if company_info: + company_context = f""" + +This {input_doc_type} is received by the following company: +{company_info} + +Use this to correctly identify the company as the buyer/recipient and the other party as the vendor/supplier. +""" + + return f"""Generate {document_type} for the given {input_doc_type} according to above JSON schema.{company_context} Document data is given below: {document_data}""" diff --git a/transaction_parser/transaction_parser/controllers/transaction.py b/transaction_parser/transaction_parser/controllers/transaction.py index 178ffed..d257331 100644 --- a/transaction_parser/transaction_parser/controllers/transaction.py +++ b/transaction_parser/transaction_parser/controllers/transaction.py @@ -85,6 +85,7 @@ def _parse_file_content( document_schema=schema, document_data=content, file_doc_name=self.file.name, + company=self.company, ) ################################### From df95e1d3c0bb738280822f8c89457e6f40bf05f7 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 1 Apr 2026 11:44:36 +0530 Subject: [PATCH 77/88] fix: enhance prompts with company details and improve email account matching logic --- .../ai_integration/prompts.py | 5 +--- .../transaction_parser_party_email.json | 2 +- .../overrides/communication.py | 25 ++++++++++--------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/transaction_parser/transaction_parser/ai_integration/prompts.py b/transaction_parser/transaction_parser/ai_integration/prompts.py index 8db715b..19f134d 100644 --- a/transaction_parser/transaction_parser/ai_integration/prompts.py +++ b/transaction_parser/transaction_parser/ai_integration/prompts.py @@ -5,8 +5,7 @@ def get_system_prompt(document_schema: dict) -> str: - prompt = f"""You are a JSON data extraction and validation expert for an ERP platform. - + return f"""You are a JSON data extraction and validation expert for your company's ERP platform. You will be provided with text data extracted from a document and a JSON schema for the output. Your role is to: @@ -34,8 +33,6 @@ def get_system_prompt(document_schema: dict) -> str: JSON schema is given below: {document_schema}""" - return prompt - def get_user_prompt( document_type: str, document_data: str, company_info: str = "" diff --git a/transaction_parser/transaction_parser/doctype/transaction_parser_party_email/transaction_parser_party_email.json b/transaction_parser/transaction_parser/doctype/transaction_parser_party_email/transaction_parser_party_email.json index c8aab80..72e3372 100644 --- a/transaction_parser/transaction_parser/doctype/transaction_parser_party_email/transaction_parser_party_email.json +++ b/transaction_parser/transaction_parser/doctype/transaction_parser_party_email/transaction_parser_party_email.json @@ -50,4 +50,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} \ No newline at end of file +} diff --git a/transaction_parser/transaction_parser/overrides/communication.py b/transaction_parser/transaction_parser/overrides/communication.py index 679ae95..9cc0687 100644 --- a/transaction_parser/transaction_parser/overrides/communication.py +++ b/transaction_parser/transaction_parser/overrides/communication.py @@ -13,6 +13,18 @@ def on_update(doc, method=None): if not (settings.enabled and settings.parse_incoming_emails): return + matched_account = next( + ( + row + for row in settings.incoming_email_accounts + if row.to_email in doc.recipients + ), + None, + ) + + if not matched_account: + return + if settings.parse_party_emails: matched_party_config = next( (row for row in settings.party_emails if row.party_email == doc.sender), @@ -41,21 +53,10 @@ def on_update(doc, method=None): settings, default_user, matched_party_config.party, + matched_party_config.company, ) return - matched_account = next( - ( - row - for row in settings.incoming_email_accounts - if row.to_email in doc.recipients - ), - None, - ) - - if not matched_account: - return - # Attachments are not available when the Communication doc is created. # Next time the doc is updated, we will check for attachments, # and update the flag `is_processed_by_transaction_parser` accordingly. From 5d45c3d622f3f29a5abec830c592b8b235802f07 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 1 Apr 2026 12:29:24 +0530 Subject: [PATCH 78/88] fix: ensure company address is checked before accessing display property and update company context role hints in prompts --- .../transaction_parser/ai_integration/parser.py | 2 +- .../transaction_parser/ai_integration/prompts.py | 7 ++++++- .../transaction_parser/overrides/communication.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/transaction_parser/transaction_parser/ai_integration/parser.py b/transaction_parser/transaction_parser/ai_integration/parser.py index 8349288..262bdcf 100644 --- a/transaction_parser/transaction_parser/ai_integration/parser.py +++ b/transaction_parser/transaction_parser/ai_integration/parser.py @@ -78,7 +78,7 @@ def _get_company_info(company: str) -> str: info = f"Company: {company}" address = get_company_address(company) - if address.company_address_display: + if address and address.company_address_display: address_text = strip_html(address.company_address_display).strip() info += f"\nLocated at: {address_text}" diff --git a/transaction_parser/transaction_parser/ai_integration/prompts.py b/transaction_parser/transaction_parser/ai_integration/prompts.py index 19f134d..8fd4d91 100644 --- a/transaction_parser/transaction_parser/ai_integration/prompts.py +++ b/transaction_parser/transaction_parser/ai_integration/prompts.py @@ -41,12 +41,17 @@ def get_user_prompt( company_context = "" if company_info: + if document_type == "Sales Order": + role_hint = "Use this to correctly identify the company as the seller/vendor and the other party as the customer/buyer." + else: + role_hint = "Use this to correctly identify the company as the buyer/recipient and the other party as the vendor/supplier." + company_context = f""" This {input_doc_type} is received by the following company: {company_info} -Use this to correctly identify the company as the buyer/recipient and the other party as the vendor/supplier. +{role_hint} """ return f"""Generate {document_type} for the given {input_doc_type} according to above JSON schema.{company_context} diff --git a/transaction_parser/transaction_parser/overrides/communication.py b/transaction_parser/transaction_parser/overrides/communication.py index 9cc0687..6ba0397 100644 --- a/transaction_parser/transaction_parser/overrides/communication.py +++ b/transaction_parser/transaction_parser/overrides/communication.py @@ -53,7 +53,7 @@ def on_update(doc, method=None): settings, default_user, matched_party_config.party, - matched_party_config.company, + matched_account.company, ) return From 64807749d34c9e8b4e129bbdbc46f98e455ecee5 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 1 Apr 2026 12:37:37 +0530 Subject: [PATCH 79/88] fix: ensure company address is only appended if it is not empty in the info string --- .../transaction_parser/ai_integration/parser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/transaction_parser/transaction_parser/ai_integration/parser.py b/transaction_parser/transaction_parser/ai_integration/parser.py index 262bdcf..bd70cfd 100644 --- a/transaction_parser/transaction_parser/ai_integration/parser.py +++ b/transaction_parser/transaction_parser/ai_integration/parser.py @@ -80,7 +80,9 @@ def _get_company_info(company: str) -> str: address = get_company_address(company) if address and address.company_address_display: address_text = strip_html(address.company_address_display).strip() - info += f"\nLocated at: {address_text}" + + if address_text: + info += f"\nLocated at: {address_text}" return info From a7d019432bb810c2a023f699c4b33ea2408ae285 Mon Sep 17 00:00:00 2001 From: Karm Soni Date: Wed, 1 Apr 2026 12:39:03 +0530 Subject: [PATCH 80/88] feat: enhance response merging and attachment processing for improved file handling --- .../transaction_parser/__init__.py | 19 +- .../controllers/transaction.py | 7 + .../overrides/communication.py | 9 +- .../utils/response_merger.py | 206 +++--------------- 4 files changed, 53 insertions(+), 188 deletions(-) diff --git a/transaction_parser/transaction_parser/__init__.py b/transaction_parser/transaction_parser/__init__.py index fcc2c56..a327757 100644 --- a/transaction_parser/transaction_parser/__init__.py +++ b/transaction_parser/transaction_parser/__init__.py @@ -54,7 +54,24 @@ def _parse( file_urls = [file_urls] file_names = frappe.get_list( - "File", filters={"file_url": ("in", file_urls)}, pluck="name" + "File", + filters={"file_url": ("in", file_urls)}, + fields=["name", "file_type"], + order_by="creation desc", + group_by="file_url", + ) + + # xlsx/xls first, then pdf, then csv. If no xlsx/xls, csv takes its place. + file_types = {(f.file_type or "").lower() for f in file_names} + has_spreadsheet = file_types & {"xlsx", "xls"} + + if has_spreadsheet: + FILE_TYPE_PRIORITY = {"xlsx": 0, "xls": 0, "pdf": 1, "csv": 2} + else: + FILE_TYPE_PRIORITY = {"csv": 0, "pdf": 1} + + file_names.sort( + key=lambda f: FILE_TYPE_PRIORITY.get((f.file_type or "").lower(), 99) ) files = [] diff --git a/transaction_parser/transaction_parser/controllers/transaction.py b/transaction_parser/transaction_parser/controllers/transaction.py index 69c5fd7..e3cc57b 100644 --- a/transaction_parser/transaction_parser/controllers/transaction.py +++ b/transaction_parser/transaction_parser/controllers/transaction.py @@ -113,6 +113,7 @@ def _parse_multiple_files( merger = ResponseMerger( response, schema=self.get_schema(), + match_keys=self.get_match_keys(), ) for file in self.files[1:]: @@ -128,6 +129,12 @@ def _parse_multiple_files( ########## Output Schema ########## ################################### + def get_match_keys(self) -> dict[str, list[str]]: + """Return list field name -> key fields used to match items during merge.""" + return { + "item_list": ["party_item_code", "quantity", "rate", "description"], + } + def get_schema(self) -> dict: if not self.schema: self.schema = self._get_schema() diff --git a/transaction_parser/transaction_parser/overrides/communication.py b/transaction_parser/transaction_parser/overrides/communication.py index 99c1b7e..5ddc78d 100644 --- a/transaction_parser/transaction_parser/overrides/communication.py +++ b/transaction_parser/transaction_parser/overrides/communication.py @@ -108,19 +108,12 @@ def _process_attachments( if not filtered_attachments: return - sorted_attachments = sorted( - filtered_attachments, - key=lambda attachment: {"xlsx": 0, "csv": 1, "pdf": 2}.get( - attachment.file_url.split(".")[-1].lower(), 3 - ), - ) - frappe.enqueue( "transaction_parser.transaction_parser.overrides.communication._parse_attachments", doc=doc, country=country, transaction_type=transaction_type, - attachments=sorted_attachments, + attachments=filtered_attachments, ai_model=settings.default_ai_model, user=user, party=party, diff --git a/transaction_parser/transaction_parser/utils/response_merger.py b/transaction_parser/transaction_parser/utils/response_merger.py index 0398a41..826750c 100644 --- a/transaction_parser/transaction_parser/utils/response_merger.py +++ b/transaction_parser/transaction_parser/utils/response_merger.py @@ -11,7 +11,6 @@ class FieldType: required: bool def is_empty(self, value: Any) -> bool: - """Check if a value is considered empty.""" return ( value is None or value == "" @@ -41,18 +40,10 @@ class ListField(FieldType): class SchemaParser: - """Parses document schema and builds structured field type hierarchy.""" + """Parses a schema dict into a structured FieldType hierarchy.""" def parse(self, schema: dict) -> dict[str, FieldType]: - """ - Parse schema dictionary into structured field types. - - Args: - schema: Schema dictionary to parse - - Returns: - Dictionary mapping field names to FieldType instances - """ + """Parse schema dict into field name -> FieldType mapping.""" fields = {} for key, value in schema.items(): @@ -61,101 +52,55 @@ def parse(self, schema: dict) -> dict[str, FieldType]: return fields def _parse_field(self, schema_value: Any) -> FieldType: - """ - Parse a single field value into appropriate FieldType. - - Args: - schema_value: Value from schema (string, list, or dict) - - Returns: - Appropriate FieldType instance - """ - # List field: schema value is a list + """Determine and return the appropriate FieldType for a schema value.""" if isinstance(schema_value, list): - required = self._is_required(schema_value) - - # Empty list or primitive list items if not schema_value: - return ListField( - required=required, item_type=PrimitiveField(required=True) - ) + return ListField(required=True, item_type=PrimitiveField(required=True)) item_schema = schema_value[0] - # List of objects (e.g., item_list) if isinstance(item_schema, dict): item_fields = self.parse(item_schema) return ListField( - required=required, + required=True, item_type=ObjectField(required=True, children=item_fields), ) - - # List of primitives (e.g., emails, phones) else: return ListField( - required=required, - item_type=PrimitiveField(required=self._is_required(item_schema)), + required=True, + item_type=PrimitiveField(required=True), ) - # Object field: schema value is a dict elif isinstance(schema_value, dict): children = self.parse(schema_value) return ObjectField(required=True, children=children) - # Primitive field: schema value is a string else: - return PrimitiveField(required=self._is_required(schema_value)) - - def _is_required(self, schema_value: Any) -> bool: - """ - Check if field is required based on schema notation. - - Args: - schema_value: Schema value (string or other type) - - Returns: - True if required, False if optional - """ - if isinstance(schema_value, str): - return "| null" not in schema_value and "| 0" not in schema_value - - return True + return PrimitiveField(required=True) class ResponseMerger: """Schema-driven merger for AI responses from multiple attachments.""" - def __init__(self, response: dict, schema: dict): - """ - Initialize ResponseMerger with a base response and optional schema. - - Args: - response: Initial response dict from first file parsing - schema: Document schema dict for automatic field detection - """ + def __init__( + self, + response: dict, + schema: dict, + match_keys: dict[str, list[str]] | None = None, + ): self.response = _dict(response) if isinstance(response, dict) else response self.schema = schema + self.match_keys = match_keys or {} - # Parse schema into structured field types parser = SchemaParser() self.fields = parser.parse(schema) def is_complete(self) -> bool: - """ - Check if all required fields are filled. - - Returns: - True if complete, False otherwise - """ + """Return True if all required fields are filled.""" return len(self.get_missing_fields()) == 0 def get_missing_fields(self) -> list[str]: - """ - Get list of missing required field paths. - - Returns: - List of dot-separated field paths that are missing - """ + """Return dot-separated paths of missing required fields.""" missing = [] self._check_missing_fields(self.fields, self.response, "", missing) return missing @@ -167,44 +112,26 @@ def _check_missing_fields( path_prefix: str, missing: list[str], ) -> None: - """ - Recursively check for missing required fields. - - Args: - fields: Field type definitions - data: Current data dict to check - path_prefix: Current path prefix for nested fields - missing: List to append missing field paths to - """ for key, field_type in fields.items(): field_path = f"{path_prefix}.{key}" if path_prefix else key value = data.get(key) if isinstance(data, dict) else None - # Check primitive fields if isinstance(field_type, PrimitiveField): if field_type.required and field_type.is_empty(value): missing.append(field_path) - # Check object fields recursively elif isinstance(field_type, ObjectField): if field_type.required and field_type.is_empty(value): missing.append(field_path) - elif value: self._check_missing_fields( field_type.children, value, field_path, missing ) # List fields are checked during merging, not at top level - # (we care about item contents, not just list existence) def merge(self, new_response: dict) -> None: - """ - Merge new response into existing response. - - Args: - new_response: New response dict to merge from - """ + """Merge new_response into the existing response.""" self._merge_fields(self.fields, self.response, new_response) def _merge_fields( @@ -213,43 +140,22 @@ def _merge_fields( target: dict, source: dict, ) -> None: - """ - Merge source data into target based on field definitions. - - Args: - fields: Field type definitions - target: Target dict to merge into - source: Source dict to merge from - """ for key, field_type in fields.items(): source_value = source.get(key) - # Skip if source doesn't have this field if source_value is None: continue - # Handle primitive fields if isinstance(field_type, PrimitiveField): self._merge_primitive(target, key, source_value) - # Handle object fields elif isinstance(field_type, ObjectField): self._merge_object(field_type, target, key, source_value) - # Handle list fields elif isinstance(field_type, ListField): self._merge_list(field_type, target, key, source_value) def _merge_primitive(self, target: dict, key: str, source_value: Any) -> None: - """ - Merge primitive field value. - - Args: - target: Target dict - key: Field key - source_value: Value from source - """ - # Only fill if target is empty if not target.get(key) and source_value: target[key] = source_value @@ -260,20 +166,9 @@ def _merge_object( key: str, source_value: dict, ) -> None: - """ - Merge object field recursively. - - Args: - field_type: ObjectField definition - target: Target dict - key: Field key - source_value: Object value from source - """ - # Ensure target has the object - if key not in target: + if key not in target or target[key] is None: target[key] = {} - # Recursively merge children self._merge_fields(field_type.children, target[key], source_value) def _merge_list( @@ -283,34 +178,25 @@ def _merge_list( key: str, source_value: list, ) -> None: - """ - Merge list field by matching items. - - Args: - field_type: ListField definition - target: Target dict - key: Field key - source_value: List value from source - """ target_list = target.get(key, []) - # If target list is empty, use source list if not target_list: target[key] = source_value return - # If source list is empty, nothing to merge if not source_value: return - # Only merge if items are objects (not primitive lists) if not isinstance(field_type.item_type, ObjectField): return - # Match and merge items + key_fields = self.match_keys.get(key, []) + if not key_fields: + return + for target_item in target_list: matched_item = self._find_matching_item( - target_item, source_value, field_type.item_type + target_item, source_value, key_fields ) if matched_item: self._merge_fields( @@ -321,34 +207,11 @@ def _find_matching_item( self, target_item: dict, source_items: list[dict], - item_type: ObjectField, + key_fields: list[str], ) -> dict | None: - """ - Find matching item in source list using intelligent matching. - - Matching strategy: - - Extract key fields from item schema (party_item_code, quantity, rate, description) - - Require at least 2 matching fields for a match - - Args: - target_item: Item to find match for - source_items: List of candidate items - item_type: ObjectField describing item structure - - Returns: - Matching item or None - """ - # Define priority key fields for matching - key_field_names = ["party_item_code", "quantity", "rate", "description"] - - # Filter to only fields that exist in schema - available_keys = [ - name for name in key_field_names if name in item_type.children - ] - - # Try to find match + # Require at least 2 matching key fields to avoid false positives for source_item in source_items: - if self._items_match(target_item, source_item, available_keys): + if self._items_match(target_item, source_item, key_fields): return source_item return None @@ -359,28 +222,13 @@ def _items_match( item2: dict, key_fields: list[str], ) -> bool: - """ - Check if two items match based on key fields. - - Requires at least 2 matching fields. - - Args: - item1: First item - item2: Second item - key_fields: List of key field names to check - - Returns: - True if items match, False otherwise - """ matches = 0 for field in key_fields: value1 = item1.get(field) value2 = item2.get(field) - # Both must exist and be equal if value1 and value2 and value1 == value2: matches += 1 - # Require at least 2 matching fields return matches >= 2 From 6b760f05b23b8d9469b2a5279815eb63e5cd845b Mon Sep 17 00:00:00 2001 From: Karm Soni Date: Wed, 1 Apr 2026 13:19:35 +0530 Subject: [PATCH 81/88] fix: update file name reference and enhance AI model options in settings --- .../transaction_parser/controllers/transaction.py | 2 +- .../transaction_parser_settings.json | 10 +++++----- .../transaction_parser_settings.py | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/transaction_parser/transaction_parser/controllers/transaction.py b/transaction_parser/transaction_parser/controllers/transaction.py index 5fdb551..63d13ad 100644 --- a/transaction_parser/transaction_parser/controllers/transaction.py +++ b/transaction_parser/transaction_parser/controllers/transaction.py @@ -104,7 +104,7 @@ def _parse_single_file( document_type=self.DOCTYPE, document_schema=schema, document_data=content, - file_doc_name=self.file.name, + file_doc_name=file.name, company=self.company, ) diff --git a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json index 0d9bd4c..1980923 100644 --- a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json +++ b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json @@ -92,16 +92,16 @@ "fieldtype": "Select", "label": "Default AI Model", "mandatory_depends_on": "eval: doc.enabled", - "options": "DeepSeek Chat\nDeepSeek Reasoner\nOpenAI gpt-4o\nOpenAI gpt-4o-mini\nOpenAI gpt-5\nOpenAI gpt-5-mini\nGoogle Gemini Pro\nGoogle Gemini Flash" + "options": "DeepSeek Chat\nDeepSeek Reasoner\nOpenAI gpt-4o\nOpenAI gpt-4o-mini\nOpenAI gpt-5\nOpenAI gpt-5-mini\nGoogle Gemini Pro-2.5\nGoogle Gemini Flash-2.5" }, { - "default": "OCR", + "default": "OCRMyPDF", "depends_on": "eval: doc.enabled", - "description": "Select the library to use for PDF text extraction. OCR uses PyMuPDF + OCRmyPDF. Docling provides advanced document understanding.", + "description": "Select the library to use for PDF text extraction", "fieldname": "pdf_processor", "fieldtype": "Select", "label": "PDF Processor", - "options": "OCR\nDocling" + "options": "OCRMyPDF\nDocling" }, { "depends_on": "eval: doc.enabled", @@ -174,7 +174,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2026-04-01 09:23:24.896779", + "modified": "2026-04-01 09:44:18.808088", "modified_by": "Administrator", "module": "Transaction Parser", "name": "Transaction Parser Settings", diff --git a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py index 8a0f887..8ff3911 100644 --- a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py +++ b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.py @@ -40,8 +40,8 @@ class TransactionParserSettings(Document): "OpenAI gpt-4o-mini", "OpenAI gpt-5", "OpenAI gpt-5-mini", - "Google Gemini Pro", - "Google Gemini Flash", + "Google Gemini Pro-2.5", + "Google Gemini Flash-2.5", ] enabled: DF.Check incoming_email_accounts: DF.Table[TransactionParserEmailAccount] @@ -51,7 +51,7 @@ class TransactionParserSettings(Document): parse_party_emails: DF.Check party_emails: DF.Table[TransactionParserPartyEmail] party_schema: DF.JSON | None - pdf_processor: DF.Literal["OCR", "Docling"] + pdf_processor: DF.Literal["OCRMyPDF", "Docling"] process_one_document_per_communication: DF.Check tax_schema: DF.JSON | None From d62f6707891398126ecfe04517bd01d3f2cf431f Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 1 Apr 2026 15:20:06 +0530 Subject: [PATCH 82/88] refactor: add seller document type and update user prompt role hint logic --- .../transaction_parser/ai_integration/prompts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/transaction_parser/transaction_parser/ai_integration/prompts.py b/transaction_parser/transaction_parser/ai_integration/prompts.py index 8fd4d91..9d63d50 100644 --- a/transaction_parser/transaction_parser/ai_integration/prompts.py +++ b/transaction_parser/transaction_parser/ai_integration/prompts.py @@ -3,6 +3,8 @@ # Mapping of output document types to their corresponding input document types INPUT_DOCUMENTS = {"Sales Order": "Purchase Order", "Purchase Invoice": "Sales Invoice"} +SELLER_DOCUMENT_TYPES = {"Sales Order"} + def get_system_prompt(document_schema: dict) -> str: return f"""You are a JSON data extraction and validation expert for your company's ERP platform. @@ -41,7 +43,7 @@ def get_user_prompt( company_context = "" if company_info: - if document_type == "Sales Order": + if document_type in SELLER_DOCUMENT_TYPES: role_hint = "Use this to correctly identify the company as the seller/vendor and the other party as the customer/buyer." else: role_hint = "Use this to correctly identify the company as the buyer/recipient and the other party as the vendor/supplier." From a59697892806df8a6d9e8b6fe9f2f25970f7e5e4 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 1 Apr 2026 18:19:07 +0530 Subject: [PATCH 83/88] refactor: enhance file handling and AI parsing logic to support multiple files --- transaction_parser/parser_benchmark/runner.py | 126 +++++++++++++++--- 1 file changed, 111 insertions(+), 15 deletions(-) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index a4f59a0..dcb2af0 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -20,6 +20,7 @@ from transaction_parser.transaction_parser.controllers.transaction import Transaction from transaction_parser.transaction_parser.utils.file_processor import FileProcessor from transaction_parser.transaction_parser.utils.pdf_processor import get_pdf_processor +from transaction_parser.transaction_parser.utils.response_merger import ResponseMerger class BenchmarkRunner: @@ -59,10 +60,10 @@ def run(self): try: file_docs: list[File] = self._get_file_docs() - self.controller: Transaction = self._get_controller(file_docs[0]) + self.controller: Transaction = self._get_controller(file_docs) - file_content = self._run_file_parsing(file_docs) - ai_content = self._run_ai_parsing(file_content, file_docs[0].name) + file_contents = self._run_file_parsing(file_docs) + ai_content = self._run_ai_parsing(file_contents, file_docs) self._calculate_cost() self._score_response(ai_content) @@ -85,15 +86,38 @@ def _get_file_docs(self) -> list[File]: file_docs = self.dataset.get_file_docs() if not file_docs: frappe.throw(_("No files in dataset {0}").format(self.dataset.name)) + + # sort by file type priority to match the actual parser behavior + file_docs = self._sort_files_by_priority(file_docs) return file_docs - def _get_controller(self, file_doc: File) -> Transaction: + def _sort_files_by_priority(self, file_docs: list[File]) -> list[File]: + """Sort files by type priority: xlsx/xls first, then pdf, then csv. + + If no xlsx/xls files exist, csv takes priority over pdf. + This mirrors the sorting logic in _parse() to ensure consistent file ordering. + """ + # TODO: Too many code repetation with transaction_parser/transaction_parser/__init__.py. Refactor to centralize file sorting logic. + file_types = {(f.file_type or "").lower() for f in file_docs} + has_spreadsheet = file_types & {"xlsx", "xls"} + + if has_spreadsheet: + FILE_TYPE_PRIORITY = {"xlsx": 0, "xls": 0, "pdf": 1, "csv": 2} + else: + FILE_TYPE_PRIORITY = {"csv": 0, "pdf": 1} + + return sorted( + file_docs, + key=lambda f: FILE_TYPE_PRIORITY.get((f.file_type or "").lower(), 99), + ) + + def _get_controller(self, file_docs: list[File]) -> Transaction: ds = self.dataset cls = get_controller(ds.country, ds.transaction_type) controller: Transaction = cls(company=ds.company, party=ds.party) controller.initialize() - controller.file = file_doc + controller.files = file_docs controller.ai_model = self.log.ai_model return controller @@ -107,7 +131,7 @@ def _get_cost_row(self): # ── step 1: file parsing ──────────────────────────────── - def _run_file_parsing(self, file_docs: list[File]) -> str: + def _run_file_parsing(self, file_docs: list[File]) -> list[str]: # to prevent stopping an already running tracemalloc instance was_tracing = tracemalloc.is_tracing() if not was_tracing: @@ -127,12 +151,6 @@ def _run_file_parsing(self, file_docs: list[File]) -> str: pdf_processor, ) contents.append(content) - - combined = ( - "\n\n--- Document Separator ---\n\n".join(contents) - if len(contents) > 1 - else contents[0] - ) finally: self.log.file_parse_time = flt(default_timer() - start, self.precision) _, peak = tracemalloc.get_traced_memory() @@ -142,12 +160,23 @@ def _run_file_parsing(self, file_docs: list[File]) -> str: peak / 1024 / 1024, self.precision ) # bytes → MB - self.log.file_content = combined - return combined + self.log.file_content = ( + "\n\n--- Document Separator ---\n\n".join(contents) + if len(contents) > 1 + else contents[0] + ) + return contents # ── step 2: AI parsing ────────────────────────────────── - def _run_ai_parsing(self, file_content: str, file_name: str) -> dict: + def _run_ai_parsing(self, file_contents: list[str], file_docs: list[File]) -> dict: + if len(file_contents) == 1: + return self._run_single_ai_parse(file_contents[0], file_docs[0].name) + + return self._run_multi_ai_parse(file_contents, file_docs) + + def _run_single_ai_parse(self, file_content: str, file_name: str) -> dict: + """Parse a single file with AI.""" parser = AIParser(self.log.ai_model) start = default_timer() @@ -168,6 +197,73 @@ def _run_ai_parsing(self, file_content: str, file_name: str) -> dict: return ai_content + def _run_multi_ai_parse( + self, file_contents: list[str], file_docs: list[File] + ) -> dict: + """Parse multiple files individually with AI and merge responses. + + Mirrors the controller's _parse_multiple_files flow: + parse each file → merge with ResponseMerger → aggregate tokens. + """ + total_prompt = 0 + total_completion = 0 + total_tokens_count = 0 + schema = self.controller.get_schema() + + start = default_timer() + + # parse first file + parser = AIParser(self.log.ai_model) + response = parser.parse( + document_type=self.controller.DOCTYPE, + document_schema=schema, + document_data=file_contents[0], + file_doc_name=file_docs[0].name, + company=self.dataset.company, + ) + + usage = parser.ai_response.get("usage", {}) + total_prompt += usage.get("prompt_tokens", 0) + total_completion += usage.get("completion_tokens", 0) + total_tokens_count += usage.get("total_tokens", 0) + + # merge remaining files + merger = ResponseMerger( + response, + schema=schema, + match_keys=self.controller.get_match_keys(), + ) + + for i, file_content in enumerate(file_contents[1:], 1): + if merger.is_complete(): + break + + parser = AIParser(self.log.ai_model) + new_response = parser.parse( + document_type=self.controller.DOCTYPE, + document_schema=schema, + document_data=file_content, + file_doc_name=file_docs[i].name, + company=self.dataset.company, + ) + + usage = parser.ai_response.get("usage", {}) + total_prompt += usage.get("prompt_tokens", 0) + total_completion += usage.get("completion_tokens", 0) + total_tokens_count += usage.get("total_tokens", 0) + + merger.merge(new_response) + + self.log.ai_parse_time = flt(default_timer() - start, self.precision) + self.log.prompt_tokens = total_prompt + self.log.completion_tokens = total_completion + self.log.total_tokens = total_tokens_count + + ai_content = merger.response + self.log.ai_response = frappe.as_json(ai_content, indent=2) + + return ai_content + # ── step 3: cost calculation ──────────────────────────── def _calculate_cost(self): From 21ec294b117480864131bea576e95616f7566e10 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 1 Apr 2026 18:48:35 +0530 Subject: [PATCH 84/88] refactor: patch for process one document per communication --- transaction_parser/patches.txt | 1 + .../patches/enable_one_document_per_communication.py | 7 +++++++ .../transaction_parser_settings.json | 4 ++-- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 transaction_parser/patches/enable_one_document_per_communication.py diff --git a/transaction_parser/patches.txt b/transaction_parser/patches.txt index 9107383..cad4a73 100644 --- a/transaction_parser/patches.txt +++ b/transaction_parser/patches.txt @@ -10,3 +10,4 @@ execute:from transaction_parser.install import after_install; after_install() #2 transaction_parser.patches.set_default_pdf_processor #1 transaction_parser.patches.recalculate_accuracy transaction_parser.patches.populate_dataset_files_table +transaction_parser.patches.enable_one_document_per_communication diff --git a/transaction_parser/patches/enable_one_document_per_communication.py b/transaction_parser/patches/enable_one_document_per_communication.py new file mode 100644 index 0000000..103b8b8 --- /dev/null +++ b/transaction_parser/patches/enable_one_document_per_communication.py @@ -0,0 +1,7 @@ +import frappe + + +def execute(): + frappe.db.set_single_value( + "Transaction Parser Settings", "process_one_document_per_communication", 1 + ) diff --git a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json index 1980923..f5499d3 100644 --- a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json +++ b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json @@ -165,7 +165,7 @@ }, { "default": "0", - "description": "When enabled, all attachments from a communication are combined to create a single document. When disabled, each attachment creates a separate document.", + "description": "All attachments from a communication are combined to create a single document", "fieldname": "process_one_document_per_communication", "fieldtype": "Check", "label": "Process One Document Per Communication" @@ -174,7 +174,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2026-04-01 09:44:18.808088", + "modified": "2026-04-01 15:11:22.650340", "modified_by": "Administrator", "module": "Transaction Parser", "name": "Transaction Parser Settings", From f35a05d703123be1c78e9aa19f9dd5a896e8098c Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 1 Apr 2026 19:01:33 +0530 Subject: [PATCH 85/88] refactor: add dependency for process_one_document_per_communication setting and update modified timestamp --- transaction_parser/parser_benchmark/runner.py | 1 + .../transaction_parser_settings.json | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/transaction_parser/parser_benchmark/runner.py b/transaction_parser/parser_benchmark/runner.py index dcb2af0..ff93e7c 100644 --- a/transaction_parser/parser_benchmark/runner.py +++ b/transaction_parser/parser_benchmark/runner.py @@ -132,6 +132,7 @@ def _get_cost_row(self): # ── step 1: file parsing ──────────────────────────────── def _run_file_parsing(self, file_docs: list[File]) -> list[str]: + # TODO: It is assumed that Process One Document Per Communication is enabled # to prevent stopping an already running tracemalloc instance was_tracing = tracemalloc.is_tracing() if not was_tracing: diff --git a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json index f5499d3..d32ab8b 100644 --- a/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json +++ b/transaction_parser/transaction_parser/doctype/transaction_parser_settings/transaction_parser_settings.json @@ -165,6 +165,7 @@ }, { "default": "0", + "depends_on": "eval: doc.parse_incoming_emails", "description": "All attachments from a communication are combined to create a single document", "fieldname": "process_one_document_per_communication", "fieldtype": "Check", @@ -174,7 +175,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2026-04-01 15:11:22.650340", + "modified": "2026-04-01 15:29:35.262995", "modified_by": "Administrator", "module": "Transaction Parser", "name": "Transaction Parser Settings", From 7a2e974df3fd99f09d3f90fba6ec77e9f6aae0a4 Mon Sep 17 00:00:00 2001 From: Abdeali Chharchhoda Date: Wed, 1 Apr 2026 19:05:35 +0530 Subject: [PATCH 86/88] fix: correct file_url to file_urls in parse function and handle None case in notification --- transaction_parser/transaction_parser/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transaction_parser/transaction_parser/__init__.py b/transaction_parser/transaction_parser/__init__.py index a655c9e..ecdc89e 100644 --- a/transaction_parser/transaction_parser/__init__.py +++ b/transaction_parser/transaction_parser/__init__.py @@ -24,7 +24,7 @@ def parse(transaction, country, file_url, ai_model=None, page_limit=None, compan _parse, country=cstr(country), transaction=cstr(transaction), - file_url=cstr(file_url), + file_urls=cstr(file_url), ai_model=cstr(ai_model), page_limit=cint(page_limit), company=cstr(company) if company else None, @@ -109,7 +109,7 @@ def _parse( ): notification = { "document_type": "File", - "document_name": file, + "document_name": file.name if file else None, "subject": _("Duplicate entry found for {0}").format(file_urls), "message": str(e), } From 7110c5e3431d3f14bcbc81edc0323ea6769d749c Mon Sep 17 00:00:00 2001 From: Karm Soni Date: Fri, 3 Apr 2026 13:36:47 +0530 Subject: [PATCH 87/88] feat: implement custom exception handling for file processing errors and enhance error logging --- transaction_parser/exceptions.py | 5 +++ .../transaction_parser/__init__.py | 44 +++++++++++-------- .../ai_integration/parser.py | 11 ++--- .../controllers/transaction.py | 43 ++++++++++++------ .../overrides/communication.py | 1 - .../utils/file_processor.py | 18 +++++--- 6 files changed, 75 insertions(+), 47 deletions(-) create mode 100644 transaction_parser/exceptions.py diff --git a/transaction_parser/exceptions.py b/transaction_parser/exceptions.py new file mode 100644 index 0000000..a6458d7 --- /dev/null +++ b/transaction_parser/exceptions.py @@ -0,0 +1,5 @@ +import frappe + + +class FileProcessingError(frappe.ValidationError): + """Custom exception for file processing errors.""" diff --git a/transaction_parser/transaction_parser/__init__.py b/transaction_parser/transaction_parser/__init__.py index a327757..b6b16e3 100644 --- a/transaction_parser/transaction_parser/__init__.py +++ b/transaction_parser/transaction_parser/__init__.py @@ -80,7 +80,7 @@ def _parse( files.append(file) controller = get_controller(country, transaction)(party=party, company=company) - doc = controller.generate(files, ai_model, page_limit, communication_name) + doc = controller.generate(files, ai_model, page_limit) filenames = ( ", ".join([f.file_name for f in files]) @@ -99,33 +99,39 @@ def _parse( except Exception as e: notification = None + reference_doctype = "Communication" if communication_name else "File" + reference_docname = ( + communication_name + if communication_name + else (files[0].name if files else None) + ) if ( isinstance(e, frappe.DuplicateEntryError) and frappe.flags.skip_duplicate_error ): + subject = _("Duplicate {0} found for {1}").format( + _(TRANSACTION_MAP[transaction]), + f"{reference_doctype} {reference_docname}", + ) + notification = { - "document_type": "Communication" if communication_name else "File", - "document_name": ( - communication_name if communication_name else files[0].name - ), - "subject": _("Duplicate entry found for {0}").format(file_urls), + "document_type": reference_doctype, + "document_name": reference_docname, + "subject": subject, "message": str(e), } - return - - error_log = frappe.log_error( - "Transaction Parser API Error", - reference_doctype="Communication" if communication_name else "File", - reference_name=( - communication_name - if communication_name - else files[0].name - if files - else None - ), + + if not (error_log := getattr(e, "error_log", None)): + error_log = frappe.log_error( + "Transaction Parser Error", + reference_doctype=reference_doctype, + reference_name=reference_docname, + ) + + message = _("Failed to generate {0} from {1}").format( + TRANSACTION_MAP[transaction], f"{reference_doctype} {reference_docname}" ) - message = _("Failed to generate {0} from {1}").format(_(transaction), file_urls) notification = { "document_type": error_log.doctype, diff --git a/transaction_parser/transaction_parser/ai_integration/parser.py b/transaction_parser/transaction_parser/ai_integration/parser.py index 7cbe23e..84d1343 100644 --- a/transaction_parser/transaction_parser/ai_integration/parser.py +++ b/transaction_parser/transaction_parser/ai_integration/parser.py @@ -35,14 +35,12 @@ def parse( document_schema: dict, document_data: str, doc_name: str | None = None, - is_communication: bool = False, ) -> dict: messages = self._build_messages(document_type, document_schema, document_data) response = self.send_message( messages=messages, doc_name=doc_name, - is_communication=is_communication, ) return self.get_content(response) @@ -72,10 +70,9 @@ def send_message( self, messages: tuple, doc_name: str | None = None, - is_communication: bool = False, ) -> dict: """Send messages to AI API and handle the response.""" - log = self._create_log_entry(doc_name, is_communication) + log = self._create_log_entry(doc_name) try: response = self._make_api_call(messages) @@ -96,15 +93,13 @@ def send_message( finally: enqueue_integration_request(**log) - def _create_log_entry( - self, doc_name: str | None, is_communication: bool = False - ) -> frappe._dict: + def _create_log_entry(self, doc_name: str | None) -> frappe._dict: """Create a log entry for the API call.""" log = frappe._dict(url=self.model.base_url) log.update( { - "reference_doctype": "Communication" if is_communication else "File", + "reference_doctype": "File", "reference_name": doc_name, } ) diff --git a/transaction_parser/transaction_parser/controllers/transaction.py b/transaction_parser/transaction_parser/controllers/transaction.py index e3cc57b..86f53ac 100644 --- a/transaction_parser/transaction_parser/controllers/transaction.py +++ b/transaction_parser/transaction_parser/controllers/transaction.py @@ -2,8 +2,10 @@ import frappe from erpnext.setup.utils import get_exchange_rate from erpnext.stock.get_item_details import get_item_details +from httpx import HTTPError from rapidfuzz import fuzz, process +from transaction_parser.exceptions import FileProcessingError from transaction_parser.transaction_parser.ai_integration.parser import AIParser from transaction_parser.transaction_parser.utils import to_dict from transaction_parser.transaction_parser.utils.file_processor import FileProcessor @@ -36,7 +38,6 @@ def generate( files, ai_model: str | None = None, page_limit: int | None = None, - communication_name: str | None = None, ): self.initialize() @@ -45,7 +46,6 @@ def generate( self.files = files self.ai_model = ai_model - self.communication_name = communication_name self.data = self._parse_file_content(ai_model, page_limit) self.doc = frappe.get_doc({"doctype": self.DOCTYPE}) self.doc.is_created_by_transaction_parser = 1 @@ -61,7 +61,6 @@ def generate( def initialize(self) -> None: # file processing self.files = None - self.communication_name = None # output schema self.schema = None @@ -95,16 +94,34 @@ def _parse_single_file( ai_model: str | None = None, page_limit: int | None = None, ) -> dict: - content = FileProcessor().get_content(file, page_limit) - schema = self.get_schema() - - return AIParser(ai_model, self.settings).parse( - document_type=self.DOCTYPE, - document_schema=schema, - document_data=content, - doc_name=self.communication_name or file.name, - is_communication=bool(self.communication_name), - ) + try: + content = FileProcessor().get_content(file, page_limit) + schema = self.get_schema() + + return AIParser(ai_model, self.settings).parse( + document_type=self.DOCTYPE, + document_schema=schema, + document_data=content, + doc_name=file.name, + ) + + except FileProcessingError as e: + error_log = frappe.log_error( + title="File processing error in Transaction Parser", + reference_doctype="File", + reference_name=file.name, + ) + e.error_log = error_log + raise e + + except HTTPError as e: + error_log = frappe.log_error( + title="Transaction Parser API error", + reference_doctype="File", + reference_name=file.name, + ) + e.error_log = error_log + raise e def _parse_multiple_files( self, ai_model: str | None = None, page_limit: int | None = None diff --git a/transaction_parser/transaction_parser/overrides/communication.py b/transaction_parser/transaction_parser/overrides/communication.py index 5ddc78d..7579fe2 100644 --- a/transaction_parser/transaction_parser/overrides/communication.py +++ b/transaction_parser/transaction_parser/overrides/communication.py @@ -151,7 +151,6 @@ def _parse_attachments( user=user, party=party, company=company, - communication_name=doc.name, ) frappe.db.commit() diff --git a/transaction_parser/transaction_parser/utils/file_processor.py b/transaction_parser/transaction_parser/utils/file_processor.py index 8a3ccd9..72351bc 100644 --- a/transaction_parser/transaction_parser/utils/file_processor.py +++ b/transaction_parser/transaction_parser/utils/file_processor.py @@ -10,17 +10,23 @@ read_xlsx_file_from_attached_file, ) +from transaction_parser.exceptions import FileProcessingError + class FileProcessor: """Process files: PDF (trim pages, apply OCR), CSV/Excel (parse data), extract content.""" def get_content(self, doc, page_limit=None): - if doc.file_type == "PDF": - return self._process_pdf(doc, page_limit) - elif doc.file_type in ["CSV", "XLSX", "XLS"]: - return self._process_spreadsheet(doc) - else: - frappe.throw(_("Only PDF, CSV, and Excel files are supported")) + try: + if doc.file_type == "PDF": + return self._process_pdf(doc, page_limit) + elif doc.file_type in ["CSV", "XLSX", "XLS"]: + return self._process_spreadsheet(doc) + else: + frappe.throw(_("Only PDF, CSV, and Excel files are supported")) + + except Exception as e: + raise FileProcessingError from e def _process_pdf(self, doc, page_limit=None): """Process PDF files with OCR and page limiting.""" From 84d825870ad8cfa197aaa67214035735fc73881c Mon Sep 17 00:00:00 2001 From: Karm Soni Date: Fri, 3 Apr 2026 13:54:51 +0530 Subject: [PATCH 88/88] fix: update parameter name in _create_log_entry method for consistency --- .../transaction_parser/ai_integration/parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transaction_parser/transaction_parser/ai_integration/parser.py b/transaction_parser/transaction_parser/ai_integration/parser.py index 47fb6e3..f07a7b9 100644 --- a/transaction_parser/transaction_parser/ai_integration/parser.py +++ b/transaction_parser/transaction_parser/ai_integration/parser.py @@ -112,14 +112,14 @@ def send_message(self, messages: tuple, file_doc_name: str | None = None) -> dic finally: enqueue_integration_request(**log) - def _create_log_entry(self, doc_name: str | None) -> frappe._dict: + def _create_log_entry(self, file_doc_name: str | None) -> frappe._dict: """Create a log entry for the API call.""" log = frappe._dict(url=self.model.base_url) log.update( { "reference_doctype": "File", - "reference_name": doc_name, + "reference_name": file_doc_name, } )