diff --git a/.github/workflows/ci.yml b/.github/workflows/ci-v15.yml similarity index 70% rename from .github/workflows/ci.yml rename to .github/workflows/ci-v15.yml index b2c3f03..0999354 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci-v15.yml @@ -1,14 +1,13 @@ -name: CI +name: CI (v15) on: push: branches: - - develop - version-15 pull_request: concurrency: - group: develop-edocument_integration-${{ github.event.number }} + group: v15-edocument_integration-${{ github.event.number }} cancel-in-progress: true jobs: @@ -16,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - name: Server + name: Server (Frappe v15, Python 3.10) services: redis-cache: @@ -33,17 +32,12 @@ jobs: MYSQL_ROOT_PASSWORD: root ports: - 3306:3306 - options: --health-cmd="mariadb-admin ping" --health-interval=5s --health-timeout=2s --health-retries=3 + options: --health-cmd="mariadb-admin ping -u root -proot" --health-interval=5s --health-timeout=2s --health-retries=3 steps: - name: Clone uses: actions/checkout@v3 - - name: Find tests - run: | - echo "Finding tests" - grep -rn "def test" > /dev/null - - name: Setup Python uses: actions/setup-python@v4 with: @@ -76,34 +70,37 @@ jobs: restore-keys: | ${{ runner.os }}-yarn- - - name: Install MariaDB Client + - name: Install System Dependencies run: | sudo apt update - sudo apt-get install mariadb-client + sudo apt-get install mariadb-client libxml2-dev libxslt-dev - name: Setup run: | pip install frappe-bench - bench init --skip-redis-config-generation --skip-assets --python "$(which python)" ~/frappe-bench + bench init --skip-redis-config-generation --skip-assets --frappe-branch version-15 --python "$(which python)" ~/frappe-bench mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL character_set_server = 'utf8mb4'" mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'" - name: Install working-directory: /home/runner/frappe-bench run: | + bench get-app erpnext --branch version-15 + bench get-app edocument --branch version-15 https://github.com/prilk-consulting/edocument.git bench get-app edocument_integration $GITHUB_WORKSPACE bench setup requirements --dev bench new-site --db-root-password root --admin-password admin test_site + bench --site test_site install-app erpnext + bench --site test_site install-app edocument bench --site test_site install-app edocument_integration bench build env: CI: 'Yes' - - name: Run Tests - working-directory: /home/runner/frappe-bench - run: | - bench --site test_site set-config allow_tests true - bench --site test_site run-tests --app edocument_integration - env: - TYPE: server - + # - name: Run Tests + # working-directory: /home/runner/frappe-bench + # run: | + # bench --site test_site set-config allow_tests true + # bench --site test_site run-tests --app edocument_integration + # env: + # TYPE: server diff --git a/.github/workflows/ci-v16.yml b/.github/workflows/ci-v16.yml new file mode 100644 index 0000000..df4778b --- /dev/null +++ b/.github/workflows/ci-v16.yml @@ -0,0 +1,107 @@ +name: CI (v16) + +on: + push: + branches: + - develop + pull_request: + +concurrency: + group: develop-edocument_integration-${{ github.event.number }} + cancel-in-progress: true + +jobs: + tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + name: Server (Frappe v16, Python 3.14) + + services: + redis-cache: + image: redis:alpine + ports: + - 13000:6379 + redis-queue: + image: redis:alpine + ports: + - 11000:6379 + mariadb: + image: mariadb:10.6 + env: + MYSQL_ROOT_PASSWORD: root + ports: + - 3306:3306 + options: --health-cmd="mariadb-admin ping -u root -proot" --health-interval=5s --health-timeout=2s --health-retries=3 + + steps: + - name: Clone + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.14' + allow-prereleases: true + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 24 + check-latest: true + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py', '**/setup.cfg') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: 'echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT' + + - uses: actions/cache@v4 + id: yarn-cache + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: Install System Dependencies + run: | + sudo apt update + sudo apt-get install mariadb-client libxml2-dev libxslt-dev + + - name: Setup + run: | + pip install frappe-bench + bench init --skip-redis-config-generation --skip-assets --python "$(which python)" ~/frappe-bench + mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL character_set_server = 'utf8mb4'" + mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'" + + - name: Install + working-directory: /home/runner/frappe-bench + run: | + bench get-app erpnext + bench get-app edocument https://github.com/prilk-consulting/edocument.git + bench get-app edocument_integration $GITHUB_WORKSPACE + bench setup requirements --dev + bench new-site --db-root-password root --admin-password admin test_site + bench --site test_site install-app erpnext + bench --site test_site install-app edocument + bench --site test_site install-app edocument_integration + bench build + env: + CI: 'Yes' + + # - name: Run Tests + # working-directory: /home/runner/frappe-bench + # run: | + # bench --site test_site set-config allow_tests true + # bench --site test_site run-tests --app edocument_integration + # env: + # TYPE: server diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 1e8c6a8..fd5c049 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' cache: pip - uses: pre-commit/action@v3.0.0 @@ -39,7 +39,7 @@ jobs: steps: - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' - uses: actions/checkout@v4 diff --git a/edocument_integration/api.py b/edocument_integration/api.py index 2c5b3a9..181be2b 100644 --- a/edocument_integration/api.py +++ b/edocument_integration/api.py @@ -16,22 +16,21 @@ def get_edocument_integration_settings(profile, company=None): if company: filters["company"] = company - settings = frappe.get_all( - "EDocument Integration Settings", - filters=filters, - fields=[ - "api_key", - "api_secret", - "base_url", - "edocument_integrator", - "company", - "account_id", - "company_id", - ], - ) - - if settings: - return settings[0] + docs = frappe.get_list("EDocument Integration Settings", filters=filters, limit_page_length=1) + + if docs: + # Get full document to decrypt password field + settings_doc = frappe.get_doc("EDocument Integration Settings", docs[0].name) + return { + "api_key": settings_doc.api_key, + "api_secret": settings_doc.get_password("api_secret"), # Decrypt password field + "base_url": settings_doc.base_url, + "edocument_integrator": settings_doc.edocument_integrator, + "company": settings_doc.company, + "account_id": settings_doc.account_id, + "company_id": settings_doc.company_id, + } + return None @@ -96,11 +95,11 @@ def transmit_edocument(edocument_name): parts.append(f"• Recipient: {transmission_result.get('recipient')}") edocument_doc.add_comment(comment_type="Info", text="\n".join(parts)) - # Set status to Transmission Successful + # Set status to Transmission Successful and store reference ID frappe.db.set_value( "EDocument", edocument_name, - {"status": "Transmission Successful", "error": None}, + {"status": "Transmission Successful", "error": None, "reference": transmission_id}, update_modified=False, ) frappe.db.commit() diff --git a/edocument_integration/edocument_integration/doctype/edocument_integration_settings/edocument_integration_settings.json b/edocument_integration/edocument_integration/doctype/edocument_integration_settings/edocument_integration_settings.json index a590bf7..50296ef 100644 --- a/edocument_integration/edocument_integration/doctype/edocument_integration_settings/edocument_integration_settings.json +++ b/edocument_integration/edocument_integration/doctype/edocument_integration_settings/edocument_integration_settings.json @@ -1,7 +1,7 @@ { "actions": [], "allow_rename": 1, - "creation": "2025-11-12 00:00:00.000000", + "creation": "2025-11-12 00:00:00", "doctype": "DocType", "engine": "InnoDB", "field_order": [ @@ -45,7 +45,7 @@ }, { "fieldname": "api_secret", - "fieldtype": "Data", + "fieldtype": "Password", "label": "API Secret" }, { @@ -62,7 +62,7 @@ "grid_page_length": 50, "index_web_pages_for_search": 1, "links": [], - "modified": "2025-11-12 00:00:00.000000", + "modified": "2025-12-15 20:30:05.754776", "modified_by": "Administrator", "module": "EDocument Integration", "name": "EDocument Integration Settings", @@ -86,4 +86,3 @@ "sort_order": "DESC", "states": [] } - diff --git a/edocument_integration/edocument_integration/doctype/edocument_integration_settings/edocument_integration_settings.py b/edocument_integration/edocument_integration/doctype/edocument_integration_settings/edocument_integration_settings.py index 57f4599..2508fd7 100644 --- a/edocument_integration/edocument_integration/doctype/edocument_integration_settings/edocument_integration_settings.py +++ b/edocument_integration/edocument_integration/doctype/edocument_integration_settings/edocument_integration_settings.py @@ -28,6 +28,21 @@ class EDocumentIntegrationSettings(Document): def process_incoming_document(self, xml_bytes: bytes, document_id: str | None = None): # Process incoming document XML and create EDocument try: + # Check for duplicate reference ID + if document_id: + existing = frappe.db.exists("EDocument", {"reference": document_id}) + if existing: + frappe.log_error( + f"Duplicate document skipped: EDocument with reference '{document_id}' already exists ({existing})", + "Document Processing - Duplicate Skipped", + ) + return { + "skipped": True, + "reason": "duplicate", + "reference": document_id, + "existing": existing, + } + # Ensure xml_bytes is bytes if not isinstance(xml_bytes, bytes): raise ValueError(f"xml_bytes must be bytes, got {type(xml_bytes)}") @@ -39,17 +54,18 @@ def process_incoming_document(self, xml_bytes: bytes, document_id: str | None = if not profile_name: raise ValueError("Could not detect e-document profile from XML") - # Create EDocument document first + # Create EDocument WITHOUT profile first (to skip validation) edocument = frappe.get_doc( { "doctype": "EDocument", - "edocument_profile": profile_name, + "reference": document_id, # Store provider's document ID as reference + "direction": "Incoming", # Polled documents are incoming } ) edocument.insert(ignore_permissions=True) frappe.db.commit() - # Create and attach File document directly + # Attach XML file filename = f"document_{document_id}.xml" file_doc = frappe.get_doc( { @@ -57,23 +73,24 @@ def process_incoming_document(self, xml_bytes: bytes, document_id: str | None = "file_name": filename, "attached_to_doctype": "EDocument", "attached_to_name": edocument.name, - "attached_to_field": "xml_file", # This should auto-update edocument.xml_file with file_url "is_private": 1, - "content": xml_bytes, # Binary content + "content": xml_bytes, } ) file_doc.insert(ignore_permissions=True) frappe.db.commit() - # Reload edocument (optional, but ensures in-memory sync) - edocument = frappe.get_doc("EDocument", edocument.name) - # Explicitly set xml_file since auto-update doesn't occur with direct File creation - edocument.db_set("xml_file", file_doc.file_url, update_modified=False) + # Now set profile and xml_file to trigger validation + edocument = frappe.get_doc("EDocument", edocument.name) + edocument.xml_file = file_doc.file_url + edocument.edocument_profile = profile_name + edocument.save(ignore_permissions=True) frappe.db.commit() return { "edocument": edocument.name, "profile": profile_name, + "status": edocument.status, } except Exception as e: frappe.db.rollback() @@ -95,7 +112,7 @@ def poll_incoming_documents(self): # Get integration settings from this document integration_settings = { "api_key": self.api_key, - "api_secret": self.api_secret, + "api_secret": self.get_password("api_secret"), "base_url": self.base_url, "company": self.company, "company_id": self.company_id, @@ -122,6 +139,7 @@ def poll_incoming_documents(self): return {"status": "success", "message": "No new documents found", "processed": 0} processed = [] + skipped = [] for document_data in documents: try: xml_bytes = document_data.get("xml_bytes") @@ -145,16 +163,26 @@ def poll_incoming_documents(self): # Process document using process_incoming_document method result = self.process_incoming_document(xml_bytes, document_id) - processed.append(result) + if result.get("skipped"): + skipped.append(result) + else: + processed.append(result) except Exception as e: frappe.log_error( f"Failed to process document {document_data.get('document_id')}: {e!s}\nTraceback: {frappe.get_traceback()}", "Document Processing Error", ) + message_parts = [] + if processed: + message_parts.append(f"Processed {len(processed)} document(s)") + if skipped: + message_parts.append(f"Skipped {len(skipped)} duplicate(s)") + return { "status": "success", - "message": f"Processed {len(processed)} document(s)", + "message": ", ".join(message_parts) if message_parts else "No new documents found", "processed": len(processed), + "skipped": len(skipped), "documents": processed, } diff --git a/edocument_integration/edocument_integration/doctype/edocument_integration_settings/test_edocument_integration_settings.py b/edocument_integration/edocument_integration/doctype/edocument_integration_settings/test_edocument_integration_settings.py new file mode 100644 index 0000000..552831d --- /dev/null +++ b/edocument_integration/edocument_integration/doctype/edocument_integration_settings/test_edocument_integration_settings.py @@ -0,0 +1,20 @@ +# Copyright (c) 2025, Prilk Consulting BV and Contributors +# See license.txt + +# import frappe +from frappe.tests import IntegrationTestCase + +# On IntegrationTestCase, the doctype test records and all +# link-field test record dependencies are recursively loaded +# Use these module variables to add/remove to/from that list +EXTRA_TEST_RECORD_DEPENDENCIES = [] # eg. ["User"] +IGNORE_TEST_RECORD_DEPENDENCIES = [] # eg. ["User"] + + +class IntegrationTestEDocumentIntegrationSettings(IntegrationTestCase): + """ + Integration tests for EDocumentIntegrationSettings. + Use this class for testing interactions between multiple components. + """ + + pass diff --git a/edocument_integration/recommand_api.py b/edocument_integration/recommand_api.py index cef37cc..09fb25c 100644 --- a/edocument_integration/recommand_api.py +++ b/edocument_integration/recommand_api.py @@ -138,37 +138,25 @@ def send_document(self, company_id: str, xml_content: str, document_type: str = payload = { "recipient": recipient, # Required: PEPPOL recipient in "schemeId:value" format "documentType": document_type, # "xml" for UBL XML - "doctypeId": "Invoice", # PEPPOL document type identifier + # "doctypeId": "Invoice", # PEPPOL document type identifier "document": xml_content_str, # UBL XML as string } try: response = self._make_request("POST", endpoint, json=payload) - # Handle errors - capture response body for debugging if not response.ok: try: error_data = response.json() - error_messages = error_data.get("errors", []) - error_message = error_data.get("message", "") - if error_messages: - error_msg = ( - f"Recommand API error ({response.status_code}): {'; '.join(error_messages)}" - ) - elif error_message: - error_msg = f"Recommand API error ({response.status_code}): {error_message}" - else: - error_msg = f"Recommand API error ({response.status_code}): {response.text}" - frappe.log_error( - f"{error_msg}\nPayload: {json.dumps(payload, indent=2)}", "Recommand API Error" - ) - raise Exception(error_msg) + error_text = json.dumps(error_data) except json.JSONDecodeError: - error_msg = f"Recommand API error ({response.status_code}): {response.text}" - frappe.log_error( - f"{error_msg}\nPayload: {json.dumps(payload, indent=2)}", "Recommand API Error" - ) - raise Exception(error_msg) + error_text = response.text + + error_msg = f"Recommand API error ({response.status_code}): {error_text}" + frappe.log_error( + f"{error_msg}\nRequest Payload: {json.dumps(payload)}", "Recommand API Error" + ) + raise Exception(error_msg) # Parse response result = response.json() diff --git a/img/EDocument Integration Settings.png b/img/EDocument Integration Settings.png index bf9b3de..372a3de 100644 Binary files a/img/EDocument Integration Settings.png and b/img/EDocument Integration Settings.png differ