From daad4ac1efbb325a227591be33c20c71b0d8de2e Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 27 Nov 2020 10:34:24 +0100 Subject: [PATCH 01/68] Add edi_storage --- edi_storage_oca/README.rst | 105 ++++ edi_storage_oca/__init__.py | 2 + edi_storage_oca/__manifest__.py | 17 + edi_storage_oca/components/__init__.py | 3 + edi_storage_oca/components/base.py | 54 +++ edi_storage_oca/components/check.py | 83 ++++ edi_storage_oca/components/send.py | 39 ++ edi_storage_oca/demo/edi_backend_demo.xml | 14 + edi_storage_oca/models/__init__.py | 1 + edi_storage_oca/models/edi_backend.py | 67 +++ edi_storage_oca/readme/CONTRIBUTORS.rst | 1 + edi_storage_oca/readme/DESCRIPTION.rst | 23 + edi_storage_oca/readme/USAGE.rst | 1 + edi_storage_oca/security/ir_model_access.xml | 12 + edi_storage_oca/static/description/icon.png | Bin 0 -> 9455 bytes edi_storage_oca/static/description/index.html | 450 ++++++++++++++++++ .../tests/_TODO_test_components_base.py | 43 ++ edi_storage_oca/tests/__init__.py | 1 + edi_storage_oca/tests/common.py | 116 +++++ .../tests/test_edi_backend_storage.py | 204 ++++++++ edi_storage_oca/views/edi_backend_views.xml | 21 + 21 files changed, 1257 insertions(+) create mode 100644 edi_storage_oca/README.rst create mode 100644 edi_storage_oca/__init__.py create mode 100644 edi_storage_oca/__manifest__.py create mode 100644 edi_storage_oca/components/__init__.py create mode 100644 edi_storage_oca/components/base.py create mode 100644 edi_storage_oca/components/check.py create mode 100644 edi_storage_oca/components/send.py create mode 100644 edi_storage_oca/demo/edi_backend_demo.xml create mode 100644 edi_storage_oca/models/__init__.py create mode 100644 edi_storage_oca/models/edi_backend.py create mode 100644 edi_storage_oca/readme/CONTRIBUTORS.rst create mode 100644 edi_storage_oca/readme/DESCRIPTION.rst create mode 100644 edi_storage_oca/readme/USAGE.rst create mode 100644 edi_storage_oca/security/ir_model_access.xml create mode 100644 edi_storage_oca/static/description/icon.png create mode 100644 edi_storage_oca/static/description/index.html create mode 100644 edi_storage_oca/tests/_TODO_test_components_base.py create mode 100644 edi_storage_oca/tests/__init__.py create mode 100644 edi_storage_oca/tests/common.py create mode 100644 edi_storage_oca/tests/test_edi_backend_storage.py create mode 100644 edi_storage_oca/views/edi_backend_views.xml diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst new file mode 100644 index 000000000..fffe95837 --- /dev/null +++ b/edi_storage_oca/README.rst @@ -0,0 +1,105 @@ +=========================== +EDI Storage backend support +=========================== + +.. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png + :target: https://odoo-community.org/page/development-status + :alt: Alpha +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi-lightgray.png?logo=github + :target: https://github.com/OCA/edi/tree/13.0/edi_storage + :alt: OCA/edi +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/edi-13-0/edi-13-0-edi_storage + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png + :target: https://runbot.odoo-community.org/runbot/226/13.0 + :alt: Try me on Runbot + +|badge1| |badge2| |badge3| |badge4| |badge5| + +Allow exchange files using storage backends from `OCA/storage`. + +This module adds a storage backend relation on the EDI backend. +There you can configure the backend to be used (most often and SFTP) +and the paths where to read or put files. + +Often the convention when exchanging files via SFTP +is to have one input forder (to receive files) +and an output folder (to send files). + +Inside this folder you have this hierarchy:: + + input/output folder + |- pending + |- done + |- error + +* `pending` folder contains files that have been just sent +* `done` folder contains files that have been processes successfully +* `error` folder contains files with errors and cannot be processed + +The storage handlers take care of reading files and putting files +in/from the right place and update exchange records data accordingly. + +.. IMPORTANT:: + This is an alpha version, the data model and design can change at any time without warning. + Only for development or testing purpose, do not use in production. + `More details on development status `_ + +**Table of contents** + +.. contents:: + :local: + +Usage +===== + +Go to "EDI -> EDI backend" then configure your backend to use a storage backend. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +~~~~~~~ + +* ACSONE + +Contributors +~~~~~~~~~~~~ + +* Simone Orsi + +Maintainers +~~~~~~~~~~~ + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/edi `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/edi_storage_oca/__init__.py b/edi_storage_oca/__init__.py new file mode 100644 index 000000000..f24d3e242 --- /dev/null +++ b/edi_storage_oca/__init__.py @@ -0,0 +1,2 @@ +from . import components +from . import models diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py new file mode 100644 index 000000000..9d4ff373f --- /dev/null +++ b/edi_storage_oca/__manifest__.py @@ -0,0 +1,17 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +{ + "name": "EDI Storage backend support", + "summary": """ + Base module to allow exchanging files via storage backend (eg: SFTP). + """, + "version": "13.0.1.0.0", + "development_status": "Alpha", + "license": "AGPL-3", + "author": "ACSONE,Odoo Community Association (OCA)", + "depends": ["edi", "storage_backend", "component"], + "data": ["security/ir_model_access.xml", "views/edi_backend_views.xml"], + "demo": ["demo/edi_backend_demo.xml"], +} diff --git a/edi_storage_oca/components/__init__.py b/edi_storage_oca/components/__init__.py new file mode 100644 index 000000000..9f64b792b --- /dev/null +++ b/edi_storage_oca/components/__init__.py @@ -0,0 +1,3 @@ +from . import base +from . import check +from . import send diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py new file mode 100644 index 000000000..2a337fed1 --- /dev/null +++ b/edi_storage_oca/components/base.py @@ -0,0 +1,54 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from pathlib import PurePath + +from odoo.addons.component.core import AbstractComponent + + +class EDIStorageSendComponentMixin(AbstractComponent): + + _name = "edi.storage.component.mixin" + + @property + def storage(self): + return self.backend.storage_id + + def _dir_by_state(self, direction, state): + """Return remote directory path by direction and state. + + :param direction: string stating direction of the exchange + :param state: string stating state of the exchange + :return: PurePath object + """ + assert direction in ("input", "output") + assert state in ("pending", "done", "error") + return PurePath((self.backend[direction + "_dir_" + state] or "").strip(" /")) + + def _remote_file_path(self, direction, state, filename): + """Return remote file path by direction and state for give filename. + + :param direction: string stating direction of the exchange + :param state: string stating state of the exchange + :param filename: string for file name + :return: PurePath object + """ + return self._dir_by_state(direction, state) / filename.strip("/ ") + + def _get_remote_file(self, state, filename=None, binary=False): + """Get file for current exchange_record in the given destination state. + + :param state: string ("pending", "done", "error") + :param filename: custom file name, exchange_record filename used by default + :return: remote file content as string + """ + filename = filename or self.exchange_record.exchange_filename + path = self._remote_file_path(self.exchange_record.direction, state, filename) + try: + # TODO: support match via pattern (eg: filename-prefix-*) + # otherwise is impossible to retrieve input files and acks + # (the date will never match) + return self.storage.get(path.as_posix(), binary=binary) + except FileNotFoundError: + return None diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py new file mode 100644 index 000000000..e76ec75aa --- /dev/null +++ b/edi_storage_oca/components/check.py @@ -0,0 +1,83 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import logging + +from odoo.tools import pycompat + +from odoo.addons.component.core import Component + +_logger = logging.getLogger(__name__) + + +class EDIStorageCheckComponentMixin(Component): + + _name = "edi.storage.component.check" + _inherit = [ + "edi.component.check.mixin", + "edi.storage.component.mixin", + ] + _usage = "edi.storage.check" + + def check(self): + return self._exchange_output_check() + + def _exchange_output_check(self): + """Check status output exchange and update record. + + 1. check if the file has been processed already (done) + 2. if yes, post message and exit + 3. if not, check for errors + 4. if no errors, return + + :return: boolean + * False if there's nothing else to be done + * True if file still need action + """ + if self._get_remote_file("done"): + _logger.info( + "%s done for: %s", + self.exchange_record.model, + self.exchange_record.name, + ) + if ( + not self.exchange_record.edi_exchange_state + == "output_sent_and_processed" + ): + self.exchange_record.edi_exchange_state = "output_sent_and_processed" + self.backend._notify_done(self.exchange_record) + if self.exchange_record.type_id.ack_needed: + self._exchange_output_handle_ack() + return False + + error = self._get_remote_file("error") + if error: + _logger.info( + "%s error for: %s", + self.exchange_record.model, + self.exchange_record.name, + ) + # Assume a text file will be placed there w/ the same name and error suffix + err_filename = self.exchange_record.exchange_filename + ".error" + error_report = self._get_remote_file("error", filename=err_filename) + if self.exchange_record.edi_exchange_state == "output_sent": + self.exchange_record.update( + { + "edi_exchange_state": "output_sent_and_error", + "exchange_error": pycompat.to_text(error_report), + } + ) + self.backend._notify_error(self.exchange_record, "process_ko") + return False + return True + + def _exchange_output_handle_ack(self): + ack_file = self._get_remote_file( + "done", filename=self.exchange_record.ack_filename + ) + if ack_file: + self.exchange_record._set_file_content(ack_file, field_name="ack_file") + self.backend._notify_ack_received(self.exchange_record) + else: + self.backend._notify_ack_missing(self.exchange_record) diff --git a/edi_storage_oca/components/send.py b/edi_storage_oca/components/send.py new file mode 100644 index 000000000..4d3f37675 --- /dev/null +++ b/edi_storage_oca/components/send.py @@ -0,0 +1,39 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo.addons.component.core import Component + + +class EDIStorageSendComponent(Component): + + _name = "edi.storage.component.send" + _inherit = [ + "edi.component.send.mixin", + "edi.storage.component.mixin", + ] + _usage = "edi.storage.send" + + def send(self): + # If the file has been sent already, refresh its state + # TODO: double check if this is useless + # since the backend checks the state already + checker = self.component(usage="edi.storage.check") + result = checker.check() + if not result: + # all good here + return True + + direction = self.exchange_record.direction + filename = self.exchange_record.exchange_filename + filedata = self.exchange_record.exchange_file + path = self._remote_file_path(direction, "pending", filename) + self.storage.add(path.as_posix(), filedata, binary=False) + # TODO: delegate this to generic storage backend + # except paramiko.ssh_exception.AuthenticationException: + # # TODO this exc handling should be moved to sftp backend IMO + # error = _("Authentication error") + # state = "error_on_send" + # TODO: catch other specific exceptions + # this will swallow all the exceptions! + return True diff --git a/edi_storage_oca/demo/edi_backend_demo.xml b/edi_storage_oca/demo/edi_backend_demo.xml new file mode 100644 index 000000000..d8ff5d972 --- /dev/null +++ b/edi_storage_oca/demo/edi_backend_demo.xml @@ -0,0 +1,14 @@ + + + + Storage Demo EDI backend + + + demo_in/pending + demo_in/done + demo_in/error + demo_out/pending + demo_out/done + demo_out/error + + diff --git a/edi_storage_oca/models/__init__.py b/edi_storage_oca/models/__init__.py new file mode 100644 index 000000000..4b2c54ac3 --- /dev/null +++ b/edi_storage_oca/models/__init__.py @@ -0,0 +1 @@ +from . import edi_backend diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py new file mode 100644 index 000000000..5ba933639 --- /dev/null +++ b/edi_storage_oca/models/edi_backend.py @@ -0,0 +1,67 @@ +# Copyright 2020 ACSONE SA +# @author Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import fields, models + + +class EDIBackend(models.Model): + + _inherit = "edi.backend" + + storage_id = fields.Many2one( + string="Storage backend", + comodel_name="storage.backend", + help="Storage for in-out files", + ondelete="restrict", + ) + """ + We assume the exchanges happen it 2 ways (input, output) + and we have a hierarchy of directory like: + + from_A_to_B + |- pending + |- done + |- error + from_B_to_A + |- pending + |- done + |- error + + where A and B are the partners exchanging data and they are in turn + sender and receiver and vice versa. + """ + # TODO: these paths should probably be by type instead + # Here we can maybe set a common root folder for this exchange. + input_dir_pending = fields.Char( + "Input pending directory", help="Path to folder for pending operations" + ) + input_dir_done = fields.Char( + "Input done directory", help="Path to folder for doneful operations" + ) + input_dir_error = fields.Char( + "Input error directory", help="Path to folder for error operations" + ) + output_dir_pending = fields.Char( + "Output pending directory", help="Path to folder for pending operations" + ) + output_dir_done = fields.Char( + "Output done directory", help="Path to folder for doneful operations" + ) + output_dir_error = fields.Char( + "Output error directory", help="Path to folder for error operations" + ) + + def _get_component_usage_candidates(self, exchange_record, key): + candidates = super()._get_component_usage_candidates(exchange_record, key) + if not self.storage_id: + return candidates + storage_generic = "edi.storage.{}".format(key) + storage_by_backend_type = storage_generic + "." + self.storage_id.backend_type + type_code = exchange_record.type_id.code + return [ + storage_by_backend_type + "." + type_code, + storage_by_backend_type, + storage_generic + "." + exchange_record.direction, + storage_generic, + ] + candidates diff --git a/edi_storage_oca/readme/CONTRIBUTORS.rst b/edi_storage_oca/readme/CONTRIBUTORS.rst new file mode 100644 index 000000000..f583948be --- /dev/null +++ b/edi_storage_oca/readme/CONTRIBUTORS.rst @@ -0,0 +1 @@ +* Simone Orsi diff --git a/edi_storage_oca/readme/DESCRIPTION.rst b/edi_storage_oca/readme/DESCRIPTION.rst new file mode 100644 index 000000000..bdef14a8a --- /dev/null +++ b/edi_storage_oca/readme/DESCRIPTION.rst @@ -0,0 +1,23 @@ +Allow exchange files using storage backends from `OCA/storage`. + +This module adds a storage backend relation on the EDI backend. +There you can configure the backend to be used (most often and SFTP) +and the paths where to read or put files. + +Often the convention when exchanging files via SFTP +is to have one input forder (to receive files) +and an output folder (to send files). + +Inside this folder you have this hierarchy:: + + input/output folder + |- pending + |- done + |- error + +* `pending` folder contains files that have been just sent +* `done` folder contains files that have been processes successfully +* `error` folder contains files with errors and cannot be processed + +The storage handlers take care of reading files and putting files +in/from the right place and update exchange records data accordingly. diff --git a/edi_storage_oca/readme/USAGE.rst b/edi_storage_oca/readme/USAGE.rst new file mode 100644 index 000000000..a47c64c84 --- /dev/null +++ b/edi_storage_oca/readme/USAGE.rst @@ -0,0 +1 @@ +Go to "EDI -> EDI backend" then configure your backend to use a storage backend. diff --git a/edi_storage_oca/security/ir_model_access.xml b/edi_storage_oca/security/ir_model_access.xml new file mode 100644 index 000000000..b7d7b8498 --- /dev/null +++ b/edi_storage_oca/security/ir_model_access.xml @@ -0,0 +1,12 @@ + + + + access_storage_backend EDI manager + + + + + + + + diff --git a/edi_storage_oca/static/description/icon.png b/edi_storage_oca/static/description/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..3a0328b516c4980e8e44cdb63fd945757ddd132d GIT binary patch literal 9455 zcmW++2RxMjAAjx~&dlBk9S+%}OXg)AGE&Cb*&}d0jUxM@u(PQx^-s)697TX`ehR4?GS^qbkof1cslKgkU)h65qZ9Oc=ml_0temigYLJfnz{IDzUf>bGs4N!v3=Z3jMq&A#7%rM5eQ#dc?k~! zVpnB`o+K7|Al`Q_U;eD$B zfJtP*jH`siUq~{KE)`jP2|#TUEFGRryE2`i0**z#*^6~AI|YzIWy$Cu#CSLW3q=GA z6`?GZymC;dCPk~rBS%eCb`5OLr;RUZ;D`}um=H)BfVIq%7VhiMr)_#G0N#zrNH|__ zc+blN2UAB0=617@>_u;MPHN;P;N#YoE=)R#i$k_`UAA>WWCcEVMh~L_ zj--gtp&|K1#58Yz*AHCTMziU1Jzt_jG0I@qAOHsk$2}yTmVkBp_eHuY$A9)>P6o~I z%aQ?!(GqeQ-Y+b0I(m9pwgi(IIZZzsbMv+9w{PFtd_<_(LA~0H(xz{=FhLB@(1&qHA5EJw1>>=%q2f&^X>IQ{!GJ4e9U z&KlB)z(84HmNgm2hg2C0>WM{E(DdPr+EeU_N@57;PC2&DmGFW_9kP&%?X4}+xWi)( z;)z%wI5>D4a*5XwD)P--sPkoY(a~WBw;E~AW`Yue4kFa^LM3X`8x|}ZUeMnqr}>kH zG%WWW>3ml$Yez?i%)2pbKPI7?5o?hydokgQyZsNEr{a|mLdt;X2TX(#B1j35xPnPW z*bMSSOauW>o;*=kO8ojw91VX!qoOQb)zHJ!odWB}d+*K?#sY_jqPdg{Sm2HdYzdEx zOGVPhVRTGPtv0o}RfVP;Nd(|CB)I;*t&QO8h zFfekr30S!-LHmV_Su-W+rEwYXJ^;6&3|L$mMC8*bQptyOo9;>Qb9Q9`ySe3%V$A*9 zeKEe+b0{#KWGp$F+tga)0RtI)nhMa-K@JS}2krK~n8vJ=Ngm?R!9G<~RyuU0d?nz# z-5EK$o(!F?hmX*2Yt6+coY`6jGbb7tF#6nHA zuKk=GGJ;ZwON1iAfG$E#Y7MnZVmrY|j0eVI(DN_MNFJmyZ|;w4tf@=CCDZ#5N_0K= z$;R~bbk?}TpfDjfB&aiQ$VA}s?P}xPERJG{kxk5~R`iRS(SK5d+Xs9swCozZISbnS zk!)I0>t=A<-^z(cmSFz3=jZ23u13X><0b)P)^1T_))Kr`e!-pb#q&J*Q`p+B6la%C zuVl&0duN<;uOsB3%T9Fp8t{ED108<+W(nOZd?gDnfNBC3>M8WE61$So|P zVvqH0SNtDTcsUdzaMDpT=Ty0pDHHNL@Z0w$Y`XO z2M-_r1S+GaH%pz#Uy0*w$Vdl=X=rQXEzO}d6J^R6zjM1u&c9vYLvLp?W7w(?np9x1 zE_0JSAJCPB%i7p*Wvg)pn5T`8k3-uR?*NT|J`eS#_#54p>!p(mLDvmc-3o0mX*mp_ zN*AeS<>#^-{S%W<*mz^!X$w_2dHWpcJ6^j64qFBft-o}o_Vx80o0>}Du;>kLts;$8 zC`7q$QI(dKYG`Wa8#wl@V4jVWBRGQ@1dr-hstpQL)Tl+aqVpGpbSfN>5i&QMXfiZ> zaA?T1VGe?rpQ@;+pkrVdd{klI&jVS@I5_iz!=UMpTsa~mBga?1r}aRBm1WS;TT*s0f0lY=JBl66Upy)-k4J}lh=P^8(SXk~0xW=T9v*B|gzIhN z>qsO7dFd~mgxAy4V?&)=5ieYq?zi?ZEoj)&2o)RLy=@hbCRcfT5jigwtQGE{L*8<@Yd{zg;CsL5mvzfDY}P-wos_6PfprFVaeqNE%h zKZhLtcQld;ZD+>=nqN~>GvROfueSzJD&BE*}XfU|H&(FssBqY=hPCt`d zH?@s2>I(|;fcW&YM6#V#!kUIP8$Nkdh0A(bEVj``-AAyYgwY~jB zT|I7Bf@%;7aL7Wf4dZ%VqF$eiaC38OV6oy3Z#TER2G+fOCd9Iaoy6aLYbPTN{XRPz z;U!V|vBf%H!}52L2gH_+j;`bTcQRXB+y9onc^wLm5wi3-Be}U>k_u>2Eg$=k!(l@I zcCg+flakT2Nej3i0yn+g+}%NYb?ta;R?(g5SnwsQ49U8Wng8d|{B+lyRcEDvR3+`O{zfmrmvFrL6acVP%yG98X zo&+VBg@px@i)%o?dG(`T;n*$S5*rnyiR#=wW}}GsAcfyQpE|>a{=$Hjg=-*_K;UtD z#z-)AXwSRY?OPefw^iI+ z)AXz#PfEjlwTes|_{sB?4(O@fg0AJ^g8gP}ex9Ucf*@_^J(s_5jJV}c)s$`Myn|Kd z$6>}#q^n{4vN@+Os$m7KV+`}c%4)4pv@06af4-x5#wj!KKb%caK{A&Y#Rfs z-po?Dcb1({W=6FKIUirH&(yg=*6aLCekcKwyfK^JN5{wcA3nhO(o}SK#!CINhI`-I z1)6&n7O&ZmyFMuNwvEic#IiOAwNkR=u5it{B9n2sAJV5pNhar=j5`*N!Na;c7g!l$ z3aYBqUkqqTJ=Re-;)s!EOeij=7SQZ3Hq}ZRds%IM*PtM$wV z@;rlc*NRK7i3y5BETSKuumEN`Xu_8GP1Ri=OKQ$@I^ko8>H6)4rjiG5{VBM>B|%`&&s^)jS|-_95&yc=GqjNo{zFkw%%HHhS~e=s zD#sfS+-?*t|J!+ozP6KvtOl!R)@@-z24}`9{QaVLD^9VCSR2b`b!KC#o;Ki<+wXB6 zx3&O0LOWcg4&rv4QG0)4yb}7BFSEg~=IR5#ZRj8kg}dS7_V&^%#Do==#`u zpy6{ox?jWuR(;pg+f@mT>#HGWHAJRRDDDv~@(IDw&R>9643kK#HN`!1vBJHnC+RM&yIh8{gG2q zA%e*U3|N0XSRa~oX-3EAneep)@{h2vvd3Xvy$7og(sayr@95+e6~Xvi1tUqnIxoIH zVWo*OwYElb#uyW{Imam6f2rGbjR!Y3`#gPqkv57dB6K^wRGxc9B(t|aYDGS=m$&S!NmCtrMMaUg(c zc2qC=2Z`EEFMW-me5B)24AqF*bV5Dr-M5ig(l-WPS%CgaPzs6p_gnCIvTJ=Y<6!gT zVt@AfYCzjjsMEGi=rDQHo0yc;HqoRNnNFeWZgcm?f;cp(6CNylj36DoL(?TS7eU#+ z7&mfr#y))+CJOXQKUMZ7QIdS9@#-}7y2K1{8)cCt0~-X0O!O?Qx#E4Og+;A2SjalQ zs7r?qn0H044=sDN$SRG$arw~n=+T_DNdSrarmu)V6@|?1-ZB#hRn`uilTGPJ@fqEy zGt(f0B+^JDP&f=r{#Y_wi#AVDf-y!RIXU^0jXsFpf>=Ji*TeqSY!H~AMbJdCGLhC) zn7Rx+sXw6uYj;WRYrLd^5IZq@6JI1C^YkgnedZEYy<&4(z%Q$5yv#Boo{AH8n$a zhb4Y3PWdr269&?V%uI$xMcUrMzl=;w<_nm*qr=c3Rl@i5wWB;e-`t7D&c-mcQl7x! zZWB`UGcw=Y2=}~wzrfLx=uet<;m3~=8I~ZRuzvMQUQdr+yTV|ATf1Uuomr__nDf=X zZ3WYJtHp_ri(}SQAPjv+Y+0=fH4krOP@S&=zZ-t1jW1o@}z;xk8 z(Nz1co&El^HK^NrhVHa-_;&88vTU>_J33=%{if;BEY*J#1n59=07jrGQ#IP>@u#3A z;!q+E1Rj3ZJ+!4bq9F8PXJ@yMgZL;>&gYA0%_Kbi8?S=XGM~dnQZQ!yBSgcZhY96H zrWnU;k)qy`rX&&xlDyA%(a1Hhi5CWkmg(`Gb%m(HKi-7Z!LKGRP_B8@`7&hdDy5n= z`OIxqxiVfX@OX1p(mQu>0Ai*v_cTMiw4qRt3~NBvr9oBy0)r>w3p~V0SCm=An6@3n)>@z!|o-$HvDK z|3D2ZMJkLE5loMKl6R^ez@Zz%S$&mbeoqH5`Bb){Ei21q&VP)hWS2tjShfFtGE+$z zzCR$P#uktu+#!w)cX!lWN1XU%K-r=s{|j?)Akf@q#3b#{6cZCuJ~gCxuMXRmI$nGtnH+-h z+GEi!*X=AP<|fG`1>MBdTb?28JYc=fGvAi2I<$B(rs$;eoJCyR6_bc~p!XR@O-+sD z=eH`-ye})I5ic1eL~TDmtfJ|8`0VJ*Yr=hNCd)G1p2MMz4C3^Mj?7;!w|Ly%JqmuW zlIEW^Ft%z?*|fpXda>Jr^1noFZEwFgVV%|*XhH@acv8rdGxeEX{M$(vG{Zw+x(ei@ zmfXb22}8-?Fi`vo-YVrTH*C?a8%M=Hv9MqVH7H^J$KsD?>!SFZ;ZsvnHr_gn=7acz z#W?0eCdVhVMWN12VV^$>WlQ?f;P^{(&pYTops|btm6aj>_Uz+hqpGwB)vWp0Cf5y< zft8-je~nn?W11plq}N)4A{l8I7$!ks_x$PXW-2XaRFswX_BnF{R#6YIwMhAgd5F9X zGmwdadS6(a^fjHtXg8=l?Rc0Sm%hk6E9!5cLVloEy4eh(=FwgP`)~I^5~pBEWo+F6 zSf2ncyMurJN91#cJTy_u8Y}@%!bq1RkGC~-bV@SXRd4F{R-*V`bS+6;W5vZ(&+I<9$;-V|eNfLa5n-6% z2(}&uGRF;p92eS*sE*oR$@pexaqr*meB)VhmIg@h{uzkk$9~qh#cHhw#>O%)b@+(| z^IQgqzuj~Sk(J;swEM-3TrJAPCq9k^^^`q{IItKBRXYe}e0Tdr=Huf7da3$l4PdpwWDop%^}n;dD#K4s#DYA8SHZ z&1!riV4W4R7R#C))JH1~axJ)RYnM$$lIR%6fIVA@zV{XVyx}C+a-Dt8Y9M)^KU0+H zR4IUb2CJ{Hg>CuaXtD50jB(_Tcx=Z$^WYu2u5kubqmwp%drJ6 z?Fo40g!Qd<-l=TQxqHEOuPX0;^z7iX?Ke^a%XT<13TA^5`4Xcw6D@Ur&VT&CUe0d} z1GjOVF1^L@>O)l@?bD~$wzgf(nxX1OGD8fEV?TdJcZc2KoUe|oP1#=$$7ee|xbY)A zDZq+cuTpc(fFdj^=!;{k03C69lMQ(|>uhRfRu%+!k&YOi-3|1QKB z z?n?eq1XP>p-IM$Z^C;2L3itnbJZAip*Zo0aw2bs8@(s^~*8T9go!%dHcAz2lM;`yp zD=7&xjFV$S&5uDaiScyD?B-i1ze`+CoRtz`Wn+Zl&#s4&}MO{@N!ufrzjG$B79)Y2d3tBk&)TxUTw@QS0TEL_?njX|@vq?Uz(nBFK5Pq7*xj#u*R&i|?7+6# z+|r_n#SW&LXhtheZdah{ZVoqwyT{D>MC3nkFF#N)xLi{p7J1jXlmVeb;cP5?e(=f# zuT7fvjSbjS781v?7{)-X3*?>tq?)Yd)~|1{BDS(pqC zC}~H#WXlkUW*H5CDOo<)#x7%RY)A;ShGhI5s*#cRDA8YgqG(HeKDx+#(ZQ?386dv! zlXCO)w91~Vw4AmOcATuV653fa9R$fyK8ul%rG z-wfS zihugoZyr38Im?Zuh6@RcF~t1anQu7>#lPpb#}4cOA!EM11`%f*07RqOVkmX{p~KJ9 z^zP;K#|)$`^Rb{rnHGH{~>1(fawV0*Z#)}M`m8-?ZJV<+e}s9wE# z)l&az?w^5{)`S(%MRzxdNqrs1n*-=jS^_jqE*5XDrA0+VE`5^*p3CuM<&dZEeCjoz zR;uu_H9ZPZV|fQq`Cyw4nscrVwi!fE6ciMmX$!_hN7uF;jjKG)d2@aC4ropY)8etW=xJvni)8eHi`H$%#zn^WJ5NLc-rqk|u&&4Z6fD_m&JfSI1Bvb?b<*n&sfl0^t z=HnmRl`XrFvMKB%9}>PaA`m-fK6a0(8=qPkWS5bb4=v?XcWi&hRY?O5HdulRi4?fN zlsJ*N-0Qw+Yic@s0(2uy%F@ib;GjXt01Fmx5XbRo6+n|pP(&nodMoap^z{~q ziEeaUT@Mxe3vJSfI6?uLND(CNr=#^W<1b}jzW58bIfyWTDle$mmS(|x-0|2UlX+9k zQ^EX7Nw}?EzVoBfT(-LT|=9N@^hcn-_p&sqG z&*oVs2JSU+N4ZD`FhCAWaS;>|wH2G*Id|?pa#@>tyxX`+4HyIArWDvVrX)2WAOQff z0qyHu&-S@i^MS-+j--!pr4fPBj~_8({~e1bfcl0wI1kaoN>mJL6KUPQm5N7lB(ui1 zE-o%kq)&djzWJ}ob<-GfDlkB;F31j-VHKvQUGQ3sp`CwyGJk_i!y^sD0fqC@$9|jO zOqN!r!8-p==F@ZVP=U$qSpY(gQ0)59P1&t@y?5rvg<}E+GB}26NYPp4f2YFQrQtot5mn3wu_qprZ=>Ig-$ zbW26Ws~IgY>}^5w`vTB(G`PTZaDiGBo5o(tp)qli|NeV( z@H_=R8V39rt5J5YB2Ky?4eJJ#b`_iBe2ot~6%7mLt5t8Vwi^Jy7|jWXqa3amOIoRb zOr}WVFP--DsS`1WpN%~)t3R!arKF^Q$e12KEqU36AWwnCBICpH4XCsfnyrHr>$I$4 z!DpKX$OKLWarN7nv@!uIA+~RNO)l$$w}p(;b>mx8pwYvu;dD_unryX_NhT8*Tj>BTrTTL&!?O+%Rv;b?B??gSzdp?6Uug9{ zd@V08Z$BdI?fpoCS$)t4mg4rT8Q_I}h`0d-vYZ^|dOB*Q^S|xqTV*vIg?@fVFSmMpaw0qtTRbx} z({Pg?#{2`sc9)M5N$*N|4;^t$+QP?#mov zGVC@I*lBVrOU-%2y!7%)fAKjpEFsgQc4{amtiHb95KQEwvf<(3T<9-Zm$xIew#P22 zc2Ix|App^>v6(3L_MCU0d3W##AB0M~3D00EWoKZqsJYT(#@w$Y_H7G22M~ApVFTRHMI_3be)Lkn#0F*V8Pq zc}`Cjy$bE;FJ6H7p=0y#R>`}-m4(0F>%@P|?7fx{=R^uFdISRnZ2W_xQhD{YuR3t< z{6yxu=4~JkeA;|(J6_nv#>Nvs&FuLA&PW^he@t(UwFFE8)|a!R{`E`K`i^ZnyE4$k z;(749Ix|oi$c3QbEJ3b~D_kQsPz~fIUKym($a_7dJ?o+40*OLl^{=&oq$<#Q(yyrp z{J-FAniyAw9tPbe&IhQ|a`DqFTVQGQ&Gq3!C2==4x{6EJwiPZ8zub-iXoUtkJiG{} zPaR&}_fn8_z~(=;5lD-aPWD3z8PZS@AaUiomF!G8I}Mf>e~0g#BelA-5#`cj;O5>N Xviia!U7SGha1wx#SCgwmn*{w2TRX*I literal 0 HcmV?d00001 diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html new file mode 100644 index 000000000..70dda24be --- /dev/null +++ b/edi_storage_oca/static/description/index.html @@ -0,0 +1,450 @@ + + + + + + +EDI Storage backend support + + + +
+

EDI Storage backend support

+ + +

Alpha License: AGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Allow exchange files using storage backends from OCA/storage.

+

This module adds a storage backend relation on the EDI backend. +There you can configure the backend to be used (most often and SFTP) +and the paths where to read or put files.

+

Often the convention when exchanging files via SFTP +is to have one input forder (to receive files) +and an output folder (to send files).

+

Inside this folder you have this hierarchy:

+
+input/output folder
+    |- pending
+    |- done
+    |- error
+
+
    +
  • pending folder contains files that have been just sent
    • +
  • done folder contains files that have been processes successfully
    • +
  • error folder contains files with errors and cannot be processed
    • +
+

The storage handlers take care of reading files and putting files +in/from the right place and update exchange records data accordingly.

+
+

Important

+

This is an alpha version, the data model and design can change at any time without warning. +Only for development or testing purpose, do not use in production. +More details on development status

+
+

Table of contents

+ +
+

Usage

+

Go to “EDI -> EDI backend” then configure your backend to use a storage backend.

+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • ACSONE
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+Odoo Community Association +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/edi project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/edi_storage_oca/tests/_TODO_test_components_base.py b/edi_storage_oca/tests/_TODO_test_components_base.py new file mode 100644 index 000000000..ede7d946e --- /dev/null +++ b/edi_storage_oca/tests/_TODO_test_components_base.py @@ -0,0 +1,43 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from freezegun import freeze_time + +from .common import EDIBackendCommonTestCase + + +class EDIBackendTestCase(EDIBackendCommonTestCase): + def test_remote_file_path(self): + to_test = ( + (("input", "pending", "foo.csv"), "demo_in/pending/foo.csv"), + (("input", "done", "foo.csv"), "demo_in/done/foo.csv"), + (("input", "error", "foo.csv"), "demo_in/error/foo.csv"), + (("output", "pending", "foo.csv"), "demo_out/pending/foo.csv"), + (("output", "done", "foo.csv"), "demo_out/done/foo.csv"), + (("output", "error", "foo.csv"), "demo_out/error/foo.csv"), + ) + for _args, expected in to_test: + path_obj = self.backend._remote_file_path(*_args) + self.assertEqual(path_obj.as_posix(), expected) + + with self.assertRaises(AssertionError): + self.backend._remote_file_path("WHATEVER", "error", "foo.csv") + + with self.assertRaises(AssertionError): + self.backend._remote_file_path("input", "WHATEVER", "foo.csv") + + @freeze_time("2020-10-21 10:00:00") + def test_create_record(self): + vals = { + "model": self.partner._name, + "res_id": self.partner.id, + } + record = self.backend.create_record("test_csv_input", vals) + expected = { + "type_id": self.exchange_type_in.id, + "record_id": self.partner, + "edi_exchange_state": "new", + "exchange_filename": "EDI_EXC_TEST-test_csv_input-2020-10-21-10-00-00.csv", + } + self.assertRecordValues(record, [expected]) diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py new file mode 100644 index 000000000..d81ae344d --- /dev/null +++ b/edi_storage_oca/tests/__init__.py @@ -0,0 +1 @@ +from . import test_edi_backend_storage diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py new file mode 100644 index 000000000..2d02206ab --- /dev/null +++ b/edi_storage_oca/tests/common.py @@ -0,0 +1,116 @@ +# Copyright 2020 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import base64 +import functools + +import mock + +from odoo.addons.edi.tests.common import EDIBackendCommonComponentTestCase + + +class TestEDIStorageBase(EDIBackendCommonComponentTestCase): + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage.demo_edi_backend_storage") + + @classmethod + def _setup_records(cls): + super()._setup_records() + cls.filedata = base64.b64encode(b"This is a simple file") + vals = { + "model": cls.partner._name, + "res_id": cls.partner.id, + "exchange_file": cls.filedata, + } + cls.record = cls.backend.create_record("test_csv_output", vals) + + cls.fakepath = "/tmp/{}".format(cls._filename(cls)) + with open(cls.fakepath, "w+b") as fakefile: + fakefile.write(b"filecontent") + + cls.fakepath_ack = "/tmp/{}.ack".format(cls._filename(cls)) + with open(cls.fakepath_ack, "w+b") as fakefile: + fakefile.write(b"ACK filecontent") + + cls.fakepath_error = "/tmp/{}.error".format(cls._filename(cls)) + with open(cls.fakepath_error, "w+b") as fakefile: + fakefile.write(b"ERROR XYZ: line 2 broken on bla bla") + + cls.checker = cls.backend._get_component( + ["edi.storage.check"], work_ctx={"exchange_record": cls.record} + ) + cls.sender = cls.backend._get_component( + ["edi.storage.send"], work_ctx={"exchange_record": cls.record} + ) + + def setUp(self): + super().setUp() + self._storage_backend_calls = [] + + def _filename(self, record=None, ack=False): + record = record or self.record + return record.exchange_filename if not ack else record.ack_filename + + def _file_fullpath(self, state, record=None, ack=False): + record = record or self.record + fname = self._filename(record, ack=ack) + if state == "error-report": + # Exception as we read from the same path but w/ error suffix + state = "error" + fname += ".error" + return ( + self.checker._remote_file_path(record.direction, state, fname) + ).as_posix() + + def _mocked_backend_get(self, mocked_paths, path, **kwargs): + self._storage_backend_calls.append(path) + if mocked_paths.get(path): + with open(mocked_paths.get(path), "rb") as remote_file: + return remote_file.read() + raise FileNotFoundError() + + def _mocked_backend_add(self, path, data, **kwargs): + self._storage_backend_calls.append(path) + + def _mock_storage_backend_get(self, mocked_paths): + mock_path = "odoo.addons.storage_backend.models.storage_backend.StorageBackend" + mocked = functools.partial(self._mocked_backend_get, mocked_paths) + return mock.patch(mock_path + ".get", mocked) + + def _mock_storage_backend_add(self): + mock_path = "odoo.addons.storage_backend.models.storage_backend.StorageBackend" + return mock.patch(mock_path + ".add", self._mocked_backend_add) + + def _test_result( + self, record, expected_values, expected_messages=None, state_paths=None, + ): + state_paths = state_paths or ("done", "pending", "error") + # Paths will be something like: + # [ + # 'demo_out/pending/$filename.csv', + # 'demo_out/pending/$filename.csv', + # 'demo_out/error/$filename.csv', + # ] + for state in state_paths: + path = self._file_fullpath(state, record=record) + self.assertIn(path, self._storage_backend_calls) + self.assertRecordValues(record, [expected_values]) + if expected_messages: + self.assertEqual(len(record.record.message_ids), len(expected_messages)) + for msg_rec, expected in zip(record.record.message_ids, expected_messages): + self.assertIn(expected["message"], msg_rec.body) + self.assertIn("level-" + expected["level"], msg_rec.body) + # TODO: test content of file sent + + def _test_send(self, record, mocked_paths=None): + with self._mock_storage_backend_add(): + if mocked_paths: + with self._mock_storage_backend_get(mocked_paths): + self.backend.exchange_send(record) + else: + self.backend.exchange_send(record) + + def _test_run_cron(self, mocked_paths): + with self._mock_storage_backend_add(): + with self._mock_storage_backend_get(mocked_paths): + self.backend._cron_check_output_exchange_sync() diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py new file mode 100644 index 000000000..de240c955 --- /dev/null +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -0,0 +1,204 @@ +# Copyright 2020 ACSONE SA/NV () +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +import base64 + +from freezegun import freeze_time + +from odoo.tools import mute_logger + +from .common import TestEDIStorageBase + +LOGGERS = ( + "odoo.addons.edi_storage.components.check", + "odoo.addons.edi.models.edi_backend", +) + + +@freeze_time("2020-10-21 10:30:00") +class TestEDIBackendOutput(TestEDIStorageBase): + @mute_logger(*LOGGERS) + def test_export_file_sent(self): + """Send, no errors.""" + self.record.edi_exchange_state = "output_pending" + mocked_paths = {self._file_fullpath("pending"): self.fakepath} + # TODO: test send only w/out cron (make sure check works) + # self._test_send(self.record, mocked_paths=mocked_paths) + self._test_run_cron(mocked_paths) + self._test_result( + self.record, + {"edi_exchange_state": "output_sent"}, + expected_messages=[ + { + "message": self.record._exchange_status_message("send_ok"), + "level": "info", + } + ], + ) + + @mute_logger(*LOGGERS) + def test_export_file_already_done(self): + """Already sent, successfully.""" + self.record.edi_exchange_state = "output_sent" + mocked_paths = {self._file_fullpath("done"): self.fakepath} + # TODO: test send only w/out cron (make sure check works) + self._test_run_cron(mocked_paths) + # As we simulate to find a file in `done` folder, + # we should get the final good state + # and only one call to ftp + self._test_result( + self.record, + {"edi_exchange_state": "output_sent_and_processed", "ack_file": False}, + state_paths=("done",), + expected_messages=[ + { + "message": self.record._exchange_status_message("process_ok"), + "level": "info", + } + ], + ) + + @mute_logger(*LOGGERS) + def test_export_file_already_done_ack_needed_not_found(self): + self.record.edi_exchange_state = "output_sent" + self.record.type_id.ack_needed = True + mocked_paths = { + self._file_fullpath("done"): self.fakepath, + } + self._test_run_cron(mocked_paths) + # No ack file found, warning message is posted + self._test_result( + self.record, + {"edi_exchange_state": "output_sent_and_processed"}, + state_paths=("done",), + expected_messages=[ + { + "message": self.record._exchange_status_message("ack_missing"), + "level": "warning", + }, + { + "message": self.record._exchange_status_message("process_ok"), + "level": "info", + }, + ], + ) + + @mute_logger(*LOGGERS) + def test_export_file_already_done_ack_needed_found(self): + self.record.edi_exchange_state = "output_sent" + self.record.type_id.ack_needed = True + mocked_paths = { + self._file_fullpath("done"): self.fakepath, + self._file_fullpath("done", ack=True): self.fakepath_ack, + } + self._test_run_cron(mocked_paths) + # Found ack file, set on record + self._test_result( + self.record, + { + "edi_exchange_state": "output_sent_and_processed", + "ack_file": base64.b64encode(b"ACK filecontent"), + }, + state_paths=("done",), + expected_messages=[ + { + "message": self.record._exchange_status_message("ack_received"), + "level": "info", + }, + { + "message": self.record._exchange_status_message("process_ok"), + "level": "info", + }, + ], + ) + + @mute_logger(*LOGGERS) + def test_already_sent_process_error(self): + """Already sent, error process.""" + self.record.edi_exchange_state = "output_sent" + mocked_paths = { + self._file_fullpath("error"): self.fakepath, + self._file_fullpath("error-report"): self.fakepath_error, + } + self._test_run_cron(mocked_paths) + # As we simulate to find a file in `error` folder, + # we should get a call for: done, error and then the read of the report. + self._test_result( + self.record, + { + "edi_exchange_state": "output_sent_and_error", + "exchange_error": "ERROR XYZ: line 2 broken on bla bla", + }, + state_paths=("done", "error", "error-report"), + expected_messages=[ + { + "message": self.record._exchange_status_message("process_ko"), + "level": "error", + } + ], + ) + + @mute_logger(*LOGGERS) + def test_cron_full_flow(self): + """Already sent, update the state via cron.""" + self.record.edi_exchange_state = "output_sent" + rec1 = self.record + partner2 = self.env.ref("base.res_partner_2") + partner3 = self.env.ref("base.res_partner_3") + rec2 = self.record.copy( + { + "model": partner2._name, + "res_id": partner2.id, + "exchange_filename": "rec2.csv", + } + ) + rec3 = self.record.copy( + { + "model": partner3._name, + "res_id": partner3.id, + "exchange_filename": "rec3.csv", + "edi_exchange_state": "output_sent_and_error", + } + ) + mocked_paths = { + self._file_fullpath("done", record=rec1): self.fakepath, + self._file_fullpath("error", record=rec2): self.fakepath, + self._file_fullpath("error-report", record=rec2): self.fakepath_error, + self._file_fullpath("done", record=rec3): self.fakepath, + } + self._test_run_cron(mocked_paths) + self._test_result( + rec1, + {"edi_exchange_state": "output_sent_and_processed"}, + state_paths=("done",), + expected_messages=[ + { + "message": rec1._exchange_status_message("process_ok"), + "level": "info", + } + ], + ) + self._test_result( + rec2, + { + "edi_exchange_state": "output_sent_and_error", + "exchange_error": "ERROR XYZ: line 2 broken on bla bla", + }, + state_paths=("done", "error", "error-report"), + expected_messages=[ + { + "message": rec2._exchange_status_message("process_ko"), + "level": "error", + } + ], + ) + self._test_result( + rec3, + {"edi_exchange_state": "output_sent_and_processed"}, + state_paths=("done",), + expected_messages=[ + { + "message": rec3._exchange_status_message("process_ok"), + "level": "info", + } + ], + ) diff --git a/edi_storage_oca/views/edi_backend_views.xml b/edi_storage_oca/views/edi_backend_views.xml new file mode 100644 index 000000000..5c7916eb1 --- /dev/null +++ b/edi_storage_oca/views/edi_backend_views.xml @@ -0,0 +1,21 @@ + + + + edi.backend + + + + + + + + + + + + + + + + + From d8500ac395d34f9aa16885ef050eca306805c6d9 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 27 Nov 2020 13:50:23 +0100 Subject: [PATCH 02/68] edi_storage: add base tests for component --- .../tests/_TODO_test_components_base.py | 43 ------------------- edi_storage_oca/tests/__init__.py | 1 + edi_storage_oca/tests/common.py | 10 +++-- edi_storage_oca/tests/test_components_base.py | 43 +++++++++++++++++++ 4 files changed, 50 insertions(+), 47 deletions(-) delete mode 100644 edi_storage_oca/tests/_TODO_test_components_base.py create mode 100644 edi_storage_oca/tests/test_components_base.py diff --git a/edi_storage_oca/tests/_TODO_test_components_base.py b/edi_storage_oca/tests/_TODO_test_components_base.py deleted file mode 100644 index ede7d946e..000000000 --- a/edi_storage_oca/tests/_TODO_test_components_base.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2020 ACSONE -# @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). - -from freezegun import freeze_time - -from .common import EDIBackendCommonTestCase - - -class EDIBackendTestCase(EDIBackendCommonTestCase): - def test_remote_file_path(self): - to_test = ( - (("input", "pending", "foo.csv"), "demo_in/pending/foo.csv"), - (("input", "done", "foo.csv"), "demo_in/done/foo.csv"), - (("input", "error", "foo.csv"), "demo_in/error/foo.csv"), - (("output", "pending", "foo.csv"), "demo_out/pending/foo.csv"), - (("output", "done", "foo.csv"), "demo_out/done/foo.csv"), - (("output", "error", "foo.csv"), "demo_out/error/foo.csv"), - ) - for _args, expected in to_test: - path_obj = self.backend._remote_file_path(*_args) - self.assertEqual(path_obj.as_posix(), expected) - - with self.assertRaises(AssertionError): - self.backend._remote_file_path("WHATEVER", "error", "foo.csv") - - with self.assertRaises(AssertionError): - self.backend._remote_file_path("input", "WHATEVER", "foo.csv") - - @freeze_time("2020-10-21 10:00:00") - def test_create_record(self): - vals = { - "model": self.partner._name, - "res_id": self.partner.id, - } - record = self.backend.create_record("test_csv_input", vals) - expected = { - "type_id": self.exchange_type_in.id, - "record_id": self.partner, - "edi_exchange_state": "new", - "exchange_filename": "EDI_EXC_TEST-test_csv_input-2020-10-21-10-00-00.csv", - } - self.assertRecordValues(record, [expected]) diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py index d81ae344d..b788a0071 100644 --- a/edi_storage_oca/tests/__init__.py +++ b/edi_storage_oca/tests/__init__.py @@ -1 +1,2 @@ from . import test_edi_backend_storage +from . import test_components_base diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 2d02206ab..a6af8cb00 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -7,6 +7,10 @@ from odoo.addons.edi.tests.common import EDIBackendCommonComponentTestCase +STORAGE_BACKEND_MOCK_PATH = ( + "odoo.addons.storage_backend.models.storage_backend.StorageBackend" +) + class TestEDIStorageBase(EDIBackendCommonComponentTestCase): @classmethod @@ -73,13 +77,11 @@ def _mocked_backend_add(self, path, data, **kwargs): self._storage_backend_calls.append(path) def _mock_storage_backend_get(self, mocked_paths): - mock_path = "odoo.addons.storage_backend.models.storage_backend.StorageBackend" mocked = functools.partial(self._mocked_backend_get, mocked_paths) - return mock.patch(mock_path + ".get", mocked) + return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".get", mocked) def _mock_storage_backend_add(self): - mock_path = "odoo.addons.storage_backend.models.storage_backend.StorageBackend" - return mock.patch(mock_path + ".add", self._mocked_backend_add) + return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".add", self._mocked_backend_add) def _test_result( self, record, expected_values, expected_messages=None, state_paths=None, diff --git a/edi_storage_oca/tests/test_components_base.py b/edi_storage_oca/tests/test_components_base.py new file mode 100644 index 000000000..c2ba7275f --- /dev/null +++ b/edi_storage_oca/tests/test_components_base.py @@ -0,0 +1,43 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import mock + +from .common import STORAGE_BACKEND_MOCK_PATH, TestEDIStorageBase + + +class EDIBackendTestCase(TestEDIStorageBase): + def test_remote_file_path(self): + to_test = ( + (("input", "pending", "foo.csv"), "demo_in/pending/foo.csv"), + (("input", "done", "foo.csv"), "demo_in/done/foo.csv"), + (("input", "error", "foo.csv"), "demo_in/error/foo.csv"), + (("output", "pending", "foo.csv"), "demo_out/pending/foo.csv"), + (("output", "done", "foo.csv"), "demo_out/done/foo.csv"), + (("output", "error", "foo.csv"), "demo_out/error/foo.csv"), + ) + for _args, expected in to_test: + path_obj = self.checker._remote_file_path(*_args) + self.assertEqual(path_obj.as_posix(), expected) + + with self.assertRaises(AssertionError): + self.checker._remote_file_path("WHATEVER", "error", "foo.csv") + + with self.assertRaises(AssertionError): + self.checker._remote_file_path("input", "WHATEVER", "foo.csv") + + def test_get_remote_file(self): + with mock.patch(STORAGE_BACKEND_MOCK_PATH + ".get") as mocked: + self.checker._get_remote_file("pending") + mocked.assert_called_with( + "demo_out/pending/{}".format(self._filename(self.record)), binary=False + ) + self.checker._get_remote_file("done") + mocked.assert_called_with( + "demo_out/done/{}".format(self._filename(self.record)), binary=False + ) + self.checker._get_remote_file("error") + mocked.assert_called_with( + "demo_out/error/{}".format(self._filename(self.record)), binary=False + ) From 7552e9f39723be53393e57e03d01daa170ba2145 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Sun, 29 Nov 2020 18:58:26 +0100 Subject: [PATCH 03/68] edi_storage: fix messages test isolation --- edi_storage_oca/tests/common.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index a6af8cb00..d49b443a5 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -98,8 +98,12 @@ def _test_result( self.assertIn(path, self._storage_backend_calls) self.assertRecordValues(record, [expected_values]) if expected_messages: - self.assertEqual(len(record.record.message_ids), len(expected_messages)) - for msg_rec, expected in zip(record.record.message_ids, expected_messages): + # consider only edi related messages + messages = record.record.message_ids.filtered( + lambda x: "edi-exchange" in x.body + ) + self.assertEqual(len(messages), len(expected_messages)) + for msg_rec, expected in zip(messages, expected_messages): self.assertIn(expected["message"], msg_rec.body) self.assertIn("level-" + expected["level"], msg_rec.body) # TODO: test content of file sent From ff6a0504034376ad0530f02b6507d448222e65ac Mon Sep 17 00:00:00 2001 From: oca-travis Date: Mon, 30 Nov 2020 11:50:12 +0000 Subject: [PATCH 04/68] [UPD] Update edi_storage.pot --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/i18n/edi_storage.pot | 77 ++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 edi_storage_oca/i18n/edi_storage.pot diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 9d4ff373f..0d0500a2f 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.0.0", + "version": "13.0.1.1.0", "development_status": "Alpha", "license": "AGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", diff --git a/edi_storage_oca/i18n/edi_storage.pot b/edi_storage_oca/i18n/edi_storage.pot new file mode 100644 index 000000000..d6944e83b --- /dev/null +++ b/edi_storage_oca/i18n/edi_storage.pot @@ -0,0 +1,77 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * edi_storage +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 13.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: edi_storage +#: model:ir.model,name:edi_storage.model_edi_backend +msgid "EDI Backend" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_done +msgid "Input done directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_error +msgid "Input error directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_pending +msgid "Input pending directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_done +msgid "Output done directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_error +msgid "Output error directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_pending +msgid "Output pending directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_done +#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_done +msgid "Path to folder for doneful operations" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_error +#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_error +msgid "Path to folder for error operations" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_pending +#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_pending +msgid "Path to folder for pending operations" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__storage_id +msgid "Storage backend" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__storage_id +msgid "Storage for in-out files" +msgstr "" From 6954962f49f3b4776528927f2ff83919021208d2 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Wed, 2 Dec 2020 08:04:25 +0100 Subject: [PATCH 05/68] edi_storage: adapt to edi api changes --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/check.py | 8 ++++---- edi_storage_oca/tests/common.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 0d0500a2f..494055c6c 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.1.0", + "version": "13.0.1.2.0", "development_status": "Alpha", "license": "AGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index e76ec75aa..835f979b8 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -46,7 +46,7 @@ def _exchange_output_check(self): == "output_sent_and_processed" ): self.exchange_record.edi_exchange_state = "output_sent_and_processed" - self.backend._notify_done(self.exchange_record) + self.exchange_record._notify_done() if self.exchange_record.type_id.ack_needed: self._exchange_output_handle_ack() return False @@ -68,7 +68,7 @@ def _exchange_output_check(self): "exchange_error": pycompat.to_text(error_report), } ) - self.backend._notify_error(self.exchange_record, "process_ko") + self.exchange_record._notify_error("process_ko") return False return True @@ -78,6 +78,6 @@ def _exchange_output_handle_ack(self): ) if ack_file: self.exchange_record._set_file_content(ack_file, field_name="ack_file") - self.backend._notify_ack_received(self.exchange_record) + self.exchange_record._notify_ack_received() else: - self.backend._notify_ack_missing(self.exchange_record) + self.exchange_record._notify_ack_missing() diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index d49b443a5..d55fe6774 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -40,10 +40,10 @@ def _setup_records(cls): with open(cls.fakepath_error, "w+b") as fakefile: fakefile.write(b"ERROR XYZ: line 2 broken on bla bla") - cls.checker = cls.backend._get_component( + cls.checker = cls.backend._find_component( ["edi.storage.check"], work_ctx={"exchange_record": cls.record} ) - cls.sender = cls.backend._get_component( + cls.sender = cls.backend._find_component( ["edi.storage.send"], work_ctx={"exchange_record": cls.record} ) From 11e0fbe4b787df3fcf630391ffad39e676d16a6f Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Wed, 2 Dec 2020 08:38:55 +0100 Subject: [PATCH 06/68] edi_storage: put ack handling on hold Ack files should be treated as incoming records. Receive files is not handled yet by the framework and will be added soon. --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/check.py | 21 +++- edi_storage_oca/tests/common.py | 4 +- .../tests/test_edi_backend_storage.py | 109 +++++++++--------- 4 files changed, 73 insertions(+), 63 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 494055c6c..838e6579a 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.2.0", + "version": "13.0.1.3.0", "development_status": "Alpha", "license": "AGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index 835f979b8..965b01dfe 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -47,8 +47,6 @@ def _exchange_output_check(self): ): self.exchange_record.edi_exchange_state = "output_sent_and_processed" self.exchange_record._notify_done() - if self.exchange_record.type_id.ack_needed: - self._exchange_output_handle_ack() return False error = self._get_remote_file("error") @@ -72,12 +70,23 @@ def _exchange_output_check(self): return False return True + # FIXME: this is not used ATM -> should be refactored + # into an incoming exchange. + # The backend will look for records needing an ack + # and generate and ack record. def _exchange_output_handle_ack(self): - ack_file = self._get_remote_file( - "done", filename=self.exchange_record.ack_filename - ) + ack_type = self.exchange_record.type_id.ack_type_id + filename = ack_type._make_exchange_filename(self.exchange_record) + ack_file = self._get_remote_file("done", filename=filename) if ack_file: - self.exchange_record._set_file_content(ack_file, field_name="ack_file") + self.backend.create_record( + ack_type.code, + { + "parent_id": self.exchange_record.id, + "exchange_file": ack_file, + "edi_exchange_state": "input_received", + }, + ) self.exchange_record._notify_ack_received() else: self.exchange_record._notify_ack_missing() diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index d55fe6774..992c4b44f 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -53,7 +53,9 @@ def setUp(self): def _filename(self, record=None, ack=False): record = record or self.record - return record.exchange_filename if not ack else record.ack_filename + if ack: + record.type_id.ack_type_id._make_exchange_filename(record) + return record.exchange_filename def _file_fullpath(self, state, record=None, ack=False): record = record or self.record diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index de240c955..1eda66121 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -1,7 +1,5 @@ # Copyright 2020 ACSONE SA/NV () # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). -import base64 - from freezegun import freeze_time from odoo.tools import mute_logger @@ -47,7 +45,7 @@ def test_export_file_already_done(self): # and only one call to ftp self._test_result( self.record, - {"edi_exchange_state": "output_sent_and_processed", "ack_file": False}, + {"edi_exchange_state": "output_sent_and_processed"}, state_paths=("done",), expected_messages=[ { @@ -57,59 +55,60 @@ def test_export_file_already_done(self): ], ) - @mute_logger(*LOGGERS) - def test_export_file_already_done_ack_needed_not_found(self): - self.record.edi_exchange_state = "output_sent" - self.record.type_id.ack_needed = True - mocked_paths = { - self._file_fullpath("done"): self.fakepath, - } - self._test_run_cron(mocked_paths) - # No ack file found, warning message is posted - self._test_result( - self.record, - {"edi_exchange_state": "output_sent_and_processed"}, - state_paths=("done",), - expected_messages=[ - { - "message": self.record._exchange_status_message("ack_missing"), - "level": "warning", - }, - { - "message": self.record._exchange_status_message("process_ok"), - "level": "info", - }, - ], - ) + # FIXME: ack should be handle as an incoming record (new machinery to be added) + # @mute_logger(*LOGGERS) + # def test_export_file_already_done_ack_needed_not_found(self): + # self.record.edi_exchange_state = "output_sent" + # self.record.type_id.ack_needed = True + # mocked_paths = { + # self._file_fullpath("done"): self.fakepath, + # } + # self._test_run_cron(mocked_paths) + # # No ack file found, warning message is posted + # self._test_result( + # self.record, + # {"edi_exchange_state": "output_sent_and_processed"}, + # state_paths=("done",), + # expected_messages=[ + # { + # "message": self.record._exchange_status_message("ack_missing"), + # "level": "warning", + # }, + # { + # "message": self.record._exchange_status_message("process_ok"), + # "level": "info", + # }, + # ], + # ) - @mute_logger(*LOGGERS) - def test_export_file_already_done_ack_needed_found(self): - self.record.edi_exchange_state = "output_sent" - self.record.type_id.ack_needed = True - mocked_paths = { - self._file_fullpath("done"): self.fakepath, - self._file_fullpath("done", ack=True): self.fakepath_ack, - } - self._test_run_cron(mocked_paths) - # Found ack file, set on record - self._test_result( - self.record, - { - "edi_exchange_state": "output_sent_and_processed", - "ack_file": base64.b64encode(b"ACK filecontent"), - }, - state_paths=("done",), - expected_messages=[ - { - "message": self.record._exchange_status_message("ack_received"), - "level": "info", - }, - { - "message": self.record._exchange_status_message("process_ok"), - "level": "info", - }, - ], - ) + # @mute_logger(*LOGGERS) + # def test_export_file_already_done_ack_needed_found(self): + # self.record.edi_exchange_state = "output_sent" + # self.record.type_id.ack_needed = True + # mocked_paths = { + # self._file_fullpath("done"): self.fakepath, + # self._file_fullpath("done", ack=True): self.fakepath_ack, + # } + # self._test_run_cron(mocked_paths) + # # Found ack file, set on record + # self._test_result( + # self.record, + # { + # "edi_exchange_state": "output_sent_and_processed", + # "ack_file": base64.b64encode(b"ACK filecontent"), + # }, + # state_paths=("done",), + # expected_messages=[ + # { + # "message": self.record._exchange_status_message("ack_received"), + # "level": "info", + # }, + # { + # "message": self.record._exchange_status_message("process_ok"), + # "level": "info", + # }, + # ], + # ) @mute_logger(*LOGGERS) def test_already_sent_process_error(self): From f35bdbeb3f9a29e93c59af8a03dc37e7834dc83a Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Mon, 8 Feb 2021 09:47:58 +0100 Subject: [PATCH 07/68] edi_storage: adapt to new components lookup --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/base.py | 14 ++- edi_storage_oca/components/check.py | 2 +- edi_storage_oca/components/send.py | 4 +- edi_storage_oca/models/edi_backend.py | 29 ++++--- edi_storage_oca/tests/common.py | 8 +- edi_storage_oca/tests/test_component_match.py | 86 +++++++++++++++++++ edi_storage_oca/tests/test_components_base.py | 2 +- 8 files changed, 129 insertions(+), 18 deletions(-) create mode 100644 edi_storage_oca/tests/test_component_match.py diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 838e6579a..20db765f1 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.3.0", + "version": "13.0.1.4.0", "development_status": "Alpha", "license": "AGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index 2a337fed1..c6e67c148 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -7,9 +7,21 @@ from odoo.addons.component.core import AbstractComponent -class EDIStorageSendComponentMixin(AbstractComponent): +class EDIStorageComponentMixin(AbstractComponent): _name = "edi.storage.component.mixin" + _inherit = "edi.component.mixin" + # Components having `_storage_backend_type` will have precedence. + # If the value is not set, generic components will be used. + _storage_backend_type = None + + @classmethod + def _component_match(cls, work, usage=None, model_name=None, **kw): + res = super()._component_match(work, usage=usage, model_name=model_name, **kw) + storage_type = kw.get("storage_backend_type") + if storage_type and cls._storage_backend_type: + return cls._storage_backend_type == storage_type + return res @property def storage(self): diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index 965b01dfe..6e3cfa99a 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -18,7 +18,7 @@ class EDIStorageCheckComponentMixin(Component): "edi.component.check.mixin", "edi.storage.component.mixin", ] - _usage = "edi.storage.check" + _usage = "storage.check" def check(self): return self._exchange_output_check() diff --git a/edi_storage_oca/components/send.py b/edi_storage_oca/components/send.py index 4d3f37675..b5ec45b5c 100644 --- a/edi_storage_oca/components/send.py +++ b/edi_storage_oca/components/send.py @@ -12,13 +12,13 @@ class EDIStorageSendComponent(Component): "edi.component.send.mixin", "edi.storage.component.mixin", ] - _usage = "edi.storage.send" + _usage = "storage.send" def send(self): # If the file has been sent already, refresh its state # TODO: double check if this is useless # since the backend checks the state already - checker = self.component(usage="edi.storage.check") + checker = self.component(usage="storage.check") result = checker.check() if not result: # all good here diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 5ba933639..63aa10249 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -52,16 +52,25 @@ class EDIBackend(models.Model): "Output error directory", help="Path to folder for error operations" ) + _storage_actions = ("check", "send", "receive") + def _get_component_usage_candidates(self, exchange_record, key): candidates = super()._get_component_usage_candidates(exchange_record, key) - if not self.storage_id: + if not self.storage_id or key not in self._storage_actions: return candidates - storage_generic = "edi.storage.{}".format(key) - storage_by_backend_type = storage_generic + "." + self.storage_id.backend_type - type_code = exchange_record.type_id.code - return [ - storage_by_backend_type + "." + type_code, - storage_by_backend_type, - storage_generic + "." + exchange_record.direction, - storage_generic, - ] + candidates + return ["storage.{}".format(key)] + candidates + + def _component_match_attrs(self, exchange_record, key): + # Override to inject storage_backend_type + res = super()._component_match_attrs(exchange_record, key) + if not self.storage_id or key not in self._storage_actions: + return res + res["storage_backend_type"] = self.storage_id.backend_type + return res + + def _component_sort_key(self, component_class): + res = super()._component_sort_key(component_class) + # Override to give precedence by storage_backend_type when needed. + if not self.storage_id: + return res + return (1 if component_class._storage_backend_type else 0,) + res diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 992c4b44f..9cd239422 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -41,10 +41,14 @@ def _setup_records(cls): fakefile.write(b"ERROR XYZ: line 2 broken on bla bla") cls.checker = cls.backend._find_component( - ["edi.storage.check"], work_ctx={"exchange_record": cls.record} + cls.partner._name, + ["storage.check"], + work_ctx={"exchange_record": cls.record}, ) cls.sender = cls.backend._find_component( - ["edi.storage.send"], work_ctx={"exchange_record": cls.record} + cls.partner._name, + ["storage.send"], + work_ctx={"exchange_record": cls.record}, ) def setUp(self): diff --git a/edi_storage_oca/tests/test_component_match.py b/edi_storage_oca/tests/test_component_match.py new file mode 100644 index 000000000..4cd1bad87 --- /dev/null +++ b/edi_storage_oca/tests/test_component_match.py @@ -0,0 +1,86 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo.addons.component.core import Component +from odoo.addons.edi.tests.common import EDIBackendCommonComponentRegistryTestCase + + +class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls._load_module_components(cls, "edi_storage") + + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage.demo_edi_backend_storage") + + def test_component_match(self): + """Lookup with special match method.""" + + class SFTPCheck(Component): + _name = "sftp.check" + _inherit = "edi.storage.component.check" + _usage = "storage.check" + # _backend_type = "demo_backend" + _storage_backend_type = "sftp" + + class SFTPSend(Component): + _name = "sftp.send" + _inherit = "edi.storage.component.send" + _usage = "storage.send" + # _backend_type = "demo_backend" + _storage_backend_type = "sftp" + + class S3Check(Component): + _name = "s3.check" + _inherit = "edi.storage.component.check" + _usage = "storage.check" + # _exchange_type = "test_csv_output" + _storage_backend_type = "s3" + + class S3Send(Component): + _name = "s3.send" + _inherit = "edi.storage.component.send" + _usage = "storage.send" + # _exchange_type = "test_csv_output" + _storage_backend_type = "s3" + + self._build_components(SFTPCheck, SFTPSend, S3Check, S3Send) + + component = self.backend._find_component( + "res.partner", + ["storage.check"], + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="s3", + ) + self.assertEqual(component._name, S3Check._name) + + component = self.backend._find_component( + "res.partner", + ["storage.check"], + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="sftp", + ) + self.assertEqual(component._name, SFTPCheck._name) + + component = self.backend._find_component( + "res.partner", + ["storage.send"], + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="sftp", + ) + self.assertEqual(component._name, SFTPSend._name) + + component = self.backend._find_component( + "res.partner", + ["storage.send"], + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="s3", + ) + self.assertEqual(component._name, S3Send._name) diff --git a/edi_storage_oca/tests/test_components_base.py b/edi_storage_oca/tests/test_components_base.py index c2ba7275f..27969a605 100644 --- a/edi_storage_oca/tests/test_components_base.py +++ b/edi_storage_oca/tests/test_components_base.py @@ -7,7 +7,7 @@ from .common import STORAGE_BACKEND_MOCK_PATH, TestEDIStorageBase -class EDIBackendTestCase(TestEDIStorageBase): +class EDIStorageComponentTestCase(TestEDIStorageBase): def test_remote_file_path(self): to_test = ( (("input", "pending", "foo.csv"), "demo_in/pending/foo.csv"), From d6d73e142c8fb782df572adf09bcc8a76d50ed4c Mon Sep 17 00:00:00 2001 From: Enric Tobella Date: Wed, 10 Mar 2021 13:56:20 +0100 Subject: [PATCH 08/68] [CHG] edi: Use more permissive licence: AGPL-> LGPL --- edi_storage_oca/README.rst | 8 ++++---- edi_storage_oca/__manifest__.py | 4 ++-- edi_storage_oca/components/base.py | 2 +- edi_storage_oca/components/check.py | 2 +- edi_storage_oca/components/send.py | 2 +- edi_storage_oca/models/edi_backend.py | 2 +- edi_storage_oca/static/description/index.html | 2 +- edi_storage_oca/tests/common.py | 2 +- edi_storage_oca/tests/test_component_match.py | 2 +- edi_storage_oca/tests/test_components_base.py | 2 +- edi_storage_oca/tests/test_edi_backend_storage.py | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index fffe95837..cc387fb2d 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -10,9 +10,9 @@ EDI Storage backend support .. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png :target: https://odoo-community.org/page/development-status :alt: Alpha -.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png - :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html - :alt: License: AGPL-3 +.. |badge2| image:: https://img.shields.io/badge/licence-LGPL--3-blue.png + :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html + :alt: License: LGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi-lightgray.png?logo=github :target: https://github.com/OCA/edi/tree/13.0/edi_storage :alt: OCA/edi @@ -23,7 +23,7 @@ EDI Storage backend support :target: https://runbot.odoo-community.org/runbot/226/13.0 :alt: Try me on Runbot -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| |badge5| Allow exchange files using storage backends from `OCA/storage`. diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 20db765f1..503ae5cfe 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). { "name": "EDI Storage backend support", @@ -9,7 +9,7 @@ """, "version": "13.0.1.4.0", "development_status": "Alpha", - "license": "AGPL-3", + "license": "LGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", "depends": ["edi", "storage_backend", "component"], "data": ["security/ir_model_access.xml", "views/edi_backend_views.xml"], diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index c6e67c148..01d0c8b2c 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). from pathlib import PurePath diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index 6e3cfa99a..878cfffcb 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). import logging diff --git a/edi_storage_oca/components/send.py b/edi_storage_oca/components/send.py index b5ec45b5c..cbb6ed309 100644 --- a/edi_storage_oca/components/send.py +++ b/edi_storage_oca/components/send.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). from odoo.addons.component.core import Component diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 63aa10249..2437a6bc1 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE SA # @author Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). from odoo import fields, models diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index 70dda24be..b034d89fe 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -367,7 +367,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Alpha License: AGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 9cd239422..134c807a9 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -1,5 +1,5 @@ # Copyright 2020 ACSONE SA/NV () -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). import base64 import functools diff --git a/edi_storage_oca/tests/test_component_match.py b/edi_storage_oca/tests/test_component_match.py index 4cd1bad87..e422c9698 100644 --- a/edi_storage_oca/tests/test_component_match.py +++ b/edi_storage_oca/tests/test_component_match.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). from odoo.addons.component.core import Component from odoo.addons.edi.tests.common import EDIBackendCommonComponentRegistryTestCase diff --git a/edi_storage_oca/tests/test_components_base.py b/edi_storage_oca/tests/test_components_base.py index 27969a605..57abd1385 100644 --- a/edi_storage_oca/tests/test_components_base.py +++ b/edi_storage_oca/tests/test_components_base.py @@ -1,6 +1,6 @@ # Copyright 2020 ACSONE # @author: Simone Orsi -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). import mock diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index 1eda66121..f49138fc0 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -1,5 +1,5 @@ # Copyright 2020 ACSONE SA/NV () -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). from freezegun import freeze_time from odoo.tools import mute_logger From 1188affd805ae87329abdaca6edcb5cda3391ba0 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Mon, 15 Mar 2021 21:46:44 +0000 Subject: [PATCH 09/68] [UPD] README.rst --- edi_storage_oca/README.rst | 2 +- edi_storage_oca/__manifest__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index cc387fb2d..0cceccfbf 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -23,7 +23,7 @@ EDI Storage backend support :target: https://runbot.odoo-community.org/runbot/226/13.0 :alt: Try me on Runbot -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| |badge5| Allow exchange files using storage backends from `OCA/storage`. diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 503ae5cfe..0df82391d 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.4.0", + "version": "13.0.1.4.1", "development_status": "Alpha", "license": "LGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", From 35232a8ba962648649090b9d1ec512cba8ce1753 Mon Sep 17 00:00:00 2001 From: Enric Tobella Date: Wed, 7 Apr 2021 10:35:07 +0200 Subject: [PATCH 10/68] [FIX] edi_storage: sort key was not working properly --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/models/edi_backend.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 0df82391d..0243ab8cf 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.4.1", + "version": "13.0.1.5.0", "development_status": "Alpha", "license": "LGPL-3", "author": "ACSONE,Odoo Community Association (OCA)", diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 2437a6bc1..0ba578fa0 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -73,4 +73,6 @@ def _component_sort_key(self, component_class): # Override to give precedence by storage_backend_type when needed. if not self.storage_id: return res - return (1 if component_class._storage_backend_type else 0,) + res + return ( + 1 if getattr(component_class, "_storage_backend_type", False) else 0, + ) + res From 9aaa4f12ec91f72a3e7e961d0cf180eee39442a5 Mon Sep 17 00:00:00 2001 From: fshah Date: Wed, 5 May 2021 14:43:32 +0530 Subject: [PATCH 11/68] [IMP] edi_storage_oca: isort, black, prettier. --- edi_storage_oca/__manifest__.py | 1 + edi_storage_oca/tests/common.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 0243ab8cf..488f60613 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -10,6 +10,7 @@ "version": "13.0.1.5.0", "development_status": "Alpha", "license": "LGPL-3", + "website": "https://github.com/OCA/edi", "author": "ACSONE,Odoo Community Association (OCA)", "depends": ["edi", "storage_backend", "component"], "data": ["security/ir_model_access.xml", "views/edi_backend_views.xml"], diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 134c807a9..c5ed77c45 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -90,7 +90,11 @@ def _mock_storage_backend_add(self): return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".add", self._mocked_backend_add) def _test_result( - self, record, expected_values, expected_messages=None, state_paths=None, + self, + record, + expected_values, + expected_messages=None, + state_paths=None, ): state_paths = state_paths or ("done", "pending", "error") # Paths will be something like: From 0e2dafa63daa57c4af34e63660ee47eb150d37ca Mon Sep 17 00:00:00 2001 From: fshah Date: Wed, 5 May 2021 15:07:19 +0530 Subject: [PATCH 12/68] [MIG] Migrate module edi_storage_oca to v14. --- edi_storage_oca/README.rst | 11 ++++++----- edi_storage_oca/__manifest__.py | 4 ++-- edi_storage_oca/demo/edi_backend_demo.xml | 2 +- edi_storage_oca/readme/CONTRIBUTORS.rst | 1 + edi_storage_oca/static/description/index.html | 6 +++--- edi_storage_oca/tests/common.py | 4 ++-- edi_storage_oca/tests/test_component_match.py | 4 ++-- edi_storage_oca/tests/test_edi_backend_storage.py | 4 ++-- edi_storage_oca/views/edi_backend_views.xml | 2 +- 9 files changed, 20 insertions(+), 18 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 0cceccfbf..14f8cd692 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -14,13 +14,13 @@ EDI Storage backend support :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html :alt: License: LGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi-lightgray.png?logo=github - :target: https://github.com/OCA/edi/tree/13.0/edi_storage + :target: https://github.com/OCA/edi/tree/14.0/edi_storage_oca :alt: OCA/edi .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png - :target: https://translation.odoo-community.org/projects/edi-13-0/edi-13-0-edi_storage + :target: https://translation.odoo-community.org/projects/edi-14-0/edi-14-0-edi_storage_oca :alt: Translate me on Weblate .. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png - :target: https://runbot.odoo-community.org/runbot/226/13.0 + :target: https://runbot.odoo-community.org/runbot/226/14.0 :alt: Try me on Runbot |badge1| |badge2| |badge3| |badge4| |badge5| @@ -70,7 +70,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. @@ -86,6 +86,7 @@ Contributors ~~~~~~~~~~~~ * Simone Orsi +* Foram Shah Maintainers ~~~~~~~~~~~ @@ -100,6 +101,6 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use. -This module is part of the `OCA/edi `_ project on GitHub. +This module is part of the `OCA/edi `_ project on GitHub. You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 488f60613..b8606ae4c 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,12 +7,12 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "13.0.1.5.0", + "version": "14.0.1.0.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", "author": "ACSONE,Odoo Community Association (OCA)", - "depends": ["edi", "storage_backend", "component"], + "depends": ["edi_oca", "storage_backend", "component"], "data": ["security/ir_model_access.xml", "views/edi_backend_views.xml"], "demo": ["demo/edi_backend_demo.xml"], } diff --git a/edi_storage_oca/demo/edi_backend_demo.xml b/edi_storage_oca/demo/edi_backend_demo.xml index d8ff5d972..5b003460f 100644 --- a/edi_storage_oca/demo/edi_backend_demo.xml +++ b/edi_storage_oca/demo/edi_backend_demo.xml @@ -2,7 +2,7 @@ Storage Demo EDI backend - + demo_in/pending demo_in/done diff --git a/edi_storage_oca/readme/CONTRIBUTORS.rst b/edi_storage_oca/readme/CONTRIBUTORS.rst index f583948be..e3fa6d8e9 100644 --- a/edi_storage_oca/readme/CONTRIBUTORS.rst +++ b/edi_storage_oca/readme/CONTRIBUTORS.rst @@ -1 +1,2 @@ * Simone Orsi +* Foram Shah diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index b034d89fe..da7347ce6 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -367,7 +367,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) @@ -417,7 +417,7 @@

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -feedback.

+feedback.

Do not contact contributors directly about support or help with technical issues.

@@ -441,7 +441,7 @@

Maintainers

OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.

-

This module is part of the OCA/edi project on GitHub.

+

This module is part of the OCA/edi project on GitHub.

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index c5ed77c45..4be4c1daf 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -5,7 +5,7 @@ import mock -from odoo.addons.edi.tests.common import EDIBackendCommonComponentTestCase +from odoo.addons.edi_oca.tests.common import EDIBackendCommonComponentTestCase STORAGE_BACKEND_MOCK_PATH = ( "odoo.addons.storage_backend.models.storage_backend.StorageBackend" @@ -15,7 +15,7 @@ class TestEDIStorageBase(EDIBackendCommonComponentTestCase): @classmethod def _get_backend(cls): - return cls.env.ref("edi_storage.demo_edi_backend_storage") + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") @classmethod def _setup_records(cls): diff --git a/edi_storage_oca/tests/test_component_match.py b/edi_storage_oca/tests/test_component_match.py index e422c9698..e4fbfbe82 100644 --- a/edi_storage_oca/tests/test_component_match.py +++ b/edi_storage_oca/tests/test_component_match.py @@ -10,11 +10,11 @@ class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): @classmethod def setUpClass(cls): super().setUpClass() - cls._load_module_components(cls, "edi_storage") + cls._load_module_components(cls, "edi_storage_oca") @classmethod def _get_backend(cls): - return cls.env.ref("edi_storage.demo_edi_backend_storage") + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") def test_component_match(self): """Lookup with special match method.""" diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index f49138fc0..319d6e1cd 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -7,8 +7,8 @@ from .common import TestEDIStorageBase LOGGERS = ( - "odoo.addons.edi_storage.components.check", - "odoo.addons.edi.models.edi_backend", + "odoo.addons.edi_storage_oca.components.check", + "odoo.addons.edi_oca.models.edi_backend", ) diff --git a/edi_storage_oca/views/edi_backend_views.xml b/edi_storage_oca/views/edi_backend_views.xml index 5c7916eb1..6d2e4d5d3 100644 --- a/edi_storage_oca/views/edi_backend_views.xml +++ b/edi_storage_oca/views/edi_backend_views.xml @@ -2,7 +2,7 @@ edi.backend - + From 4c3ae23c7be43911782ebab3dc0bce1f127c9f0f Mon Sep 17 00:00:00 2001 From: oca-travis Date: Thu, 27 May 2021 13:11:31 +0000 Subject: [PATCH 13/68] [UPD] Update edi_storage_oca.pot --- edi_storage_oca/i18n/edi_storage_oca.pot | 92 ++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 edi_storage_oca/i18n/edi_storage_oca.pot diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot new file mode 100644 index 000000000..fdd8638b1 --- /dev/null +++ b/edi_storage_oca/i18n/edi_storage_oca.pot @@ -0,0 +1,92 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * edi_storage_oca +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 14.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__display_name +msgid "Display Name" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model,name:edi_storage_oca.model_edi_backend +msgid "EDI Backend" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__id +msgid "ID" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_done +msgid "Input done directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_error +msgid "Input error directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_pending +msgid "Input pending directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend____last_update +msgid "Last Modified on" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_done +msgid "Output done directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_error +msgid "Output error directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_pending +msgid "Output pending directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_done +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_done +msgid "Path to folder for doneful operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_error +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_error +msgid "Path to folder for error operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_pending +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_pending +msgid "Path to folder for pending operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__storage_id +msgid "Storage backend" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__storage_id +msgid "Storage for in-out files" +msgstr "" From a12f08eeb28a34cdfd219c5de36947027aab5db6 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Thu, 27 May 2021 13:16:44 +0000 Subject: [PATCH 14/68] [UPD] README.rst --- edi_storage_oca/README.rst | 2 +- edi_storage_oca/static/description/index.html | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 14f8cd692..516004710 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -70,7 +70,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index da7347ce6..a52f95fda 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -367,7 +367,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) @@ -417,7 +417,7 @@

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -feedback.

+feedback.

Do not contact contributors directly about support or help with technical issues.

@@ -432,6 +432,7 @@

Authors

Contributors

From 1a5793e55ebe96d25d0de8a0d9527b0936ae8dec Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Tue, 22 Jun 2021 13:32:31 +0200 Subject: [PATCH 15/68] [IMP] edi_storage_oca: implement basis of the input process. * A cron job check related storage on edi backends to create the needed exchange files. * Add basic receive component. --- edi_storage_oca/__manifest__.py | 6 ++- edi_storage_oca/components/__init__.py | 1 + edi_storage_oca/components/base.py | 8 ++-- edi_storage_oca/components/receive.py | 27 ++++++++++++++ edi_storage_oca/data/cron.xml | 15 ++++++++ edi_storage_oca/models/edi_backend.py | 49 +++++++++++++++++++++++++ edi_storage_oca/readme/CONTRIBUTORS.rst | 1 + 7 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 edi_storage_oca/components/receive.py create mode 100644 edi_storage_oca/data/cron.xml diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index b8606ae4c..c15efdd67 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -13,6 +13,10 @@ "website": "https://github.com/OCA/edi", "author": "ACSONE,Odoo Community Association (OCA)", "depends": ["edi_oca", "storage_backend", "component"], - "data": ["security/ir_model_access.xml", "views/edi_backend_views.xml"], + "data": [ + "data/cron.xml", + "security/ir_model_access.xml", + "views/edi_backend_views.xml", + ], "demo": ["demo/edi_backend_demo.xml"], } diff --git a/edi_storage_oca/components/__init__.py b/edi_storage_oca/components/__init__.py index 9f64b792b..7333ccfb3 100644 --- a/edi_storage_oca/components/__init__.py +++ b/edi_storage_oca/components/__init__.py @@ -1,3 +1,4 @@ from . import base from . import check from . import send +from . import receive diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index 01d0c8b2c..ca0fa5cef 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -27,7 +27,7 @@ def _component_match(cls, work, usage=None, model_name=None, **kw): def storage(self): return self.backend.storage_id - def _dir_by_state(self, direction, state): + def _dir_by_state(self, direction, state, absolute=False): """Return remote directory path by direction and state. :param direction: string stating direction of the exchange @@ -36,9 +36,11 @@ def _dir_by_state(self, direction, state): """ assert direction in ("input", "output") assert state in ("pending", "done", "error") + if absolute: + return PurePath(self.backend[direction + "_dir_" + state] or "") return PurePath((self.backend[direction + "_dir_" + state] or "").strip(" /")) - def _remote_file_path(self, direction, state, filename): + def _remote_file_path(self, direction, state, filename, absolute=False): """Return remote file path by direction and state for give filename. :param direction: string stating direction of the exchange @@ -46,7 +48,7 @@ def _remote_file_path(self, direction, state, filename): :param filename: string for file name :return: PurePath object """ - return self._dir_by_state(direction, state) / filename.strip("/ ") + return self._dir_by_state(direction, state, absolute) / filename.strip("/ ") def _get_remote_file(self, state, filename=None, binary=False): """Get file for current exchange_record in the given destination state. diff --git a/edi_storage_oca/components/receive.py b/edi_storage_oca/components/receive.py new file mode 100644 index 000000000..e421df38a --- /dev/null +++ b/edi_storage_oca/components/receive.py @@ -0,0 +1,27 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo.addons.component.core import Component + + +class EDIStorageReceiveComponent(Component): + + _name = "edi.storage.component.receive" + _inherit = [ + "edi.component.receive.mixin", + "edi.storage.component.mixin", + ] + _usage = "storage.receive" + + def receive(self): + checker = self.component(usage="storage.check") + result = checker.check() + if not result: + # all good here + return True + + direction = self.exchange_record.direction + filename = self.exchange_record.exchange_filename + path = self._remote_file_path(direction, "pending", filename, absolute=True) + filedata = self.storage.get(path.as_posix(), binary=True) + return filedata diff --git a/edi_storage_oca/data/cron.xml b/edi_storage_oca/data/cron.xml new file mode 100644 index 000000000..0dad93786 --- /dev/null +++ b/edi_storage_oca/data/cron.xml @@ -0,0 +1,15 @@ + + + + EDI backend storage check pending input + + + 1 + hours + -1 + + + code + model.search([])._cron_check_storage_pending_input() + + diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 0ba578fa0..315b63b16 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -1,9 +1,14 @@ # Copyright 2020 ACSONE SA # @author Simone Orsi +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). +import logging + from odoo import fields, models +_logger = logging.getLogger(__name__) + class EDIBackend(models.Model): @@ -76,3 +81,47 @@ def _component_sort_key(self, component_class): return ( 1 if getattr(component_class, "_storage_backend_type", False) else 0, ) + res + + def _cron_check_storage_pending_input(self, **kw): + for backend in self.filtered(lambda b: b.storage_id): + backend._check_storage_pending_input(**kw) + + def _check_storage_pending_input(self, **kw): + self.ensure_one() + if not self.storage_id or not self.input_dir_pending: + _logger.info( + "%s ignored. No storage and/or input directory specified.", self.name + ) + return False + + pending_files = self.storage_id.list_files(self.input_dir_pending) + exchange_type = self.env["edi.exchange.type"].search( + self._domain_exchange_type_storage_pending_input() + ) + if not len(exchange_type) == 1: + _logger.info("%s ignored. More than one exchange type found.", self.name) + return False + for file_name in pending_files: + existing = self.env["edi.exchange.record"].search( + [ + ("backend_id", "=", self.id), + ("type_id", "=", exchange_type.id), + ("exchange_filename", "=", file_name), + ] + ) + if existing: + continue + self.create_record( + exchange_type.code, self._get_exchange_record_vals(file_name) + ) + _logger.debug("%s: new exchange record generated.", self.name) + return True + + def _domain_exchange_type_storage_pending_input(self): + return [ + ("backend_type_id", "=", self.backend_type_id.id), + ("direction", "=", "input"), + ] + + def _get_exchange_record_vals(self, file_name): + return {"exchange_filename": file_name, "edi_exchange_state": "input_pending"} diff --git a/edi_storage_oca/readme/CONTRIBUTORS.rst b/edi_storage_oca/readme/CONTRIBUTORS.rst index e3fa6d8e9..55ca1c7e0 100644 --- a/edi_storage_oca/readme/CONTRIBUTORS.rst +++ b/edi_storage_oca/readme/CONTRIBUTORS.rst @@ -1,2 +1,3 @@ * Simone Orsi * Foram Shah +* Lois Rilo From 5f361de6b138326f04b578c403b65f4d1b2d76f1 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Tue, 22 Jun 2021 13:34:51 +0200 Subject: [PATCH 16/68] [FIX] edi_oca: show the exchange file name while there is no file yet. --- edi_storage_oca/components/base.py | 12 +++--- edi_storage_oca/components/receive.py | 4 +- edi_storage_oca/models/edi_backend.py | 4 +- edi_storage_oca/tests/common.py | 38 ++++++++++++++++--- .../tests/test_edi_backend_storage.py | 29 ++++++++++++++ 5 files changed, 72 insertions(+), 15 deletions(-) diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index ca0fa5cef..5010e6844 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -27,7 +27,7 @@ def _component_match(cls, work, usage=None, model_name=None, **kw): def storage(self): return self.backend.storage_id - def _dir_by_state(self, direction, state, absolute=False): + def _dir_by_state(self, direction, state): """Return remote directory path by direction and state. :param direction: string stating direction of the exchange @@ -36,11 +36,11 @@ def _dir_by_state(self, direction, state, absolute=False): """ assert direction in ("input", "output") assert state in ("pending", "done", "error") - if absolute: - return PurePath(self.backend[direction + "_dir_" + state] or "") - return PurePath((self.backend[direction + "_dir_" + state] or "").strip(" /")) + return PurePath( + (self.backend[direction + "_dir_" + state] or "").strip().rstrip("/") + ) - def _remote_file_path(self, direction, state, filename, absolute=False): + def _remote_file_path(self, direction, state, filename): """Return remote file path by direction and state for give filename. :param direction: string stating direction of the exchange @@ -48,7 +48,7 @@ def _remote_file_path(self, direction, state, filename, absolute=False): :param filename: string for file name :return: PurePath object """ - return self._dir_by_state(direction, state, absolute) / filename.strip("/ ") + return self._dir_by_state(direction, state) / filename.strip("/ ") def _get_remote_file(self, state, filename=None, binary=False): """Get file for current exchange_record in the given destination state. diff --git a/edi_storage_oca/components/receive.py b/edi_storage_oca/components/receive.py index e421df38a..695c40778 100644 --- a/edi_storage_oca/components/receive.py +++ b/edi_storage_oca/components/receive.py @@ -22,6 +22,6 @@ def receive(self): direction = self.exchange_record.direction filename = self.exchange_record.exchange_filename - path = self._remote_file_path(direction, "pending", filename, absolute=True) - filedata = self.storage.get(path.as_posix(), binary=True) + path = self._remote_file_path(direction, "pending", filename) + filedata = self.storage.get(path.as_posix()) return filedata diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 315b63b16..451495876 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -83,7 +83,7 @@ def _component_sort_key(self, component_class): ) + res def _cron_check_storage_pending_input(self, **kw): - for backend in self.filtered(lambda b: b.storage_id): + for backend in self: backend._check_storage_pending_input(**kw) def _check_storage_pending_input(self, **kw): @@ -102,7 +102,7 @@ def _check_storage_pending_input(self, **kw): _logger.info("%s ignored. More than one exchange type found.", self.name) return False for file_name in pending_files: - existing = self.env["edi.exchange.record"].search( + existing = self.env["edi.exchange.record"].search_count( [ ("backend_id", "=", self.id), ("type_id", "=", exchange_type.id), diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 4be4c1daf..7519ccfba 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -40,6 +40,14 @@ def _setup_records(cls): with open(cls.fakepath_error, "w+b") as fakefile: fakefile.write(b"ERROR XYZ: line 2 broken on bla bla") + cls.fakepath_input_pending_1 = "/tmp/test-input-001.csv" + with open(cls.fakepath_input_pending_1, "w+b") as fakefile: + fakefile.write(b"I received this in my storage.") + + cls.fakepath_input_pending_2 = "/tmp/test-input-002.csv" + with open(cls.fakepath_input_pending_2, "w+b") as fakefile: + fakefile.write(b"I received that in my storage.") + cls.checker = cls.backend._find_component( cls.partner._name, ["storage.check"], @@ -61,16 +69,19 @@ def _filename(self, record=None, ack=False): record.type_id.ack_type_id._make_exchange_filename(record) return record.exchange_filename - def _file_fullpath(self, state, record=None, ack=False): + def _file_fullpath( + self, state, record=None, ack=False, direction=False, fname=None + ): record = record or self.record - fname = self._filename(record, ack=ack) + if not fname: + fname = self._filename(record, ack=ack) + if not direction: + direction = record.direction if state == "error-report": # Exception as we read from the same path but w/ error suffix state = "error" fname += ".error" - return ( - self.checker._remote_file_path(record.direction, state, fname) - ).as_posix() + return (self.checker._remote_file_path(direction, state, fname)).as_posix() def _mocked_backend_get(self, mocked_paths, path, **kwargs): self._storage_backend_calls.append(path) @@ -82,6 +93,14 @@ def _mocked_backend_get(self, mocked_paths, path, **kwargs): def _mocked_backend_add(self, path, data, **kwargs): self._storage_backend_calls.append(path) + def _mocked_backend_list_files(self, mocked_paths, path, **kwargs): + files = [] + path_length = len(path) + for p in mocked_paths.keys(): + if path in p and path != p: + files.append(p[path_length:]) + return files + def _mock_storage_backend_get(self, mocked_paths): mocked = functools.partial(self._mocked_backend_get, mocked_paths) return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".get", mocked) @@ -89,6 +108,10 @@ def _mock_storage_backend_get(self, mocked_paths): def _mock_storage_backend_add(self): return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".add", self._mocked_backend_add) + def _mock_storage_backend_list_files(self, mocked_paths): + mocked = functools.partial(self._mocked_backend_list_files, mocked_paths) + return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".list_files", mocked) + def _test_result( self, record, @@ -130,3 +153,8 @@ def _test_run_cron(self, mocked_paths): with self._mock_storage_backend_add(): with self._mock_storage_backend_get(mocked_paths): self.backend._cron_check_output_exchange_sync() + + def _test_run_cron_pending_input(self, mocked_paths): + with self._mock_storage_backend_add(): + with self._mock_storage_backend_list_files(mocked_paths): + self.backend._cron_check_storage_pending_input() diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index 319d6e1cd..d4779a704 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -201,3 +201,32 @@ def test_cron_full_flow(self): } ], ) + + @mute_logger(*LOGGERS) + def test_create_input_exchange_file_from_file_received(self): + exch_type = self.exchange_type_in + input_dir = "/test_input/pending/" + file_names = ["some-file.csv", "another-file.csv"] + self.backend.input_dir_pending = input_dir + mocked_paths = { + input_dir: "/tmp/", + self._file_fullpath( + "pending", direction="input", fname=file_names[0] + ): self.fakepath_input_pending_1, + self._file_fullpath( + "pending", direction="input", fname=file_names[1] + ): self.fakepath_input_pending_2, + } + existing = self.env["edi.exchange.record"].search_count( + [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] + ) + self.assertEqual(existing, 0) + # Run cron action: + self._test_run_cron_pending_input(mocked_paths) + new_records = self.env["edi.exchange.record"].search( + [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] + ) + self.assertEqual(len(new_records), 2) + for rec in new_records: + self.assertIn(rec.exchange_filename, file_names) + self.assertEqual(rec.edi_exchange_state, "input_pending") From b2bf7e53617fce6433413c77aa0cea244761e57f Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Wed, 23 Jun 2021 15:46:00 +0200 Subject: [PATCH 17/68] edi_storage: improve input process --- edi_storage_oca/data/cron.xml | 2 +- edi_storage_oca/models/edi_backend.py | 78 ++++++++++++++++++--------- 2 files changed, 54 insertions(+), 26 deletions(-) diff --git a/edi_storage_oca/data/cron.xml b/edi_storage_oca/data/cron.xml index 0dad93786..d57316cd4 100644 --- a/edi_storage_oca/data/cron.xml +++ b/edi_storage_oca/data/cron.xml @@ -10,6 +10,6 @@ code - model.search([])._cron_check_storage_pending_input() + model.search([('storage_backend_id', '!=', False)])._cron_check_storage_pending_input() diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 451495876..4f14f9935 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -4,6 +4,7 @@ # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). import logging +import os from odoo import fields, models @@ -82,46 +83,73 @@ def _component_sort_key(self, component_class): 1 if getattr(component_class, "_storage_backend_type", False) else 0, ) + res - def _cron_check_storage_pending_input(self, **kw): + def _storage_cron_check_pending_input(self, **kw): for backend in self: - backend._check_storage_pending_input(**kw) + backend._storage_check_pending_input(**kw) - def _check_storage_pending_input(self, **kw): + def _storage_check_pending_input(self, **kw): + """Create new exchange records if new files found. + + Collect input exchange types and for each of them, + check by pattern if the a new exchange record is required. + """ self.ensure_one() if not self.storage_id or not self.input_dir_pending: _logger.info( - "%s ignored. No storage and/or input directory specified.", self.name + "%s ignored: no storage and/or input directory specified.", self.name ) return False - pending_files = self.storage_id.list_files(self.input_dir_pending) - exchange_type = self.env["edi.exchange.type"].search( - self._domain_exchange_type_storage_pending_input() + exchange_types = self.env["edi.exchange.type"].search( + self._storage_exchange_type_pending_input_domain() ) - if not len(exchange_type) == 1: - _logger.info("%s ignored. More than one exchange type found.", self.name) - return False - for file_name in pending_files: - existing = self.env["edi.exchange.record"].search_count( - [ - ("backend_id", "=", self.id), - ("type_id", "=", exchange_type.id), - ("exchange_filename", "=", file_name), - ] - ) - if existing: - continue - self.create_record( - exchange_type.code, self._get_exchange_record_vals(file_name) + for exchange_type in exchange_types: + # NOTE: this call might keep hanging the cron + # if the remote storage is slow (eg: too many files) + # We should probably run this code in a separate job per exchange type. + file_names = self._storage_get_input_filenames(exchange_type) + _logger.info( + "Processing exchange type '%s': found %s files to process", + exchange_type.display_name, + len(file_names), ) - _logger.debug("%s: new exchange record generated.", self.name) + for file_name in file_names: + # TODO: add config for job + self.with_delay()._storage_create_record_if_missing( + exchange_type, file_name + ) return True - def _domain_exchange_type_storage_pending_input(self): + def _storage_exchange_type_pending_input_domain(self): + """Domain for retrieving input exchange types. + """ return [ ("backend_type_id", "=", self.backend_type_id.id), ("direction", "=", "input"), ] - def _get_exchange_record_vals(self, file_name): + def _storage_create_record_if_missing(self, exchange_type, remote_file_name): + """Create a new exchange record for given type and file name if missing. + """ + file_name = os.path.basename(remote_file_name) + extra_domain = [("exchange_filename", "=", file_name)] + existing = self._find_existing_exchange_records( + exchange_type, extra_domain=extra_domain, count_only=True + ) + if existing: + return + record = self.create_record( + exchange_type.code, self._storage_new_exchange_record_vals(file_name) + ) + _logger.debug("%s: new exchange record generated.", self.name) + return record.identifier + + def _storage_get_input_filenames(self, exchange_type): + bits = [exchange_type.exchange_filename_pattern] + if exchange_type.exchange_file_ext: + bits.append("*." + exchange_type.exchange_file_ext) + pattern = "".join(bits) + return self.storage_id.find_files(self.input_dir_pending, pattern=pattern) + + def _storage_new_exchange_record_vals(self, file_name): return {"exchange_filename": file_name, "edi_exchange_state": "input_pending"} From f529856d8c3841d12b821ed49cf8a7844e019da3 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Wed, 23 Jun 2021 17:10:11 +0200 Subject: [PATCH 18/68] [FIX] edi_storage_oca: support the use or not of filename patern + small runtime fixes. --- edi_storage_oca/data/cron.xml | 4 +++- edi_storage_oca/models/edi_backend.py | 7 ++++++- edi_storage_oca/tests/common.py | 2 +- edi_storage_oca/tests/test_edi_backend_storage.py | 3 ++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/edi_storage_oca/data/cron.xml b/edi_storage_oca/data/cron.xml index d57316cd4..df21892a0 100644 --- a/edi_storage_oca/data/cron.xml +++ b/edi_storage_oca/data/cron.xml @@ -10,6 +10,8 @@ code - model.search([('storage_backend_id', '!=', False)])._cron_check_storage_pending_input() + model.search([('storage_id', '!=', False)])._storage_cron_check_pending_input() diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 4f14f9935..2b9674a53 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -145,11 +145,16 @@ def _storage_create_record_if_missing(self, exchange_type, remote_file_name): return record.identifier def _storage_get_input_filenames(self, exchange_type): + if not exchange_type.exchange_filename_pattern: + # If there is not patter, return everything + return self.storage_id.list_files(self.input_dir_pending) + + # TODO: validate pattern usage. bits = [exchange_type.exchange_filename_pattern] if exchange_type.exchange_file_ext: bits.append("*." + exchange_type.exchange_file_ext) pattern = "".join(bits) - return self.storage_id.find_files(self.input_dir_pending, pattern=pattern) + return self.storage_id.find_files(pattern, self.input_dir_pending) def _storage_new_exchange_record_vals(self, file_name): return {"exchange_filename": file_name, "edi_exchange_state": "input_pending"} diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 7519ccfba..5aaa9951e 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -157,4 +157,4 @@ def _test_run_cron(self, mocked_paths): def _test_run_cron_pending_input(self, mocked_paths): with self._mock_storage_backend_add(): with self._mock_storage_backend_list_files(mocked_paths): - self.backend._cron_check_storage_pending_input() + self.backend._storage_cron_check_pending_input() diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index d4779a704..06448474b 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -203,8 +203,9 @@ def test_cron_full_flow(self): ) @mute_logger(*LOGGERS) - def test_create_input_exchange_file_from_file_received(self): + def test_create_input_exchange_file_from_file_received_no_pattern(self): exch_type = self.exchange_type_in + exch_type.exchange_filename_pattern = "" input_dir = "/test_input/pending/" file_names = ["some-file.csv", "another-file.csv"] self.backend.input_dir_pending = input_dir From 241242b103f4744355b29c334b6a0e37afb6d739 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Thu, 1 Jul 2021 09:29:10 +0200 Subject: [PATCH 19/68] [IMP] edi_storage_oca: add specific channel and job function definitions --- edi_storage_oca/__manifest__.py | 2 ++ edi_storage_oca/data/job_channel_data.xml | 6 ++++++ edi_storage_oca/data/queue_job_function_data.xml | 7 +++++++ edi_storage_oca/models/edi_backend.py | 1 - 4 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 edi_storage_oca/data/job_channel_data.xml create mode 100644 edi_storage_oca/data/queue_job_function_data.xml diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index c15efdd67..c950bd37f 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -15,6 +15,8 @@ "depends": ["edi_oca", "storage_backend", "component"], "data": [ "data/cron.xml", + "data/job_channel_data.xml", + "data/queue_job_function_data.xml", "security/ir_model_access.xml", "views/edi_backend_views.xml", ], diff --git a/edi_storage_oca/data/job_channel_data.xml b/edi_storage_oca/data/job_channel_data.xml new file mode 100644 index 000000000..b3b3b770d --- /dev/null +++ b/edi_storage_oca/data/job_channel_data.xml @@ -0,0 +1,6 @@ + + + edi_storage + + + diff --git a/edi_storage_oca/data/queue_job_function_data.xml b/edi_storage_oca/data/queue_job_function_data.xml new file mode 100644 index 000000000..196528419 --- /dev/null +++ b/edi_storage_oca/data/queue_job_function_data.xml @@ -0,0 +1,7 @@ + + + + _storage_create_record_if_missing + + + diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 2b9674a53..5e229a8a4 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -114,7 +114,6 @@ def _storage_check_pending_input(self, **kw): len(file_names), ) for file_name in file_names: - # TODO: add config for job self.with_delay()._storage_create_record_if_missing( exchange_type, file_name ) From 9a8538bb04144179f74a74a2d53ef4d5b7a8cf1c Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Thu, 1 Jul 2021 12:57:49 +0200 Subject: [PATCH 20/68] [FIX] edi_storage_oca: support regex pattern searching when fetching filenames from storage. Also extend help in `exchange_filename_pattern` field to clarify usage. --- edi_storage_oca/models/__init__.py | 1 + edi_storage_oca/models/edi_backend.py | 7 ++++--- edi_storage_oca/models/edi_exchange_type.py | 21 +++++++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 edi_storage_oca/models/edi_exchange_type.py diff --git a/edi_storage_oca/models/__init__.py b/edi_storage_oca/models/__init__.py index 4b2c54ac3..f34a72163 100644 --- a/edi_storage_oca/models/__init__.py +++ b/edi_storage_oca/models/__init__.py @@ -1 +1,2 @@ from . import edi_backend +from . import edi_exchange_type diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 5e229a8a4..21d34095e 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -148,12 +148,13 @@ def _storage_get_input_filenames(self, exchange_type): # If there is not patter, return everything return self.storage_id.list_files(self.input_dir_pending) - # TODO: validate pattern usage. bits = [exchange_type.exchange_filename_pattern] if exchange_type.exchange_file_ext: - bits.append("*." + exchange_type.exchange_file_ext) + bits.append(r"\." + exchange_type.exchange_file_ext) pattern = "".join(bits) - return self.storage_id.find_files(pattern, self.input_dir_pending) + full_paths = self.storage_id.find_files(pattern, self.input_dir_pending) + pending_path_len = len(self.input_dir_pending) + return [p[pending_path_len:].strip("/") for p in full_paths] def _storage_new_exchange_record_vals(self, file_name): return {"exchange_filename": file_name, "edi_exchange_state": "input_pending"} diff --git a/edi_storage_oca/models/edi_exchange_type.py b/edi_storage_oca/models/edi_exchange_type.py new file mode 100644 index 000000000..42a7f7008 --- /dev/null +++ b/edi_storage_oca/models/edi_exchange_type.py @@ -0,0 +1,21 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo import fields, models + + +class EDIExchangeType(models.Model): + _inherit = "edi.exchange.type" + + # Extend help to explain new usage. + exchange_filename_pattern = fields.Char( + help="For output exchange types this should be a formatting string " + "with the following variables available (to be used between " + "brackets, `{}`): `exchange_record`, `record_name`, `type` and " + "`dt`. For instance, a valid string would be " + "{record_name}-{type.code}-{dt}\n" + "For input exchange types related to storage backends " + "it should be a regex expression to filter " + "the files to be fetched from the pending directory in the related " + "storage. E.g: `.*my-type-[0-9]*.\\.csv`" + ) From 2310a9ed12359425213a4b79d7d051107161f720 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Fri, 2 Jul 2021 09:23:52 +0200 Subject: [PATCH 21/68] [FIX] edi_storage: consider if the exchange type has a specific backend assigned. --- edi_storage_oca/models/edi_backend.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 21d34095e..0f8196be7 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -125,6 +125,9 @@ def _storage_exchange_type_pending_input_domain(self): return [ ("backend_type_id", "=", self.backend_type_id.id), ("direction", "=", "input"), + "|", + ("backend_id", "=", False), + ("backend_id", "=", self.id), ] def _storage_create_record_if_missing(self, exchange_type, remote_file_name): From 77c01c37eb08187206a9c36671148cd0da8e35c3 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Mon, 5 Jul 2021 15:56:58 +0200 Subject: [PATCH 22/68] [IMP] : black, isort, prettier --- edi_storage_oca/models/edi_backend.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 0f8196be7..5f95e3382 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -120,8 +120,7 @@ def _storage_check_pending_input(self, **kw): return True def _storage_exchange_type_pending_input_domain(self): - """Domain for retrieving input exchange types. - """ + """Domain for retrieving input exchange types.""" return [ ("backend_type_id", "=", self.backend_type_id.id), ("direction", "=", "input"), @@ -131,8 +130,7 @@ def _storage_exchange_type_pending_input_domain(self): ] def _storage_create_record_if_missing(self, exchange_type, remote_file_name): - """Create a new exchange record for given type and file name if missing. - """ + """Create a new exchange record for given type and file name if missing.""" file_name = os.path.basename(remote_file_name) extra_domain = [("exchange_filename", "=", file_name)] existing = self._find_existing_exchange_records( From d75ddea0564f875d9a7570076c983d9f45deb178 Mon Sep 17 00:00:00 2001 From: oca-travis Date: Tue, 6 Jul 2021 09:43:30 +0000 Subject: [PATCH 23/68] [UPD] Update edi_storage_oca.pot --- edi_storage_oca/i18n/edi_storage_oca.pot | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot index fdd8638b1..784ca4c68 100644 --- a/edi_storage_oca/i18n/edi_storage_oca.pot +++ b/edi_storage_oca/i18n/edi_storage_oca.pot @@ -15,6 +15,7 @@ msgstr "" #. module: edi_storage_oca #: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__display_name +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__display_name msgid "Display Name" msgstr "" @@ -23,8 +24,33 @@ msgstr "" msgid "EDI Backend" msgstr "" +#. module: edi_storage_oca +#: model:ir.model,name:edi_storage_oca.model_edi_exchange_type +msgid "EDI Exchange Type" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.actions.server,name:edi_storage_oca.cron_check_storage_pending_input_ir_actions_server +#: model:ir.cron,cron_name:edi_storage_oca.cron_check_storage_pending_input +#: model:ir.cron,name:edi_storage_oca.cron_check_storage_pending_input +msgid "EDI backend storage check pending input" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern +msgid "Exchange Filename Pattern" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern +msgid "" +"For output exchange types this should be a formatting string with the following variables available (to be used between brackets, `{}`): `exchange_record`, `record_name`, `type` and `dt`. For instance, a valid string would be {record_name}-{type.code}-{dt}\n" +"For input exchange types related to storage backends it should be a regex expression to filter the files to be fetched from the pending directory in the related storage. E.g: `.*my-type-[0-9]*.\\.csv`" +msgstr "" + #. module: edi_storage_oca #: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__id +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__id msgid "ID" msgstr "" @@ -45,6 +71,7 @@ msgstr "" #. module: edi_storage_oca #: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend____last_update +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type____last_update msgid "Last Modified on" msgstr "" From 1af53c59eb0de986f64bbc051851755e63a834c0 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Tue, 6 Jul 2021 09:49:11 +0000 Subject: [PATCH 24/68] [UPD] README.rst --- edi_storage_oca/README.rst | 1 + edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/static/description/index.html | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 516004710..dc3a04a12 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -87,6 +87,7 @@ Contributors * Simone Orsi * Foram Shah +* Lois Rilo Maintainers ~~~~~~~~~~~ diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index c950bd37f..e25ee456d 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.0.0", + "version": "14.0.1.1.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index a52f95fda..257c2d221 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -433,6 +433,7 @@

Contributors

From 28502682336dfd70a671178dd8044baea54861ba Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Mon, 5 Jul 2021 18:36:03 +0200 Subject: [PATCH 25/68] [IMP] edi_storage: add listener to move files to done/error dirs when a file is processed the listener will move it to the done directory or the error directory based on the result of processing the file. --- edi_storage_oca/components/__init__.py | 1 + edi_storage_oca/components/listener.py | 59 +++++++++++++++ edi_storage_oca/tests/__init__.py | 1 + .../tests/test_edi_storage_listener.py | 71 +++++++++++++++++++ 4 files changed, 132 insertions(+) create mode 100644 edi_storage_oca/components/listener.py create mode 100644 edi_storage_oca/tests/test_edi_storage_listener.py diff --git a/edi_storage_oca/components/__init__.py b/edi_storage_oca/components/__init__.py index 7333ccfb3..7bff9ddfc 100644 --- a/edi_storage_oca/components/__init__.py +++ b/edi_storage_oca/components/__init__.py @@ -2,3 +2,4 @@ from . import check from . import send from . import receive +from . import listener diff --git a/edi_storage_oca/components/listener.py b/edi_storage_oca/components/listener.py new file mode 100644 index 000000000..a1f248b7a --- /dev/null +++ b/edi_storage_oca/components/listener.py @@ -0,0 +1,59 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import functools +from pathlib import PurePath + +from odoo.addons.component.core import Component + + +class EdiStorageListener(Component): + _name = "edi.storage.component.listener" + _inherit = "base.event.listener" + + def _move_file(self, storage, from_dir_str, to_dir_str, filename): + from_dir = PurePath(from_dir_str) + to_dir = PurePath(to_dir_str) + if filename not in storage.list_files(from_dir): + # The file might have been moved after a previous error. + return False + self._add_after_commit_hook( + storage._move_files, [(from_dir / filename).as_posix()], to_dir.as_posix() + ) + return True + + def _add_after_commit_hook(self, move_func, sftp_filepath, sftp_destination_path): + """Add hook after commit to move the file when transaction is over.""" + self.env.cr.after( + "commit", + functools.partial(move_func, sftp_filepath, sftp_destination_path), + ) + + def on_edi_exchange_done(self, record): + storage = record.backend_id.storage_id + res = False + if record.direction == "input" and storage: + file = record.exchange_filename + pending_dir = record.backend_id.input_dir_pending + done_dir = record.backend_id.input_dir_done + error_dir = record.backend_id.input_dir_error + if not done_dir: + return res + res = self._move_file(storage, pending_dir, done_dir, file) + if not res: + # If a file previously failed it should have been previously + # moved to the error dir, therefore it is not present in the + # pending dir and we need to retry from error dir. + res = self._move_file(storage, error_dir, done_dir, file) + return res + + def on_edi_exchange_error(self, record): + storage = record.backend_id.storage_id + res = False + if record.direction == "input" and storage: + file = record.exchange_filename + pending_dir = record.backend_id.input_dir_pending + error_dir = record.backend_id.input_dir_error + if error_dir: + res = self._move_file(storage, pending_dir, error_dir, file) + return res diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py index b788a0071..a91d5f238 100644 --- a/edi_storage_oca/tests/__init__.py +++ b/edi_storage_oca/tests/__init__.py @@ -1,2 +1,3 @@ from . import test_edi_backend_storage from . import test_components_base +from . import test_edi_storage_listener diff --git a/edi_storage_oca/tests/test_edi_storage_listener.py b/edi_storage_oca/tests/test_edi_storage_listener.py new file mode 100644 index 000000000..4f12f3311 --- /dev/null +++ b/edi_storage_oca/tests/test_edi_storage_listener.py @@ -0,0 +1,71 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import base64 + +import mock + +from odoo.addons.edi_oca.tests.common import EDIBackendCommonComponentRegistryTestCase +from odoo.addons.edi_oca.tests.fake_components import FakeInputProcess + +LISTENER_MOCK_PATH = ( + "odoo.addons.edi_storage_oca.components.listener.EdiStorageListener" +) + + +class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls._load_module_components(cls, "edi_storage_oca") + cls._build_components( + cls, + FakeInputProcess, + ) + vals = { + "model": cls.partner._name, + "res_id": cls.partner.id, + "exchange_file": base64.b64encode(b"1234"), + } + cls.record = cls.backend.create_record("test_csv_input", vals) + cls.fake_move_args = None + + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") + + def setUp(self): + super().setUp() + FakeInputProcess.reset_faked() + + def _move_file_mocked(self, *args): + self.fake_move_args = [*args] + if not all([*args]): + return False + return True + + def _mock_listener_move_file(self): + return mock.patch(LISTENER_MOCK_PATH + "._move_file", self._move_file_mocked) + + def test_01_process_record_success(self): + with self._mock_listener_move_file(): + self.record.write({"edi_exchange_state": "input_received"}) + self.record.action_exchange_process() + storage, from_dir_str, to_dir_str, filename = self.fake_move_args + self.assertEqual(storage, self.backend.storage_id) + self.assertEqual(from_dir_str, self.backend.input_dir_pending) + self.assertEqual(to_dir_str, self.backend.input_dir_done) + self.assertEqual(filename, self.record.exchange_filename) + + def test_02_process_record_with_error(self): + with self._mock_listener_move_file(): + self.record.write({"edi_exchange_state": "input_received"}) + self.record._set_file_content("TEST %d" % self.record.id) + self.record.with_context( + test_break_process="OOPS! Something went wrong :(" + ).action_exchange_process() + storage, from_dir_str, to_dir_str, filename = self.fake_move_args + self.assertEqual(storage, self.backend.storage_id) + self.assertEqual(from_dir_str, self.backend.input_dir_pending) + self.assertEqual(to_dir_str, self.backend.input_dir_error) + self.assertEqual(filename, self.record.exchange_filename) From 50d37ccfcdbb39e95fd74a977ec0b2f106177989 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Fri, 9 Jul 2021 11:25:05 +0200 Subject: [PATCH 26/68] [FIX] edi_storage: `list_files` expects a string. --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/listener.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index e25ee456d..fb84ffcbb 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.1.0", + "version": "14.0.1.2.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/components/listener.py b/edi_storage_oca/components/listener.py index a1f248b7a..7b551c81e 100644 --- a/edi_storage_oca/components/listener.py +++ b/edi_storage_oca/components/listener.py @@ -14,7 +14,7 @@ class EdiStorageListener(Component): def _move_file(self, storage, from_dir_str, to_dir_str, filename): from_dir = PurePath(from_dir_str) to_dir = PurePath(to_dir_str) - if filename not in storage.list_files(from_dir): + if filename not in storage.list_files(from_dir.as_posix()): # The file might have been moved after a previous error. return False self._add_after_commit_hook( From 5c87d3002e8d81b3734bfad36f050a6a0324084e Mon Sep 17 00:00:00 2001 From: JordiMForgeFlow Date: Wed, 4 Aug 2021 09:23:35 +0200 Subject: [PATCH 27/68] [14.0][FIX] edi_storage_oca: remove duplicate backend_type_id field in edi backend form --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/views/edi_backend_views.xml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index fb84ffcbb..7fd2405b1 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.2.0", + "version": "14.0.1.2.1", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/views/edi_backend_views.xml b/edi_storage_oca/views/edi_backend_views.xml index 6d2e4d5d3..520965fb3 100644 --- a/edi_storage_oca/views/edi_backend_views.xml +++ b/edi_storage_oca/views/edi_backend_views.xml @@ -6,7 +6,6 @@ - From ec16215e0d9093e0378a76ecc919599ea409c0f7 Mon Sep 17 00:00:00 2001 From: Lois Rilo Date: Mon, 6 Sep 2021 12:04:31 +0200 Subject: [PATCH 28/68] [FIX] edi_storage: incorrect call to output checker in receive component. checker is only relevant for output direction. --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/receive.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 7fd2405b1..efe19ecca 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.2.1", + "version": "14.0.1.2.2", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/components/receive.py b/edi_storage_oca/components/receive.py index 695c40778..0061ac1d9 100644 --- a/edi_storage_oca/components/receive.py +++ b/edi_storage_oca/components/receive.py @@ -14,12 +14,6 @@ class EDIStorageReceiveComponent(Component): _usage = "storage.receive" def receive(self): - checker = self.component(usage="storage.check") - result = checker.check() - if not result: - # all good here - return True - direction = self.exchange_record.direction filename = self.exchange_record.exchange_filename path = self._remote_file_path(direction, "pending", filename) From e61e767250a75a023aee7acd01cccb04c4e39a46 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 17 Sep 2021 17:14:19 +0200 Subject: [PATCH 29/68] edi: improve backend view extension --- edi_storage_oca/views/edi_backend_views.xml | 24 +++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/edi_storage_oca/views/edi_backend_views.xml b/edi_storage_oca/views/edi_backend_views.xml index 520965fb3..ead51ef51 100644 --- a/edi_storage_oca/views/edi_backend_views.xml +++ b/edi_storage_oca/views/edi_backend_views.xml @@ -4,17 +4,19 @@ edi.backend - - - - - - - - - - - + + + + + + + + + + + + + From 18bbaae64ee3ec7b64f7c8ec1f4add7a70293b74 Mon Sep 17 00:00:00 2001 From: oca-travis Date: Wed, 13 Oct 2021 14:22:09 +0000 Subject: [PATCH 30/68] [UPD] Update edi_storage_oca.pot --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/i18n/edi_storage_oca.pot | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index efe19ecca..3481bef65 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.2.2", + "version": "14.0.1.3.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot index 784ca4c68..eb9e80688 100644 --- a/edi_storage_oca/i18n/edi_storage_oca.pot +++ b/edi_storage_oca/i18n/edi_storage_oca.pot @@ -108,6 +108,11 @@ msgstr "" msgid "Path to folder for pending operations" msgstr "" +#. module: edi_storage_oca +#: model_terms:ir.ui.view,arch_db:edi_storage_oca.edi_backend_view_form +msgid "Storage" +msgstr "" + #. module: edi_storage_oca #: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__storage_id msgid "Storage backend" From 7b48cb6755b5f8632f97d0947f87293aef455408 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Wed, 22 Sep 2021 20:07:37 +0200 Subject: [PATCH 31/68] edi: load main vars at init --- edi_storage_oca/tests/__init__.py | 1 + edi_storage_oca/tests/test_component_match.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py index a91d5f238..678979f8c 100644 --- a/edi_storage_oca/tests/__init__.py +++ b/edi_storage_oca/tests/__init__.py @@ -1,3 +1,4 @@ from . import test_edi_backend_storage from . import test_components_base +from . import test_component_match from . import test_edi_storage_listener diff --git a/edi_storage_oca/tests/test_component_match.py b/edi_storage_oca/tests/test_component_match.py index e4fbfbe82..f0a776ae9 100644 --- a/edi_storage_oca/tests/test_component_match.py +++ b/edi_storage_oca/tests/test_component_match.py @@ -3,7 +3,7 @@ # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). from odoo.addons.component.core import Component -from odoo.addons.edi.tests.common import EDIBackendCommonComponentRegistryTestCase +from odoo.addons.edi_oca.tests.common import EDIBackendCommonComponentRegistryTestCase class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): @@ -49,9 +49,13 @@ class S3Send(Component): self._build_components(SFTPCheck, SFTPSend, S3Check, S3Send) + # Record not relevant for these tests + work_ctx = {"exchange_record": self.env["edi.exchange.record"].browse()} + component = self.backend._find_component( "res.partner", ["storage.check"], + work_ctx=work_ctx, backend_type="demo_backend", exchange_type="test_csv_output", storage_backend_type="s3", @@ -61,6 +65,7 @@ class S3Send(Component): component = self.backend._find_component( "res.partner", ["storage.check"], + work_ctx=work_ctx, backend_type="demo_backend", exchange_type="test_csv_output", storage_backend_type="sftp", @@ -70,6 +75,7 @@ class S3Send(Component): component = self.backend._find_component( "res.partner", ["storage.send"], + work_ctx=work_ctx, backend_type="demo_backend", exchange_type="test_csv_output", storage_backend_type="sftp", @@ -79,6 +85,7 @@ class S3Send(Component): component = self.backend._find_component( "res.partner", ["storage.send"], + work_ctx=work_ctx, backend_type="demo_backend", exchange_type="test_csv_output", storage_backend_type="s3", From f6a3ea4fce77340f58821657e3e935da6c6e678f Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Sat, 30 Oct 2021 08:52:28 +0200 Subject: [PATCH 32/68] edi_storage: fix find files test --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/tests/common.py | 6 ++++++ edi_storage_oca/tests/test_edi_backend_storage.py | 4 +++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 3481bef65..d0cb44158 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.3.0", + "version": "14.0.1.4.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 5aaa9951e..31e4077c0 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -112,6 +112,12 @@ def _mock_storage_backend_list_files(self, mocked_paths): mocked = functools.partial(self._mocked_backend_list_files, mocked_paths) return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".list_files", mocked) + def _mock_storage_backend_find_files(self, result): + def _result(self, pattern, relative_path=None, **kw): + return result + + return mock.patch(STORAGE_BACKEND_MOCK_PATH + ".find_files", _result) + def _test_result( self, record, diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index 06448474b..2f396d39d 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -223,7 +223,9 @@ def test_create_input_exchange_file_from_file_received_no_pattern(self): ) self.assertEqual(existing, 0) # Run cron action: - self._test_run_cron_pending_input(mocked_paths) + found_files = [input_dir + fname for fname in file_names] + with self._mock_storage_backend_find_files(found_files): + self._test_run_cron_pending_input(mocked_paths) new_records = self.env["edi.exchange.record"].search( [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] ) From 70d2d5c9ea848594d941ea1a8a7f53ab44c14623 Mon Sep 17 00:00:00 2001 From: Iryna Vyshnevska Date: Mon, 6 Dec 2021 00:28:47 +0200 Subject: [PATCH 33/68] [MIG][15.0] edi_storage_oca --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/i18n/edi_storage.pot | 77 -------------- edi_storage_oca/i18n/edi_storage_oca.pot | 124 ----------------------- edi_storage_oca/tests/common.py | 1 + 4 files changed, 2 insertions(+), 202 deletions(-) delete mode 100644 edi_storage_oca/i18n/edi_storage.pot delete mode 100644 edi_storage_oca/i18n/edi_storage_oca.pot diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index d0cb44158..a7480dafd 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "14.0.1.4.0", + "version": "15.0.1.0.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/i18n/edi_storage.pot b/edi_storage_oca/i18n/edi_storage.pot deleted file mode 100644 index d6944e83b..000000000 --- a/edi_storage_oca/i18n/edi_storage.pot +++ /dev/null @@ -1,77 +0,0 @@ -# Translation of Odoo Server. -# This file contains the translation of the following modules: -# * edi_storage -# -msgid "" -msgstr "" -"Project-Id-Version: Odoo Server 13.0\n" -"Report-Msgid-Bugs-To: \n" -"Last-Translator: \n" -"Language-Team: \n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=UTF-8\n" -"Content-Transfer-Encoding: \n" -"Plural-Forms: \n" - -#. module: edi_storage -#: model:ir.model,name:edi_storage.model_edi_backend -msgid "EDI Backend" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_done -msgid "Input done directory" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_error -msgid "Input error directory" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_pending -msgid "Input pending directory" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_done -msgid "Output done directory" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_error -msgid "Output error directory" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_pending -msgid "Output pending directory" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_done -#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_done -msgid "Path to folder for doneful operations" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_error -#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_error -msgid "Path to folder for error operations" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_pending -#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_pending -msgid "Path to folder for pending operations" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__storage_id -msgid "Storage backend" -msgstr "" - -#. module: edi_storage -#: model:ir.model.fields,help:edi_storage.field_edi_backend__storage_id -msgid "Storage for in-out files" -msgstr "" diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot deleted file mode 100644 index eb9e80688..000000000 --- a/edi_storage_oca/i18n/edi_storage_oca.pot +++ /dev/null @@ -1,124 +0,0 @@ -# Translation of Odoo Server. -# This file contains the translation of the following modules: -# * edi_storage_oca -# -msgid "" -msgstr "" -"Project-Id-Version: Odoo Server 14.0\n" -"Report-Msgid-Bugs-To: \n" -"Last-Translator: \n" -"Language-Team: \n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=UTF-8\n" -"Content-Transfer-Encoding: \n" -"Plural-Forms: \n" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__display_name -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__display_name -msgid "Display Name" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model,name:edi_storage_oca.model_edi_backend -msgid "EDI Backend" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model,name:edi_storage_oca.model_edi_exchange_type -msgid "EDI Exchange Type" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.actions.server,name:edi_storage_oca.cron_check_storage_pending_input_ir_actions_server -#: model:ir.cron,cron_name:edi_storage_oca.cron_check_storage_pending_input -#: model:ir.cron,name:edi_storage_oca.cron_check_storage_pending_input -msgid "EDI backend storage check pending input" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern -msgid "Exchange Filename Pattern" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,help:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern -msgid "" -"For output exchange types this should be a formatting string with the following variables available (to be used between brackets, `{}`): `exchange_record`, `record_name`, `type` and `dt`. For instance, a valid string would be {record_name}-{type.code}-{dt}\n" -"For input exchange types related to storage backends it should be a regex expression to filter the files to be fetched from the pending directory in the related storage. E.g: `.*my-type-[0-9]*.\\.csv`" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__id -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__id -msgid "ID" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_done -msgid "Input done directory" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_error -msgid "Input error directory" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_pending -msgid "Input pending directory" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend____last_update -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type____last_update -msgid "Last Modified on" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_done -msgid "Output done directory" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_error -msgid "Output error directory" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_pending -msgid "Output pending directory" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_done -#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_done -msgid "Path to folder for doneful operations" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_error -#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_error -msgid "Path to folder for error operations" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_pending -#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_pending -msgid "Path to folder for pending operations" -msgstr "" - -#. module: edi_storage_oca -#: model_terms:ir.ui.view,arch_db:edi_storage_oca.edi_backend_view_form -msgid "Storage" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__storage_id -msgid "Storage backend" -msgstr "" - -#. module: edi_storage_oca -#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__storage_id -msgid "Storage for in-out files" -msgstr "" diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 31e4077c0..385b1a5f5 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -58,6 +58,7 @@ def _setup_records(cls): ["storage.send"], work_ctx={"exchange_record": cls.record}, ) + return def setUp(self): super().setUp() From b05039680f1df1eaf2bd39cdacb38b74a25e4caa Mon Sep 17 00:00:00 2001 From: oca-ci Date: Tue, 15 Mar 2022 12:12:00 +0000 Subject: [PATCH 34/68] [UPD] Update edi_storage_oca.pot --- edi_storage_oca/i18n/edi_storage_oca.pot | 106 +++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 edi_storage_oca/i18n/edi_storage_oca.pot diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot new file mode 100644 index 000000000..3458c862e --- /dev/null +++ b/edi_storage_oca/i18n/edi_storage_oca.pot @@ -0,0 +1,106 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * edi_storage_oca +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 15.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: edi_storage_oca +#: model:ir.model,name:edi_storage_oca.model_edi_backend +msgid "EDI Backend" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model,name:edi_storage_oca.model_edi_exchange_type +msgid "EDI Exchange Type" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.actions.server,name:edi_storage_oca.cron_check_storage_pending_input_ir_actions_server +#: model:ir.cron,cron_name:edi_storage_oca.cron_check_storage_pending_input +#: model:ir.cron,name:edi_storage_oca.cron_check_storage_pending_input +msgid "EDI backend storage check pending input" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern +msgid "Exchange Filename Pattern" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern +msgid "" +"For output exchange types this should be a formatting string with the following variables available (to be used between brackets, `{}`): `exchange_record`, `record_name`, `type` and `dt`. For instance, a valid string would be {record_name}-{type.code}-{dt}\n" +"For input exchange types related to storage backends it should be a regex expression to filter the files to be fetched from the pending directory in the related storage. E.g: `.*my-type-[0-9]*.\\.csv`" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_done +msgid "Input done directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_error +msgid "Input error directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_pending +msgid "Input pending directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_done +msgid "Output done directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_error +msgid "Output error directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_pending +msgid "Output pending directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_done +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_done +msgid "Path to folder for doneful operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_error +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_error +msgid "Path to folder for error operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_pending +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_pending +msgid "Path to folder for pending operations" +msgstr "" + +#. module: edi_storage_oca +#: model_terms:ir.ui.view,arch_db:edi_storage_oca.edi_backend_view_form +msgid "Storage" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__storage_id +msgid "Storage backend" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__storage_id +msgid "Storage for in-out files" +msgstr "" From 4a9709221fac4532cde081cc58f08ad6c9f64eac Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Tue, 15 Mar 2022 12:15:16 +0000 Subject: [PATCH 35/68] [UPD] README.rst --- edi_storage_oca/README.rst | 10 +++++----- edi_storage_oca/static/description/index.html | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index dc3a04a12..e75484b38 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -14,13 +14,13 @@ EDI Storage backend support :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html :alt: License: LGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi-lightgray.png?logo=github - :target: https://github.com/OCA/edi/tree/14.0/edi_storage_oca + :target: https://github.com/OCA/edi/tree/15.0/edi_storage_oca :alt: OCA/edi .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png - :target: https://translation.odoo-community.org/projects/edi-14-0/edi-14-0-edi_storage_oca + :target: https://translation.odoo-community.org/projects/edi-15-0/edi-15-0-edi_storage_oca :alt: Translate me on Weblate .. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png - :target: https://runbot.odoo-community.org/runbot/226/14.0 + :target: https://runbot.odoo-community.org/runbot/226/15.0 :alt: Try me on Runbot |badge1| |badge2| |badge3| |badge4| |badge5| @@ -70,7 +70,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. @@ -102,6 +102,6 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use. -This module is part of the `OCA/edi `_ project on GitHub. +This module is part of the `OCA/edi `_ project on GitHub. You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index 257c2d221..8fa1825de 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -367,7 +367,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) @@ -417,7 +417,7 @@

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us smashing it by providing a detailed and welcomed -feedback.

+feedback.

Do not contact contributors directly about support or help with technical issues.

@@ -443,7 +443,7 @@

Maintainers

OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.

-

This module is part of the OCA/edi project on GitHub.

+

This module is part of the OCA/edi project on GitHub.

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

From 573b590a8c1e075f219ab728ac7a7b696168be6e Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Thu, 10 Mar 2022 12:41:35 +0100 Subject: [PATCH 36/68] edi: get rid of name, use identifier Name contained redundant information like type name and related record. On for and tree view is totally useless. To still provide information on linked resource name the 'related_name' field has been added. --- edi_storage_oca/components/check.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index 878cfffcb..90f13c951 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -37,9 +37,8 @@ def _exchange_output_check(self): """ if self._get_remote_file("done"): _logger.info( - "%s done for: %s", - self.exchange_record.model, - self.exchange_record.name, + "%s done", + self.exchange_record.identifier, ) if ( not self.exchange_record.edi_exchange_state @@ -52,9 +51,8 @@ def _exchange_output_check(self): error = self._get_remote_file("error") if error: _logger.info( - "%s error for: %s", - self.exchange_record.model, - self.exchange_record.name, + "%s error", + self.exchange_record.identifier, ) # Assume a text file will be placed there w/ the same name and error suffix err_filename = self.exchange_record.exchange_filename + ".error" From 2d5ead5f7a5ea898cfb56f7be24cb39c678e3f01 Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Thu, 10 Mar 2022 16:28:30 +0100 Subject: [PATCH 37/68] edi_storage: fail gracefully w/o error report --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/check.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index a7480dafd..f9b30ed42 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "15.0.1.0.0", + "version": "15.0.1.1.0", "development_status": "Alpha", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py index 90f13c951..2cd97fb4a 100644 --- a/edi_storage_oca/components/check.py +++ b/edi_storage_oca/components/check.py @@ -56,7 +56,9 @@ def _exchange_output_check(self): ) # Assume a text file will be placed there w/ the same name and error suffix err_filename = self.exchange_record.exchange_filename + ".error" - error_report = self._get_remote_file("error", filename=err_filename) + error_report = ( + self._get_remote_file("error", filename=err_filename) or "no-report" + ) if self.exchange_record.edi_exchange_state == "output_sent": self.exchange_record.update( { From 05c482574cfa8de9b6722955e7920fad8b89ca61 Mon Sep 17 00:00:00 2001 From: Jasmin Solanki Date: Tue, 12 Apr 2022 16:45:22 +0530 Subject: [PATCH 38/68] [IMP] edi_storage_oca: Update development status --- edi_storage_oca/README.rst | 6 +++--- edi_storage_oca/__manifest__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index e75484b38..335f991fe 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -7,9 +7,9 @@ EDI Storage backend support !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png :target: https://odoo-community.org/page/development-status - :alt: Alpha + :alt: Beta .. |badge2| image:: https://img.shields.io/badge/licence-LGPL--3-blue.png :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html :alt: License: LGPL-3 @@ -23,7 +23,7 @@ EDI Storage backend support :target: https://runbot.odoo-community.org/runbot/226/15.0 :alt: Try me on Runbot -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| |badge5| Allow exchange files using storage backends from `OCA/storage`. diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index f9b30ed42..92310bce1 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -8,7 +8,7 @@ Base module to allow exchanging files via storage backend (eg: SFTP). """, "version": "15.0.1.1.0", - "development_status": "Alpha", + "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi", "author": "ACSONE,Odoo Community Association (OCA)", From df06cdd17c9a87ec37bfda1730cd2889e0fe9c1c Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Tue, 12 Apr 2022 14:39:57 +0000 Subject: [PATCH 39/68] [UPD] README.rst --- edi_storage_oca/README.rst | 7 +------ edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/static/description/index.html | 8 +------- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 335f991fe..c182ec9be 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -23,7 +23,7 @@ EDI Storage backend support :target: https://runbot.odoo-community.org/runbot/226/15.0 :alt: Try me on Runbot -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| |badge5| Allow exchange files using storage backends from `OCA/storage`. @@ -49,11 +49,6 @@ Inside this folder you have this hierarchy:: The storage handlers take care of reading files and putting files in/from the right place and update exchange records data accordingly. -.. IMPORTANT:: - This is an alpha version, the data model and design can change at any time without warning. - Only for development or testing purpose, do not use in production. - `More details on development status `_ - **Table of contents** .. contents:: diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 92310bce1..6eeee4e89 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "15.0.1.1.0", + "version": "15.0.1.1.1", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index 8fa1825de..d08dd56bb 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -367,7 +367,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Alpha License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

+

Beta License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runbot

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) @@ -389,12 +389,6 @@

EDI Storage backend support

The storage handlers take care of reading files and putting files in/from the right place and update exchange records data accordingly.

-
-

Important

-

This is an alpha version, the data model and design can change at any time without warning. -Only for development or testing purpose, do not use in production. -More details on development status

-

Table of contents

    From 8b4b78a8693f15ae8a5dedaf6273043587fbd22a Mon Sep 17 00:00:00 2001 From: jcoux Date: Tue, 17 May 2022 16:50:06 +0200 Subject: [PATCH 40/68] edi_storage_oca: improve error handling When trying to get done/failed directories, if we don't found it on SFTP server, we ignore the error. Current implementation only ignores FileNotFoundError, but depending on the SFTP server, we can have OSError. --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/base.py | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 6eeee4e89..d9715bb91 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "15.0.1.1.1", + "version": "15.0.1.1.2", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index 5010e6844..40e1dcb2f 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -1,11 +1,13 @@ # Copyright 2020 ACSONE # @author: Simone Orsi # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). - +import logging from pathlib import PurePath from odoo.addons.component.core import AbstractComponent +_logger = logging.getLogger(__file__) + class EDIStorageComponentMixin(AbstractComponent): @@ -65,4 +67,19 @@ def _get_remote_file(self, state, filename=None, binary=False): # (the date will never match) return self.storage.get(path.as_posix(), binary=binary) except FileNotFoundError: + _logger.info( + "Ignored FileNotFoundError when trying " + "to get file %s into path %s for state %s", + filename, + path, + state, + ) + return None + except OSError: + _logger.info( + "Ignored OSError when trying to get file %s into path %s for state %s", + filename, + path, + state, + ) return None From c3cea9fa9512dca6cba8da77b2f6d1913376de4e Mon Sep 17 00:00:00 2001 From: Simone Orsi Date: Fri, 6 May 2022 12:20:57 +0200 Subject: [PATCH 41/68] edi_storage: path configurable by type and param By setting specific storage path on exchange type advanced settings, we can configure specific path for each exchange type (input or output). Path can also be configured by config parameters to allow to easily set different values depending of the environment. --- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/components/base.py | 18 ++++- edi_storage_oca/components/listener.py | 20 ++++-- edi_storage_oca/components/receive.py | 4 +- edi_storage_oca/components/send.py | 5 +- edi_storage_oca/models/edi_backend.py | 16 +++-- edi_storage_oca/models/edi_exchange_type.py | 41 ++++++++++++ edi_storage_oca/tests/__init__.py | 1 + edi_storage_oca/tests/common.py | 15 +++-- edi_storage_oca/tests/test_components_base.py | 12 ++-- .../tests/test_edi_backend_storage.py | 13 ++-- edi_storage_oca/tests/test_exchange_type.py | 67 +++++++++++++++++++ 12 files changed, 179 insertions(+), 35 deletions(-) create mode 100644 edi_storage_oca/tests/test_exchange_type.py diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index d9715bb91..2e490068d 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "15.0.1.1.2", + "version": "15.0.1.2.0", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi", diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py index 40e1dcb2f..db1a2c8a3 100644 --- a/edi_storage_oca/components/base.py +++ b/edi_storage_oca/components/base.py @@ -1,4 +1,5 @@ # Copyright 2020 ACSONE +# Copyright 2022 Camptocamp # @author: Simone Orsi # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). import logging @@ -50,8 +51,22 @@ def _remote_file_path(self, direction, state, filename): :param filename: string for file name :return: PurePath object """ + _logger.warning( + "Call of deprecated function `_remote_file_path`. " + "Please use `_get_remote_file_path` instead.", + ) return self._dir_by_state(direction, state) / filename.strip("/ ") + def _get_remote_file_path(self, state, filename=None): + """Retrieve remote path for current exchange record.""" + filename = filename or self.exchange_record.exchange_filename + direction = self.exchange_record.direction + directory = self._dir_by_state(direction, state).as_posix() + path = self.exchange_record.type_id._storage_fullpath( + directory=directory, filename=filename + ) + return path + def _get_remote_file(self, state, filename=None, binary=False): """Get file for current exchange_record in the given destination state. @@ -59,8 +74,7 @@ def _get_remote_file(self, state, filename=None, binary=False): :param filename: custom file name, exchange_record filename used by default :return: remote file content as string """ - filename = filename or self.exchange_record.exchange_filename - path = self._remote_file_path(self.exchange_record.direction, state, filename) + path = self._get_remote_file_path(state, filename=filename) try: # TODO: support match via pattern (eg: filename-prefix-*) # otherwise is impossible to retrieve input files and acks diff --git a/edi_storage_oca/components/listener.py b/edi_storage_oca/components/listener.py index 7b551c81e..54ad1ec01 100644 --- a/edi_storage_oca/components/listener.py +++ b/edi_storage_oca/components/listener.py @@ -34,9 +34,15 @@ def on_edi_exchange_done(self, record): res = False if record.direction == "input" and storage: file = record.exchange_filename - pending_dir = record.backend_id.input_dir_pending - done_dir = record.backend_id.input_dir_done - error_dir = record.backend_id.input_dir_error + pending_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_pending + ).as_posix() + done_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_done + ).as_posix() + error_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_error + ).as_posix() if not done_dir: return res res = self._move_file(storage, pending_dir, done_dir, file) @@ -52,8 +58,12 @@ def on_edi_exchange_error(self, record): res = False if record.direction == "input" and storage: file = record.exchange_filename - pending_dir = record.backend_id.input_dir_pending - error_dir = record.backend_id.input_dir_error + pending_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_pending + ).as_posix() + error_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_error + ).as_posix() if error_dir: res = self._move_file(storage, pending_dir, error_dir, file) return res diff --git a/edi_storage_oca/components/receive.py b/edi_storage_oca/components/receive.py index 0061ac1d9..b0ab8257e 100644 --- a/edi_storage_oca/components/receive.py +++ b/edi_storage_oca/components/receive.py @@ -14,8 +14,6 @@ class EDIStorageReceiveComponent(Component): _usage = "storage.receive" def receive(self): - direction = self.exchange_record.direction - filename = self.exchange_record.exchange_filename - path = self._remote_file_path(direction, "pending", filename) + path = self._get_remote_file_path("pending") filedata = self.storage.get(path.as_posix()) return filedata diff --git a/edi_storage_oca/components/send.py b/edi_storage_oca/components/send.py index cbb6ed309..95c6ecdc6 100644 --- a/edi_storage_oca/components/send.py +++ b/edi_storage_oca/components/send.py @@ -23,11 +23,8 @@ def send(self): if not result: # all good here return True - - direction = self.exchange_record.direction - filename = self.exchange_record.exchange_filename filedata = self.exchange_record.exchange_file - path = self._remote_file_path(direction, "pending", filename) + path = self._get_remote_file_path("pending") self.storage.add(path.as_posix(), filedata, binary=False) # TODO: delegate this to generic storage backend # except paramiko.ssh_exception.AuthenticationException: diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index 5f95e3382..3669eabf9 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -145,16 +145,24 @@ def _storage_create_record_if_missing(self, exchange_type, remote_file_name): return record.identifier def _storage_get_input_filenames(self, exchange_type): + full_input_dir_pending = exchange_type._storage_fullpath( + self.input_dir_pending + ).as_posix() if not exchange_type.exchange_filename_pattern: - # If there is not patter, return everything - return self.storage_id.list_files(self.input_dir_pending) + # If there is not pattern, return everything + filenames = [ + x + for x in self.storage_id.list_files(full_input_dir_pending) + if x.strip("/") + ] + return filenames bits = [exchange_type.exchange_filename_pattern] if exchange_type.exchange_file_ext: bits.append(r"\." + exchange_type.exchange_file_ext) pattern = "".join(bits) - full_paths = self.storage_id.find_files(pattern, self.input_dir_pending) - pending_path_len = len(self.input_dir_pending) + full_paths = self.storage_id.find_files(pattern, full_input_dir_pending) + pending_path_len = len(full_input_dir_pending) return [p[pending_path_len:].strip("/") for p in full_paths] def _storage_new_exchange_record_vals(self, file_name): diff --git a/edi_storage_oca/models/edi_exchange_type.py b/edi_storage_oca/models/edi_exchange_type.py index 42a7f7008..258576edc 100644 --- a/edi_storage_oca/models/edi_exchange_type.py +++ b/edi_storage_oca/models/edi_exchange_type.py @@ -1,6 +1,8 @@ # Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). +from pathlib import PurePath + from odoo import fields, models @@ -19,3 +21,42 @@ class EDIExchangeType(models.Model): "the files to be fetched from the pending directory in the related " "storage. E.g: `.*my-type-[0-9]*.\\.csv`" ) + + def _storage_path(self): + """Retrieve specific path for current exchange type. + + In your exchange type you can pass this config: + + storage: + # simple string + path: path/to/file + + Or + + storage: + # name of the param containing the path + path_config_param: path/to/file + + Thanks to the param you could even configure it by env. + """ + self.ensure_one() + storage_settings = self.advanced_settings.get("storage", {}) + path = storage_settings.get("path") + if path: + return PurePath(path) + path_config_param = storage_settings.get("path_config_param") + if path_config_param: + icp = self.env["ir.config_parameter"].sudo() + path = icp.get_param(path_config_param) + if path: + return PurePath(path) + + def _storage_fullpath(self, directory=None, filename=None): + self.ensure_one() + path_prefix = self._storage_path() + path = PurePath((directory or "").strip().rstrip("/")) + if path_prefix: + path = path_prefix / path + if filename: + path = path / filename.strip("/") + return path diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py index 678979f8c..18bc95300 100644 --- a/edi_storage_oca/tests/__init__.py +++ b/edi_storage_oca/tests/__init__.py @@ -2,3 +2,4 @@ from . import test_components_base from . import test_component_match from . import test_edi_storage_listener +from . import test_exchange_type diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index 385b1a5f5..cdca0f2e0 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -27,6 +27,7 @@ def _setup_records(cls): "exchange_file": cls.filedata, } cls.record = cls.backend.create_record("test_csv_output", vals) + cls.record_input = cls.backend.create_record("test_csv_input", vals) cls.fakepath = "/tmp/{}".format(cls._filename(cls)) with open(cls.fakepath, "w+b") as fakefile: @@ -53,6 +54,11 @@ def _setup_records(cls): ["storage.check"], work_ctx={"exchange_record": cls.record}, ) + cls.checker_input = cls.backend._find_component( + cls.partner._name, + ["storage.check"], + work_ctx={"exchange_record": cls.record_input}, + ) cls.sender = cls.backend._find_component( cls.partner._name, ["storage.send"], @@ -70,19 +76,16 @@ def _filename(self, record=None, ack=False): record.type_id.ack_type_id._make_exchange_filename(record) return record.exchange_filename - def _file_fullpath( - self, state, record=None, ack=False, direction=False, fname=None - ): + def _file_fullpath(self, state, record=None, ack=False, fname=None, checker=None): record = record or self.record + checker = checker or self.checker if not fname: fname = self._filename(record, ack=ack) - if not direction: - direction = record.direction if state == "error-report": # Exception as we read from the same path but w/ error suffix state = "error" fname += ".error" - return (self.checker._remote_file_path(direction, state, fname)).as_posix() + return checker._get_remote_file_path(state, filename=fname).as_posix() def _mocked_backend_get(self, mocked_paths, path, **kwargs): self._storage_backend_calls.append(path) diff --git a/edi_storage_oca/tests/test_components_base.py b/edi_storage_oca/tests/test_components_base.py index 57abd1385..9388993b0 100644 --- a/edi_storage_oca/tests/test_components_base.py +++ b/edi_storage_oca/tests/test_components_base.py @@ -18,14 +18,16 @@ def test_remote_file_path(self): (("output", "error", "foo.csv"), "demo_out/error/foo.csv"), ) for _args, expected in to_test: - path_obj = self.checker._remote_file_path(*_args) + direction, state, filename = _args + if direction == "input": + checker = self.checker_input + else: + checker = self.checker + path_obj = checker._get_remote_file_path(state, filename) self.assertEqual(path_obj.as_posix(), expected) with self.assertRaises(AssertionError): - self.checker._remote_file_path("WHATEVER", "error", "foo.csv") - - with self.assertRaises(AssertionError): - self.checker._remote_file_path("input", "WHATEVER", "foo.csv") + self.checker_input._get_remote_file_path("WHATEVER", "foo.csv") def test_get_remote_file(self): with mock.patch(STORAGE_BACKEND_MOCK_PATH + ".get") as mocked: diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index 2f396d39d..0d4793d3f 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -212,22 +212,25 @@ def test_create_input_exchange_file_from_file_received_no_pattern(self): mocked_paths = { input_dir: "/tmp/", self._file_fullpath( - "pending", direction="input", fname=file_names[0] + "pending", fname=file_names[0], checker=self.checker_input ): self.fakepath_input_pending_1, self._file_fullpath( - "pending", direction="input", fname=file_names[1] + "pending", fname=file_names[1], checker=self.checker_input ): self.fakepath_input_pending_2, } - existing = self.env["edi.exchange.record"].search_count( + existing_records = self.env["edi.exchange.record"].search( [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] ) - self.assertEqual(existing, 0) # Run cron action: found_files = [input_dir + fname for fname in file_names] with self._mock_storage_backend_find_files(found_files): self._test_run_cron_pending_input(mocked_paths) new_records = self.env["edi.exchange.record"].search( - [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] + [ + ("backend_id", "=", self.backend.id), + ("type_id", "=", exch_type.id), + ("id", "not in", existing_records.ids), + ] ) self.assertEqual(len(new_records), 2) for rec in new_records: diff --git a/edi_storage_oca/tests/test_exchange_type.py b/edi_storage_oca/tests/test_exchange_type.py new file mode 100644 index 000000000..6b886eab3 --- /dev/null +++ b/edi_storage_oca/tests/test_exchange_type.py @@ -0,0 +1,67 @@ +# Copyright 2022 Camptocamp SA (https://www.camptocamp.com). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo.addons.edi_oca.tests.common import EDIBackendCommonTestCase + + +class EDIExchangeTypeTestCase(EDIBackendCommonTestCase): + def _check_test_storage_fullpath(self, wanted_fullpath, directory, filename): + fullpath = self.exchange_type_out._storage_fullpath(directory, filename) + self.assertEqual(fullpath.as_posix(), wanted_fullpath) + + def _do_test_storage_fullpath(self, prefix=""): + # Test with no directory and no filename + wanted_fullpath = prefix or "." + self._check_test_storage_fullpath(wanted_fullpath, None, None) + + # Test with directory + directory = "test_directory" + wanted_fullpath = f"{prefix}/{directory}" if prefix else directory + self._check_test_storage_fullpath(wanted_fullpath, directory, None) + + # Test with filename + filename = "test_filename.csv" + wanted_fullpath = f"{prefix}/{filename}" if prefix else filename + self._check_test_storage_fullpath(wanted_fullpath, None, filename) + + # Test with directory and filename + wanted_fullpath = ( + f"{prefix}/{directory}/{filename}" if prefix else f"{directory}/{filename}" + ) + self._check_test_storage_fullpath(wanted_fullpath, directory, filename) + + def test_storage_fullpath(self): + """ + Test storage fullpath defined into advanced settings. + Example of pattern: + storage: + # simple string + path: path/to/file + # name of the param containing the path + path_config_param: path/to/file + """ + + # Test without any prefix + self._do_test_storage_fullpath() + + # Force path on advanced settings + prefix = "prefix/path" + self.exchange_type_out.advanced_settings_edit = f""" + storage: + path: {prefix} + """ + self._do_test_storage_fullpath(prefix=prefix) + + # Force path on advanced settings using config param, but not defined + self.exchange_type_out.advanced_settings_edit = """ + storage: + path_config_param: prefix_path_config_param + """ + self._do_test_storage_fullpath() + + # Define config param + prefix = "prefix/path/by/config/param" + self.env["ir.config_parameter"].sudo().set_param( + "prefix_path_config_param", prefix + ) + self._do_test_storage_fullpath(prefix=prefix) From 242ffe0333d08c20a8cd4d71b6fbc91854a645f6 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Sun, 3 Sep 2023 12:44:19 +0000 Subject: [PATCH 42/68] [UPD] README.rst --- edi_storage_oca/README.rst | 15 +++++--- edi_storage_oca/static/description/index.html | 38 ++++++++++--------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index c182ec9be..dcd6d47fd 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -2,10 +2,13 @@ EDI Storage backend support =========================== -.. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:f2718fe12ff53362a2da3e253d22e0641ebcd6a01195db14f8d9c8c71878080d + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png :target: https://odoo-community.org/page/development-status @@ -19,11 +22,11 @@ EDI Storage backend support .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png :target: https://translation.odoo-community.org/projects/edi-15-0/edi-15-0-edi_storage_oca :alt: Translate me on Weblate -.. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png - :target: https://runbot.odoo-community.org/runbot/226/15.0 - :alt: Try me on Runbot +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/edi&target_branch=15.0 + :alt: Try me on Runboat -|badge1| |badge2| |badge3| |badge4| |badge5| +|badge1| |badge2| |badge3| |badge4| |badge5| Allow exchange files using storage backends from `OCA/storage`. @@ -64,7 +67,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. -If you spotted it first, help us smashing it by providing a detailed and welcomed +If you spotted it first, help us to smash it by providing a detailed and welcomed `feedback `_. Do not contact contributors directly about support or help with technical issues. diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index d08dd56bb..3a323e627 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -1,20 +1,20 @@ - + - + EDI Storage backend support -
    -

    EDI Storage backend support

    +
    + + +Odoo Community Association + +
    +

    EDI Storage backend support

    -

    Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

    +

    Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

    Allow exchange files using storage backends from OCA/storage.

    This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) and the @@ -406,18 +411,18 @@

    EDI Storage backend support

-

Usage

+

Usage

Go to “EDI -> EDI backend” then configure your backend to use a storage backend.

-

Known issues / Roadmap

+

Known issues / Roadmap

  • clean deprecated methods in the storage
-

Bug Tracker

+

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed @@ -425,15 +430,15 @@

Bug Tracker

Do not contact contributors directly about support or help with technical issues.

-

Credits

+

Credits

-

Authors

+

Authors

  • ACSONE
-

Contributors

+

Contributors

-

Other credits

+

Other credits

The migration of this module from 15.0 to 16.0 was financially supported by Camptocamp.

-

Maintainers

+

Maintainers

This module is maintained by the OCA.

Odoo Community Association @@ -464,5 +469,6 @@

Maintainers

+ From 4766c6f309c2937c231caebabe117aaf44f370de Mon Sep 17 00:00:00 2001 From: Hector Vior Date: Sat, 6 Dec 2025 21:21:46 +0100 Subject: [PATCH 63/68] [FIX] edi_storage_oca: Remove legacy storage component code --- edi_storage_oca/models/edi_backend.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py index ba983524f..e6767a789 100644 --- a/edi_storage_oca/models/edi_backend.py +++ b/edi_storage_oca/models/edi_backend.py @@ -59,27 +59,6 @@ class EDIBackend(models.Model): "Output error directory", help="Path to folder for error operations" ) - def _get_component_usage_candidates(self, exchange_record, key): - candidates = super()._get_component_usage_candidates(exchange_record, key) - if not self.storage_id or key not in self._storage_actions: - return candidates - return [f"storage.{key}"] + candidates - - def _component_match_attrs(self, exchange_record, key): - # Override to inject storage_type - res = super()._component_match_attrs(exchange_record, key) - if not self.storage_id or key not in self._storage_actions: - return res - res["storage_type"] = self.sudo().storage_id.protocol - return res - - def _component_sort_key(self, component_class): - res = super()._component_sort_key(component_class) - # Override to give precedence by storage_type when needed. - if not self.storage_id: - return res - return (1 if getattr(component_class, "_storage_type", False) else 0,) + res - def _storage_cron_check_pending_input(self, **kw): for backend in self: backend._storage_check_pending_input(**kw) From f95abe75b20b2a29f64abc6657898cb1e86eb56d Mon Sep 17 00:00:00 2001 From: JordiMForgeFlow Date: Tue, 9 Dec 2025 11:35:36 +0100 Subject: [PATCH 64/68] [FIX] edi_storage_oca: fix migration script --- edi_storage_oca/migrations/18.0.1.0.0/post-migration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py b/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py index 7f1454af1..245c80bc4 100644 --- a/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py +++ b/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py @@ -42,6 +42,7 @@ def migrate(env, version): AND edi_backend.backend_type_id = edi_exchange_type.backend_type_id ) AND edi_backend.active IS TRUE + ) ) """, ) From b303c696ffdc649475eb5d718ee5d6607bc67ab8 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Tue, 9 Dec 2025 11:16:17 +0000 Subject: [PATCH 65/68] [BOT] post-merge updates --- edi_storage_oca/README.rst | 2 +- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/static/description/index.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 7457af899..b3fb70c5c 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -11,7 +11,7 @@ EDI Storage backend support !! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !! source digest: sha256:c9c2898d23ca2c445e2088cefb226956d525da3fb8928068c1c4bcbb9d668f3c + !! source digest: sha256:a485ae05d5e7c51de633c8061f680348cad8cac308760bffb793e179e70c5c38 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 151985031..95c5186ce 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "18.0.1.0.0", + "version": "18.0.1.0.1", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi-framework", diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index 080e90e83..fb16aaae0 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -372,7 +372,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!! source digest: sha256:c9c2898d23ca2c445e2088cefb226956d525da3fb8928068c1c4bcbb9d668f3c +!! source digest: sha256:a485ae05d5e7c51de633c8061f680348cad8cac308760bffb793e179e70c5c38 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -->

Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

Allow exchange files using storage backends from OCA/storage.

From 7741490a80ce4b767d2b6a168cb1b41c5262f079 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Tue, 9 Dec 2025 12:44:24 +0000 Subject: [PATCH 66/68] [BOT] post-merge updates --- edi_storage_oca/README.rst | 2 +- edi_storage_oca/__manifest__.py | 2 +- edi_storage_oca/static/description/index.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index b3fb70c5c..52b9f7ece 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -11,7 +11,7 @@ EDI Storage backend support !! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !! source digest: sha256:a485ae05d5e7c51de633c8061f680348cad8cac308760bffb793e179e70c5c38 + !! source digest: sha256:ae52c0238c8bab278e4e2d6a48d86d4222672b893df1f171be046f3b7a3c58ae !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 95c5186ce..39f47e247 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "18.0.1.0.1", + "version": "18.0.1.0.2", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi-framework", diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index fb16aaae0..87b66f49d 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -372,7 +372,7 @@

EDI Storage backend support

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!! source digest: sha256:a485ae05d5e7c51de633c8061f680348cad8cac308760bffb793e179e70c5c38 +!! source digest: sha256:ae52c0238c8bab278e4e2d6a48d86d4222672b893df1f171be046f3b7a3c58ae !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -->

Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

Allow exchange files using storage backends from OCA/storage.

From a96a886429c1a487abd74f3cfba354435d6ea725 Mon Sep 17 00:00:00 2001 From: Sergij Pfaifer Date: Thu, 26 Mar 2026 14:16:33 +0200 Subject: [PATCH 67/68] [IMP] edi_storage_oca: pre-commit auto fixes --- edi_storage_oca/migrations/18.0.1.0.0/post-migration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py b/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py index 245c80bc4..f33bd2d76 100644 --- a/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py +++ b/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py @@ -8,7 +8,7 @@ def migrate(env, version): env.cr, f""" UPDATE edi_exchange_type - SET send_model_id = { model.id } + SET send_model_id = {model.id} WHERE direction = 'output' AND send_model_id IS NULL AND EXISTS ( From 0f1b71ff18ad9ae153170765794768bf12f2bf6c Mon Sep 17 00:00:00 2001 From: Sergij Pfaifer Date: Thu, 26 Mar 2026 15:37:14 +0200 Subject: [PATCH 68/68] [MIG] edi_storage_oca: Migration to 19.0 --- edi_storage_oca/README.rst | 10 ++-- edi_storage_oca/__manifest__.py | 3 +- edi_storage_oca/demo/edi_backend_demo.xml | 14 ------ .../migrations/18.0.1.0.0/post-migration.py | 48 ------------------- edi_storage_oca/static/description/index.html | 6 +-- edi_storage_oca/tests/common.py | 27 ++++++++++- .../tests/test_edi_backend_storage.py | 14 +++++- 7 files changed, 47 insertions(+), 75 deletions(-) delete mode 100644 edi_storage_oca/demo/edi_backend_demo.xml delete mode 100644 edi_storage_oca/migrations/18.0.1.0.0/post-migration.py diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst index 52b9f7ece..7ab2089d0 100644 --- a/edi_storage_oca/README.rst +++ b/edi_storage_oca/README.rst @@ -21,13 +21,13 @@ EDI Storage backend support :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html :alt: License: LGPL-3 .. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi--framework-lightgray.png?logo=github - :target: https://github.com/OCA/edi-framework/tree/18.0/edi_storage_oca + :target: https://github.com/OCA/edi-framework/tree/19.0/edi_storage_oca :alt: OCA/edi-framework .. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png - :target: https://translation.odoo-community.org/projects/edi-framework-18-0/edi-framework-18-0-edi_storage_oca + :target: https://translation.odoo-community.org/projects/edi-framework-19-0/edi-framework-19-0-edi_storage_oca :alt: Translate me on Weblate .. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png - :target: https://runboat.odoo-community.org/builds?repo=OCA/edi-framework&target_branch=18.0 + :target: https://runboat.odoo-community.org/builds?repo=OCA/edi-framework&target_branch=19.0 :alt: Try me on Runboat |badge1| |badge2| |badge3| |badge4| |badge5| @@ -79,7 +79,7 @@ Bug Tracker Bugs are tracked on `GitHub Issues `_. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed -`feedback `_. +`feedback `_. Do not contact contributors directly about support or help with technical issues. @@ -121,6 +121,6 @@ OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use. -This module is part of the `OCA/edi-framework `_ project on GitHub. +This module is part of the `OCA/edi-framework `_ project on GitHub. You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py index 39f47e247..cad836fc1 100644 --- a/edi_storage_oca/__manifest__.py +++ b/edi_storage_oca/__manifest__.py @@ -7,7 +7,7 @@ "summary": """ Base module to allow exchanging files via storage backend (eg: SFTP). """, - "version": "18.0.1.0.2", + "version": "19.0.1.0.0", "development_status": "Beta", "license": "LGPL-3", "website": "https://github.com/OCA/edi-framework", @@ -18,5 +18,4 @@ "security/ir_model_access.xml", "views/edi_backend_views.xml", ], - "demo": ["demo/edi_backend_demo.xml"], } diff --git a/edi_storage_oca/demo/edi_backend_demo.xml b/edi_storage_oca/demo/edi_backend_demo.xml deleted file mode 100644 index a8740b0d4..000000000 --- a/edi_storage_oca/demo/edi_backend_demo.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - Storage Demo EDI backend - - - demo_in/pending - demo_in/done - demo_in/error - demo_out/pending - demo_out/done - demo_out/error - - diff --git a/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py b/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py deleted file mode 100644 index f33bd2d76..000000000 --- a/edi_storage_oca/migrations/18.0.1.0.0/post-migration.py +++ /dev/null @@ -1,48 +0,0 @@ -from openupgradelib import openupgrade - - -@openupgrade.migrate() -def migrate(env, version): - model = env.ref("edi_storage_oca.model_edi_oca_storage_handler") - openupgrade.logged_query( - env.cr, - f""" - UPDATE edi_exchange_type - SET send_model_id = {model.id} - WHERE direction = 'output' - AND send_model_id IS NULL - AND EXISTS ( - SELECT 1 - FROM edi_backend - WHERE ( - edi_backend.id = edi_exchange_type.backend_id - OR ( - edi_exchange_type.backend_id IS NULL - AND edi_backend.backend_type_id = edi_exchange_type.backend_type_id - ) - AND edi_backend.active IS TRUE - ) - ) - """, - ) - openupgrade.logged_query( - env.cr, - f""" - UPDATE edi_exchange_type - SET receive_model_id = {model.id} - WHERE direction = 'input' - AND receive_model_id IS NULL - AND EXISTS ( - SELECT 1 - FROM edi_backend - WHERE ( - edi_backend.id = edi_exchange_type.backend_id - OR ( - edi_exchange_type.backend_id IS NULL - AND edi_backend.backend_type_id = edi_exchange_type.backend_type_id - ) - AND edi_backend.active IS TRUE - ) - ) - """, - ) diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html index 87b66f49d..80902a540 100644 --- a/edi_storage_oca/static/description/index.html +++ b/edi_storage_oca/static/description/index.html @@ -374,7 +374,7 @@

EDI Storage backend support

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !! source digest: sha256:ae52c0238c8bab278e4e2d6a48d86d4222672b893df1f171be046f3b7a3c58ae !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! --> -

Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

+

Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

Allow exchange files using storage backends from OCA/storage.

This module adds a storage backend relation on the EDI backend. There you can configure the backend to be used (most often and SFTP) and the @@ -426,7 +426,7 @@

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed -feedback.

+feedback.

Do not contact contributors directly about support or help with technical issues.

@@ -464,7 +464,7 @@

Maintainers

OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.

-

This module is part of the OCA/edi-framework project on GitHub.

+

This module is part of the OCA/edi-framework project on GitHub.

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py index d8606d4ba..64e8a159d 100644 --- a/edi_storage_oca/tests/common.py +++ b/edi_storage_oca/tests/common.py @@ -12,7 +12,32 @@ class TestEDIStorageBase(EDIBackendCommonTestCase): @classmethod def _get_backend(cls): - return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") + backend_type = cls.env["edi.backend.type"].create( + { + "name": "Demo EDI backend type", + "code": "demo_backend", + } + ) + storage = cls.env["fs.storage"].create( + { + "name": "Odoo Filesystem Backend", + "protocol": "odoofs", + "code": "odoofs", + } + ) + return cls.env["edi.backend"].create( + { + "name": "Storage Demo EDI backend", + "backend_type_id": backend_type.id, + "storage_id": storage.id, + "input_dir_pending": "demo_in/pending", + "input_dir_done": "demo_in/done", + "input_dir_error": "demo_in/error", + "output_dir_pending": "demo_out/pending", + "output_dir_done": "demo_out/done", + "output_dir_error": "demo_out/error", + } + ) @classmethod def _setup_records(cls): diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py index e5fbe0a21..b84bd7bed 100644 --- a/edi_storage_oca/tests/test_edi_backend_storage.py +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -141,8 +141,18 @@ def test_cron_full_flow(self): """Already sent, update the state via cron.""" self.record.edi_exchange_state = "output_sent" rec1 = self.record - partner2 = self.env.ref("base.res_partner_2") - partner3 = self.env.ref("base.res_partner_3") + partner2, partner3 = self.env["res.partner"].create( + [ + { + "name": "Acme Corporation", + "is_company": True, + }, + { + "name": "Gemini Furniture", + "is_company": True, + }, + ] + ) rec2 = self.record.copy( { "model": partner2._name,