From e5504e77e9424684f4a61ac6f9125f15ec8d601f Mon Sep 17 00:00:00 2001 From: GuillemCForgeFlow Date: Mon, 19 Oct 2026 11:59:39 +0200 Subject: [PATCH] [ADD] edi_exchange_file_db_oca Databases are often restored without their filestore, which leaves every exchange record without its file, so exchanges can no longer be inspected, reprocessed or replayed. Keeping the content of the exchange file attachments in the database fixes that at a small cost, since EDI payloads are usually tiny, and the exchange record form now shows that content. --- edi_exchange_file_db_oca/README.rst | 230 +++++++ edi_exchange_file_db_oca/__init__.py | 2 + edi_exchange_file_db_oca/__manifest__.py | 21 + .../data/ir_config_parameter.xml | 7 + edi_exchange_file_db_oca/hooks.py | 118 ++++ edi_exchange_file_db_oca/models/__init__.py | 3 + .../models/edi_exchange_record.py | 171 ++++++ .../models/edi_exchange_type.py | 49 ++ .../models/ir_attachment.py | 14 + edi_exchange_file_db_oca/pyproject.toml | 3 + edi_exchange_file_db_oca/readme/CONFIGURE.md | 18 + edi_exchange_file_db_oca/readme/CONTEXT.md | 8 + .../readme/CONTRIBUTORS.md | 2 + .../readme/DESCRIPTION.md | 10 + edi_exchange_file_db_oca/readme/DEVELOP.md | 38 ++ edi_exchange_file_db_oca/readme/ROADMAP.md | 7 + edi_exchange_file_db_oca/readme/USAGE.md | 40 ++ .../static/description/icon.png | Bin 0 -> 9455 bytes .../static/description/index.html | 563 ++++++++++++++++++ edi_exchange_file_db_oca/tests/__init__.py | 1 + .../tests/test_exchange_file_db.py | 259 ++++++++ .../views/edi_exchange_record_views.xml | 65 ++ .../views/edi_exchange_type_views.xml | 17 + 23 files changed, 1646 insertions(+) create mode 100644 edi_exchange_file_db_oca/README.rst create mode 100644 edi_exchange_file_db_oca/__init__.py create mode 100644 edi_exchange_file_db_oca/__manifest__.py create mode 100644 edi_exchange_file_db_oca/data/ir_config_parameter.xml create mode 100644 edi_exchange_file_db_oca/hooks.py create mode 100644 edi_exchange_file_db_oca/models/__init__.py create mode 100644 edi_exchange_file_db_oca/models/edi_exchange_record.py create mode 100644 edi_exchange_file_db_oca/models/edi_exchange_type.py create mode 100644 edi_exchange_file_db_oca/models/ir_attachment.py create mode 100644 edi_exchange_file_db_oca/pyproject.toml create mode 100644 edi_exchange_file_db_oca/readme/CONFIGURE.md create mode 100644 edi_exchange_file_db_oca/readme/CONTEXT.md create mode 100644 edi_exchange_file_db_oca/readme/CONTRIBUTORS.md create mode 100644 edi_exchange_file_db_oca/readme/DESCRIPTION.md create mode 100644 edi_exchange_file_db_oca/readme/DEVELOP.md create mode 100644 edi_exchange_file_db_oca/readme/ROADMAP.md create mode 100644 edi_exchange_file_db_oca/readme/USAGE.md create mode 100644 edi_exchange_file_db_oca/static/description/icon.png create mode 100644 edi_exchange_file_db_oca/static/description/index.html create mode 100644 edi_exchange_file_db_oca/tests/__init__.py create mode 100644 edi_exchange_file_db_oca/tests/test_exchange_file_db.py create mode 100644 edi_exchange_file_db_oca/views/edi_exchange_record_views.xml create mode 100644 edi_exchange_file_db_oca/views/edi_exchange_type_views.xml diff --git a/edi_exchange_file_db_oca/README.rst b/edi_exchange_file_db_oca/README.rst new file mode 100644 index 000000000..b7457e196 --- /dev/null +++ b/edi_exchange_file_db_oca/README.rst @@ -0,0 +1,230 @@ +============================= +EDI Exchange File in Database +============================= + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:8e72ea86ccfb18fc3d93d3f584a29464a8538c75b2f56e0ed471067e3fd41571 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-LGPL--3-blue.png + :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html + :alt: License: LGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi--framework-lightgray.png?logo=github + :target: https://github.com/OCA/edi-framework/tree/17.0/edi_exchange_file_db_oca + :alt: OCA/edi-framework +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/edi-framework-17-0/edi-framework-17-0-edi_exchange_file_db_oca + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/edi-framework&target_branch=17.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +This module keeps the content of EDI exchange files in the database +instead of the filestore, so a database restored without its filestore +still holds every exchange payload. The **Exchange file** of an exchange +record is still an attachment: only the place where that attachment +keeps its content changes, from a filestore file to the database, so +everything that reads, writes or searches exchange files works as +before. It is enabled for every exchange type on installation, can be +disabled per type, and a size limit keeps big files in the filestore. +The exchange record form also shows the content of text files in a +**File Content** tab, with JSON indented. + +**Table of contents** + +.. contents:: + :local: + +Use Cases / Context +=================== + +Databases are often restored without their filestore for development, +debugging or testing: on a busy instance it can weigh far more than the +database, and copying it makes every restore slower and heavier. Such a +database keeps every exchange record but none of their files, so an +exchange can no longer be inspected, reprocessed or replayed to +reproduce an issue. EDI payloads are usually small and a negligible +share of the filestore, so keeping their content in the database costs +little. + +Configuration +============= + +No configuration is required: on installation every exchange type stores +the content of its files in the database. + +To change it for one exchange type: + +1. Go to *EDI > Config > Exchange types* and open the type. +2. Untick **Store File Content in Database** to keep the content of its + files in the filestore, or set **Max File Content Size in Database + (KB)** so that bigger files keep their content in the filestore. 0 + uses the global limit. + +To set the global limit, go to *Settings > Technical > Parameters > +System Parameters* (developer mode) and edit +``edi_exchange_file_db_oca.max_size_kb``, in KB. The module creates it +with ``0``, which means no limit. + +Both settings apply when a file is written: the files already stored +keep their content where it is (see **Usage**). + +Usage +===== + +Exchange files are handled as before: the module only changes where +their content is kept. + +To read the content of an exchange file: + +1. Go to *EDI > Exchanges* and open an exchange record. +2. Open the **File Content** tab. JSON is indented, and JSON and XML are + highlighted; other text, such as CSV or EDIFACT, is shown as it is. + Text that does not match the encoding of the exchange type is decoded + with the encoding detected from its content. + +Binary files, such as PDF, images or spreadsheets, and files above 1 MB +are not displayed: the tab shows their type, or their size limit, and +they are downloaded from the **Exchange file** field. + +On installation, the content of the existing exchange files is moved +from the filestore to the database, following the type settings. Files +missing from the filestore, or whose content does not match their +checksum, are left untouched. The server log reports how many files were +moved and skipped. The moved filestore files are deleted by the regular +filestore garbage collection. + +To move the existing files again later, for instance after enabling a +type or raising its limit, run from an Odoo shell: + +.. code:: python + + from odoo.addons.edi_exchange_file_db_oca.hooks import ( + move_exchange_file_contents_to_db, + ) + + move_exchange_file_contents_to_db(env) + env.cr.commit() + +Uninstalling the module keeps every file readable, since Odoo reads the +content of an attachment from the database or from the filestore, +wherever it is; new files go to the filestore again. To move the stored +content back to the filestore as well, run +``env["ir.attachment"].force_storage()`` with the system parameter +``ir_attachment.location`` set to ``file``. + +Development +=========== + +Odoo decides where an attachment keeps its content in +``ir.attachment._get_datas_related_values``, from ``_storage()``, which +knows neither the attachment's model nor its field. The decision is +therefore taken by the exchange record: ``edi.exchange.record`` writes +its **Exchange file** under the ``edi_exchange_file_content_in_db`` +context key, for which ``ir.attachment._storage()`` returns ``db``. +Records are grouped by decision and each group is written with the key +set to ``True`` or ``False``, so a value inherited from the caller never +applies. + +The key is scoped to the writing of the file only: + +- on creation, ``_create`` is overridden rather than ``create``, so the + key covers the insertion of the rows and of their attachments, but + not the quick execution of the exchange that ``edi_oca`` runs right + after; +- on update, ``write`` writes the other values first, then the file + alone. + +``edi.exchange.type._should_store_file_content_in_db(size)`` is the +single decision point: it combines the type's toggle with its size limit +(the type's own one, otherwise the global system parameter). + +Content written to the attachment without going through the field, such +as ``ir.attachment.force_storage()``, follows the configured storage +instead. + +``hooks.move_exchange_file_contents_to_db`` moves the existing content. +It selects the candidates in SQL, by batches, and asks their exchange +type. It reads each file, checks it against the stored checksum, and +writes the content with a batched SQL update: the checksum, size, +mimetype and index of the attachment stay valid since the content does +not change, and going through the ORM would recompute them for every +attachment. + +The **File Content** tab decides between text and binary before +decoding: a file is binary when ``guess_mimetype`` recognises a binary +signature (PDF, images, Office documents, archives), or when its first +KB holds a NUL byte (except for UTF-16 and UTF-32 types). Decoding +success alone is not a reliable signal, since a type in ``latin-1``, or +with the ``replace`` error handler, decodes any byte sequence. The +display limit is ``_get_exchange_file_display_max_size()``. + +Known issues / Roadmap +====================== + +- A default on the backend, to configure every type of a backend at + once. +- From Odoo 19.0, writing an attachment with the database storage no + longer marks its previous filestore file for garbage collection: the + forward port must mark it itself. +- A preview of PDF and image exchange files in the exchange record + form. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* ForgeFlow + +Contributors +------------ + +- `ForgeFlow `__: + + - Guillem Casassas + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +.. |maintainer-GuillemCForgeFlow| image:: https://github.com/GuillemCForgeFlow.png?size=40px + :target: https://github.com/GuillemCForgeFlow + :alt: GuillemCForgeFlow + +Current `maintainer `__: + +|maintainer-GuillemCForgeFlow| + +This module is part of the `OCA/edi-framework `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/edi_exchange_file_db_oca/__init__.py b/edi_exchange_file_db_oca/__init__.py new file mode 100644 index 000000000..cc6b6354a --- /dev/null +++ b/edi_exchange_file_db_oca/__init__.py @@ -0,0 +1,2 @@ +from . import models +from .hooks import post_init_hook diff --git a/edi_exchange_file_db_oca/__manifest__.py b/edi_exchange_file_db_oca/__manifest__.py new file mode 100644 index 000000000..9f3440d66 --- /dev/null +++ b/edi_exchange_file_db_oca/__manifest__.py @@ -0,0 +1,21 @@ +# Copyright 2026 ForgeFlow S.L. +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). +{ + "name": "EDI Exchange File in Database", + "summary": """ + Store the files of EDI exchange records in the database instead of the + filestore. + """, + "version": "17.0.1.0.0", + "license": "LGPL-3", + "website": "https://github.com/OCA/edi-framework", + "author": "ForgeFlow, Odoo Community Association (OCA)", + "maintainers": ["GuillemCForgeFlow"], + "depends": ["edi_oca"], + "data": [ + "data/ir_config_parameter.xml", + "views/edi_exchange_record_views.xml", + "views/edi_exchange_type_views.xml", + ], + "post_init_hook": "post_init_hook", +} diff --git a/edi_exchange_file_db_oca/data/ir_config_parameter.xml b/edi_exchange_file_db_oca/data/ir_config_parameter.xml new file mode 100644 index 000000000..93b8f6ff2 --- /dev/null +++ b/edi_exchange_file_db_oca/data/ir_config_parameter.xml @@ -0,0 +1,7 @@ + + + + edi_exchange_file_db_oca.max_size_kb + 0 + + diff --git a/edi_exchange_file_db_oca/hooks.py b/edi_exchange_file_db_oca/hooks.py new file mode 100644 index 000000000..ee8e5d3c7 --- /dev/null +++ b/edi_exchange_file_db_oca/hooks.py @@ -0,0 +1,118 @@ +# Copyright 2026 ForgeFlow S.L. +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import logging +import os + +import psycopg2 + +from odoo.tools import SQL, human_size + +_logger = logging.getLogger(__name__) + + +def _get_exchange_files_to_move(env, last_id, limit): + record_model = env["edi.exchange.record"] + env.cr.execute( + SQL( + """ + SELECT a.id, r.type_id, a.file_size, a.store_fname, a.checksum + FROM ir_attachment a + JOIN %(record_table)s r ON r.id = a.res_id + WHERE a.res_model = %(res_model)s + AND a.res_field = %(res_field)s + AND a.type = 'binary' + AND a.store_fname IS NOT NULL + AND a.id > %(last_id)s + ORDER BY a.id + LIMIT %(limit)s + """, + record_table=SQL.identifier(record_model._table), + res_model=record_model._name, + res_field="exchange_file", + last_id=last_id, + limit=limit, + ) + ) + return env.cr.fetchall() + + +def _read_exchange_file(env, fname, checksum): + attachment_model = env["ir.attachment"] + full_path = attachment_model._full_path(fname) + if not os.path.isfile(full_path): + return None, "missing" + with open(full_path, "rb") as file: + data = file.read() + if attachment_model._compute_checksum(data) != checksum: + return None, "mismatched" + return data, "moved" + + +def move_exchange_file_contents_to_db(env, batch_size=1000): + attachment_model = env["ir.attachment"] + attachment_model.flush_model( + [ + "res_model", + "res_field", + "res_id", + "type", + "file_size", + "store_fname", + "checksum", + ] + ) + env["edi.exchange.record"].flush_model(["type_id"]) + counts = {"moved": 0, "excluded": 0, "missing": 0, "mismatched": 0} + moved_size = 0 + last_id = 0 + while rows := _get_exchange_files_to_move(env, last_id, batch_size): + last_id = rows[-1][0] + exchange_types = { + exchange_type.id: exchange_type + for exchange_type in env["edi.exchange.type"].browse( + list({row[1] for row in rows}) + ) + } + values, fnames = [], [] + for attachment_id, type_id, file_size, fname, checksum in rows: + if not exchange_types[type_id]._should_store_file_content_in_db(file_size): + counts["excluded"] += 1 + continue + data, status = _read_exchange_file(env, fname, checksum) + counts[status] += 1 + if data is None: + continue + values.append((attachment_id, psycopg2.Binary(data))) + fnames.append(fname) + moved_size += len(data) + if not values: + continue + env.cr.execute_values( + """ + UPDATE ir_attachment a + SET db_datas = v.data, store_fname = NULL + FROM (VALUES %s) AS v(id, data) + WHERE a.id = v.id + """, + values, + page_size=batch_size, + ) + for fname in fnames: + attachment_model._file_delete(fname) + attachment_model.invalidate_model(["db_datas", "store_fname", "raw", "datas"]) + _logger.info( + "EDI exchange file contents moved to the database: %s (%s). Skipped: %s " + "excluded by their exchange type, %s missing from the filestore, %s not " + "matching their checksum.", + counts["moved"], + human_size(moved_size) or "0 bytes", + counts["excluded"], + counts["missing"], + counts["mismatched"], + ) + return counts + + +def post_init_hook(env): + move_exchange_file_contents_to_db(env) diff --git a/edi_exchange_file_db_oca/models/__init__.py b/edi_exchange_file_db_oca/models/__init__.py new file mode 100644 index 000000000..5c25fda65 --- /dev/null +++ b/edi_exchange_file_db_oca/models/__init__.py @@ -0,0 +1,3 @@ +from . import edi_exchange_record +from . import edi_exchange_type +from . import ir_attachment diff --git a/edi_exchange_file_db_oca/models/edi_exchange_record.py b/edi_exchange_file_db_oca/models/edi_exchange_record.py new file mode 100644 index 000000000..c6a88aee9 --- /dev/null +++ b/edi_exchange_file_db_oca/models/edi_exchange_record.py @@ -0,0 +1,171 @@ +# Copyright 2026 ForgeFlow S.L. +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import base64 +import codecs +import contextlib +import json + +import chardet + +from odoo import api, fields, models +from odoo.tools.mimetypes import guess_mimetype + +TEXT_MIMETYPES = ("application/octet-stream", "application/xml", "image/svg+xml") +DISPLAY_MAX_SIZE = 1024 * 1024 + + +class EDIExchangeRecord(models.Model): + _inherit = "edi.exchange.record" + + exchange_file_content = fields.Text( + string="File Content", + compute="_compute_exchange_file_content", + help="Content of the exchange file, when it is text. JSON is indented.", + ) + exchange_file_content_format = fields.Selection( + selection=[ + ("json", "JSON"), + ("xml", "XML"), + ("text", "Text"), + ("binary", "Binary"), + ], + compute="_compute_exchange_file_content", + help="Empty when the file is too large or missing.", + ) + exchange_file_content_mimetype = fields.Char( + string="File Type", + compute="_compute_exchange_file_content", + ) + exchange_file_content_too_large = fields.Boolean( + compute="_compute_exchange_file_content", + ) + + @api.depends("exchange_file") + def _compute_exchange_file_content(self): + for record in self: + record.update( + record.with_context(bin_size=False)._get_exchange_file_display_values() + ) + + def _create(self, data_list): + in_db = self._get_exchange_file_content_in_db_flags(data_list) + if not any(in_db): + return super()._create(data_list) + records = {} + for flag in (True, False): + group = [ + data for data, db in zip(data_list, in_db, strict=True) if db == flag + ] + if group: + records[flag] = iter( + super( + EDIExchangeRecord, + self.with_context(edi_exchange_file_content_in_db=flag), + ) + ._create(group) + .ids + ) + return self.browse([next(records[db]) for db in in_db]) + + def write(self, vals): + if "exchange_file" not in vals: + return super().write(vals) + vals = dict(vals) + file = vals.pop("exchange_file") + if vals: + super().write(vals) + size = self._get_exchange_file_size(file) + db_records = self.filtered( + lambda record: record.type_id._should_store_file_content_in_db(size) + ) + for records, in_db in ((db_records, True), (self - db_records, False)): + if records: + super( + EDIExchangeRecord, + records.with_context(edi_exchange_file_content_in_db=in_db), + ).write({"exchange_file": file}) + return True + + @api.model + def _get_exchange_file_content_in_db_flags(self, data_list): + stored_list = [data["stored"] for data in data_list] + type_ids = { + stored["type_id"] + for stored in stored_list + if stored.get("exchange_file") and stored.get("type_id") + } + exchange_types = { + exchange_type.id: exchange_type + for exchange_type in self.env["edi.exchange.type"].browse(list(type_ids)) + } + empty_type = self.env["edi.exchange.type"] + return [ + bool(stored.get("exchange_file")) + and exchange_types.get( + stored.get("type_id"), empty_type + )._should_store_file_content_in_db( + self._get_exchange_file_size(stored["exchange_file"]) + ) + for stored in stored_list + ] + + @api.model + def _get_exchange_file_size(self, file): + return len(base64.b64decode(file or b"")) + + @api.model + def _get_exchange_file_display_max_size(self): + return DISPLAY_MAX_SIZE + + def _get_exchange_file_encoding(self): + self.ensure_one() + return codecs.lookup(self.type_id.encoding or "utf-8").name + + def _is_exchange_file_binary(self, raw): + mimetype = guess_mimetype(raw) + if mimetype not in TEXT_MIMETYPES and not mimetype.startswith("text/"): + return True + encoding = self._get_exchange_file_encoding() + return not encoding.startswith(("utf-16", "utf-32")) and b"\x00" in raw[:1024] + + def _decode_exchange_file(self, raw): + encoding = self._get_exchange_file_encoding() + with contextlib.suppress(UnicodeDecodeError): + return raw.decode(encoding) + detected_encoding = chardet.detect(raw)["encoding"] + if detected_encoding: + with contextlib.suppress(UnicodeDecodeError, LookupError): + return raw.decode(detected_encoding) + return raw.decode(encoding, errors="replace") + + def _get_exchange_file_display_values(self): + self.ensure_one() + values = { + "exchange_file_content": False, + "exchange_file_content_format": False, + "exchange_file_content_too_large": False, + "exchange_file_content_mimetype": False, + } + raw = self._get_file_content(as_bytes=True) + if not raw: + return values + if self._is_exchange_file_binary(raw): + values["exchange_file_content_format"] = "binary" + values["exchange_file_content_mimetype"] = guess_mimetype(raw) + return values + if len(raw) > self._get_exchange_file_display_max_size(): + values["exchange_file_content_too_large"] = True + return values + content = self._decode_exchange_file(raw) + try: + values["exchange_file_content"] = json.dumps( + json.loads(content), indent=2, ensure_ascii=False + ) + values["exchange_file_content_format"] = "json" + except ValueError: + values["exchange_file_content"] = content + values["exchange_file_content_format"] = ( + "xml" if guess_mimetype(raw) == "application/xml" else "text" + ) + return values diff --git a/edi_exchange_file_db_oca/models/edi_exchange_type.py b/edi_exchange_file_db_oca/models/edi_exchange_type.py new file mode 100644 index 000000000..58795a5f1 --- /dev/null +++ b/edi_exchange_file_db_oca/models/edi_exchange_type.py @@ -0,0 +1,49 @@ +# Copyright 2026 ForgeFlow S.L. +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo import api, fields, models + +MAX_SIZE_PARAM = "edi_exchange_file_db_oca.max_size_kb" + + +class EDIExchangeType(models.Model): + _inherit = "edi.exchange.type" + + store_file_content_in_db = fields.Boolean( + string="Store File Content in Database", + default=True, + help="Keep the content of the exchange files of this type in the " + "database instead of the filestore. The file is still an attachment " + "of the exchange record: only the place where the attachment keeps " + "its content changes, so a database restored without its filestore " + "still holds it.", + ) + file_content_db_max_size = fields.Integer( + string="Max File Content Size in Database (KB)", + help="Files whose content is bigger than this size keep it in the " + "filestore. 0 uses the global limit, set in the system parameter " + "'edi_exchange_file_db_oca.max_size_kb' (no limit when it is not set).", + ) + + @api.model + def _get_file_content_db_global_max_size(self): + value = self.env["ir.config_parameter"].sudo().get_param(MAX_SIZE_PARAM) + try: + return int(value or 0) + except ValueError: + return 0 + + def _get_file_content_db_max_size(self): + self.ensure_one() + return ( + self.file_content_db_max_size or self._get_file_content_db_global_max_size() + ) + + def _should_store_file_content_in_db(self, size): + if not self: + return False + self.ensure_one() + if not self.store_file_content_in_db: + return False + max_size = self._get_file_content_db_max_size() + return not max_size or size <= max_size * 1024 diff --git a/edi_exchange_file_db_oca/models/ir_attachment.py b/edi_exchange_file_db_oca/models/ir_attachment.py new file mode 100644 index 000000000..b92179dea --- /dev/null +++ b/edi_exchange_file_db_oca/models/ir_attachment.py @@ -0,0 +1,14 @@ +# Copyright 2026 ForgeFlow S.L. +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo import api, models + + +class IrAttachment(models.Model): + _inherit = "ir.attachment" + + @api.model + def _storage(self): + if self.env.context.get("edi_exchange_file_content_in_db"): + return "db" + return super()._storage() diff --git a/edi_exchange_file_db_oca/pyproject.toml b/edi_exchange_file_db_oca/pyproject.toml new file mode 100644 index 000000000..4231d0ccc --- /dev/null +++ b/edi_exchange_file_db_oca/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/edi_exchange_file_db_oca/readme/CONFIGURE.md b/edi_exchange_file_db_oca/readme/CONFIGURE.md new file mode 100644 index 000000000..6e4b7e65d --- /dev/null +++ b/edi_exchange_file_db_oca/readme/CONFIGURE.md @@ -0,0 +1,18 @@ +No configuration is required: on installation every exchange type +stores the content of its files in the database. + +To change it for one exchange type: + +1. Go to *EDI \> Config \> Exchange types* and open the type. +2. Untick **Store File Content in Database** to keep the content of its + files in the filestore, or set **Max File Content Size in Database + (KB)** so that bigger files keep their content in the filestore. 0 + uses the global limit. + +To set the global limit, go to *Settings \> Technical \> Parameters \> +System Parameters* (developer mode) and edit +`edi_exchange_file_db_oca.max_size_kb`, in KB. The module creates it +with `0`, which means no limit. + +Both settings apply when a file is written: the files already stored +keep their content where it is (see **Usage**). diff --git a/edi_exchange_file_db_oca/readme/CONTEXT.md b/edi_exchange_file_db_oca/readme/CONTEXT.md new file mode 100644 index 000000000..dcd802962 --- /dev/null +++ b/edi_exchange_file_db_oca/readme/CONTEXT.md @@ -0,0 +1,8 @@ +Databases are often restored without their filestore for development, +debugging or testing: on a busy instance it can weigh far more than the +database, and copying it makes every restore slower and heavier. Such a +database keeps every exchange record but none of their files, so an +exchange can no longer be inspected, reprocessed or replayed to +reproduce an issue. EDI payloads are usually small and a negligible +share of the filestore, so keeping their content in the database costs +little. diff --git a/edi_exchange_file_db_oca/readme/CONTRIBUTORS.md b/edi_exchange_file_db_oca/readme/CONTRIBUTORS.md new file mode 100644 index 000000000..60ea8f6bd --- /dev/null +++ b/edi_exchange_file_db_oca/readme/CONTRIBUTORS.md @@ -0,0 +1,2 @@ +- [ForgeFlow](https://www.forgeflow.com): + - Guillem Casassas \<\> diff --git a/edi_exchange_file_db_oca/readme/DESCRIPTION.md b/edi_exchange_file_db_oca/readme/DESCRIPTION.md new file mode 100644 index 000000000..23640112a --- /dev/null +++ b/edi_exchange_file_db_oca/readme/DESCRIPTION.md @@ -0,0 +1,10 @@ +This module keeps the content of EDI exchange files in the database +instead of the filestore, so a database restored without its filestore +still holds every exchange payload. The **Exchange file** of an exchange +record is still an attachment: only the place where that attachment +keeps its content changes, from a filestore file to the database, so +everything that reads, writes or searches exchange files works as +before. It is enabled for every exchange type on installation, can be +disabled per type, and a size limit keeps big files in the filestore. +The exchange record form also shows the content of text files in a +**File Content** tab, with JSON indented. diff --git a/edi_exchange_file_db_oca/readme/DEVELOP.md b/edi_exchange_file_db_oca/readme/DEVELOP.md new file mode 100644 index 000000000..14fd4ad2c --- /dev/null +++ b/edi_exchange_file_db_oca/readme/DEVELOP.md @@ -0,0 +1,38 @@ +Odoo decides where an attachment keeps its content in +`ir.attachment._get_datas_related_values`, from `_storage()`, which knows +neither the attachment's model nor its field. The decision is therefore +taken by the exchange record: `edi.exchange.record` writes its +**Exchange file** under the `edi_exchange_file_content_in_db` context +key, for which `ir.attachment._storage()` returns `db`. Records are +grouped by decision and each group is written with the key set to +`True` or `False`, so a value inherited from the caller never applies. + +The key is scoped to the writing of the file only: + +- on creation, `_create` is overridden rather than `create`, so the key + covers the insertion of the rows and of their attachments, but not the + quick execution of the exchange that `edi_oca` runs right after; +- on update, `write` writes the other values first, then the file alone. + +`edi.exchange.type._should_store_file_content_in_db(size)` is the single +decision point: it combines the type's toggle with its size limit (the +type's own one, otherwise the global system parameter). + +Content written to the attachment without going through the field, such +as `ir.attachment.force_storage()`, follows the configured storage +instead. + +`hooks.move_exchange_file_contents_to_db` moves the existing content. It +selects the candidates in SQL, by batches, and asks their exchange type. +It reads each file, checks it against the stored checksum, and writes +the content with a batched SQL update: the checksum, size, mimetype and +index of the attachment stay valid since the content does not change, +and going through the ORM would recompute them for every attachment. + +The **File Content** tab decides between text and binary before +decoding: a file is binary when `guess_mimetype` recognises a binary +signature (PDF, images, Office documents, archives), or when its first +KB holds a NUL byte (except for UTF-16 and UTF-32 types). Decoding +success alone is not a reliable signal, since a type in `latin-1`, or +with the `replace` error handler, decodes any byte sequence. The display +limit is `_get_exchange_file_display_max_size()`. diff --git a/edi_exchange_file_db_oca/readme/ROADMAP.md b/edi_exchange_file_db_oca/readme/ROADMAP.md new file mode 100644 index 000000000..8b83eb236 --- /dev/null +++ b/edi_exchange_file_db_oca/readme/ROADMAP.md @@ -0,0 +1,7 @@ +- A default on the backend, to configure every type of a backend at + once. +- From Odoo 19.0, writing an attachment with the database storage no + longer marks its previous filestore file for garbage collection: the + forward port must mark it itself. +- A preview of PDF and image exchange files in the exchange record + form. diff --git a/edi_exchange_file_db_oca/readme/USAGE.md b/edi_exchange_file_db_oca/readme/USAGE.md new file mode 100644 index 000000000..9e8c9007e --- /dev/null +++ b/edi_exchange_file_db_oca/readme/USAGE.md @@ -0,0 +1,40 @@ +Exchange files are handled as before: the module only changes where +their content is kept. + +To read the content of an exchange file: + +1. Go to *EDI \> Exchanges* and open an exchange record. +2. Open the **File Content** tab. JSON is indented, and JSON and XML + are highlighted; other text, such as CSV or EDIFACT, is shown as it + is. Text that does not match the encoding of the exchange type is + decoded with the encoding detected from its content. + +Binary files, such as PDF, images or spreadsheets, and files above +1 MB are not displayed: the tab shows their type, or their size limit, +and they are downloaded from the **Exchange file** field. + +On installation, the content of the existing exchange files is moved +from the filestore to the database, following the type settings. Files +missing from the filestore, or whose content does not match their +checksum, are left untouched. The server log reports how many files +were moved and skipped. The moved filestore files are deleted by the +regular filestore garbage collection. + +To move the existing files again later, for instance after enabling a +type or raising its limit, run from an Odoo shell: + +``` python +from odoo.addons.edi_exchange_file_db_oca.hooks import ( + move_exchange_file_contents_to_db, +) + +move_exchange_file_contents_to_db(env) +env.cr.commit() +``` + +Uninstalling the module keeps every file readable, since Odoo reads the +content of an attachment from the database or from the filestore, +wherever it is; new files go to the filestore again. To move the stored +content back to the filestore as well, run +`env["ir.attachment"].force_storage()` with the system parameter +`ir_attachment.location` set to `file`. diff --git a/edi_exchange_file_db_oca/static/description/icon.png b/edi_exchange_file_db_oca/static/description/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..3a0328b516c4980e8e44cdb63fd945757ddd132d GIT binary patch literal 9455 zcmW++2RxMjAAjx~&dlBk9S+%}OXg)AGE&Cb*&}d0jUxM@u(PQx^-s)697TX`ehR4?GS^qbkof1cslKgkU)h65qZ9Oc=ml_0temigYLJfnz{IDzUf>bGs4N!v3=Z3jMq&A#7%rM5eQ#dc?k~! zVpnB`o+K7|Al`Q_U;eD$B zfJtP*jH`siUq~{KE)`jP2|#TUEFGRryE2`i0**z#*^6~AI|YzIWy$Cu#CSLW3q=GA z6`?GZymC;dCPk~rBS%eCb`5OLr;RUZ;D`}um=H)BfVIq%7VhiMr)_#G0N#zrNH|__ zc+blN2UAB0=617@>_u;MPHN;P;N#YoE=)R#i$k_`UAA>WWCcEVMh~L_ zj--gtp&|K1#58Yz*AHCTMziU1Jzt_jG0I@qAOHsk$2}yTmVkBp_eHuY$A9)>P6o~I z%aQ?!(GqeQ-Y+b0I(m9pwgi(IIZZzsbMv+9w{PFtd_<_(LA~0H(xz{=FhLB@(1&qHA5EJw1>>=%q2f&^X>IQ{!GJ4e9U z&KlB)z(84HmNgm2hg2C0>WM{E(DdPr+EeU_N@57;PC2&DmGFW_9kP&%?X4}+xWi)( z;)z%wI5>D4a*5XwD)P--sPkoY(a~WBw;E~AW`Yue4kFa^LM3X`8x|}ZUeMnqr}>kH zG%WWW>3ml$Yez?i%)2pbKPI7?5o?hydokgQyZsNEr{a|mLdt;X2TX(#B1j35xPnPW z*bMSSOauW>o;*=kO8ojw91VX!qoOQb)zHJ!odWB}d+*K?#sY_jqPdg{Sm2HdYzdEx zOGVPhVRTGPtv0o}RfVP;Nd(|CB)I;*t&QO8h zFfekr30S!-LHmV_Su-W+rEwYXJ^;6&3|L$mMC8*bQptyOo9;>Qb9Q9`ySe3%V$A*9 zeKEe+b0{#KWGp$F+tga)0RtI)nhMa-K@JS}2krK~n8vJ=Ngm?R!9G<~RyuU0d?nz# z-5EK$o(!F?hmX*2Yt6+coY`6jGbb7tF#6nHA zuKk=GGJ;ZwON1iAfG$E#Y7MnZVmrY|j0eVI(DN_MNFJmyZ|;w4tf@=CCDZ#5N_0K= z$;R~bbk?}TpfDjfB&aiQ$VA}s?P}xPERJG{kxk5~R`iRS(SK5d+Xs9swCozZISbnS zk!)I0>t=A<-^z(cmSFz3=jZ23u13X><0b)P)^1T_))Kr`e!-pb#q&J*Q`p+B6la%C zuVl&0duN<;uOsB3%T9Fp8t{ED108<+W(nOZd?gDnfNBC3>M8WE61$So|P zVvqH0SNtDTcsUdzaMDpT=Ty0pDHHNL@Z0w$Y`XO z2M-_r1S+GaH%pz#Uy0*w$Vdl=X=rQXEzO}d6J^R6zjM1u&c9vYLvLp?W7w(?np9x1 zE_0JSAJCPB%i7p*Wvg)pn5T`8k3-uR?*NT|J`eS#_#54p>!p(mLDvmc-3o0mX*mp_ zN*AeS<>#^-{S%W<*mz^!X$w_2dHWpcJ6^j64qFBft-o}o_Vx80o0>}Du;>kLts;$8 zC`7q$QI(dKYG`Wa8#wl@V4jVWBRGQ@1dr-hstpQL)Tl+aqVpGpbSfN>5i&QMXfiZ> zaA?T1VGe?rpQ@;+pkrVdd{klI&jVS@I5_iz!=UMpTsa~mBga?1r}aRBm1WS;TT*s0f0lY=JBl66Upy)-k4J}lh=P^8(SXk~0xW=T9v*B|gzIhN z>qsO7dFd~mgxAy4V?&)=5ieYq?zi?ZEoj)&2o)RLy=@hbCRcfT5jigwtQGE{L*8<@Yd{zg;CsL5mvzfDY}P-wos_6PfprFVaeqNE%h zKZhLtcQld;ZD+>=nqN~>GvROfueSzJD&BE*}XfU|H&(FssBqY=hPCt`d zH?@s2>I(|;fcW&YM6#V#!kUIP8$Nkdh0A(bEVj``-AAyYgwY~jB zT|I7Bf@%;7aL7Wf4dZ%VqF$eiaC38OV6oy3Z#TER2G+fOCd9Iaoy6aLYbPTN{XRPz z;U!V|vBf%H!}52L2gH_+j;`bTcQRXB+y9onc^wLm5wi3-Be}U>k_u>2Eg$=k!(l@I zcCg+flakT2Nej3i0yn+g+}%NYb?ta;R?(g5SnwsQ49U8Wng8d|{B+lyRcEDvR3+`O{zfmrmvFrL6acVP%yG98X zo&+VBg@px@i)%o?dG(`T;n*$S5*rnyiR#=wW}}GsAcfyQpE|>a{=$Hjg=-*_K;UtD z#z-)AXwSRY?OPefw^iI+ z)AXz#PfEjlwTes|_{sB?4(O@fg0AJ^g8gP}ex9Ucf*@_^J(s_5jJV}c)s$`Myn|Kd z$6>}#q^n{4vN@+Os$m7KV+`}c%4)4pv@06af4-x5#wj!KKb%caK{A&Y#Rfs z-po?Dcb1({W=6FKIUirH&(yg=*6aLCekcKwyfK^JN5{wcA3nhO(o}SK#!CINhI`-I z1)6&n7O&ZmyFMuNwvEic#IiOAwNkR=u5it{B9n2sAJV5pNhar=j5`*N!Na;c7g!l$ z3aYBqUkqqTJ=Re-;)s!EOeij=7SQZ3Hq}ZRds%IM*PtM$wV z@;rlc*NRK7i3y5BETSKuumEN`Xu_8GP1Ri=OKQ$@I^ko8>H6)4rjiG5{VBM>B|%`&&s^)jS|-_95&yc=GqjNo{zFkw%%HHhS~e=s zD#sfS+-?*t|J!+ozP6KvtOl!R)@@-z24}`9{QaVLD^9VCSR2b`b!KC#o;Ki<+wXB6 zx3&O0LOWcg4&rv4QG0)4yb}7BFSEg~=IR5#ZRj8kg}dS7_V&^%#Do==#`u zpy6{ox?jWuR(;pg+f@mT>#HGWHAJRRDDDv~@(IDw&R>9643kK#HN`!1vBJHnC+RM&yIh8{gG2q zA%e*U3|N0XSRa~oX-3EAneep)@{h2vvd3Xvy$7og(sayr@95+e6~Xvi1tUqnIxoIH zVWo*OwYElb#uyW{Imam6f2rGbjR!Y3`#gPqkv57dB6K^wRGxc9B(t|aYDGS=m$&S!NmCtrMMaUg(c zc2qC=2Z`EEFMW-me5B)24AqF*bV5Dr-M5ig(l-WPS%CgaPzs6p_gnCIvTJ=Y<6!gT zVt@AfYCzjjsMEGi=rDQHo0yc;HqoRNnNFeWZgcm?f;cp(6CNylj36DoL(?TS7eU#+ z7&mfr#y))+CJOXQKUMZ7QIdS9@#-}7y2K1{8)cCt0~-X0O!O?Qx#E4Og+;A2SjalQ zs7r?qn0H044=sDN$SRG$arw~n=+T_DNdSrarmu)V6@|?1-ZB#hRn`uilTGPJ@fqEy zGt(f0B+^JDP&f=r{#Y_wi#AVDf-y!RIXU^0jXsFpf>=Ji*TeqSY!H~AMbJdCGLhC) zn7Rx+sXw6uYj;WRYrLd^5IZq@6JI1C^YkgnedZEYy<&4(z%Q$5yv#Boo{AH8n$a zhb4Y3PWdr269&?V%uI$xMcUrMzl=;w<_nm*qr=c3Rl@i5wWB;e-`t7D&c-mcQl7x! zZWB`UGcw=Y2=}~wzrfLx=uet<;m3~=8I~ZRuzvMQUQdr+yTV|ATf1Uuomr__nDf=X zZ3WYJtHp_ri(}SQAPjv+Y+0=fH4krOP@S&=zZ-t1jW1o@}z;xk8 z(Nz1co&El^HK^NrhVHa-_;&88vTU>_J33=%{if;BEY*J#1n59=07jrGQ#IP>@u#3A z;!q+E1Rj3ZJ+!4bq9F8PXJ@yMgZL;>&gYA0%_Kbi8?S=XGM~dnQZQ!yBSgcZhY96H zrWnU;k)qy`rX&&xlDyA%(a1Hhi5CWkmg(`Gb%m(HKi-7Z!LKGRP_B8@`7&hdDy5n= z`OIxqxiVfX@OX1p(mQu>0Ai*v_cTMiw4qRt3~NBvr9oBy0)r>w3p~V0SCm=An6@3n)>@z!|o-$HvDK z|3D2ZMJkLE5loMKl6R^ez@Zz%S$&mbeoqH5`Bb){Ei21q&VP)hWS2tjShfFtGE+$z zzCR$P#uktu+#!w)cX!lWN1XU%K-r=s{|j?)Akf@q#3b#{6cZCuJ~gCxuMXRmI$nGtnH+-h z+GEi!*X=AP<|fG`1>MBdTb?28JYc=fGvAi2I<$B(rs$;eoJCyR6_bc~p!XR@O-+sD z=eH`-ye})I5ic1eL~TDmtfJ|8`0VJ*Yr=hNCd)G1p2MMz4C3^Mj?7;!w|Ly%JqmuW zlIEW^Ft%z?*|fpXda>Jr^1noFZEwFgVV%|*XhH@acv8rdGxeEX{M$(vG{Zw+x(ei@ zmfXb22}8-?Fi`vo-YVrTH*C?a8%M=Hv9MqVH7H^J$KsD?>!SFZ;ZsvnHr_gn=7acz z#W?0eCdVhVMWN12VV^$>WlQ?f;P^{(&pYTops|btm6aj>_Uz+hqpGwB)vWp0Cf5y< zft8-je~nn?W11plq}N)4A{l8I7$!ks_x$PXW-2XaRFswX_BnF{R#6YIwMhAgd5F9X zGmwdadS6(a^fjHtXg8=l?Rc0Sm%hk6E9!5cLVloEy4eh(=FwgP`)~I^5~pBEWo+F6 zSf2ncyMurJN91#cJTy_u8Y}@%!bq1RkGC~-bV@SXRd4F{R-*V`bS+6;W5vZ(&+I<9$;-V|eNfLa5n-6% z2(}&uGRF;p92eS*sE*oR$@pexaqr*meB)VhmIg@h{uzkk$9~qh#cHhw#>O%)b@+(| z^IQgqzuj~Sk(J;swEM-3TrJAPCq9k^^^`q{IItKBRXYe}e0Tdr=Huf7da3$l4PdpwWDop%^}n;dD#K4s#DYA8SHZ z&1!riV4W4R7R#C))JH1~axJ)RYnM$$lIR%6fIVA@zV{XVyx}C+a-Dt8Y9M)^KU0+H zR4IUb2CJ{Hg>CuaXtD50jB(_Tcx=Z$^WYu2u5kubqmwp%drJ6 z?Fo40g!Qd<-l=TQxqHEOuPX0;^z7iX?Ke^a%XT<13TA^5`4Xcw6D@Ur&VT&CUe0d} z1GjOVF1^L@>O)l@?bD~$wzgf(nxX1OGD8fEV?TdJcZc2KoUe|oP1#=$$7ee|xbY)A zDZq+cuTpc(fFdj^=!;{k03C69lMQ(|>uhRfRu%+!k&YOi-3|1QKB z z?n?eq1XP>p-IM$Z^C;2L3itnbJZAip*Zo0aw2bs8@(s^~*8T9go!%dHcAz2lM;`yp zD=7&xjFV$S&5uDaiScyD?B-i1ze`+CoRtz`Wn+Zl&#s4&}MO{@N!ufrzjG$B79)Y2d3tBk&)TxUTw@QS0TEL_?njX|@vq?Uz(nBFK5Pq7*xj#u*R&i|?7+6# z+|r_n#SW&LXhtheZdah{ZVoqwyT{D>MC3nkFF#N)xLi{p7J1jXlmVeb;cP5?e(=f# zuT7fvjSbjS781v?7{)-X3*?>tq?)Yd)~|1{BDS(pqC zC}~H#WXlkUW*H5CDOo<)#x7%RY)A;ShGhI5s*#cRDA8YgqG(HeKDx+#(ZQ?386dv! zlXCO)w91~Vw4AmOcATuV653fa9R$fyK8ul%rG z-wfS zihugoZyr38Im?Zuh6@RcF~t1anQu7>#lPpb#}4cOA!EM11`%f*07RqOVkmX{p~KJ9 z^zP;K#|)$`^Rb{rnHGH{~>1(fawV0*Z#)}M`m8-?ZJV<+e}s9wE# z)l&az?w^5{)`S(%MRzxdNqrs1n*-=jS^_jqE*5XDrA0+VE`5^*p3CuM<&dZEeCjoz zR;uu_H9ZPZV|fQq`Cyw4nscrVwi!fE6ciMmX$!_hN7uF;jjKG)d2@aC4ropY)8etW=xJvni)8eHi`H$%#zn^WJ5NLc-rqk|u&&4Z6fD_m&JfSI1Bvb?b<*n&sfl0^t z=HnmRl`XrFvMKB%9}>PaA`m-fK6a0(8=qPkWS5bb4=v?XcWi&hRY?O5HdulRi4?fN zlsJ*N-0Qw+Yic@s0(2uy%F@ib;GjXt01Fmx5XbRo6+n|pP(&nodMoap^z{~q ziEeaUT@Mxe3vJSfI6?uLND(CNr=#^W<1b}jzW58bIfyWTDle$mmS(|x-0|2UlX+9k zQ^EX7Nw}?EzVoBfT(-LT|=9N@^hcn-_p&sqG z&*oVs2JSU+N4ZD`FhCAWaS;>|wH2G*Id|?pa#@>tyxX`+4HyIArWDvVrX)2WAOQff z0qyHu&-S@i^MS-+j--!pr4fPBj~_8({~e1bfcl0wI1kaoN>mJL6KUPQm5N7lB(ui1 zE-o%kq)&djzWJ}ob<-GfDlkB;F31j-VHKvQUGQ3sp`CwyGJk_i!y^sD0fqC@$9|jO zOqN!r!8-p==F@ZVP=U$qSpY(gQ0)59P1&t@y?5rvg<}E+GB}26NYPp4f2YFQrQtot5mn3wu_qprZ=>Ig-$ zbW26Ws~IgY>}^5w`vTB(G`PTZaDiGBo5o(tp)qli|NeV( z@H_=R8V39rt5J5YB2Ky?4eJJ#b`_iBe2ot~6%7mLt5t8Vwi^Jy7|jWXqa3amOIoRb zOr}WVFP--DsS`1WpN%~)t3R!arKF^Q$e12KEqU36AWwnCBICpH4XCsfnyrHr>$I$4 z!DpKX$OKLWarN7nv@!uIA+~RNO)l$$w}p(;b>mx8pwYvu;dD_unryX_NhT8*Tj>BTrTTL&!?O+%Rv;b?B??gSzdp?6Uug9{ zd@V08Z$BdI?fpoCS$)t4mg4rT8Q_I}h`0d-vYZ^|dOB*Q^S|xqTV*vIg?@fVFSmMpaw0qtTRbx} z({Pg?#{2`sc9)M5N$*N|4;^t$+QP?#mov zGVC@I*lBVrOU-%2y!7%)fAKjpEFsgQc4{amtiHb95KQEwvf<(3T<9-Zm$xIew#P22 zc2Ix|App^>v6(3L_MCU0d3W##AB0M~3D00EWoKZqsJYT(#@w$Y_H7G22M~ApVFTRHMI_3be)Lkn#0F*V8Pq zc}`Cjy$bE;FJ6H7p=0y#R>`}-m4(0F>%@P|?7fx{=R^uFdISRnZ2W_xQhD{YuR3t< z{6yxu=4~JkeA;|(J6_nv#>Nvs&FuLA&PW^he@t(UwFFE8)|a!R{`E`K`i^ZnyE4$k z;(749Ix|oi$c3QbEJ3b~D_kQsPz~fIUKym($a_7dJ?o+40*OLl^{=&oq$<#Q(yyrp z{J-FAniyAw9tPbe&IhQ|a`DqFTVQGQ&Gq3!C2==4x{6EJwiPZ8zub-iXoUtkJiG{} zPaR&}_fn8_z~(=;5lD-aPWD3z8PZS@AaUiomF!G8I}Mf>e~0g#BelA-5#`cj;O5>N Xviia!U7SGha1wx#SCgwmn*{w2TRX*I literal 0 HcmV?d00001 diff --git a/edi_exchange_file_db_oca/static/description/index.html b/edi_exchange_file_db_oca/static/description/index.html new file mode 100644 index 000000000..f007eb84d --- /dev/null +++ b/edi_exchange_file_db_oca/static/description/index.html @@ -0,0 +1,563 @@ + + + + + +EDI Exchange File in Database + + + +
+

EDI Exchange File in Database

+ + +

Beta License: LGPL-3 OCA/edi-framework Translate me on Weblate Try me on Runboat

+

This module keeps the content of EDI exchange files in the database +instead of the filestore, so a database restored without its filestore +still holds every exchange payload. The Exchange file of an exchange +record is still an attachment: only the place where that attachment +keeps its content changes, from a filestore file to the database, so +everything that reads, writes or searches exchange files works as +before. It is enabled for every exchange type on installation, can be +disabled per type, and a size limit keeps big files in the filestore. +The exchange record form also shows the content of text files in a +File Content tab, with JSON indented.

+

Table of contents

+ +
+

Use Cases / Context

+

Databases are often restored without their filestore for development, +debugging or testing: on a busy instance it can weigh far more than the +database, and copying it makes every restore slower and heavier. Such a +database keeps every exchange record but none of their files, so an +exchange can no longer be inspected, reprocessed or replayed to +reproduce an issue. EDI payloads are usually small and a negligible +share of the filestore, so keeping their content in the database costs +little.

+
+
+

Configuration

+

No configuration is required: on installation every exchange type stores +the content of its files in the database.

+

To change it for one exchange type:

+
    +
  1. Go to EDI > Config > Exchange types and open the type.
    2. +
  2. Untick Store File Content in Database to keep the content of its +files in the filestore, or set Max File Content Size in Database +(KB) so that bigger files keep their content in the filestore. 0 +uses the global limit.
    3. +
+

To set the global limit, go to Settings > Technical > Parameters > +System Parameters (developer mode) and edit +edi_exchange_file_db_oca.max_size_kb, in KB. The module creates it +with 0, which means no limit.

+

Both settings apply when a file is written: the files already stored +keep their content where it is (see Usage).

+
+
+

Usage

+

Exchange files are handled as before: the module only changes where +their content is kept.

+

To read the content of an exchange file:

+
    +
  1. Go to EDI > Exchanges and open an exchange record.
    2. +
  2. Open the File Content tab. JSON is indented, and JSON and XML are +highlighted; other text, such as CSV or EDIFACT, is shown as it is. +Text that does not match the encoding of the exchange type is decoded +with the encoding detected from its content.
    3. +
+

Binary files, such as PDF, images or spreadsheets, and files above 1 MB +are not displayed: the tab shows their type, or their size limit, and +they are downloaded from the Exchange file field.

+

On installation, the content of the existing exchange files is moved +from the filestore to the database, following the type settings. Files +missing from the filestore, or whose content does not match their +checksum, are left untouched. The server log reports how many files were +moved and skipped. The moved filestore files are deleted by the regular +filestore garbage collection.

+

To move the existing files again later, for instance after enabling a +type or raising its limit, run from an Odoo shell:

+
+from odoo.addons.edi_exchange_file_db_oca.hooks import (
+    move_exchange_file_contents_to_db,
+)
+
+move_exchange_file_contents_to_db(env)
+env.cr.commit()
+
+

Uninstalling the module keeps every file readable, since Odoo reads the +content of an attachment from the database or from the filestore, +wherever it is; new files go to the filestore again. To move the stored +content back to the filestore as well, run +env["ir.attachment"].force_storage() with the system parameter +ir_attachment.location set to file.

+
+
+

Development

+

Odoo decides where an attachment keeps its content in +ir.attachment._get_datas_related_values, from _storage(), which +knows neither the attachment’s model nor its field. The decision is +therefore taken by the exchange record: edi.exchange.record writes +its Exchange file under the edi_exchange_file_content_in_db +context key, for which ir.attachment._storage() returns db. +Records are grouped by decision and each group is written with the key +set to True or False, so a value inherited from the caller never +applies.

+

The key is scoped to the writing of the file only:

+
    +
  • on creation, _create is overridden rather than create, so the +key covers the insertion of the rows and of their attachments, but +not the quick execution of the exchange that edi_oca runs right +after;
    • +
  • on update, write writes the other values first, then the file +alone.
    • +
+

edi.exchange.type._should_store_file_content_in_db(size) is the +single decision point: it combines the type’s toggle with its size limit +(the type’s own one, otherwise the global system parameter).

+

Content written to the attachment without going through the field, such +as ir.attachment.force_storage(), follows the configured storage +instead.

+

hooks.move_exchange_file_contents_to_db moves the existing content. +It selects the candidates in SQL, by batches, and asks their exchange +type. It reads each file, checks it against the stored checksum, and +writes the content with a batched SQL update: the checksum, size, +mimetype and index of the attachment stay valid since the content does +not change, and going through the ORM would recompute them for every +attachment.

+

The File Content tab decides between text and binary before +decoding: a file is binary when guess_mimetype recognises a binary +signature (PDF, images, Office documents, archives), or when its first +KB holds a NUL byte (except for UTF-16 and UTF-32 types). Decoding +success alone is not a reliable signal, since a type in latin-1, or +with the replace error handler, decodes any byte sequence. The +display limit is _get_exchange_file_display_max_size().

+
+
+

Known issues / Roadmap

+
    +
  • A default on the backend, to configure every type of a backend at +once.
    • +
  • From Odoo 19.0, writing an attachment with the database storage no +longer marks its previous filestore file for garbage collection: the +forward port must mark it itself.
    • +
  • A preview of PDF and image exchange files in the exchange record +form.
    • +
+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • ForgeFlow
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

Current maintainer:

+

GuillemCForgeFlow

+

This module is part of the OCA/edi-framework project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/edi_exchange_file_db_oca/tests/__init__.py b/edi_exchange_file_db_oca/tests/__init__.py new file mode 100644 index 000000000..c4717f52d --- /dev/null +++ b/edi_exchange_file_db_oca/tests/__init__.py @@ -0,0 +1 @@ +from . import test_exchange_file_db diff --git a/edi_exchange_file_db_oca/tests/test_exchange_file_db.py b/edi_exchange_file_db_oca/tests/test_exchange_file_db.py new file mode 100644 index 000000000..68d3fe96a --- /dev/null +++ b/edi_exchange_file_db_oca/tests/test_exchange_file_db.py @@ -0,0 +1,259 @@ +# Copyright 2026 ForgeFlow S.L. +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import base64 +import os +from unittest.mock import patch + +from odoo.addons.edi_oca.tests.common import EDIBackendCommonTestCase + +from ..hooks import move_exchange_file_contents_to_db +from ..models.edi_exchange_type import MAX_SIZE_PARAM + + +class TestExchangeFileDb(EDIBackendCommonTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.Attachment = cls.env["ir.attachment"] + cls.ExchangeRecord = cls.env["edi.exchange.record"] + + def _create_record(self, content=None, exchange_type=None): + exchange_type = exchange_type or self.exchange_type_out + vals = {"model": self.partner._name, "res_id": self.partner.id} + if content is not None: + vals["exchange_file"] = base64.b64encode(content) + return self.backend.create_record(exchange_type.code, vals) + + def _get_attachment(self, record): + return self.Attachment.search( + [ + ("res_model", "=", record._name), + ("res_field", "=", "exchange_file"), + ("res_id", "=", record.id), + ] + ) + + def assertInDb(self, record, content): + attachment = self._get_attachment(record) + self.assertFalse(attachment.store_fname) + self.assertEqual(attachment.db_datas, content) + record.invalidate_recordset(["exchange_file"]) + self.assertEqual(record._get_file_content(as_bytes=True), content) + + def assertInFilestore(self, record, content): + attachment = self._get_attachment(record) + self.assertTrue(attachment.store_fname) + self.assertFalse(attachment.db_datas) + self.assertEqual(attachment._file_read(attachment.store_fname), content) + record.invalidate_recordset(["exchange_file"]) + self.assertEqual(record._get_file_content(as_bytes=True), content) + + def test_file_in_db(self): + record = self._create_record(b"in database") + self.assertInDb(record, b"in database") + + def test_file_in_filestore_when_disabled(self): + self.exchange_type_out.store_file_content_in_db = False + record = self._create_record(b"in filestore") + self.assertInFilestore(record, b"in filestore") + + def test_global_max_size(self): + self.env["ir.config_parameter"].sudo().set_param(MAX_SIZE_PARAM, "1") + small = self._create_record(b"x" * 1024) + big = self._create_record(b"x" * 1025) + self.assertInDb(small, b"x" * 1024) + self.assertInFilestore(big, b"x" * 1025) + + def test_type_max_size_overrides_global(self): + self.env["ir.config_parameter"].sudo().set_param(MAX_SIZE_PARAM, "1") + self.exchange_type_out.file_content_db_max_size = 2 + record = self._create_record(b"x" * 2048) + self.assertInDb(record, b"x" * 2048) + record = self._create_record(b"x" * 2049) + self.assertInFilestore(record, b"x" * 2049) + + def test_rewrite(self): + record = self._create_record(b"before") + record.exchange_file = base64.b64encode(b"after") + self.assertInDb(record, b"after") + self.exchange_type_out.store_file_content_in_db = False + record.exchange_file = base64.b64encode(b"back to filestore") + self.assertInFilestore(record, b"back to filestore") + self.exchange_type_out.store_file_content_in_db = True + record.exchange_file = base64.b64encode(b"back to database") + self.assertInDb(record, b"back to database") + + def test_create_batch(self): + self.exchange_type_in.store_file_content_in_db = False + records = self.ExchangeRecord.create( + [ + { + "backend_id": self.backend.id, + "type_id": exchange_type.id, + "exchange_file": base64.b64encode(content), + } + for exchange_type, content in ( + (self.exchange_type_out, b"first"), + (self.exchange_type_in, b"second"), + (self.exchange_type_out, b"third"), + ) + ] + ) + self.assertEqual(records[1].type_id, self.exchange_type_in) + self.assertInDb(records[0], b"first") + self.assertInFilestore(records[1], b"second") + self.assertInDb(records[2], b"third") + self.assertFalse(records.env.context.get("edi_exchange_file_content_in_db")) + + def test_write_batch(self): + self.exchange_type_in.store_file_content_in_db = False + records = self._create_record() | self._create_record( + exchange_type=self.exchange_type_in + ) + records.write({"exchange_file": base64.b64encode(b"both")}) + self.assertInDb(records[0], b"both") + self.assertInFilestore(records[1], b"both") + + def test_attachment_outside_exchange_file(self): + record = self._create_record() + attachment = self.Attachment.create( + { + "name": "chatter", + "res_model": record._name, + "res_id": record.id, + "raw": b"chatter", + } + ) + self.assertTrue(attachment.store_fname) + self.assertFalse(attachment.db_datas) + + def test_other_attachment_rewrite(self): + attachment = self.Attachment.create( + { + "name": "partner", + "res_model": self.partner._name, + "res_id": self.partner.id, + "raw": b"partner", + } + ) + attachment.raw = b"partner rewritten" + self.assertTrue(attachment.store_fname) + self.assertFalse(attachment.db_datas) + self.assertEqual(attachment.raw, b"partner rewritten") + + def test_search_exchange_file(self): + with_file = self._create_record(b"content") + without_file = self._create_record() + domain = [("id", "in", (with_file | without_file).ids)] + self.assertEqual( + self.ExchangeRecord.search(domain + [("exchange_file", "=", False)]), + without_file, + ) + self.assertEqual( + self.ExchangeRecord.search(domain + [("exchange_file", "!=", False)]), + with_file, + ) + + def test_move_to_db(self): + self.exchange_type_out.store_file_content_in_db = False + record = self._create_record(b"move me") + attachment = self._get_attachment(record) + fname = attachment.store_fname + checksum = attachment.checksum + self.exchange_type_out.store_file_content_in_db = True + counts = move_exchange_file_contents_to_db(self.env) + self.assertEqual( + counts, {"moved": 1, "excluded": 0, "missing": 0, "mismatched": 0} + ) + self.assertInDb(record, b"move me") + self.assertEqual(attachment.checksum, checksum) + checklist = os.path.join(self.Attachment._full_path("checklist"), fname) + self.assertTrue(os.path.isfile(checklist)) + + def test_move_to_db_respects_max_size(self): + self.exchange_type_out.store_file_content_in_db = False + record = self._create_record(b"x" * 2048) + self.exchange_type_out.write( + {"store_file_content_in_db": True, "file_content_db_max_size": 1} + ) + counts = move_exchange_file_contents_to_db(self.env) + self.assertEqual( + counts, {"moved": 0, "excluded": 1, "missing": 0, "mismatched": 0} + ) + self.assertInFilestore(record, b"x" * 2048) + + def test_move_to_db_skips_missing_and_mismatched(self): + self.exchange_type_out.store_file_content_in_db = False + missing = self._get_attachment(self._create_record(b"missing")) + mismatched_record = self._create_record(b"mismatched") + mismatched = self._get_attachment(mismatched_record) + self.env.cr.execute( + "UPDATE ir_attachment SET store_fname = %s WHERE id = %s", + ("00/does-not-exist", missing.id), + ) + self.env.cr.execute( + "UPDATE ir_attachment SET checksum = %s WHERE id = %s", + ("0" * 40, mismatched.id), + ) + self.Attachment.invalidate_model(["store_fname", "checksum"]) + self.exchange_type_out.store_file_content_in_db = True + counts = move_exchange_file_contents_to_db(self.env) + self.assertEqual( + counts, {"moved": 0, "excluded": 0, "missing": 1, "mismatched": 1} + ) + self.assertEqual(missing.store_fname, "00/does-not-exist") + self.assertInFilestore(mismatched_record, b"mismatched") + + def test_file_content_json_indented(self): + record = self._create_record(b'{"a": 1, "b": "\xc3\xa9"}') + record = record.with_context(bin_size=True) + self.assertEqual(record.exchange_file_content_format, "json") + self.assertEqual(record.exchange_file_content, '{\n "a": 1,\n "b": "\xe9"\n}') + + def test_file_content_xml(self): + content = b'\n1' + record = self._create_record(content) + self.assertEqual(record.exchange_file_content_format, "xml") + self.assertEqual(record.exchange_file_content, content.decode()) + + def test_file_content_text(self): + record = self._create_record(b"a;b\n1;2\n") + self.assertEqual(record.exchange_file_content_format, "text") + self.assertEqual(record.exchange_file_content, "a;b\n1;2\n") + + def test_file_content_wrong_encoding(self): + content = "referencia;descripción\nCT0118;tuerca\n" + record = self._create_record(content.encode("latin-1")) + self.assertEqual(record.exchange_file_content_format, "text") + self.assertEqual(record.exchange_file_content, content) + + def test_file_content_binary_signature(self): + self.exchange_type_out.encoding = "latin-1" + record = self._create_record(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n1 0 obj") + self.assertEqual(record.exchange_file_content_format, "binary") + self.assertEqual(record.exchange_file_content_mimetype, "application/pdf") + self.assertFalse(record.exchange_file_content) + + def test_file_content_binary_nul(self): + self.exchange_type_out.encoding = "latin-1" + record = self._create_record(b"\x1f\x8b\x08\x00\x00\x00\x00\x00") + self.assertEqual(record.exchange_file_content_format, "binary") + self.assertFalse(record.exchange_file_content) + + def test_file_content_too_large(self): + record = self._create_record(b'{"a": 1}') + with patch.object( + type(self.ExchangeRecord), + "_get_exchange_file_display_max_size", + return_value=4, + ): + record.invalidate_recordset(["exchange_file_content_too_large"]) + self.assertTrue(record.exchange_file_content_too_large) + self.assertFalse(record.exchange_file_content_format) + self.assertFalse(record.exchange_file_content) + + def test_file_content_empty(self): + record = self._create_record() + self.assertFalse(record.exchange_file_content_format) + self.assertFalse(record.exchange_file_content_too_large) diff --git a/edi_exchange_file_db_oca/views/edi_exchange_record_views.xml b/edi_exchange_file_db_oca/views/edi_exchange_record_views.xml new file mode 100644 index 000000000..23a2513ac --- /dev/null +++ b/edi_exchange_file_db_oca/views/edi_exchange_record_views.xml @@ -0,0 +1,65 @@ + + + + edi.exchange.record.form - edi_exchange_file_db_oca + edi.exchange.record + + + + + + +
+ This file is binary () and cannot be displayed here: download it from the + Exchange file field. +
+
+ The file is too large to be displayed here: download it + from the Exchange file field. +
+ + + + + + + + diff --git a/edi_exchange_file_db_oca/views/edi_exchange_type_views.xml b/edi_exchange_file_db_oca/views/edi_exchange_type_views.xml new file mode 100644 index 000000000..b624c5878 --- /dev/null +++ b/edi_exchange_file_db_oca/views/edi_exchange_type_views.xml @@ -0,0 +1,17 @@ + + + + edi.exchange.type.form - edi_exchange_file_db_oca + edi.exchange.type + + + + + + + + +