diff --git a/compute_field_after_install/README.rst b/compute_field_after_install/README.rst new file mode 100644 index 00000000000..1d359c9f7d0 --- /dev/null +++ b/compute_field_after_install/README.rst @@ -0,0 +1,105 @@ +=========================== +Compute field after install +=========================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:26bad7f441b66e0de5861db7e578a3651b9e17b90748e4ca3fe6ba91025751ee + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fserver--tools-lightgray.png?logo=github + :target: https://github.com/OCA/server-tools/tree/18.0/compute_field_after_install + :alt: OCA/server-tools +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/server-tools-18-0/server-tools-18-0-compute_field_after_install + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/server-tools&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +Computed field computation can be a really long process when installing +a module and can block the migration process for a long time. + +This module gives the possibility to defer the field computation after +the installation. + +⚠️ Use with caution ⚠️ + +Not all computed fields can be deferred without risking the generation +of corrupted data. E.g. the total amount of an invoice, if differed, +could lead to wrong data computation in other stored fields that depend +on it. That's why this module should never be used in a database +migration process such as OpenUpgrade, where computed fields have to be +triggered by the framework as expected. + +**Table of contents** + +.. contents:: + :local: + +Usage +===== + +Just install any module with computed field. + +Database is available quickly whatever its size and computed fields. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* Akretion + +Contributors +------------ + +- Sébastien BEAU +- Florian Mounier + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +.. |maintainer-sebastienbeau| image:: https://github.com/sebastienbeau.png?size=40px + :target: https://github.com/sebastienbeau + :alt: sebastienbeau + +Current `maintainer `__: + +|maintainer-sebastienbeau| + +This module is part of the `OCA/server-tools `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/compute_field_after_install/__init__.py b/compute_field_after_install/__init__.py new file mode 100644 index 00000000000..0650744f6bc --- /dev/null +++ b/compute_field_after_install/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/compute_field_after_install/__manifest__.py b/compute_field_after_install/__manifest__.py new file mode 100644 index 00000000000..b84b5f741ed --- /dev/null +++ b/compute_field_after_install/__manifest__.py @@ -0,0 +1,23 @@ +# Copyright 2025 Akretion (http://www.akretion.com) +# Sébastien BEAU +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +{ + "name": "Compute field after install", + "summary": "Compute computed fields after the install process", + "version": "18.0.1.0.0", + "category": "Tools", + "website": "https://github.com/OCA/server-tools", + "author": "Akretion,Odoo Community Association (OCA)", + "maintainers": ["sebastienbeau"], + "license": "AGPL-3", + "installable": True, + "depends": [ + "base", + ], + "data": [ + "data/ir_cron.xml", + "views/recompute_field_view.xml", + "security/ir.model.access.csv", + ], +} diff --git a/compute_field_after_install/data/ir_cron.xml b/compute_field_after_install/data/ir_cron.xml new file mode 100644 index 00000000000..6c660ea952c --- /dev/null +++ b/compute_field_after_install/data/ir_cron.xml @@ -0,0 +1,18 @@ + + + + + Run Recompute field + + + 30 + minutes + + code + model._run_all() + + diff --git a/compute_field_after_install/models/__init__.py b/compute_field_after_install/models/__init__.py new file mode 100644 index 00000000000..09d08483c70 --- /dev/null +++ b/compute_field_after_install/models/__init__.py @@ -0,0 +1 @@ +from . import recompute_field diff --git a/compute_field_after_install/models/recompute_field.py b/compute_field_after_install/models/recompute_field.py new file mode 100644 index 00000000000..bfa0c159fff --- /dev/null +++ b/compute_field_after_install/models/recompute_field.py @@ -0,0 +1,137 @@ +# Copyright 2025 Akretion (http://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + + +import logging +from functools import reduce +from itertools import groupby + +from odoo import api, fields, models +from odoo.exceptions import UserError +from odoo.tools import config +from odoo.tools.translate import _ + +_logger = logging.getLogger(__name__) + + +class RecomputeField(models.Model): + _name = "recompute.field" + _description = "Recompute Field" + + model = fields.Char(required=True) + field = fields.Char(required=True) + last_id = fields.Integer( + help="Last record ID on which computing have been executed" + ) + + step = fields.Integer( + required=True, + help="Recomputing batch size.", + compute="_compute_default_step", + store=True, + readonly=False, + precompute=True, + ) + state = fields.Selection( + [ + ("todo", "Todo"), + ("done", "Done"), + ] + ) + + @api.depends("model", "field") + def _compute_default_step(self): + for recompute_field in self: + model = recompute_field.model.replace(".", "_") + recompute_field.step = config.get( + f"computed_fields_batch_size__{model}__{recompute_field.field}", + config.get( + f"computed_fields_batch_size__{model}", + config.get("computed_fields_batch_size", 1000), + ), + ) + + @api.constrains("step") + def _check_step(self): + for recompute_field in self: + if recompute_field.step <= 0: + raise UserError(_("Step must be greater than 0")) + + @api.model + def _run_all(self): + return self.search([("state", "=", "todo")]).run() + + def run(self): + # Group tasks by compute method to avoid computing multifields + # computes multiple times + + def group_key(task): + return task.model, str(self.env[task.model]._fields[task.field].compute) + + model_tasks = groupby( + self.sorted(key=group_key), + group_key, + ) + for (model, _compute_fun), tasks in model_tasks: + tasks = reduce(lambda x, y: x | y, tasks) + fields = set(tasks.mapped("field")) + last_id = max(tasks.mapped("last_id"), default=None) + step = min(tasks.mapped("step")) + + while True: + _logger.info( + "Recompute fields %s for model %s in background. Last id %d", + fields, + model, + last_id, + ) + records = self.env[model].search( + [("id", "<", last_id)] if last_id else [], + limit=step, + order="id desc", + ) + if not records: + tasks.state = "done" + self.env.cr.commit() # pylint: disable=E8102 + break + for field in fields: + field_ = records._fields[field] + self.env.add_to_compute(field_, records) + + records.flush_recordset() + last_id = records[-1].id + tasks.last_id = last_id + self.env.cr.commit() # pylint: disable=E8102 + + return True + + +ori_add_to_compute = api.Environment.add_to_compute + + +def add_to_compute(self, field, records): + if ( + "recompute.field" in self + and len(records) > config.get("computed_fields_defer_threshold", 50000) + and self.context.get("module") + and not getattr(field, "precompute", False) + ): + _logger.info( + "Deferring computation of field %s for model %s as there is %s records", + field.name, + records._name, + len(records), + ) + self["recompute.field"].create( + { + "field": field.name, + "model": records._name, + "state": "todo", + } + ) + else: + return ori_add_to_compute(self, field, records) + + +api.Environment.add_to_compute = add_to_compute diff --git a/compute_field_after_install/pyproject.toml b/compute_field_after_install/pyproject.toml new file mode 100644 index 00000000000..4231d0cccb3 --- /dev/null +++ b/compute_field_after_install/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/compute_field_after_install/readme/CONFIGURATION.rst b/compute_field_after_install/readme/CONFIGURATION.rst new file mode 100644 index 00000000000..1a319da11e0 --- /dev/null +++ b/compute_field_after_install/readme/CONFIGURATION.rst @@ -0,0 +1,20 @@ +This module can be installed just like any other Odoo module, by adding it +to Odoo *addons_path*. In order for the module to work correctly, +it needs to be loaded as a server-wide module. +This can be done with the ``server_wide_modules`` parameter in your Odoo config +file or with the ``--load`` command-line parameter. + +Additionnaly you can customize the minimum number of records that will defer the +fields computation by adding the following variable in the odoo config file +``computed_fields_defer_threshold=20000`` + +You can also customize the batch size of the fields computation by adding +the following variable in the odoo config file +``computed_fields_batch_size=1000`` or more specifically for a model +``computed_fields_batch_size__=1000`` and for a specific model field +``computed_fields_batch_size____=1000`` + +i.e.: +``computed_fields_batch_size__res_partner=2000`` +``computed_fields_batch_size__res_partner__commercial_company_name=50`` + \ No newline at end of file diff --git a/compute_field_after_install/readme/CONTRIBUTORS.md b/compute_field_after_install/readme/CONTRIBUTORS.md new file mode 100644 index 00000000000..b6503771d80 --- /dev/null +++ b/compute_field_after_install/readme/CONTRIBUTORS.md @@ -0,0 +1,2 @@ +- Sébastien BEAU \ +- Florian Mounier \ diff --git a/compute_field_after_install/readme/DESCRIPTION.md b/compute_field_after_install/readme/DESCRIPTION.md new file mode 100644 index 00000000000..2c53d8fa64e --- /dev/null +++ b/compute_field_after_install/readme/DESCRIPTION.md @@ -0,0 +1,14 @@ +Computed field computation can be a really long process when installing +a module and can block the migration process for a long time. + +This module gives the possibility to defer the field computation after +the installation. + +:warning: Use with caution :warning: + +Not all computed fields can be deferred without risking the generation of +corrupted data. E.g. the total amount of an invoice, if differed, could lead +to wrong data computation in other stored fields that depend on it. That's why +this module should never be used in a database migration process such as +OpenUpgrade, where computed fields have to be triggered by the framework +as expected. diff --git a/compute_field_after_install/readme/USAGE.md b/compute_field_after_install/readme/USAGE.md new file mode 100644 index 00000000000..9f4526d7939 --- /dev/null +++ b/compute_field_after_install/readme/USAGE.md @@ -0,0 +1,3 @@ +Just install any module with computed field. + +Database is available quickly whatever its size and computed fields. diff --git a/compute_field_after_install/security/ir.model.access.csv b/compute_field_after_install/security/ir.model.access.csv new file mode 100644 index 00000000000..5d195c5d7fa --- /dev/null +++ b/compute_field_after_install/security/ir.model.access.csv @@ -0,0 +1,2 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_recompute_field,access_recompute_field,model_recompute_field,base.group_user,1,0,0,0 diff --git a/compute_field_after_install/static/description/index.html b/compute_field_after_install/static/description/index.html new file mode 100644 index 00000000000..7a04e7fc3cf --- /dev/null +++ b/compute_field_after_install/static/description/index.html @@ -0,0 +1,442 @@ + + + + + +Compute field after install + + + +
+

Compute field after install

+ + +

Beta License: AGPL-3 OCA/server-tools Translate me on Weblate Try me on Runboat

+

Computed field computation can be a really long process when installing +a module and can block the migration process for a long time.

+

This module gives the possibility to defer the field computation after +the installation.

+

⚠️ Use with caution ⚠️

+

Not all computed fields can be deferred without risking the generation +of corrupted data. E.g. the total amount of an invoice, if differed, +could lead to wrong data computation in other stored fields that depend +on it. That’s why this module should never be used in a database +migration process such as OpenUpgrade, where computed fields have to be +triggered by the framework as expected.

+

Table of contents

+ +
+

Usage

+

Just install any module with computed field.

+

Database is available quickly whatever its size and computed fields.

+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Akretion
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

Current maintainer:

+

sebastienbeau

+

This module is part of the OCA/server-tools project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/compute_field_after_install/tests/__init__.py b/compute_field_after_install/tests/__init__.py new file mode 100644 index 00000000000..2adaf71023c --- /dev/null +++ b/compute_field_after_install/tests/__init__.py @@ -0,0 +1 @@ +from . import test_recompute diff --git a/compute_field_after_install/tests/test_recompute.py b/compute_field_after_install/tests/test_recompute.py new file mode 100644 index 00000000000..5b2ec186d0b --- /dev/null +++ b/compute_field_after_install/tests/test_recompute.py @@ -0,0 +1,252 @@ +# Copyright 2025 Akretion (https://www.akretion.com). +# @author Sébastien BEAU +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import math +from contextlib import contextmanager +from unittest.mock import Mock, patch + +from odoo.tests import TransactionCase +from odoo.tools import config + + +class TestRecompute(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + + def setUp(self): + super().setUp() + self.env.cr.commit = Mock() + + @contextmanager + def _count_computes(self, obj, method_name): + count = {"records": 0, "calls": 0} + + method = getattr(obj.__class__, method_name) + + def _compute(self, *args, **kwargs): + count["calls"] += 1 + count["records"] += len(self) + return method(self, *args, **kwargs) + + with patch.object( + obj.__class__, method_name, side_effect=_compute, autospec=True + ): + yield count + + def test_add_fields(self): + # Simulate the module installation with a computed field + records = self.env["res.partner"].search([]) + with patch.dict( + config.options, {"computed_fields_defer_threshold": 1}, clear=True + ): + self.env(context={"module": "fake_module"}).add_to_compute( + records._fields["commercial_company_name"], records + ) + + # Check that a job have been created + recompute_field = self.env["recompute.field"].search( + [ + ("model", "=", "res.partner"), + ("field", "=", "commercial_company_name"), + ] + ) + self.assertEqual(recompute_field.state, "todo") + + # Purge field commercial_company_name to simulate + # the installation of a new field + self.env.cr.execute("UPDATE res_partner SET commercial_company_name=null") + self.env.invalidate_all() + + partner = self.env.ref("base.res_partner_address_7") + self.assertFalse(partner.commercial_company_name) + + # Run the cron to process computed field + + with self._count_computes( + partner, + "_compute_commercial_company_name", + ) as computed: + self.env["recompute.field"]._run_all() + self.assertEqual(computed["records"], len(records)) + self.assertEqual(computed["calls"], 1) + self.assertEqual(recompute_field.state, "done") + + # Check that field have been recomputed correctly + self.assertEqual(partner.commercial_company_name, "Ready Mat") + + def test_add_fields_batch(self): + # Simulate the module installation with a computed field + records = self.env["res.partner"].search([]) + with patch.dict( + config.options, + {"computed_fields_defer_threshold": 1, "computed_fields_batch_size": 10}, + clear=True, + ): + self.env(context={"module": "fake_module"}).add_to_compute( + records._fields["commercial_company_name"], records + ) + + # Check that a job have been created + recompute_field = self.env["recompute.field"].search( + [ + ("model", "=", "res.partner"), + ("field", "=", "commercial_company_name"), + ] + ) + self.assertEqual(recompute_field.state, "todo") + + # Purge field commercial_company_name to simulate + # the installation of a new field + self.env.cr.execute("UPDATE res_partner SET commercial_company_name=null") + self.env.invalidate_all() + + partner = self.env.ref("base.res_partner_address_7") + self.assertFalse(partner.commercial_company_name) + + # Run the cron to process computed field + + with self._count_computes( + partner, + "_compute_commercial_company_name", + ) as computed: + self.env["recompute.field"]._run_all() + self.assertEqual(computed["records"], len(records)) + self.assertEqual(computed["calls"], math.ceil(len(records) / 10.0)) + self.assertEqual(recompute_field.state, "done") + + # Check that field have been recomputed correctly + self.assertEqual(partner.commercial_company_name, "Ready Mat") + + def test_add_precompute_fields(self): + # Precompute fields should not be deferred + records = self.env["res.partner"].search([]) + with patch.dict( + config.options, {"computed_fields_defer_threshold": 1}, clear=True + ): + self.env(context={"module": "fake_module"}).add_to_compute( + records._fields["user_id"], records + ) + + # Check that a job have NOT been created + recompute_field = self.env["recompute.field"].search( + [ + ("model", "=", "res.partner"), + ("field", "=", "user_id"), + ] + ) + self.assertFalse(recompute_field) + + def test_default_step(self): + recompute_field = self.env["recompute.field"].create( + { + "model": "res.partner", + "field": "commercial_company_name", + "state": "todo", + } + ) + self.assertEqual(recompute_field.step, 1000) + + with patch.dict( + config.options, {"computed_fields_batch_size": 2000}, clear=True + ): + recompute_field = self.env["recompute.field"].create( + { + "model": "res.partner", + "field": "commercial_company_name", + "state": "todo", + } + ) + self.assertEqual(recompute_field.step, 2000) + + with patch.dict( + config.options, + {"computed_fields_batch_size__res_partner": 3000}, + clear=True, + ): + recompute_field = self.env["recompute.field"].create( + { + "model": "res.partner", + "field": "commercial_company_name", + "state": "todo", + } + ) + self.assertEqual(recompute_field.step, 3000) + + option_key = ( + "computed_fields_batch_size" + "__res_partner__commercial_company_name" + ) + with patch.dict( + config.options, + {option_key: 4000}, + clear=True, + ): + recompute_field = self.env["recompute.field"].create( + { + "model": "res.partner", + "field": "commercial_company_name", + "state": "todo", + } + ) + self.assertEqual(recompute_field.step, 4000) + + def test_multifields(self): + if "sale.order" not in self.env: + self.skipTest("This test requires the sale module to be installed") + + records = self.env["sale.order"].search([]) + original_id_amounts = { + record.id: (record.amount_untaxed, record.amount_tax, record.amount_total) + for record in records + } + + with patch.dict( + config.options, {"computed_fields_defer_threshold": 1}, clear=True + ): + for field in ("amount_untaxed", "amount_tax", "amount_total"): + self.env(context={"module": "fake_module"}).add_to_compute( + records._fields[field], records + ) + + # Check that jobs have been created + recompute_fields = self.env["recompute.field"].search( + [ + ("model", "=", "sale.order"), + ] + ) + self.assertEqual(len(recompute_fields), 3) + self.assertEqual(recompute_fields.mapped("state"), ["todo", "todo", "todo"]) + + # Purge field commercial_company_name to simulate + # the installation of a new field + self.env.cr.execute( + """ + UPDATE sale_order + SET amount_untaxed=null, amount_tax=null, amount_total=null + """ + ) + self.env.invalidate_all() + + for record in records: + self.assertFalse(record.amount_untaxed) + self.assertFalse(record.amount_tax) + self.assertFalse(record.amount_total) + + # Run the cron to process computed field + + with self._count_computes( + self.env["sale.order"], + "_compute_amounts", + ) as computed: + self.env["recompute.field"]._run_all() + self.assertEqual(computed["records"], len(records)) + self.assertEqual(computed["calls"], 1) + + self.assertEqual(recompute_fields.mapped("state"), ["done", "done", "done"]) + + for record in records: + self.assertEqual(record.amount_untaxed, original_id_amounts[record.id][0]) + self.assertEqual(record.amount_tax, original_id_amounts[record.id][1]) + self.assertEqual(record.amount_total, original_id_amounts[record.id][2]) diff --git a/compute_field_after_install/views/recompute_field_view.xml b/compute_field_after_install/views/recompute_field_view.xml new file mode 100644 index 00000000000..def5851abf3 --- /dev/null +++ b/compute_field_after_install/views/recompute_field_view.xml @@ -0,0 +1,78 @@ + + + + + recompute.field + + + + + + + +