From ed719372226cfab415799ccb6f28fb4e4b160ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Alix?= Date: Fri, 7 Jun 2024 17:11:28 +0200 Subject: [PATCH] Add 'odoo_project_changelog' module --- odoo_project_changelog/README.rst | 61 +++ odoo_project_changelog/__init__.py | 1 + odoo_project_changelog/__manifest__.py | 22 + odoo_project_changelog/data/queue_job.xml | 31 ++ odoo_project_changelog/models/__init__.py | 2 + odoo_project_changelog/models/odoo_project.py | 85 ++++ .../models/odoo_project_repository.py | 87 ++++ .../readme/CONTRIBUTORS.rst | 2 + odoo_project_changelog/readme/DESCRIPTION.rst | 1 + .../report/ir_actions_report.xml | 17 + .../report/odoo_project_changelog.xml | 79 ++++ .../security/ir.model.access.csv | 2 + .../static/description/index.html | 416 ++++++++++++++++++ odoo_project_changelog/utils/__init__.py | 0 odoo_project_changelog/utils/scanner.py | 50 +++ odoo_project_changelog/views/odoo_project.xml | 67 +++ odoo_repository/lib/scanner.py | 178 +++++++- .../odoo/addons/odoo_project_changelog | 1 + setup/odoo_project_changelog/setup.py | 6 + 19 files changed, 1104 insertions(+), 4 deletions(-) create mode 100644 odoo_project_changelog/README.rst create mode 100644 odoo_project_changelog/__init__.py create mode 100644 odoo_project_changelog/__manifest__.py create mode 100644 odoo_project_changelog/data/queue_job.xml create mode 100644 odoo_project_changelog/models/__init__.py create mode 100644 odoo_project_changelog/models/odoo_project.py create mode 100644 odoo_project_changelog/models/odoo_project_repository.py create mode 100644 odoo_project_changelog/readme/CONTRIBUTORS.rst create mode 100644 odoo_project_changelog/readme/DESCRIPTION.rst create mode 100644 odoo_project_changelog/report/ir_actions_report.xml create mode 100644 odoo_project_changelog/report/odoo_project_changelog.xml create mode 100644 odoo_project_changelog/security/ir.model.access.csv create mode 100644 odoo_project_changelog/static/description/index.html create mode 100644 odoo_project_changelog/utils/__init__.py create mode 100644 odoo_project_changelog/utils/scanner.py create mode 100644 odoo_project_changelog/views/odoo_project.xml create mode 120000 setup/odoo_project_changelog/odoo/addons/odoo_project_changelog create mode 100644 setup/odoo_project_changelog/setup.py diff --git a/odoo_project_changelog/README.rst b/odoo_project_changelog/README.rst new file mode 100644 index 00000000..0b340c44 --- /dev/null +++ b/odoo_project_changelog/README.rst @@ -0,0 +1,61 @@ +========================= +Odoo Project - Changelogs +========================= + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:caf51f538da8b936e047ed66d1dce665e597514cd6e5da23c75d763fef84f5a3 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-camptocamp%2Fodoo--repository-lightgray.png?logo=github + :target: https://github.com/camptocamp/odoo-repository/tree/16.0/odoo_project_changelog + :alt: camptocamp/odoo-repository + +|badge1| |badge2| |badge3| + +This module allows to generate CHANGELOGs for repositories used within a project. + +**Table of contents** + +.. contents:: + :local: + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +~~~~~~~ + +* Camptocamp + +Contributors +~~~~~~~~~~~~ + +* Camptocamp + * Sébastien Alix + +Maintainers +~~~~~~~~~~~ + +This module is part of the `camptocamp/odoo-repository `_ project on GitHub. + +You are welcome to contribute. diff --git a/odoo_project_changelog/__init__.py b/odoo_project_changelog/__init__.py new file mode 100644 index 00000000..0650744f --- /dev/null +++ b/odoo_project_changelog/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/odoo_project_changelog/__manifest__.py b/odoo_project_changelog/__manifest__.py new file mode 100644 index 00000000..bb99b0ca --- /dev/null +++ b/odoo_project_changelog/__manifest__.py @@ -0,0 +1,22 @@ +# Copyright 2024 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +{ + "name": "Odoo Project - Changelogs", + "summary": "Generate Changelogs from repositories for installed modules.", + "version": "16.0.1.0.0", + "category": "Tools", + "author": "Camptocamp, Odoo Community Association (OCA)", + "website": "https://github.com/camptocamp/odoo-repository", + "data": [ + "security/ir.model.access.csv", + "data/queue_job.xml", + "views/odoo_project.xml", + "report/ir_actions_report.xml", + "report/odoo_project_changelog.xml", + ], + "installable": True, + "depends": [ + "odoo_project", + ], + "license": "AGPL-3", +} diff --git a/odoo_project_changelog/data/queue_job.xml b/odoo_project_changelog/data/queue_job.xml new file mode 100644 index 00000000..96ac6013 --- /dev/null +++ b/odoo_project_changelog/data/queue_job.xml @@ -0,0 +1,31 @@ + + + + + + odoo_project_changelog + + + + + + _generate_changelog + + + + + + + _generate_changelog_report + + + + + diff --git a/odoo_project_changelog/models/__init__.py b/odoo_project_changelog/models/__init__.py new file mode 100644 index 00000000..3efaabd9 --- /dev/null +++ b/odoo_project_changelog/models/__init__.py @@ -0,0 +1,2 @@ +from . import odoo_project_repository +from . import odoo_project diff --git a/odoo_project_changelog/models/odoo_project.py b/odoo_project_changelog/models/odoo_project.py new file mode 100644 index 00000000..dfa5321e --- /dev/null +++ b/odoo_project_changelog/models/odoo_project.py @@ -0,0 +1,85 @@ +# Copyright 2024 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from urllib.parse import urljoin + +from odoo import fields, models + +from odoo.addons.queue_job.delay import chain +from odoo.addons.queue_job.job import identity_exact + + +class OdooProject(models.Model): + _inherit = "odoo.project" + + used_repository_ids = fields.One2many( + comodel_name="odoo.project.repository", + inverse_name="odoo_project_id", + string="Used Repositories", + context={"active_test": False}, + ) + changelog_enabled_repository_ids = fields.One2many( + comodel_name="odoo.project.repository", + inverse_name="odoo_project_id", + string="Enabled Repositories for CHANGELOG", + ) + changelog_state = fields.Selection( + selection=[ + ("none", "None"), + ("in_progress", "In progress"), + ("done", "Done"), + ], + default="none", + copy=False, + ) + changelog_url = fields.Char(compute="_compute_changelog_url") + + def _compute_changelog_url(self): + for rec in self: + rec.changelog_url = urljoin( + rec.get_base_url() + "/", + f"report/html/odoo_project_changelog.report_changelog/{rec.id}", + ) + + def action_generate_changelog(self): + self.ensure_one() + self.changelog_state = "in_progress" + self.used_repository_ids.changelog = False + jobs = self._create_jobs() + chain(*jobs).delay() + + def action_open_changelog(self): + return { + "type": "ir.actions.act_url", + "url": self.changelog_url, + "target": "_new", + "target_type": "public", + } + + def _create_jobs(self): + self.ensure_one() + jobs = [] + # Spawn jobs generating a changelog for each repository + for repo in self.used_repository_ids: + if not repo.active: + continue + delayable = repo.delayable( + description=( + f"Collect CHANGELOG data for {self.display_name}, " + f"repository {repo.repository_branch_id.display_name}" + ), + identity_key=identity_exact, + ) + job = delayable._generate_changelog() + jobs.append(job) + # Spawn job updating the CHANGELOG state to done + delayable = self.delayable( + description=(f"Set CHANGELOG as ready for {self.display_name}"), + identity_key=identity_exact, + ) + job = delayable._set_changelog_done() + jobs.append(job) + return jobs + + def _set_changelog_done(self): + self.changelog_state = "done" diff --git a/odoo_project_changelog/models/odoo_project_repository.py b/odoo_project_changelog/models/odoo_project_repository.py new file mode 100644 index 00000000..fb30e737 --- /dev/null +++ b/odoo_project_changelog/models/odoo_project_repository.py @@ -0,0 +1,87 @@ +# Copyright 2024 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import fields, models + +from ..utils.scanner import ChangelogScannerOdooEnv + + +class OdooProjectRepository(models.Model): + _name = "odoo.project.repository" + _description = "Repository used in a project" + + odoo_project_id = fields.Many2one( + comodel_name="odoo.project", + ondelete="cascade", + string="Project", + required=True, + index=True, + readonly=True, + ) + repository_branch_id = fields.Many2one( + comodel_name="odoo.repository.branch", + ondelete="cascade", + string="Repository Branch", + required=True, + index=True, + ) + deployed_commit = fields.Char(help="The changelog is generated from this commit.") + target_commit = fields.Char( + help=( + "Changelog is generated until this commit. " + "If not set, the latest commit of the branch is used." + ) + ) + active = fields.Boolean(string="Include", default=True) + changelog = fields.Serialized() + + def _prepare_changelog_scanner_parameters(self): + ir_config = self.env["ir.config_parameter"] + odoo_repository = self.repository_branch_id.repository_id + repositories_path = ir_config.get_param(odoo_repository._repositories_path_key) + return { + "org": odoo_repository.org_id.name, + "name": odoo_repository.name, + "clone_url": odoo_repository.clone_url, + "odoo_project_repository_id": self.id, + "repositories_path": repositories_path, + "repo_type": odoo_repository.repo_type, + "ssh_key": odoo_repository.ssh_key_id.private_key, + "token": odoo_repository._get_token(), + "env": self.env, + } + + def _generate_changelog(self): + self.ensure_one() + params = self._prepare_changelog_scanner_parameters() + scanner = ChangelogScannerOdooEnv(**params) + scanner.scan() + + def push_changelog(self, changelog): + """Store the changelog. Called by the scanner.""" + self.ensure_one() + self.changelog = changelog + self.target_commit = self.changelog["target_commit"] + + def _get_report_data(self): + """Return data used by the CHANGELOG report.""" + self.ensure_one() + project_module_model = self.env["odoo.project.module"] + if not self.changelog.get("modules"): + return {"categories": {}, "count": 0} + # Collect all related categories and sort them by name + project_module_ids = [ + int(project_module_id) for project_module_id in self.changelog["modules"] + ] + project_modules = project_module_model.browse(project_module_ids).exists() + categories = project_modules.category_id.sorted( + key=lambda o: (o.name or "").lower() # Case insensitive + ) + data = {"categories": {categ: {} for categ in categories}} + data["categories"][self.env["odoo.module.category"]] = {} + data["count"] = len(self.changelog["modules"]) + for project_module_id, module_data in self.changelog["modules"].items(): + project_module = project_module_model.browse(int(project_module_id)) + categ = project_module.category_id + data["categories"][categ][project_module] = module_data + return data diff --git a/odoo_project_changelog/readme/CONTRIBUTORS.rst b/odoo_project_changelog/readme/CONTRIBUTORS.rst new file mode 100644 index 00000000..a0c91e35 --- /dev/null +++ b/odoo_project_changelog/readme/CONTRIBUTORS.rst @@ -0,0 +1,2 @@ +* Camptocamp + * Sébastien Alix diff --git a/odoo_project_changelog/readme/DESCRIPTION.rst b/odoo_project_changelog/readme/DESCRIPTION.rst new file mode 100644 index 00000000..8110b39d --- /dev/null +++ b/odoo_project_changelog/readme/DESCRIPTION.rst @@ -0,0 +1 @@ +This module allows to generate CHANGELOGs for repositories used within a project. diff --git a/odoo_project_changelog/report/ir_actions_report.xml b/odoo_project_changelog/report/ir_actions_report.xml new file mode 100644 index 00000000..4412fd1b --- /dev/null +++ b/odoo_project_changelog/report/ir_actions_report.xml @@ -0,0 +1,17 @@ + + + + + + CHANGELOG + odoo.project + qweb-html + odoo_project_changelog.report_changelog + odoo_project_changelog.report_changelog + CHANGELOG + + report + + + diff --git a/odoo_project_changelog/report/odoo_project_changelog.xml b/odoo_project_changelog/report/odoo_project_changelog.xml new file mode 100644 index 00000000..7dad0740 --- /dev/null +++ b/odoo_project_changelog/report/odoo_project_changelog.xml @@ -0,0 +1,79 @@ + + + + + + + diff --git a/odoo_project_changelog/security/ir.model.access.csv b/odoo_project_changelog/security/ir.model.access.csv new file mode 100644 index 00000000..1964452d --- /dev/null +++ b/odoo_project_changelog/security/ir.model.access.csv @@ -0,0 +1,2 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_odoo_project_repository_user,odoo_project_repository_user,model_odoo_project_repository,odoo_repository.group_odoo_repository_user,1,1,1,1 diff --git a/odoo_project_changelog/static/description/index.html b/odoo_project_changelog/static/description/index.html new file mode 100644 index 00000000..68da1dab --- /dev/null +++ b/odoo_project_changelog/static/description/index.html @@ -0,0 +1,416 @@ + + + + + +Odoo Project - Changelogs + + + +
+

Odoo Project - Changelogs

+ + +

Beta License: AGPL-3 camptocamp/odoo-repository

+

This module allows to generate CHANGELOGs for repositories used within a project.

+

Table of contents

+ +
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Camptocamp
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is part of the camptocamp/odoo-repository project on GitHub.

+

You are welcome to contribute.

+
+
+
+ + diff --git a/odoo_project_changelog/utils/__init__.py b/odoo_project_changelog/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/odoo_project_changelog/utils/scanner.py b/odoo_project_changelog/utils/scanner.py new file mode 100644 index 00000000..8405ca92 --- /dev/null +++ b/odoo_project_changelog/utils/scanner.py @@ -0,0 +1,50 @@ +# Copyright 2024 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo.addons.odoo_repository.lib.scanner import ChangelogScanner + + +class ChangelogScannerOdooEnv(ChangelogScanner): + """ChangelogScanner running on the same server than Odoo. + + This class takes an additional `env` parameter (`odoo.api.Environment`) + used to request Odoo, and implement required methods to use it. + """ + + def __init__(self, *args, **kwargs): + if kwargs.get("env"): + self.env = kwargs.pop("env") + super().__init__(*args, **kwargs) + + def _get_odoo_project_repository_data(self, project_repo_id): + project_repo = ( + self.env["odoo.project.repository"].browse(project_repo_id).exists() + ) + project = project_repo.odoo_project_id + data = { + "odoo_project_id": project.id, + "branch": project.odoo_version_id.name, + "source_commit": project_repo.deployed_commit, + "target_commit": project_repo.target_commit, + "modules": [ + # List of dicts {"id": PROJECT_MODULE_ID, ...} + # {"id": 1, "name": "base", "path": "odoo/addons/base"}, + # {"id": 2, "name": "account", "path": "addons/account"}, + {"id": mod.id, "name": mod.module_name, "path": mod.full_path} + for mod in project.project_module_ids.filtered_domain( + [ + ( + "repository_branch_id", + "=", + project_repo.repository_branch_id.id, + ) + ] + ) + ], + } + return data + + def _push_odoo_project_repository_changelog(self, project_repo_id, changelog): + self.env["odoo.project.repository"].browse( + project_repo_id + ).exists().push_changelog(changelog) diff --git a/odoo_project_changelog/views/odoo_project.xml b/odoo_project_changelog/views/odoo_project.xml new file mode 100644 index 00000000..fc4426ae --- /dev/null +++ b/odoo_project_changelog/views/odoo_project.xml @@ -0,0 +1,67 @@ + + + + + + odoo.project.form.inherit + odoo.project + + + + +
+ +
+ + + + + + + + + + + + + + + + + diff --git a/odoo_repository/lib/scanner.py b/odoo_repository/lib/scanner.py index f4aac4b3..dc0ce30d 100644 --- a/odoo_repository/lib/scanner.py +++ b/odoo_repository/lib/scanner.py @@ -29,6 +29,20 @@ MANIFEST_FILES = ("__manifest__.py", "__openerp__.py") +AUTHOR_EMAILS_TO_SKIP = [ + "transbot@odoo-community.org", + "noreply@weblate.org", + "oca-git-bot@odoo-community.org", + "oca+oca-travis@odoo-community.org", + "oca-ci@odoo-community.org", + "shopinvader-git-bot@shopinvader.com", +] + +SUMMARY_TERMS_TO_SKIP = [ + "Translated using Weblate", + "Added translation using Weblate", +] + @contextlib.contextmanager def set_env(**environ): @@ -376,19 +390,32 @@ def _filter_file_path(self, path): def _get_last_commit_of_git_tree(self, ref, tree): return tree.repo.git.log("--pretty=%H", "-n 1", ref, "--", tree.path) - def _get_commits_of_git_tree(self, from_, to_, tree): + def _get_commits_of_git_tree(self, from_, to_, tree, patterns=None): """Returns commits between `from_` and `to_` in chronological order. The list of commits can be limited to a `tree`. """ + if not patterns: + patterns = tuple() rev_pattern = f"{from_}..{to_}" if not from_: rev_pattern = to_ elif not to_: rev_pattern = from_ - commits = tree.repo.git.log( - "--pretty=%H", "-r", rev_pattern, "--reverse", "--", tree.path - ) + cmd = [ + "--pretty=%H", + "-r", + rev_pattern, + "--reverse", + "--", + tree.path, + *patterns, + ] + if patterns: + # It's mandatory to use shell here to leverage file patterns + commits = tree.repo.git.execute(" ".join(["git", "log"] + cmd), shell=True) + else: + commits = tree.repo.git.log(cmd) return commits.split() def _odoo_module(self, tree): @@ -1131,3 +1158,146 @@ def _push_scanned_data(self, repo_branch_id, module, data): def _update_last_scanned_commit(self, repo_branch_id, last_scanned_commit): """Update the last scanned commit for the repository/branch.""" raise NotImplementedError + + +class ChangelogScanner(BaseScanner): + """Generate a changelog for a repository used in a project.""" + + def __init__( + self, + org: str, + name: str, + clone_url: str, + odoo_project_repository_id: int, + repositories_path: str = None, + repo_type: str = None, + ssh_key: str = None, + token: str = None, + ): + self.odoo_project_repository_id = odoo_project_repository_id + data = self._get_odoo_project_repository_data(odoo_project_repository_id) + self.branch = data["branch"] + self.source_commit = data["source_commit"] + self.target_commit = data["target_commit"] or f"origin/{self.branch}" + self.modules = data["modules"] + super().__init__( + org, + name, + clone_url, + [self.branch], + repositories_path, + repo_type, + ssh_key, + token, + ) + + def scan(self): + res = self.sync() + changelog = self._generate_changelog() + self._push_odoo_project_repository_changelog( + self.odoo_project_repository_id, changelog + ) + return res + + def _generate_changelog(self): + with self.repo() as repo: + if not self._branch_exists(repo, self.branch): + return + last_commit = self._get_last_fetched_commit(repo, self.branch) + changelog = { + "source_commit": self.source_commit, + "target_commit": last_commit, + "modules": {}, + } + for module_data in self.modules: + module_path = module_data["path"] + _logger.info( + "%s#%s: generate changelog for %s", + self.full_name, + self.branch, + module_path, + ) + module_changelog = self._generate_module_changelog(repo, module_path) + if module_changelog: + changelog["modules"][module_data["id"]] = module_changelog + return changelog + + def _generate_module_changelog(self, repo, module_path): + changelog = [] + tree = self._get_subtree(repo.commit(self.source_commit).tree, module_path) + if not tree: + return changelog + # Leverage git pathspecs magic (patterns) as it is faster than checking + # the content (diffs) within Python process to get only relevant commits.. + commits = self._get_commits_of_git_tree( + self.source_commit, + self.target_commit, + tree, + patterns=( + "':^*/i18n/*'", + "':^*/i18n_extra/*'", + "':^*.html'", + "':^*.rst'", + "':^*/tests/*'", + "':^*/demo/*'", + "':^*/doc/*'", + ), + ) + for commit_sha in commits: + commit = repo.commit(commit_sha) + if self._skip_commit(commit): + continue + changelog.append(self._prepare_module_changelog(commit)) + return changelog + + @staticmethod + def _skip_commit(commit): + """Check if a commit should be skipped or not. + + E.g merge or translations commits are skipped. + """ + return ( + # Skip merge commit + len(commit.parents) > 1 + or commit.author.email in AUTHOR_EMAILS_TO_SKIP + or any([term in commit.summary for term in SUMMARY_TERMS_TO_SKIP]) + ) + + def _prepare_module_changelog(self, commit): + message = commit.message.split("\n") + message.pop(0) # Remove redundant summary (first line) + message = "\n".join(message).strip() + return { + "hexsha": commit.hexsha, + "authored_datetime": commit.authored_datetime.replace( + tzinfo=None + ).isoformat(), + "summary": commit.summary, + "message": message, + } + + def _get_odoo_project_repository_data(self, project_repo_id): + """Return required data to generate the changelog. + + Return a dictionary such as: + + { + "odoo_project_id": 10, + "branch": "17.0", + "source_commit": "7b58a288b3d79fbdc91dbf14aaeac0d69d65c327", + "target_commit": None, + "modules": [ + # List of dicts {"id": PROJECT_MODULE_ID, ...} + {"id": 1, "name": "base", "path": "odoo/addons/base"}, + {"id": 2, "name": "account", "path": "addons/account"}, + ] + } + """ + raise NotImplementedError + + def _push_odoo_project_repository_changelog(self, project_repo_id, changelog): + """Push the resulting changelog to its 'odoo.project.repository' object. + + It has to use the 'odoo.project.repository.push_changelog' RPC endpoint. + """ + raise NotImplementedError diff --git a/setup/odoo_project_changelog/odoo/addons/odoo_project_changelog b/setup/odoo_project_changelog/odoo/addons/odoo_project_changelog new file mode 120000 index 00000000..b5425a49 --- /dev/null +++ b/setup/odoo_project_changelog/odoo/addons/odoo_project_changelog @@ -0,0 +1 @@ +../../../../odoo_project_changelog \ No newline at end of file diff --git a/setup/odoo_project_changelog/setup.py b/setup/odoo_project_changelog/setup.py new file mode 100644 index 00000000..28c57bb6 --- /dev/null +++ b/setup/odoo_project_changelog/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +)