From 27fa57bd52ed91d952ee84cb2a7fe2f8592fb3d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Alix?= Date: Wed, 28 May 2025 13:46:55 +0200 Subject: [PATCH 1/6] odoo_repository: scan multiple branches of a repository On repositories, the Odoo version and branch to clone can now be set manually in the `odoo.repository.branch` relation. This allows to set different Odoo versions/branches hosted in a repository instead of one. --- odoo_repository/__manifest__.py | 2 +- odoo_repository/data/odoo_branch.xml | 15 +- odoo_repository/lib/scanner.py | 87 +++++++--- .../migrations/16.0.1.2.0/pre-migration.py | 43 +++++ odoo_repository/models/odoo_branch.py | 33 +++- odoo_repository/models/odoo_module_branch.py | 20 ++- .../models/odoo_module_branch_version.py | 4 +- odoo_repository/models/odoo_repository.py | 162 +++++++++--------- .../models/odoo_repository_branch.py | 14 +- odoo_repository/tests/common.py | 10 +- .../tests/test_odoo_repository_scan.py | 30 ++-- .../tests/test_repository_scanner.py | 16 +- odoo_repository/tests/test_sync_node.py | 2 +- odoo_repository/utils/scanner.py | 14 +- odoo_repository/views/odoo_branch.xml | 12 +- odoo_repository/views/odoo_repository.xml | 39 ++--- .../views/odoo_repository_branch.xml | 2 + 17 files changed, 303 insertions(+), 202 deletions(-) create mode 100644 odoo_repository/migrations/16.0.1.2.0/pre-migration.py diff --git a/odoo_repository/__manifest__.py b/odoo_repository/__manifest__.py index d1d990ca..a89686da 100644 --- a/odoo_repository/__manifest__.py +++ b/odoo_repository/__manifest__.py @@ -3,7 +3,7 @@ { "name": "Odoo Repositories Data", "summary": "Base module to host data collected from Odoo repositories.", - "version": "16.0.1.1.0", + "version": "16.0.1.2.0", "category": "Tools", "author": "Camptocamp, Odoo Community Association (OCA)", "website": "https://github.com/camptocamp/odoo-repository", diff --git a/odoo_repository/data/odoo_branch.xml b/odoo_repository/data/odoo_branch.xml index 382a76a6..46196f60 100644 --- a/odoo_repository/data/odoo_branch.xml +++ b/odoo_repository/data/odoo_branch.xml @@ -14,61 +14,56 @@ 8.0 - 9.0 - 10.0 - 11.0 - 12.0 - 13.0 - 14.0 - 15.0 - 16.0 - 17.0 - + + + + + 18.0 diff --git a/odoo_repository/lib/scanner.py b/odoo_repository/lib/scanner.py index b75d4737..b59c4ef2 100644 --- a/odoo_repository/lib/scanner.py +++ b/odoo_repository/lib/scanner.py @@ -421,6 +421,8 @@ def __init__( org: str, name: str, clone_url: str, + # FIXME: put specific branch names to clone in 'migration_path': + # E.g. [('14.0', 'master'), ('18.0', '18.0-mig')] migration_path: tuple[str], new_repo_name: str = None, new_repo_url: str = None, @@ -431,7 +433,7 @@ def __init__( workaround_fs_errors: bool = False, clone_name: str = None, ): - branches = sorted(migration_path) + branches = [mp[1] for mp in sorted(migration_path)] super().__init__( org, name, @@ -460,7 +462,7 @@ def sync(self, fetch=True): self._set_git_remote_url(repo, self.new_repo_name, self.new_repo_url) return res - def scan(self, addons_path=".", module_names=None): + def scan(self, addons_path=".", target_addons_path=".", module_names=None): # Clone/fetch has been done during the repository scan, the migration # scan will be processed on the current history of commits res = self.sync(fetch=False) @@ -468,7 +470,10 @@ def scan(self, addons_path=".", module_names=None): # there is nothing to scan then. if not res: return False - source_branch, target_branch = self.migration_path + source_version = self.migration_path[0][0] + source_branch = self.migration_path[0][1] + target_version = self.migration_path[1][0] + target_branch = self.migration_path[1][1] target_remote = "origin" with self.repo() as repo: if self.new_repo_name and self.new_repo_url: @@ -482,10 +487,13 @@ def scan(self, addons_path=".", module_names=None): ): return self._scan_migration_path( repo, + source_version, source_branch, target_remote, + target_version, target_branch, addons_path=addons_path, + target_addons_path=target_addons_path, module_names=module_names, ) return res @@ -493,10 +501,13 @@ def scan(self, addons_path=".", module_names=None): def _scan_migration_path( self, repo, + source_version, source_branch, target_remote, + target_version, target_branch, addons_path=".", + target_addons_path=".", module_names=None, ): repo_source_commit = self._get_last_fetched_commit(repo, source_branch) @@ -520,23 +531,27 @@ def _scan_migration_path( continue repo_id = self._get_odoo_repository_id() module_branch_id = self._get_odoo_module_branch_id( - repo_id, module, source_branch + repo_id, module, source_version ) if not module_branch_id: _logger.warning( - "Module '%s' for branch %s does not exist on Odoo, " + "Module '%s' for version %s does not exist on Odoo, " "a new scan of the repository is required. Aborted" - % (module, source_branch) + % (module, source_version) ) continue # For each module and source/target branch: # - get commit of 'module' relative to the last fetched commit # - get commit of 'module' relative to the last scanned commit + module_path = str(pathlib.Path(addons_path).joinpath(module)) + target_module_path = str( + pathlib.Path(target_addons_path).joinpath(target_module) + ) module_source_tree = self._get_subtree( - repo.commit(repo_source_commit).tree, module + repo.commit(repo_source_commit).tree, module_path ) module_target_tree = self._get_subtree( - repo.commit(repo_target_commit).tree, target_module + repo.commit(repo_target_commit).tree, target_module_path ) module_source_commit = self._get_last_commit_of_git_tree( repo_source_commit, module_source_tree @@ -550,7 +565,7 @@ def _scan_migration_path( ) # Retrieve existing migration data if any and check if it is outdated data = self._get_odoo_module_branch_migration_data( - repo_id, module, source_branch, target_branch + repo_id, module, source_version, target_version ) if ( data.get("last_source_mig_scanned_commit") != module_source_commit @@ -559,11 +574,14 @@ def _scan_migration_path( scanned_data = self._scan_module( repo, addons_path, + target_addons_path, module, target_module, module_branch_id, + source_version, source_branch, target_remote, + target_version, target_branch, module_source_commit, module_target_commit, @@ -579,11 +597,14 @@ def _scan_module( self, repo: git.Repo, addons_path: str, + target_addons_path: str, module: str, target_module: str, module_branch_id: int, + source_version: str, source_branch: str, target_remote: str, + target_version: str, target_branch: str, source_commit: str, target_commit: str, @@ -595,14 +616,19 @@ def _scan_module( """Collect the migration data of a module.""" data = { "addons_path": addons_path, + "target_addons_path": addons_path, "module": module, + "source_version": source_version, "source_branch": source_branch, + "target_version": target_version, "target_branch": target_branch, "source_commit": source_last_scanned_commit, "target_commit": target_last_scanned_commit, } module_path = str(pathlib.Path(addons_path).joinpath(module)) - target_module_path = str(pathlib.Path(addons_path).joinpath(target_module)) + target_module_path = str( + pathlib.Path(target_addons_path).joinpath(target_module) + ) # If files updated in the module since the last scan are not relevant # (e.g. all new commits are updating PO files), we skip the scan. source_scan_relevant = self._is_scan_module_relevant( @@ -631,14 +657,16 @@ def _scan_module( "%s: relevant changes detected in '%s' (%s -> %s)", self.full_name, module if source_scan_relevant else target_module, - source_branch, - target_branch, + source_version, + target_version, ) oca_port_data = self._run_oca_port( module_path, target_module_path, + source_version, source_branch, target_remote, + target_version, target_branch, ) data["report"] = oca_port_data @@ -696,8 +724,10 @@ def _run_oca_port( self, module_path, target_module_path, + source_version, source_branch, target_remote, + target_version, target_branch, ): _logger.info( @@ -710,7 +740,9 @@ def _run_oca_port( # Initialize the oca-port app params = { "source": f"origin/{source_branch}", + "source_version": source_version, "target": f"{target_remote}/{target_branch}", + "target_version": target_version, "addon_path": module_path, "target_addon_path": target_module_path, "upstream_org": self.org, @@ -756,7 +788,7 @@ def _get_odoo_module_branch_migration_id( raise NotImplementedError def _get_odoo_module_branch_migration_data( - self, repo_id, module, source_branch, target_branch + self, repo_id, module, source_version, target_version ) -> dict: """Return last scanned commits regarding `module`.""" raise NotImplementedError @@ -776,6 +808,7 @@ def __init__( org: str, name: str, clone_url: str, + version: str, branch: str, addons_paths_data: list, repositories_path: str = None, @@ -797,6 +830,7 @@ def __init__( workaround_fs_errors, clone_name, ) + self.version = version self.branch = branch self.addons_paths_data = addons_paths_data @@ -813,8 +847,13 @@ def detect_modules_to_scan(self): def _detect_modules_to_scan(self, repo, repo_id): if not self._branch_exists(repo, self.branch): return - branch_id = self._get_odoo_branch_id(repo_id, self.branch) - repo_branch_id = self._create_odoo_repository_branch(repo_id, branch_id) + branch_id = self._get_odoo_branch_id(self.version) + cloned_branch = None + if self.version != self.branch: + cloned_branch = self.branch + repo_branch_id = self._create_odoo_repository_branch( + repo_id, branch_id, cloned_branch=cloned_branch + ) last_fetched_commit = self._get_last_fetched_commit(repo, self.branch) last_scanned_commit = self._get_repo_last_scanned_commit(repo_branch_id) data = { @@ -877,8 +916,13 @@ def _detect_modules_to_scan_in_addons_path( def scan_module(self, module_path, specs): self._apply_git_global_config() repo_id = self._get_odoo_repository_id() - branch_id = self._get_odoo_branch_id(repo_id, self.branch) - repo_branch_id = self._create_odoo_repository_branch(repo_id, branch_id) + branch_id = self._get_odoo_branch_id(self.version) + cloned_branch = None + if self.version != self.branch: + cloned_branch = self.branch + repo_branch_id = self._create_odoo_repository_branch( + repo_id, branch_id, cloned_branch=cloned_branch + ) with self.repo() as repo: # Checkout the source branch to perform module code analysis branch_commit = self._get_last_fetched_commit(repo, self.branch) @@ -1049,13 +1093,8 @@ def _get_odoo_repository_id(self): """Return the ID of the 'odoo.repository' record.""" raise NotImplementedError - def _get_odoo_branch_id(self, repo_id, branch): - """Return the ID of the relevant 'odoo.branch' record. - - If the repository is cloned from a specific branch name - (like 'master' or 'main'), return the ID of the configured - Odoo version (`odoo.branch.odoo_version_id`). - """ + def _get_odoo_branch_id(self, version): + """Return the ID of the relevant 'odoo.branch' record.""" raise NotImplementedError def _get_odoo_repository_branch_id(self, repo_id, branch_id): diff --git a/odoo_repository/migrations/16.0.1.2.0/pre-migration.py b/odoo_repository/migrations/16.0.1.2.0/pre-migration.py new file mode 100644 index 00000000..158310ab --- /dev/null +++ b/odoo_repository/migrations/16.0.1.2.0/pre-migration.py @@ -0,0 +1,43 @@ +import logging + +from openupgradelib import openupgrade as ou + +from odoo.tools import sql + +_logger = logging.getLogger(__name__) + + +def migrate(cr, version): + if not version: + return + set_xml_ids_on_odoo_branch(cr) + migrate_repository_clone_branch_id_to_repository_branch(cr) + + +def set_xml_ids_on_odoo_branch(cr): + _logger.info("Add XML-ID on 'odoo.branch' 18.0...") + query = "SELECT id FROM odoo_branch WHERE name='18.0';" + cr.execute(query) + row = cr.fetchone() + if row: + branch_id = row[0] + ou.add_xmlid(cr, "odoo_repository", "odoo_branch_18", "odoo.branch", branch_id) + + +def migrate_repository_clone_branch_id_to_repository_branch(cr): + _logger.info( + "Migrate 'clone_branch_id' from 'odoo.repository' to 'odoo.repository.branch'..." + ) + # Create 'odoo_repository_branch.cloned_branch' + if not sql.column_exists(cr, "odoo_repository_branch", "cloned_branch"): + sql.create_column(cr, "odoo_repository_branch", "cloned_branch", "varchar") + # Migrate values from 'odoo_repository.clone_branch_id' to this new column + query = """ + UPDATE odoo_repository_branch + SET cloned_branch=br.name + FROM odoo_repository repo + JOIN odoo_branch br + ON repo.clone_branch_id=br.id + WHERE repo.id = odoo_repository_branch.repository_id; + """ + cr.execute(query) diff --git a/odoo_repository/models/odoo_branch.py b/odoo_repository/models/odoo_branch.py index 66b0f64d..8d26dae2 100644 --- a/odoo_repository/models/odoo_branch.py +++ b/odoo_repository/models/odoo_branch.py @@ -1,16 +1,26 @@ # Copyright 2023 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) -from odoo import api, fields, models +import re + +from odoo import _, api, fields, models +from odoo.exceptions import ValidationError class OdooBranch(models.Model): _name = "odoo.branch" - _description = "Odoo Branch" + _description = "Odoo Branch/Version" _order = "sequence, name" - name = fields.Char(required=True, index=True) - odoo_version = fields.Boolean(default=True) + name = fields.Char( + string="Version", + required=True, + index=True, + help=( + "An Odoo version is also used as an Odoo branch name in generic " + "repositories (Odoo, OCA...)." + ), + ) active = fields.Boolean(default=True) repository_branch_ids = fields.One2many( comodel_name="odoo.repository.branch", @@ -24,6 +34,14 @@ class OdooBranch(models.Model): ("name_uniq", "UNIQUE (name)", "This branch already exists."), ] + @api.constrains("name") + def _constrains_name(self): + odoo_version_pattern = r"^[0-9]+\.[0-9]$" + for rec in self: + version = re.search(odoo_version_pattern, rec.name) + if not version: + raise ValidationError(_("Version must match the pattern 'x.y'.")) + @api.model def _recompute_sequence(self): """Recompute the 'sequence' field to get release branches sorted.""" @@ -41,7 +59,6 @@ def _recompute_sequence(self): ORDER BY string_to_array(name, '.')::int[] ) AS position FROM odoo_branch - WHERE odoo_version = true ) as pos WHERE pos.id = %(id)s ) @@ -76,8 +93,6 @@ def action_force_scan(self): """ return self.action_scan(force=True) - def _get_all_odoo_versions(self): + def _get_all_odoo_versions(self, active_test=False): """Return all Odoo versions, even archived ones.""" - return self.with_context(active_test=False).search( - [("odoo_version", "=", True)] - ) + return self.with_context(active_test=active_test).search([]) diff --git a/odoo_repository/models/odoo_module_branch.py b/odoo_repository/models/odoo_module_branch.py index 4250cde9..569e6c89 100644 --- a/odoo_repository/models/odoo_module_branch.py +++ b/odoo_repository/models/odoo_module_branch.py @@ -58,8 +58,7 @@ class OdooModuleBranch(models.Model): # modules without knowing in advance what is their repo (orphaned modules). comodel_name="odoo.branch", ondelete="cascade", - string="Branch", - domain=[("odoo_version", "=", True)], + string="Odoo Version", required=True, index=True, ) @@ -243,16 +242,23 @@ def _compute_full_path(self): rec.module_name ) - @api.depends("repository_id.repo_url", "branch_name", "addons_path", "module_name") + @api.depends( + "repository_id.repo_url", + "branch_name", + "repository_branch_id.cloned_branch", + "addons_path", + "module_name", + ) def _compute_url(self): for rec in self: rec.url = False if not rec.repository_id: continue - module_path = "/".join([self.addons_path or ".", self.module_name]) - rec.url = self.repository_id._get_resource_url( - self.branch_name, module_path - ) + branch = rec.branch_name + if rec.repository_branch_id.cloned_branch: + branch = rec.repository_branch_id.cloned_branch + module_path = "/".join([rec.addons_path or ".", rec.module_name]) + rec.url = rec.repository_id._get_resource_url(branch, module_path) @api.depends("repository_branch_id.name", "module_id.name") def _compute_name(self): diff --git a/odoo_repository/models/odoo_module_branch_version.py b/odoo_repository/models/odoo_module_branch_version.py index 534ee988..e248a24e 100644 --- a/odoo_repository/models/odoo_module_branch_version.py +++ b/odoo_repository/models/odoo_module_branch_version.py @@ -105,8 +105,10 @@ def _compute_migration_script_url(self): rec.manifest_value, ] ) + rb = rec.module_branch_id.repository_branch_id + branch_name = rb.cloned_branch or rb.branch_id.name rec.migration_script_url = repo._get_resource_url( - rec.module_branch_id.branch_name, migration_path + branch_name, migration_path ) def _to_dict(self): diff --git a/odoo_repository/models/odoo_repository.py b/odoo_repository/models/odoo_repository.py index 632d372d..e7bb365d 100644 --- a/odoo_repository/models/odoo_repository.py +++ b/odoo_repository/models/odoo_repository.py @@ -77,19 +77,6 @@ class OdooRepository(models.Model): string="Token", help="Token used to clone/fetch this repository.", ) - clone_branch_id = fields.Many2one( - comodel_name="odoo.branch", - ondelete="restrict", - string="Branch to clone", - help="Branch to clone if different than configured ones", - domain=[("odoo_version", "=", False)], - ) - odoo_version_id = fields.Many2one( - comodel_name="odoo.branch", - ondelete="restrict", - string="Odoo Version", - domain=[("odoo_version", "=", True)], - ) active = fields.Boolean(default=True) addons_path_ids = fields.Many2many( comodel_name="odoo.repository.addons_path", @@ -100,7 +87,6 @@ class OdooRepository(models.Model): comodel_name="odoo.repository.branch", inverse_name="repository_id", string="Branches", - readonly=True, ) scan_weekday_ids = fields.Many2many( comodel_name="time.weekday", @@ -111,9 +97,6 @@ class OdooRepository(models.Model): ), ) specific = fields.Boolean( - compute="_compute_specific", - store=True, - readonly=False, help=( "Host specific modules. " "By default if the repository clones a specific branch, " @@ -143,8 +126,8 @@ def _compute_github_url(self): _sql_constraints = [ ( - "org_id_name_repository_id_uniq", - "UNIQUE (org_id, name, odoo_version_id)", + "org_id_name_uniq", + "UNIQUE (org_id, name)", "This repository already exists.", ), ] @@ -154,11 +137,6 @@ def _compute_display_name(self): for rec in self: rec.display_name = f"{rec.org_id.name}/{rec.name}" - @api.depends("clone_branch_id") - def _compute_specific(self): - for rec in self: - rec.specific = bool(rec.clone_branch_id) - @api.onchange("repo_url", "to_scan", "clone_url") def _onchange_repo_url(self): if not self.repo_url: @@ -171,9 +149,11 @@ def _onchange_repo_url(self): self.clone_url = self.repo_url break - @api.model - def _get_odoo_branches_to_clone(self): - return self.env["odoo.branch"].search([("odoo_version", "=", True)]) + def _get_odoo_branches_to_scan(self): + self.ensure_one() + if self.specific: + return self.branch_ids.branch_id + return self.env["odoo.branch"]._get_all_odoo_versions(active_test=True) def _cron_scanner_domain(self): today = fields.Date.today() @@ -197,12 +177,16 @@ def cron_scanner(self, branches=None, force=False): As the scanner is run on the same server than Odoo, a special class `RepositoryScannerOdooEnv` is used so the scanner can request Odoo through an environment (api.Environment). + + `branches` parameter allows to filter the `odoo.branch` to take into + account for the scan, e.g. `branches=["16.0", "18.0"]`. """ repositories = self.search(self._cron_scanner_domain()) - if not branches: - branches = self._get_odoo_branches_to_clone().mapped("name") + branches_ = self.env["odoo.branch"]._get_all_odoo_versions(active_test=True) + if branches: + branches_ = branches_.filtered(lambda br: br.name in branches) for repo in repositories: - repo.action_scan(branches=branches, force=force, raise_exc=False) + repo.action_scan(branch_ids=branches_.ids, force=force, raise_exc=False) def _check_config(self): # Check the configuration of repositories folder @@ -250,26 +234,34 @@ def _check_existing_jobs(self, raise_exc=True): return True return False - def action_scan(self, branches=None, force=False, raise_exc=True): + def action_scan(self, branch_ids=None, force=False, raise_exc=True): """Scan the whole repository.""" self._check_config() for rec in self: + if not rec.to_scan: + continue if rec._check_existing_jobs(raise_exc=raise_exc): continue - # Copy `branches` list to not override initial values - branches_ = branches and branches[:] or [] - if not rec.to_scan: - return False - if rec.clone_branch_id: - # Repository qualified with e.g. '17.0' branch but cloning a - # different branch like 'main' - branches_ = [rec.clone_branch_id.name] - if not branches_: - branches_ = rec._get_odoo_branches_to_clone().mapped("name") - if not branches_: - raise UserError(_("No branches to scan.")) + # Get branch records to scan + branches = rec._get_odoo_branches_to_scan() + if branch_ids: + branches = branches & self.env["odoo.branch"].search( + [("id", "in", branch_ids)] + ) + if not branches: + continue + # Branch names to scan could be different on specific repositories + # (e.g. 'master' or 'main' branch, representing a 16.0 Odoo version) + # => create a list of tuples ({odoo_version}, {branch_name}). + versions_branches = [(branch.name, branch.name) for branch in branches] + if rec.specific: + versions_branches = [ + (rb.branch_id.name, rb.cloned_branch or rb.branch_id.name) + for rb in rec.branch_ids + if rb.branch_id in branches + ] if force: - rec._reset_scanned_commits(branches_) + rec._reset_scanned_commits(branch_ids=branch_ids) # Scan repository branches sequentially as they need to be checked out # to perform the analysis # Here the launched job is responsible to: @@ -278,27 +270,33 @@ def action_scan(self, branches=None, force=False, raise_exc=True): # 3) spawn a job to update the last scanned commit of the repo/branch # 4) spawn the next job responsible to detect modules updated # on the next branch - branch = branches_[0] - next_branches = branches_[1:] + version_branch = versions_branches[0] + next_versions_branches = versions_branches[1:] job = rec._create_job_detect_modules_to_scan_on_branch( - branch, next_branches, branches_ + version_branch, next_versions_branches, versions_branches ) job.delay() return True def _create_job_detect_modules_to_scan_on_branch( - self, branch, next_branches, all_branches + self, version_branch, next_versions_branches, all_versions_branches ): self.ensure_one() + version, branch = version_branch + branch_str = branch + if version != branch: + branch_str = f"{branch} ({version})" delayable = self.delayable( - description=f"Detect modules to scan in {self.display_name}#{branch}", + description=f"Detect modules to scan in {self.display_name}#{branch_str}", identity_key=identity_exact, ) return delayable._detect_modules_to_scan_on_branch( - branch, next_branches, all_branches + version_branch, next_versions_branches, all_versions_branches ) - def _detect_modules_to_scan_on_branch(self, branch, next_branches, all_branches): + def _detect_modules_to_scan_on_branch( + self, version_branch, next_versions_branches, all_versions_branches + ): """Detect the modules to scan on `branch`. It will spawn a job for each module to scan, and two other jobs to: @@ -307,14 +305,15 @@ def _detect_modules_to_scan_on_branch(self, branch, next_branches, all_branches) This ensure to scan different branches sequentially for a given repository. """ + version, branch = version_branch try: # Get the list of modules updated since last scan - params = self._prepare_scanner_parameters(branch) + params = self._prepare_scanner_parameters(version, branch) scanner = RepositoryScannerOdooEnv(**params) data = scanner.detect_modules_to_scan() # Prepare all subsequent jobs based on modules to scan jobs = self._create_subsequent_jobs( - branch, next_branches, all_branches, data + version_branch, next_versions_branches, all_versions_branches, data ) # Chain them altogether if jobs: @@ -322,13 +321,16 @@ def _detect_modules_to_scan_on_branch(self, branch, next_branches, all_branches) except Exception as exc: raise RetryableJobError("Scanner error") from exc - def _create_subsequent_jobs(self, branch, next_branches, all_branches, data): + def _create_subsequent_jobs( + self, version_branch, next_versions_branches, all_versions_branches, data + ): jobs = [] + version, branch = version_branch # Spawn one job per module to scan for data_ in data.get("addons_paths", {}).values(): for module_path in data_["modules_to_scan"]: job = self._create_job_scan_module_on_branch( - branch, module_path, data_["specs"] + version, branch, module_path, data_["specs"] ) jobs.append(job) # + another one to update the last scanned commit of the repository @@ -339,28 +341,31 @@ def _create_subsequent_jobs(self, branch, next_branches, all_branches, data): ) jobs.append(job) # + another one to detect modules to scan on the next branch - branch = next_branches and next_branches[0] - next_branches = next_branches[1:] - if branch: + version_branch = next_versions_branches and next_versions_branches[0] + next_versions_branches = next_versions_branches[1:] + if version_branch: jobs.append( self._create_job_detect_modules_to_scan_on_branch( - branch, next_branches, all_branches + version_branch, next_versions_branches, all_versions_branches ) ) return jobs - def _create_job_scan_module_on_branch(self, branch, module_path, specs): + def _create_job_scan_module_on_branch(self, version, branch, module_path, specs): self.ensure_one() + branch_str = branch + if version != branch: + branch_str = f"{branch} ({version})" delayable = self.delayable( - description=f"Scan {self.display_name}#{branch} - {module_path}", + description=f"Scan {self.display_name}#{branch_str} - {module_path}", identity_key=identity_exact, ) - return delayable._scan_module_on_branch(branch, module_path, specs) + return delayable._scan_module_on_branch(version, branch, module_path, specs) - def _scan_module_on_branch(self, branch, module_path, specs): + def _scan_module_on_branch(self, version, branch, module_path, specs): """Scan `module_path` from `branch`.""" try: - params = self._prepare_scanner_parameters(branch) + params = self._prepare_scanner_parameters(version, branch) scanner = RepositoryScannerOdooEnv(**params) return scanner.scan_module(module_path, specs) except Exception as exc: @@ -378,22 +383,20 @@ def _create_job_update_last_scanned_commit( ) return delayable._update_last_scanned_commit(last_scanned_commit) - def _reset_scanned_commits(self, branches=None): + def _reset_scanned_commits(self, branch_ids=None): """Reset the scanned commits. This will make the next repository scan restarting from the beginning, and thus making it slower. """ self.ensure_one() - if branches is None: - branches = [] - branches_ = ( - self.branch_ids.filtered(lambda br: br.branch_id.name in branches) - if branches and not self.clone_branch_id - else self.branch_ids + if branch_ids is None: + branch_ids = self.branch_ids.branch_id.ids + repo_branches = self.branch_ids.filtered( + lambda rb: rb.branch_id.id in branch_ids ) - branches_.write({"last_scanned_commit": False}) - branches_.module_ids.sudo().write({"last_scanned_commit": False}) + repo_branches.write({"last_scanned_commit": False}) + repo_branches.module_ids.sudo().write({"last_scanned_commit": False}) def _get_token(self): """Return the first available token found for this repository. @@ -410,13 +413,14 @@ def _get_token(self): or os.environ.get("GITHUB_TOKEN") ) - def _prepare_scanner_parameters(self, branch): + def _prepare_scanner_parameters(self, version, branch): ir_config = self.env["ir.config_parameter"] repositories_path = ir_config.sudo().get_param(self._repositories_path_key) return { "org": self.org_id.name, "name": self.name, "clone_url": self.clone_url, + "version": version, "branch": branch, "addons_paths_data": self.addons_path_ids.read( [ @@ -437,14 +441,14 @@ def _prepare_scanner_parameters(self, branch): "env": self.env, } - def action_force_scan(self, branches=None, raise_exc=True): + def action_force_scan(self, branch_ids=None, raise_exc=True): """Force the scan of the repositories. It will restart the scan without considering the last scanned commit, overriding already collected module data if any. """ self.ensure_one() - return self.action_scan(branches=branches, force=True, raise_exc=raise_exc) + return self.action_scan(branch_ids=branch_ids, force=True, raise_exc=raise_exc) @api.model def cron_fetch_data(self, branches=None, force=False): @@ -456,7 +460,7 @@ def cron_fetch_data(self, branches=None, force=False): ) if not main_node_url: return False - branch_domain = [("odoo_version", "=", True)] + branch_domain = [] if branches: branch_domain.append(("name", "in", branches)) branches = self.env["odoo.branch"].search(branch_domain) @@ -485,9 +489,7 @@ def _import_data(self, data): def _prepare_module_branch_values(self, data): # Get branch, repository and technical module - branch = self.env["odoo.branch"].search( - [("odoo_version", "=", True), ("name", "=", data["branch"])] - ) + branch = self.env["odoo.branch"].search([("name", "=", data["branch"])]) org = self._get_repository_org(data["repository"]["org"]) repository = self._get_repository( org.id, data["repository"]["name"], data["repository"] diff --git a/odoo_repository/models/odoo_repository_branch.py b/odoo_repository/models/odoo_repository_branch.py index f6871899..1cb9c843 100644 --- a/odoo_repository/models/odoo_repository_branch.py +++ b/odoo_repository/models/odoo_repository_branch.py @@ -17,13 +17,21 @@ class OdooRepositoryBranch(models.Model): index=True, readonly=True, ) + specific = fields.Boolean( + string="Specific", + related="repository_id.specific", + store=True, + index=True, + ) branch_id = fields.Many2one( comodel_name="odoo.branch", ondelete="cascade", - string="Branch", + string="Odoo Version", required=True, index=True, - readonly=True, + ) + cloned_branch = fields.Char( + help="Force the branch to clone (optional). Used on specific repositories.", ) module_ids = fields.One2many( comodel_name="odoo.module.branch", @@ -55,7 +63,7 @@ def _compute_active(self): def action_scan(self, force=False, raise_exc=True): """Scan the repository/branch.""" return self.repository_id.action_scan( - branches=self.branch_id.mapped("name"), force=force, raise_exc=raise_exc + branch_ids=self.branch_id.ids, force=force, raise_exc=raise_exc ) def action_force_scan(self, raise_exc=True): diff --git a/odoo_repository/tests/common.py b/odoo_repository/tests/common.py index 9c91edb7..2242a1df 100644 --- a/odoo_repository/tests/common.py +++ b/odoo_repository/tests/common.py @@ -52,9 +52,9 @@ def setUp(self): self.branch = self.env["odoo.branch"].create( { "name": self.branch1_name, - "odoo_version": True, } ) + self.branch.active = True # branch2 self.branch2_name = self.source2.split("/")[1] self.branch2 = ( @@ -66,9 +66,9 @@ def setUp(self): self.branch2 = self.env["odoo.branch"].create( { "name": self.branch2_name, - "odoo_version": True, } ) + self.branch2.active = True # branch3 self.branch3_name = self.target2.split("/")[1] # technical module @@ -136,10 +136,10 @@ def _update_module_installable_on_branch(self, branch, installable=True): ) return commit.hexsha - def _run_odoo_repository_action_scan(self, branch, force=False): - """Run `action_scan` for given `branch` on the Odoo repository.""" + def _run_odoo_repository_action_scan(self, branch_id, force=False): + """Run `action_scan` for given `branch_id` on the Odoo repository.""" self.odoo_repository.with_context(queue_job__no_delay=True).action_scan( - [branch], force=force + branch_ids=[branch_id], force=force ) def _create_odoo_module(self, name): diff --git a/odoo_repository/tests/test_odoo_repository_scan.py b/odoo_repository/tests/test_odoo_repository_scan.py index 3d6c0ebb..558963ba 100644 --- a/odoo_repository/tests/test_odoo_repository_scan.py +++ b/odoo_repository/tests/test_odoo_repository_scan.py @@ -1,6 +1,8 @@ # Copyright 2024 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +from odoo import fields + from .common import Common @@ -13,7 +15,7 @@ def test_action_scan_basic(self): self.assertFalse(self.odoo_repository.specific) module = self.env["odoo.module"].search([("name", "=", self.module_name)]) self.assertFalse(module) - self._run_odoo_repository_action_scan(self.branch.name) + self._run_odoo_repository_action_scan(self.branch.id) # Check module technical name module = self.env["odoo.module"].search([("name", "=", self.module_name)]) self.assertTrue(module) @@ -54,8 +56,16 @@ def test_action_scan_basic(self): def test_action_scan_repo_specific(self): """Test the creation of a module when scanning a specific repository.""" self.odoo_repository.specific = True + self.odoo_repository.write( + { + "specific": True, + "branch_ids": [ + fields.Command.create({"branch_id": self.branch.id}), + ], + } + ) self.assertTrue(self.odoo_repository.specific) - self._run_odoo_repository_action_scan(self.branch.name) + self._run_odoo_repository_action_scan(self.branch.id) # Check module data module = self.env["odoo.module"].search([("name", "=", self.module_name)]) module_branch = self.env["odoo.module.branch"].search( @@ -71,7 +81,7 @@ def test_action_scan_repo_module_exists(self): scan will trigger an update of its data. """ # First scan, like in `test_action_scan_first_time` - self._run_odoo_repository_action_scan(self.branch.name) + self._run_odoo_repository_action_scan(self.branch.id) module = self.env["odoo.module"].search([("name", "=", self.module_name)]) module_branch = self.env["odoo.module.branch"].search( [("module_id", "=", module.id), ("branch_id", "=", self.branch.id)] @@ -79,7 +89,7 @@ def test_action_scan_repo_module_exists(self): # Change some data in the module before triggering the second scan module_branch.write({"title": False}) # Launch a second scan (force it to make it happen) - self._run_odoo_repository_action_scan(self.branch.name, force=True) + self._run_odoo_repository_action_scan(self.branch.id, force=True) self.assertEqual(module_branch.title, "Test") def test_action_scan_orphaned_module_exists(self): @@ -96,7 +106,7 @@ def test_action_scan_orphaned_module_exists(self): # Create an orphaned module. # To ease its creation, we run a scan to get the record created, and # we update it to make it orphaned. - self._run_odoo_repository_action_scan(self.branch.name) + self._run_odoo_repository_action_scan(self.branch.id) module = self.env["odoo.module"].search([("name", "=", self.module_name)]) module_branch = self.env["odoo.module.branch"].search( [("module_id", "=", module.id), ("branch_id", "=", self.branch.id)] @@ -109,7 +119,7 @@ def test_action_scan_orphaned_module_exists(self): } ) # Launch a scan - self._run_odoo_repository_action_scan(self.branch.name, force=True) + self._run_odoo_repository_action_scan(self.branch.id, force=True) self.assertEqual(module_branch.repository_id, self.odoo_repository) def _create_wrong_repo_branch(self, repo_sequence=100): @@ -134,7 +144,7 @@ def _create_wrong_repo_branch(self, repo_sequence=100): def _create_unmerged_module_branch(self): # To ease the creation of such module, we run a scan to get the record # created, and we update it to make it unmerged/pending. - self._run_odoo_repository_action_scan(self.branch.name) + self._run_odoo_repository_action_scan(self.branch.id) module = self.env["odoo.module"].search([("name", "=", self.module_name)]) module_branch = self.env["odoo.module.branch"].search( [("module_id", "=", module.id), ("branch_id", "=", self.branch.id)] @@ -175,7 +185,7 @@ def test_action_scan_repo_generic_unmerged_module_exists(self): self.assertFalse(module_branch.specific) self.assertNotEqual(module_branch.repository_id, self.odoo_repository) # Launch a scan - self._run_odoo_repository_action_scan(self.branch.name, force=True) + self._run_odoo_repository_action_scan(self.branch.id, force=True) self.assertFalse(module_branch.specific) self.assertEqual(module_branch.repository_id, self.odoo_repository) @@ -191,7 +201,7 @@ def test_action_scan_repo_specific_unmerged_module_exists(self): self.assertFalse(module_branch.specific) self.assertNotEqual(module_branch.repository_id, self.odoo_repository) # Launch a scan - self._run_odoo_repository_action_scan(self.branch.name, force=True) + self._run_odoo_repository_action_scan(self.branch.id, force=True) # Unmerged module hasn't been attached to the scanned repository self.assertNotEqual(module_branch.repository_id, self.odoo_repository) @@ -206,7 +216,7 @@ def test_action_scan_uninstallable_module(self): self._update_module_installable_on_branch(self.branch.name, installable=False) module = self.env["odoo.module"].search([("name", "=", self.module_name)]) self.assertFalse(module) - self._run_odoo_repository_action_scan(self.branch.name) + self._run_odoo_repository_action_scan(self.branch.id) module = self.env["odoo.module"].search([("name", "=", self.module_name)]) self.assertTrue(module) # Check module branch diff --git a/odoo_repository/tests/test_repository_scanner.py b/odoo_repository/tests/test_repository_scanner.py index 6740883d..7102b547 100644 --- a/odoo_repository/tests/test_repository_scanner.py +++ b/odoo_repository/tests/test_repository_scanner.py @@ -12,6 +12,7 @@ def _init_scanner(self, **params): "org": self.org.name, "name": self.repo_name, "clone_url": self.repo_upstream_path, + "version": self.branch.name, "branch": self.branch.name, "addons_paths_data": [ { @@ -46,14 +47,13 @@ def test_get_odoo_repository_id(self): def test_get_odoo_branch_id(self): scanner = self._init_scanner() - repo_id = scanner._get_odoo_repository_id() - branch_id = scanner._get_odoo_branch_id(repo_id, self.branch.name) + branch_id = scanner._get_odoo_branch_id(self.branch.name) self.assertEqual(branch_id, self.branch.id) def test_create_odoo_repository_branch(self): scanner = self._init_scanner() repo_id = scanner._get_odoo_repository_id() - branch_id = scanner._get_odoo_branch_id(repo_id, self.branch.name) + branch_id = scanner._get_odoo_branch_id(self.branch.name) # The repository branch doesn't exist yet expected_repo_branch_id = scanner._get_odoo_repository_branch_id( repo_id, branch_id @@ -69,7 +69,7 @@ def test_create_odoo_repository_branch(self): def test_get_repo_last_scanned_commit(self): scanner = self._init_scanner() repo_id = scanner._get_odoo_repository_id() - branch_id = scanner._get_odoo_branch_id(repo_id, self.branch.name) + branch_id = scanner._get_odoo_branch_id(self.branch.name) repo_branch_id = scanner._create_odoo_repository_branch(repo_id, branch_id) repo_branch = self.env["odoo.repository.branch"].browse(repo_branch_id) # Nothing has been scanned until now @@ -101,7 +101,7 @@ def test_detect_modules_to_scan_in_addons_path(self): with scanner.repo() as repo: scanner._checkout_branch(repo, self.branch.name) repo_id = scanner._get_odoo_repository_id() - branch_id = scanner._get_odoo_branch_id(repo_id, self.branch.name) + branch_id = scanner._get_odoo_branch_id(self.branch.name) repo_branch_id = scanner._create_odoo_repository_branch(repo_id, branch_id) last_fetched_commit = scanner._get_last_fetched_commit( repo, self.branch.name @@ -124,7 +124,7 @@ def test_scan_module(self): with scanner.repo() as repo: scanner._checkout_branch(repo, self.branch.name) repo_id = scanner._get_odoo_repository_id() - branch_id = scanner._get_odoo_branch_id(repo_id, self.branch.name) + branch_id = scanner._get_odoo_branch_id(self.branch.name) repo_branch_id = scanner._create_odoo_repository_branch(repo_id, branch_id) module_path = self.addon remote_branch = f"origin/{self.branch.name}" @@ -158,7 +158,7 @@ def test_push_scanned_data(self): with scanner.repo() as repo: scanner._checkout_branch(repo, self.branch.name) repo_id = scanner._get_odoo_repository_id() - branch_id = scanner._get_odoo_branch_id(repo_id, self.branch.name) + branch_id = scanner._get_odoo_branch_id(self.branch.name) repo_branch_id = scanner._create_odoo_repository_branch(repo_id, branch_id) module = self.addon remote_branch = f"origin/{self.branch.name}" @@ -201,7 +201,7 @@ def test_update_last_scanned_commit(self): scanner = self._init_scanner() scanner._clone() repo_id = scanner._get_odoo_repository_id() - branch_id = scanner._get_odoo_branch_id(repo_id, self.branch.name) + branch_id = scanner._get_odoo_branch_id(self.branch.name) repo_branch_id = scanner._create_odoo_repository_branch(repo_id, branch_id) repo_branch = self.env["odoo.repository.branch"].browse(repo_branch_id) with scanner.repo() as repo: diff --git a/odoo_repository/tests/test_sync_node.py b/odoo_repository/tests/test_sync_node.py index 078b0a02..7b2db777 100644 --- a/odoo_repository/tests/test_sync_node.py +++ b/odoo_repository/tests/test_sync_node.py @@ -8,7 +8,7 @@ class TestSyncNode(Common): def test_sync_node(self): # Scan a repository self.odoo_repository.with_context(queue_job__no_delay=True).action_scan( - [self.branch.name] + self.branch.ids ) # Check data to sync data = self.env["odoo.module.branch"]._get_modules_data() diff --git a/odoo_repository/utils/scanner.py b/odoo_repository/utils/scanner.py index 5c66ef27..954b80e0 100644 --- a/odoo_repository/utils/scanner.py +++ b/odoo_repository/utils/scanner.py @@ -25,14 +25,8 @@ def _get_odoo_repository_id(self): .id ) - def _get_odoo_branch_id(self, repo_id, branch): - repo = self.env["odoo.repository"].browse(repo_id) - if repo.clone_branch_id and repo.odoo_version_id: - return repo.odoo_version_id.id - branch = self.env["odoo.branch"].search( - [("name", "=", branch), ("odoo_version", "=", True)] - ) - return branch.id + def _get_odoo_branch_id(self, version): + return self.env["odoo.branch"].search([("name", "=", version)]).id def _get_odoo_repository_branch_id(self, repo_id, branch_id): args = [ @@ -43,13 +37,15 @@ def _get_odoo_repository_branch_id(self, repo_id, branch_id): if repo_branch: return repo_branch.id - def _create_odoo_repository_branch(self, repo_id, branch_id): + def _create_odoo_repository_branch(self, repo_id, branch_id, cloned_branch=None): repo_branch_id = self._get_odoo_repository_branch_id(repo_id, branch_id) if not repo_branch_id: values = { "repository_id": repo_id, "branch_id": branch_id, } + if cloned_branch: + values["cloned_branch"] = cloned_branch repo_branch_model = self.env["odoo.repository.branch"] repo_branch_id = repo_branch_model.create(values).id return repo_branch_id diff --git a/odoo_repository/views/odoo_branch.xml b/odoo_repository/views/odoo_branch.xml index 3085adca..806fdbfd 100644 --- a/odoo_repository/views/odoo_branch.xml +++ b/odoo_repository/views/odoo_branch.xml @@ -31,10 +31,6 @@ name="name" attrs="{'readonly': [('repository_branch_ids', '!=', [])]}" /> - @@ -56,7 +52,6 @@ - @@ -69,17 +64,12 @@ - - Branches + Versions ir.actions.act_window odoo.branch diff --git a/odoo_repository/views/odoo_repository.xml b/odoo_repository/views/odoo_repository.xml index 85ffca76..efd791df 100644 --- a/odoo_repository/views/odoo_repository.xml +++ b/odoo_repository/views/odoo_repository.xml @@ -64,22 +64,7 @@ attrs="{'invisible': [('to_scan', '=', False)]}" > - - @@ -98,14 +83,22 @@ - - - - + + + + + diff --git a/odoo_repository/views/odoo_repository_branch.xml b/odoo_repository/views/odoo_repository_branch.xml index 26711933..25640b82 100644 --- a/odoo_repository/views/odoo_repository_branch.xml +++ b/odoo_repository/views/odoo_repository_branch.xml @@ -22,6 +22,7 @@ + @@ -39,6 +40,7 @@ + From 8f853e168b76a63b35b47266eff92af5422eb28d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Alix?= Date: Wed, 28 May 2025 13:54:38 +0200 Subject: [PATCH 2/6] odoo_repository_migration: collect migration data on specific repositories Now that multiple Odoo versions/branches can be set on a specific repository, we are able to collect the migration data of specific modules hosted in it. --- .../models/odoo_migration_path.py | 2 - .../models/odoo_module_branch.py | 5 +- .../models/odoo_module_branch_migration.py | 4 +- .../models/odoo_repository.py | 122 ++++++++++++------ .../tests/test_odoo_module_branch.py | 2 + odoo_repository_migration/utils/scanner.py | 6 +- 6 files changed, 94 insertions(+), 47 deletions(-) diff --git a/odoo_repository_migration/models/odoo_migration_path.py b/odoo_repository_migration/models/odoo_migration_path.py index 7dd314be..b3db3de6 100644 --- a/odoo_repository_migration/models/odoo_migration_path.py +++ b/odoo_repository_migration/models/odoo_migration_path.py @@ -14,13 +14,11 @@ class OdooMigrationPath(models.Model): source_branch_id = fields.Many2one( comodel_name="odoo.branch", ondelete="cascade", - domain=[("odoo_version", "=", True)], required=True, ) target_branch_id = fields.Many2one( comodel_name="odoo.branch", ondelete="cascade", - domain=[("odoo_version", "=", True)], required=True, ) diff --git a/odoo_repository_migration/models/odoo_module_branch.py b/odoo_repository_migration/models/odoo_module_branch.py index a33a66c3..bf13371b 100644 --- a/odoo_repository_migration/models/odoo_module_branch.py +++ b/odoo_repository_migration/models/odoo_module_branch.py @@ -51,10 +51,7 @@ def _compute_next_odoo_version_id(self): next_odoo_version = False if rec.branch_id: next_odoo_version = self.env["odoo.branch"].search( - [ - ("odoo_version", "=", True), - ("sequence", ">", rec.branch_id.sequence), - ], + [("sequence", ">", rec.branch_id.sequence)], limit=1, ) rec.next_odoo_version_id = next_odoo_version diff --git a/odoo_repository_migration/models/odoo_module_branch_migration.py b/odoo_repository_migration/models/odoo_module_branch_migration.py index 720f4706..515b0b05 100644 --- a/odoo_repository_migration/models/odoo_module_branch_migration.py +++ b/odoo_repository_migration/models/odoo_module_branch_migration.py @@ -318,8 +318,8 @@ def _compute_migration_scan(self): def push_scanned_data(self, module_branch_id, data): migration_path = self.env["odoo.migration.path"].search( [ - ("source_branch_id", "=", data["source_branch"]), - ("target_branch_id", "=", data["target_branch"]), + ("source_branch_id", "=", data["source_version"]), + ("target_branch_id", "=", data["target_version"]), ] ) values = { diff --git a/odoo_repository_migration/models/odoo_repository.py b/odoo_repository_migration/models/odoo_repository.py index 96d2ed75..9fa0ff03 100644 --- a/odoo_repository_migration/models/odoo_repository.py +++ b/odoo_repository_migration/models/odoo_repository.py @@ -19,16 +19,25 @@ class OdooRepository(models.Model): default=False, ) - def _reset_scanned_commits(self, branches=None): - res = super()._reset_scanned_commits(branches) - if branches is None: - branches = [] - branches_ = ( - self.branch_ids.filtered(lambda br: br.branch_id.name in branches) - if branches - else self.branch_ids + def action_scan(self, branch_ids=None, force=False, raise_exc=True): + for rec in self: + # Scan only relevant branches regarding migration paths + rec_ctx = rec + if rec.specific: + rec_ctx = rec.with_context(strict_branches_scan=True) + super(OdooRepository, rec_ctx).action_scan( + branch_ids=branch_ids, force=force, raise_exc=raise_exc + ) + return True + + def _reset_scanned_commits(self, branch_ids=None): + res = super()._reset_scanned_commits(branch_ids=branch_ids) + if branch_ids is None: + branch_ids = self.branch_ids.branch_id.ids + repo_branches = self.branch_ids.filtered( + lambda rb: rb.branch_id.id in branch_ids ) - branches_.module_ids.migration_ids.sudo().write( + repo_branches.module_ids.migration_ids.sudo().write( { "last_source_scanned_commit": False, "last_target_scanned_commit": False, @@ -36,12 +45,14 @@ def _reset_scanned_commits(self, branches=None): ) return res - def _create_subsequent_jobs(self, branch, next_branches, all_branches, data): + def _create_subsequent_jobs( + self, version_branch, next_versions_branches, all_versions_branches, data + ): jobs = super()._create_subsequent_jobs( - branch, next_branches, all_branches, data + version_branch, next_versions_branches, all_versions_branches, data ) # Prepare migration scan jobs when its the last repository scan - last_scan = not next_branches + last_scan = not next_versions_branches if not last_scan: return jobs # Check if the addons_paths are compatible with 'oca_port' @@ -50,50 +61,76 @@ def _create_subsequent_jobs(self, branch, next_branches, all_branches, data): return jobs # Override to run the MigrationScanner once branches are scanned args = [] - if all_branches: + if all_versions_branches: + all_versions = [vb[0] for vb in all_versions_branches] # A strict scan of branches avoids unwanted migration scans # For instance if we are interested only by 14.0 and 17.0 branches, # this avoids to scan other migration paths like 15.0 -> 17.0 - strict_scan = self.env.context.get("strict_branches_scan") + # NOTE: a strict scan always occurs on specific repositories + strict_scan = self.env.context.get("strict_branches_scan") or self.specific args = [ "&" if strict_scan else "|", - ("source_branch_id", "in", all_branches), - ("target_branch_id", "in", all_branches), + ("source_branch_id", "in", all_versions), + ("target_branch_id", "in", all_versions), ] migration_paths = self.env["odoo.migration.path"].search(args) # Launch one job for all migration_paths if migration_paths: + # Migration paths parameter containing the migration path ID + + # the Odoo versions and branches to scan. + # E.g. {MIG_PATH_ID: [('14.0', 'master'), ('18.0', '18.0-mig')], ...} + migration_paths_param = {} + for migration_path in migration_paths: + mig_path = ( + migration_path.source_branch_id.name, + migration_path.target_branch_id.name, + ) + versions_branches = [ + vb for vb in all_versions_branches if vb[0] in mig_path + ] + migration_paths_param[migration_path.id] = versions_branches + delayable = self.delayable( description=f"Collect {self.display_name} migration data", identity_key=identity_exact, ) - job = delayable._scan_migration_paths(migration_paths.ids) + job = delayable._scan_migration_paths(migration_paths_param) jobs.append(job) return jobs - def _scan_migration_paths(self, migration_path_ids): + def _scan_migration_paths(self, migration_paths_param): """Scan repository branches to collect modules migration data. Spawn one job per module to scan. """ self.ensure_one() jobs = [] - migration_paths = ( - self.env["odoo.migration.path"].browse(migration_path_ids).exists() - ) - for migration_path in migration_paths: + for migration_path_id in migration_paths_param: + versions_branches = migration_paths_param[migration_path_id] + migration_path = ( + self.env["odoo.migration.path"] + .browse( + # Job encodes dict key as string => convert it to integer + int(migration_path_id) + ) + .exists() + ) + if not migration_path: + continue modules_to_scan = self._migration_get_modules_to_scan(migration_path) if modules_to_scan: jobs.extend( self._migration_create_jobs_scan_module( - migration_path, modules_to_scan + migration_path, versions_branches, modules_to_scan ) ) if jobs: chain(*jobs).delay() return True - def _migration_create_jobs_scan_module(self, migration_path, modules_to_scan): + def _migration_create_jobs_scan_module( + self, migration_path, versions_branches, modules_to_scan + ): jobs = [] mig_path = ( migration_path.source_branch_id.name, @@ -106,11 +143,15 @@ def _migration_create_jobs_scan_module(self, migration_path, modules_to_scan): ), identity_key=identity_exact, ) - job = delayable._scan_migration_module(migration_path.id, module.id) + job = delayable._scan_migration_module( + migration_path.id, versions_branches, module.id + ) jobs.append(job) return jobs - def _scan_migration_module(self, migration_path_id, module_branch_id): + def _scan_migration_module( + self, migration_path_id, versions_branches, module_branch_id + ): """Scan migration path for `module_branch_id`. The migration scan can only occur if: @@ -154,7 +195,7 @@ def _scan_migration_module(self, migration_path_id, module_branch_id): f"{target_repository.display_name}." ) params = self._prepare_migration_scanner_parameters( - migration_path, target_repository + versions_branches, target_repository ) module_names = [module.module_id.name] if target_module: @@ -164,7 +205,9 @@ def _scan_migration_module(self, migration_path_id, module_branch_id): try: scanner = MigrationScannerOdooEnv(**params) return scanner.scan( - addons_path=module.addons_path, module_names=module_names + addons_path=module.addons_path, + target_addons_path=target_module.addons_path or module.addons_path, + module_names=module_names, ) except Exception as exc: raise RetryableJobError("Scanner error") from exc @@ -172,7 +215,8 @@ def _scan_migration_module(self, migration_path_id, module_branch_id): def _migration_get_modules_to_scan(self, migration_path): """Return `odoo.module.branch` records that need a migration scan.""" self.ensure_one() - return self.env["odoo.module.branch"].search( + mb_model = self.env["odoo.module.branch"] + modules = mb_model.search( [ ( "repository_id", @@ -183,21 +227,27 @@ def _migration_get_modules_to_scan(self, migration_path): ("migration_scan", "=", True), ] ) + module_ids = [] + for module in modules: + migration = module.migration_ids.filtered( + lambda mig: mig.migration_path_id == migration_path + ) + if migration and not migration.migration_scan: + # Skip module that do not need a scan for the given migration path + continue + module_ids.append(module.id) + return mb_model.browse(module_ids) def _prepare_migration_scanner_parameters( self, migration_path, target_repository=None ): ir_config = self.env["ir.config_parameter"] repositories_path = ir_config.sudo().get_param(self._repositories_path_key) - mig_path = ( - migration_path.source_branch_id.name, - migration_path.target_branch_id.name, - ) params = { "org": self.org_id.name, "name": self.name, "clone_url": self.clone_url, - "migration_path": mig_path, + "migration_path": migration_path, "repositories_path": repositories_path, "repo_type": self.repo_type, "ssh_key": self.ssh_key_id.private_key, @@ -221,10 +271,10 @@ def _pre_create_or_update_module_branch(self, rec, values, raw_data): values["migration_ids"] = [] for mig in migrations: source_branch = self.env["odoo.branch"].search( - [("odoo_version", "=", True), ("name", "=", mig["source_branch"])] + [("name", "=", mig["source_branch"])] ) target_branch = self.env["odoo.branch"].search( - [("odoo_version", "=", True), ("name", "=", mig["target_branch"])] + [("name", "=", mig["target_branch"])] ) if not source_branch or not target_branch: # Such branches are not configured on this instance, skip diff --git a/odoo_repository_migration/tests/test_odoo_module_branch.py b/odoo_repository_migration/tests/test_odoo_module_branch.py index 09bf8eb8..0797d32f 100644 --- a/odoo_repository_migration/tests/test_odoo_module_branch.py +++ b/odoo_repository_migration/tests/test_odoo_module_branch.py @@ -38,7 +38,9 @@ def _simulate_migration_scan(self, target_commit, report=None): """Helper method that pushes scanned migration data.""" data = { "module": self.module_branch.module_name, + "source_version": self.branch.name, "source_branch": self.branch.name, + "target_version": self.branch2.name, "target_branch": self.branch2.name, "source_commit": self.module_branch.last_scanned_commit, "target_commit": target_commit, diff --git a/odoo_repository_migration/utils/scanner.py b/odoo_repository_migration/utils/scanner.py index c4f3eb81..a5734fcc 100644 --- a/odoo_repository_migration/utils/scanner.py +++ b/odoo_repository_migration/utils/scanner.py @@ -70,13 +70,13 @@ def _get_odoo_module_branch_migration_id( return migration.id def _get_odoo_module_branch_migration_data( - self, repo_id: int, module: str, source_branch: str, target_branch: str + self, repo_id: int, module: str, source_version: str, target_version: str ) -> dict: args = [ ("module_branch_id.repository_id", "=", repo_id), ("module_id", "=", module), - ("source_branch_id", "=", source_branch), - ("target_branch_id", "=", target_branch), + ("source_branch_id", "=", source_version), + ("target_branch_id", "=", target_version), ] migration = self.env["odoo.module.branch.migration"].search(args) if migration: From bff345ed8946db014636c86e6b04f6bb4139edea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Alix?= Date: Wed, 28 May 2025 13:58:08 +0200 Subject: [PATCH 3/6] odoo_project: minor fixes/adaptations --- odoo_project/models/odoo_project.py | 54 +++++++++++++++-------- odoo_project/tests/test_import_modules.py | 2 +- odoo_project/views/odoo_project.xml | 8 +++- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/odoo_project/models/odoo_project.py b/odoo_project/models/odoo_project.py index 55479e5c..b56d4b66 100644 --- a/odoo_project/models/odoo_project.py +++ b/odoo_project/models/odoo_project.py @@ -16,28 +16,34 @@ class OdooProject(models.Model): active = fields.Boolean(default=True) repository_id = fields.Many2one( comodel_name="odoo.repository", - ondelete="restrict", string="Repository", - domain=[ - ("clone_branch_id", "!=", False), - ("specific", "=", True), - ("odoo_version_id", "!=", False), - ], + domain=[("specific", "=", True)], + store=True, + index=True, help=( - "Repository is optional. " + "Repository this project is based on (optional). " "You can start to build/simulate a project without repository " "to get some figures." ), ) + available_odoo_version_ids = fields.One2many( + comodel_name="odoo.branch", + compute="_compute_available_odoo_version_ids", + string="Available Odoo Versions", + ) odoo_version_id = fields.Many2one( comodel_name="odoo.branch", ondelete="restrict", string="Odoo Version", - domain=[("odoo_version", "=", True)], - required=True, - compute="_compute_odoo_version_id", store=True, - readonly=False, + index=True, + ) + repository_branch_id = fields.Many2one( + comodel_name="odoo.repository.branch", + string="Repository / Branch", + compute="_compute_repository_branch_id", + store=True, + index=True, ) project_module_ids = fields.One2many( comodel_name="odoo.project.module", @@ -74,10 +80,22 @@ class OdooProject(models.Model): ) @api.depends("repository_id") - def _compute_odoo_version_id(self): + def _compute_available_odoo_version_ids(self): + all_versions = self.env["odoo.branch"]._get_all_odoo_versions() for rec in self: + rec.available_odoo_version_ids = all_versions if rec.repository_id: - rec.odoo_version_id = rec.repository_id.odoo_version_id + rec.available_odoo_version_ids = rec.repository_id.branch_ids.branch_id + + @api.depends("repository_id", "odoo_version_id") + def _compute_repository_branch_id(self): + for rec in self: + rec.repository_branch_id = False + if not rec.repository_id or not rec.odoo_version_id: + continue + rec.repository_branch_id = rec.repository_id.branch_ids.filtered( + lambda rb: rb.branch_id == rec.odoo_version_id + ) @api.depends("project_module_ids.module_id") def _compute_module_ids(self): @@ -90,11 +108,11 @@ def _compute_modules_count(self): rec.modules_count = len(rec.project_module_ids) @api.depends( - "repository_id.branch_ids.module_ids", "project_module_ids.module_branch_id" + "repository_branch_id.module_ids", "project_module_ids.module_branch_id" ) def _compute_module_not_installed_ids(self): for rec in self: - all_module_ids = set(rec.repository_id.branch_ids.module_ids.ids) + all_module_ids = set(rec.repository_branch_id.module_ids.ids) installed_module_ids = set(rec.project_module_ids.module_branch_id.ids) rec.module_not_installed_ids = list(all_module_ids - installed_module_ids) @@ -145,12 +163,12 @@ def action_find_unknown_modules(self): module.action_find_pr_url() def _get_repositories_to_scan(self): - """Returnt the repositories to scan.""" + """Return the repositories to scan.""" domain = self.env["odoo.repository"]._cron_scanner_domain() return self.project_module_ids.repository_id.filtered_domain(domain) def _get_branches_to_scan(self): - """Return the branches to scan.""" + """Return the branches/versions to scan.""" return self.project_module_ids.repository_branch_id.branch_id def action_scan(self, force=False): @@ -163,7 +181,7 @@ def action_scan(self, force=False): branches = self._get_branches_to_scan() if branches: repositories.action_scan( - branches=branches.mapped("name"), force=force, raise_exc=False + branch_ids=branches.ids, force=force, raise_exc=False ) # Scan the underlying project repository itself self.repository_id.action_scan(force=force, raise_exc=True) diff --git a/odoo_project/tests/test_import_modules.py b/odoo_project/tests/test_import_modules.py index 390fdcd4..361de268 100644 --- a/odoo_project/tests/test_import_modules.py +++ b/odoo_project/tests/test_import_modules.py @@ -134,8 +134,8 @@ def test_match_generic_module(self): def test_match_project_repo_module(self): # Assign a repository to the project - self.odoo_repository.odoo_version_id = self.branch self.project.repository_id = self.odoo_repository + self.project.odoo_version_id = self.branch mod1 = "test1" mod2 = "test2" mod1_in_repo = self.wiz_model._get_module(mod1) diff --git a/odoo_project/views/odoo_project.xml b/odoo_project/views/odoo_project.xml index 59502512..4cc06d91 100644 --- a/odoo_project/views/odoo_project.xml +++ b/odoo_project/views/odoo_project.xml @@ -44,13 +44,17 @@

+ From 1798200d105d9089df0ee8fee8f3cbe2f7c69e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Alix?= Date: Fri, 30 May 2025 10:40:59 +0200 Subject: [PATCH 4/6] odoo_project_migration: fix recomputation of project modules state --- .../models/odoo_project_module_migration.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/odoo_project_migration/models/odoo_project_module_migration.py b/odoo_project_migration/models/odoo_project_module_migration.py index 47885d25..13d8bbfb 100644 --- a/odoo_project_migration/models/odoo_project_module_migration.py +++ b/odoo_project_migration/models/odoo_project_module_migration.py @@ -6,7 +6,6 @@ class OdooProjectModuleMigration(models.Model): _name = "odoo.project.module.migration" - # TODO test to inherit from 'module_migration_id' field too _inherits = {"odoo.module.branch": "source_module_branch_id"} _description = "Module migration line of an Odoo Project" _order = ( @@ -144,7 +143,12 @@ def _compute_target_module_branch_id(self): domain=[("installable", "=", True)], ) - @api.depends("migration_path_id", "source_module_branch_id") + # NOTE: 'migration_scan' is here to re-trigger the computation + # each time the source module has its state updated regarding migration. + # FIXME: this could trigger too much computations on irrelevant records + # (one not related to the updated migration path), we should switch to + # component events to handle such cases. + @api.depends("migration_path_id", "source_module_branch_id.migration_scan") def _compute_module_migration_id(self): migration_model = self.env["odoo.module.branch.migration"] for rec in self: From 7cb50d32fd228f557c9808e933e2a17016757ad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Alix?= Date: Sun, 29 Jun 2025 15:51:20 +0200 Subject: [PATCH 5/6] fixup! odoo_repository_migration: collect migration data on specific repositories --- .../models/odoo_repository.py | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/odoo_repository_migration/models/odoo_repository.py b/odoo_repository_migration/models/odoo_repository.py index 9fa0ff03..b3a23203 100644 --- a/odoo_repository_migration/models/odoo_repository.py +++ b/odoo_repository_migration/models/odoo_repository.py @@ -81,12 +81,27 @@ def _create_subsequent_jobs( # E.g. {MIG_PATH_ID: [('14.0', 'master'), ('18.0', '18.0-mig')], ...} migration_paths_param = {} for migration_path in migration_paths: - mig_path = ( - migration_path.source_branch_id.name, - migration_path.target_branch_id.name, + source_rb = self.branch_ids.filtered( + lambda rb: rb.branch_id == migration_path.source_branch_id ) + target_rb = self.branch_ids.filtered( + lambda rb: rb.branch_id == migration_path.target_branch_id + ) + # Need the two Odoo versions of the migration path available + # in the scanned repository + if not source_rb or not target_rb: + continue + # Build list of tuples (Odoo version, branch name) corresponding + # to the migration path versions_branches = [ - vb for vb in all_versions_branches if vb[0] in mig_path + ( + source_rb.branch_id.name, + source_rb.cloned_branch or source_rb.branch_id.name, + ), + ( + target_rb.branch_id.name, + target_rb.cloned_branch or target_rb.branch_id.name, + ), ] migration_paths_param[migration_path.id] = versions_branches From a5af0a3b26601ba5309c01cee26ee228a3c82748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Alix?= Date: Mon, 7 Jul 2025 08:47:26 +0200 Subject: [PATCH 6/6] fixup! odoo_repository: scan multiple branches of a specific (project) repository --- odoo_repository/__manifest__.py | 2 +- .../migrations/16.0.1.3.0/post-migration.py | 19 ++++++++++++++++++ odoo_repository/models/odoo_repository.py | 20 +++++++++++-------- .../models/odoo_repository_branch.py | 11 +++++++--- odoo_repository/views/odoo_repository.xml | 10 +++++++--- 5 files changed, 47 insertions(+), 15 deletions(-) create mode 100644 odoo_repository/migrations/16.0.1.3.0/post-migration.py diff --git a/odoo_repository/__manifest__.py b/odoo_repository/__manifest__.py index a89686da..74cfb866 100644 --- a/odoo_repository/__manifest__.py +++ b/odoo_repository/__manifest__.py @@ -3,7 +3,7 @@ { "name": "Odoo Repositories Data", "summary": "Base module to host data collected from Odoo repositories.", - "version": "16.0.1.2.0", + "version": "16.0.1.3.0", "category": "Tools", "author": "Camptocamp, Odoo Community Association (OCA)", "website": "https://github.com/camptocamp/odoo-repository", diff --git a/odoo_repository/migrations/16.0.1.3.0/post-migration.py b/odoo_repository/migrations/16.0.1.3.0/post-migration.py new file mode 100644 index 00000000..88da69e6 --- /dev/null +++ b/odoo_repository/migrations/16.0.1.3.0/post-migration.py @@ -0,0 +1,19 @@ +import logging + +_logger = logging.getLogger(__name__) + + +def migrate(cr, version): + if not version: + return + set_manual_branches_on_odoo_repository(cr) + + +def set_manual_branches_on_odoo_repository(cr): + _logger.info("Set 'manual_branches = True' on specific repositories...") + query = """ + UPDATE odoo_repository + SET manual_branches = true + WHERE specific = true; + """ + cr.execute(query) diff --git a/odoo_repository/models/odoo_repository.py b/odoo_repository/models/odoo_repository.py index e7bb365d..521c8a58 100644 --- a/odoo_repository/models/odoo_repository.py +++ b/odoo_repository/models/odoo_repository.py @@ -96,11 +96,17 @@ class OdooRepository(models.Model): "certain days only. If not defined, the scan will happen every day." ), ) + manual_branches = fields.Boolean( + string="Configure branches manually", + help=( + "By default repository branches follows the configured Odoo versions " + "(e.g: 17.0, 18.0...). Enable this option to configure your own branches." + ), + ) specific = fields.Boolean( help=( - "Host specific modules. " - "By default if the repository clones a specific branch, " - "that means it hosts specific modules." + "Host specific modules (that are not generic). " + "Used for project repositories." ), ) @@ -151,7 +157,7 @@ def _onchange_repo_url(self): def _get_odoo_branches_to_scan(self): self.ensure_one() - if self.specific: + if self.manual_branches: return self.branch_ids.branch_id return self.env["odoo.branch"]._get_all_odoo_versions(active_test=True) @@ -250,11 +256,9 @@ def action_scan(self, branch_ids=None, force=False, raise_exc=True): ) if not branches: continue - # Branch names to scan could be different on specific repositories - # (e.g. 'master' or 'main' branch, representing a 16.0 Odoo version) - # => create a list of tuples ({odoo_version}, {branch_name}). + # Create a list of tuples ({odoo_version}, {branch_name}) versions_branches = [(branch.name, branch.name) for branch in branches] - if rec.specific: + if rec.manual_branches: versions_branches = [ (rb.branch_id.name, rb.cloned_branch or rb.branch_id.name) for rb in rec.branch_ids diff --git a/odoo_repository/models/odoo_repository_branch.py b/odoo_repository/models/odoo_repository_branch.py index 1cb9c843..d748c2a8 100644 --- a/odoo_repository/models/odoo_repository_branch.py +++ b/odoo_repository/models/odoo_repository_branch.py @@ -17,11 +17,13 @@ class OdooRepositoryBranch(models.Model): index=True, readonly=True, ) + manual_branches = fields.Boolean( + related="repository_id.manual_branches", + store=True, + ) specific = fields.Boolean( - string="Specific", related="repository_id.specific", store=True, - index=True, ) branch_id = fields.Many2one( comodel_name="odoo.branch", @@ -31,7 +33,10 @@ class OdooRepositoryBranch(models.Model): index=True, ) cloned_branch = fields.Char( - help="Force the branch to clone (optional). Used on specific repositories.", + help=( + "Force the branch to clone (optional). Used on repositories with " + "'Configure branches manually' option enabled." + ), ) module_ids = fields.One2many( comodel_name="odoo.module.branch", diff --git a/odoo_repository/views/odoo_repository.xml b/odoo_repository/views/odoo_repository.xml index efd791df..c2f9862a 100644 --- a/odoo_repository/views/odoo_repository.xml +++ b/odoo_repository/views/odoo_repository.xml @@ -64,6 +64,10 @@ attrs="{'invisible': [('to_scan', '=', False)]}" > + @@ -88,16 +92,16 @@ name="branch_ids" nolabel="1" colspan="2" - attrs="{'readonly': [('specific', '=', False)]}" + attrs="{'readonly': [('manual_branches', '=', False)]}" >