diff --git a/odoo_project_migration/__manifest__.py b/odoo_project_migration/__manifest__.py index ec2f1117..d1cd3512 100644 --- a/odoo_project_migration/__manifest__.py +++ b/odoo_project_migration/__manifest__.py @@ -3,7 +3,7 @@ { "name": "Odoo Project Migration Data", "summary": "Analyze your Odoo project migrations.", - "version": "16.0.1.0.0", + "version": "16.0.1.1.0", "category": "Tools", "author": "Camptocamp, Odoo Community Association (OCA)", "website": "https://github.com/camptocamp/odoo-repository", diff --git a/odoo_project_migration/migrations/16.0.1.1.0/post-migration.py b/odoo_project_migration/migrations/16.0.1.1.0/post-migration.py new file mode 100644 index 00000000..3a3a7d45 --- /dev/null +++ b/odoo_project_migration/migrations/16.0.1.1.0/post-migration.py @@ -0,0 +1,38 @@ +# Copyright 2025 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import logging + +from odoo import SUPERUSER_ID, api + +_logger = logging.getLogger(__name__) + + +def migrate(cr, version): + if not version: + return + env = api.Environment(cr, SUPERUSER_ID, {}) + fix_migration_states(env) + + +def fix_migration_states(env): + _logger.info("Update '.state' field...") + query = """ + SELECT mig.id + FROM odoo_module_branch_migration mig + JOIN odoo_module_branch AS source + ON mig.module_branch_id=source.id + JOIN odoo_module_branch AS target + ON mig.target_module_branch_id=target.id + WHERE source.repository_branch_id IS NOT NULL + AND target.repository_branch_id IS NOT NULL + AND source.repository_id != target.repository_id; + """ + env.cr.execute(query) + mig_ids = [row[0] for row in env.cr.fetchall()] + # Recompute migration state, especially for modules moved to another repo + project_migs = env["odoo.project.module.migration"].search( + [("module_migration_id", "in", mig_ids)] + ) + env.add_to_compute(project_migs._fields["state"], project_migs) + project_migs.modified(["state"]) diff --git a/odoo_project_migration/models/odoo_project_module_migration.py b/odoo_project_migration/models/odoo_project_module_migration.py index 5b49b1a8..5cd79a72 100644 --- a/odoo_project_migration/models/odoo_project_module_migration.py +++ b/odoo_project_migration/models/odoo_project_module_migration.py @@ -94,6 +94,9 @@ class OdooProjectModuleMigration(models.Model): ("migrate", "To migrate"), ("port_commits", "Commits to port"), ("review_migration", "Migration to review"), + ("moved_to_standard", "Moved to standard"), + ("moved_to_oca", "Moved to OCA"), + ("moved_to_generic", "Moved to generic repo"), # New states to qualify modules without migration data ("available", "Available"), ("removed", "Removed"), diff --git a/odoo_repository/lib/scanner.py b/odoo_repository/lib/scanner.py index dca7ebc3..cbd033f7 100644 --- a/odoo_repository/lib/scanner.py +++ b/odoo_repository/lib/scanner.py @@ -96,7 +96,7 @@ def sync(self, fetch=True): if self.is_cloned: with self.repo() as repo: self._apply_git_config(repo) - self._set_git_remote_url(repo) + self._set_git_remote_url(repo, "origin", self.clone_url) if fetch: res = self._fetch(repo) return res @@ -182,13 +182,16 @@ def _apply_git_config(self, repo): writer.set_value("gc", "reflogExpire", "never") writer.set_value("gc", "reflogExpireUnreachable", "never") - def _set_git_remote_url(self, repo): - """Ensure that 'origin' remote is set with the right URL.""" + def _set_git_remote_url(self, repo, remote, url): + """Ensure that `remote` has `url` set.""" # Check first the URL before setting it, as this triggers a 'chmod' # command on '.git/config' file (to protect sensitive data) that could # be not allowed on some mounted file systems. - if repo.remotes["origin"].url != self.clone_url: - repo.remotes["origin"].set_url(self.clone_url) + if remote in repo.remotes: + if repo.remotes[remote].url != url: + repo.remotes[remote].set_url(url) + else: + repo.create_remote(remote, url) @property def is_cloned(self): @@ -278,32 +281,32 @@ def _fetch(self, repo): # Return True as soon as we fetched at least one branch return bool(branches_fetched) - def _branch_exists(self, repo, branch): - refs = [r.name for r in repo.remotes.origin.refs] - branch = f"origin/{branch}" + def _branch_exists(self, repo, branch, remote="origin"): + refs = [r.name for r in repo.remotes[remote].refs] + branch = f"{remote}/{branch}" return branch in refs - def _checkout_branch(self, repo, branch): + def _checkout_branch(self, repo, branch, remote="origin"): # Ensure to clean up the repository before a checkout index_lock_path = pathlib.Path(repo.common_dir).joinpath("index.lock") if index_lock_path.exists(): index_lock_path.unlink() repo.git.reset("--hard") repo.git.clean("-xdf") - repo.git.checkout("-f", f"remotes/origin/{branch}") + repo.git.checkout("-f", f"remotes/{remote}/{branch}") - def _get_last_fetched_commit(self, repo, branch): + def _get_last_fetched_commit(self, repo, branch, remote="origin"): """Return the last fetched commit for the given `branch`.""" - return repo.rev_parse(f"remotes/origin/{branch}").hexsha + return repo.rev_parse(f"remotes/{remote}/{branch}").hexsha - def _get_module_paths(self, repo, relative_path, branch): + def _get_module_paths(self, repo, relative_path, branch, remote="origin"): """Return the list of modules available in `branch`.""" # Clean up 'relative_path' to make it compatible with 'git.Tree' object relative_tree_path = "/".join( [dir_ for dir_ in relative_path.split("/") if dir_ and dir_ != "."] ) # Return all available modules from 'relative_tree_path' - branch_commit = repo.remotes.origin.refs[branch].commit + branch_commit = repo.remotes[remote].refs[branch].commit addons_trees = branch_commit.tree.trees if relative_tree_path: addons_trees = (branch_commit.tree / relative_tree_path).trees @@ -419,6 +422,8 @@ def __init__( name: str, clone_url: str, migration_path: tuple[str], + new_repo_name: str = None, + new_repo_url: str = None, repositories_path: str = None, repo_type: str = None, ssh_key: str = None, @@ -440,6 +445,20 @@ def __init__( clone_name, ) self.migration_path = migration_path + self.new_repo_name = new_repo_name + self.new_repo_url = ( + self._prepare_clone_url(repo_type, new_repo_url, token) + if new_repo_url + else None + ) + + def sync(self, fetch=True): + res = super().sync(fetch=fetch) + # Set the new repository as remote + if self.is_cloned and self.new_repo_name and self.new_repo_url: + with self.repo() as repo: + self._set_git_remote_url(repo, self.new_repo_name, self.new_repo_url) + return res def scan(self, addons_path=".", module_names=None): # Clone/fetch has been done during the repository scan, the migration @@ -450,13 +469,21 @@ def scan(self, addons_path=".", module_names=None): if not res: return False source_branch, target_branch = self.migration_path + target_remote = "origin" with self.repo() as repo: + if self.new_repo_name and self.new_repo_url: + target_remote = self.new_repo_name + # Fetch target branch from new repo + with self._get_git_env() as git_env: + with repo.git.custom_environment(**git_env): + repo.remotes[target_remote].fetch(target_branch) if self._branch_exists(repo, source_branch) and self._branch_exists( - repo, target_branch + repo, target_branch, remote=target_remote ): return self._scan_migration_path( repo, source_branch, + target_remote, target_branch, addons_path=addons_path, module_names=module_names, @@ -464,10 +491,18 @@ def scan(self, addons_path=".", module_names=None): return res def _scan_migration_path( - self, repo, source_branch, target_branch, addons_path=".", module_names=None + self, + repo, + source_branch, + target_remote, + target_branch, + addons_path=".", + module_names=None, ): repo_source_commit = self._get_last_fetched_commit(repo, source_branch) - repo_target_commit = self._get_last_fetched_commit(repo, target_branch) + repo_target_commit = self._get_last_fetched_commit( + repo, target_branch, remote=target_remote + ) if not module_names: module_names = self._get_module_paths(repo, addons_path, source_branch) res = [] @@ -523,6 +558,7 @@ def _scan_migration_path( module, module_branch_id, source_branch, + target_remote, target_branch, module_source_commit, module_target_commit, @@ -541,6 +577,7 @@ def _scan_module( module: str, module_branch_id: int, source_branch: str, + target_remote: str, target_branch: str, source_commit: str, target_commit: str, @@ -591,7 +628,7 @@ def _scan_module( target_branch, ) oca_port_data = self._run_oca_port( - module_path, source_branch, target_branch + module_path, source_branch, target_remote, target_branch ) data["report"] = oca_port_data self._push_scanned_data(module_branch_id, data) @@ -644,7 +681,7 @@ def _check_relevant_commits(self, repo, module_path, commits): return True return False - def _run_oca_port(self, module_path, source_branch, target_branch): + def _run_oca_port(self, module_path, source_branch, target_remote, target_branch): _logger.info( "%s: collect migration data for '%s' (%s -> %s)", self.full_name, @@ -655,7 +692,7 @@ def _run_oca_port(self, module_path, source_branch, target_branch): # Initialize the oca-port app params = { "source": f"origin/{source_branch}", - "target": f"origin/{target_branch}", + "target": f"{target_remote}/{target_branch}", "addon_path": module_path, "upstream_org": self.org, "repo_path": self.path, diff --git a/odoo_repository_migration/__manifest__.py b/odoo_repository_migration/__manifest__.py index f6f6d967..9625b7aa 100644 --- a/odoo_repository_migration/__manifest__.py +++ b/odoo_repository_migration/__manifest__.py @@ -3,7 +3,7 @@ { "name": "Odoo Repository Migration Data", "summary": "Collect modules migration data for Odoo Repositories.", - "version": "16.0.1.1.0", + "version": "16.0.1.2.0", "category": "Tools", "author": "Camptocamp, Odoo Community Association (OCA)", "website": "https://github.com/camptocamp/odoo-repository", diff --git a/odoo_repository_migration/migrations/16.0.1.2.0/post-migration.py b/odoo_repository_migration/migrations/16.0.1.2.0/post-migration.py new file mode 100644 index 00000000..471bdb3e --- /dev/null +++ b/odoo_repository_migration/migrations/16.0.1.2.0/post-migration.py @@ -0,0 +1,56 @@ +# Copyright 2024 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import logging + +from odoo import SUPERUSER_ID, api + +_logger = logging.getLogger(__name__) + + +def migrate(cr, version): + if not version: + return + env = api.Environment(cr, SUPERUSER_ID, {}) + fix_migration_states(env) + + +def fix_migration_states(env): + _logger.info("Plan a scan to fix '.state' field...") + query = """ + SELECT mig.id + FROM odoo_module_branch_migration mig + JOIN odoo_module_branch AS source + ON mig.module_branch_id=source.id + JOIN odoo_module_branch AS target + ON mig.target_module_branch_id=target.id + WHERE source.repository_branch_id IS NOT NULL + AND target.repository_branch_id IS NOT NULL + AND source.repository_id != target.repository_id; + """ + env.cr.execute(query) + mig_ids = [row[0] for row in env.cr.fetchall()] + # Reset 'last_target_scanned_commit' to recompute 'migration_scan' + if mig_ids: + query = """ + UPDATE odoo_module_branch_migration + SET last_target_scanned_commit=NULL + WHERE id IN %s; + """ + args = (tuple(mig_ids),) + env.cr.execute(query, args) + # Reset collected migration data on modules that shouldn't have any + mbm_model = env["odoo.module.branch.migration"] + migs = mbm_model.search( + [ + ("results", "!=", False), + ("repository_id.collect_migration_data", "=", False), + ] + ) + migs.results = False + # Recompute migration state/flag, especially for modules moved to another repo + # that won't trigger a migration scan. + migs = mbm_model.search([("id", "in", mig_ids)]) + env.add_to_compute(migs._fields["state"], migs) + env.add_to_compute(migs._fields["migration_scan"], migs) + migs.modified(["state", "migration_scan"]) diff --git a/odoo_repository_migration/models/odoo_module_branch_migration.py b/odoo_repository_migration/models/odoo_module_branch_migration.py index b678a04d..aea3d234 100644 --- a/odoo_repository_migration/models/odoo_module_branch_migration.py +++ b/odoo_repository_migration/models/odoo_module_branch_migration.py @@ -57,12 +57,43 @@ class OdooModuleBranchMigration(models.Model): author_ids = fields.Many2many(related="module_branch_id.author_ids") maintainer_ids = fields.Many2many(related="module_branch_id.maintainer_ids") process = fields.Char(index=True) + moved_to_standard = fields.Boolean( + compute="_compute_moved_to_standard", + store=True, + help=( + "Module now available in Odoo standard code. " + "This module is maybe not exactly the same, and doesn't have the " + "same scope so it deserves a check during a migration." + ), + ) + moved_to_oca = fields.Boolean( + compute="_compute_moved_to_oca", + store=True, + help=( + "Module now available in OCA. " + "This module is maybe not exactly the same, and doesn't have the " + "same scope so it deserves a check during a migration." + ), + ) + moved_to_generic = fields.Boolean( + compute="_compute_moved_to_generic", + store=True, + string="Now generic", + help=( + "Specific module now available in a generic repository. " + "This module is maybe not exactly the same, and doesn't have the " + "same scope so it deserves a check during a migration." + ), + ) state = fields.Selection( selection=[ ("fully_ported", "Fully Ported"), ("migrate", "To migrate"), ("port_commits", "Commits to port"), ("review_migration", "Migration to review"), + ("moved_to_standard", "Moved to standard"), + ("moved_to_oca", "Moved to OCA"), + ("moved_to_generic", "Moved to generic repo"), ], string="Migration Status", compute="_compute_state", @@ -116,9 +147,58 @@ def _compute_target_module_branch_id(self): domain=[("installable", "=", True)], ) - @api.depends("process", "pr_url") + @api.depends("module_branch_id.is_standard", "target_module_branch_id.is_standard") + def _compute_moved_to_standard(self): + for rec in self: + rec.moved_to_standard = ( + not rec.module_branch_id.is_standard + and rec.target_module_branch_id.is_standard + ) + + @api.depends("org_id", "target_module_branch_id.org_id") + def _compute_moved_to_oca(self): + org_oca = self.env.ref( + "odoo_repository.odoo_repository_org_oca", raise_if_not_found=False + ) + for rec in self: + rec.moved_to_oca = False + if not org_oca: + continue + rec.moved_to_oca = ( + rec.org_id != org_oca and rec.target_module_branch_id.org_id == org_oca + ) + + @api.depends( + "repository_id.specific", "target_module_branch_id.repository_id.specific" + ) + def _compute_moved_to_generic(self): + for rec in self: + rec.moved_to_generic = ( + rec.repository_id.specific + and rec.target_module_branch_id.repository_id + and not rec.target_module_branch_id.repository_id.specific + ) + + @api.depends( + "process", "pr_url", "moved_to_standard", "moved_to_oca", "moved_to_generic" + ) def _compute_state(self): for rec in self: + if rec.moved_to_standard: + # Module moved to a standard repository (likely from OCA to + # odoo/odoo, like 'l10n_eu_oss', 'knowledge', ...). + # E.g. this could tell integrators that a module like + # 'l10n_eu_oss_oca' should now be used instead. + rec.state = "moved_to_standard" + continue + if rec.moved_to_oca: + # Module moved to an OCA repository + rec.state = "moved_to_oca" + continue + if rec.moved_to_generic: + # Specific module moved to a generic repository (public or private) + rec.state = "moved_to_generic" + continue rec.state = rec.process or "fully_ported" if rec.process == "migrate" and rec.pr_url: rec.state = "review_migration" @@ -134,17 +214,25 @@ def _compute_results_text(self): rec.results_text = pprint.pformat(rec.results) @api.depends( + "repository_id.collect_migration_data", "last_source_scanned_commit", "last_target_scanned_commit", "pr_url", "target_module_branch_id.pr_url", "target_module_branch_id.last_scanned_commit", + "state", ) def _compute_migration_scan(self): # Migration scan to do if last scanned commit doesn't match the last # migration scan, both for source and target modules. for rec in self: rec.migration_scan = False + # No migration scan if repository is not configured to do it + if not rec.repository_id.collect_migration_data: + continue + # No migration scan for modules moved to Odoo/OCA/generic repo + if rec.state and rec.state.startswith("moved_to"): + continue if ( rec.last_source_scanned_commit != rec.module_branch_id.last_scanned_commit diff --git a/odoo_repository_migration/models/odoo_repository.py b/odoo_repository_migration/models/odoo_repository.py index 521100b6..b2879e2a 100644 --- a/odoo_repository_migration/models/odoo_repository.py +++ b/odoo_repository_migration/models/odoo_repository.py @@ -117,7 +117,25 @@ def _scan_migration_module(self, migration_path_id, module_branch_id): migration_path = ( self.env["odoo.migration.path"].browse(migration_path_id).exists() ) - params = self._prepare_migration_scanner_parameters(migration_path) + # Check if module has already been migrated on target version but in a + # different repository. If so, tune the scanner parameters to perform + # the scan from current repo to new one. + target_repository = None + mig = module.migration_ids.filtered( + lambda mig: mig.migration_path_id.id == migration_path_id + ) + target_module = mig.target_module_branch_id + if target_module.repository_branch_id: + target_repository = target_module.repository_id + if not target_repository.collect_migration_data: + return ( + "Cannot collect migration data on repository " + f"{target_repository.display_name}." + ) + params = self._prepare_migration_scanner_parameters( + migration_path, target_repository + ) + # Run the migration scan try: scanner = MigrationScannerOdooEnv(**params) return scanner.scan( @@ -141,14 +159,16 @@ def _migration_get_modules_to_scan(self, migration_path): ] ) - def _prepare_migration_scanner_parameters(self, migration_path): + def _prepare_migration_scanner_parameters( + self, migration_path, target_repository=None + ): ir_config = self.env["ir.config_parameter"] repositories_path = ir_config.sudo().get_param(self._repositories_path_key) mig_path = ( migration_path.source_branch_id.name, migration_path.target_branch_id.name, ) - return { + params = { "org": self.org_id.name, "name": self.name, "clone_url": self.clone_url, @@ -163,6 +183,10 @@ def _prepare_migration_scanner_parameters(self, migration_path): "clone_name": self.clone_name, "env": self.env, } + if target_repository and target_repository != self: + params["new_repo_name"] = target_repository.name + params["new_repo_url"] = target_repository.clone_url + return params def _pre_create_or_update_module_branch(self, rec, values, raw_data): # Handle migration data diff --git a/odoo_repository_migration/tests/test_odoo_module_branch.py b/odoo_repository_migration/tests/test_odoo_module_branch.py index ebfa7254..6efabeab 100644 --- a/odoo_repository_migration/tests/test_odoo_module_branch.py +++ b/odoo_repository_migration/tests/test_odoo_module_branch.py @@ -21,6 +21,18 @@ def setUp(self): repository_branch_id=self.repo_branch.id, last_scanned_commit="sha", ) + self.std_repository = self.env.ref("odoo_repository.odoo_repository_odoo_odoo") + self.oca_repository = self.env.ref("odoo_repository.repo_oca_server_tools") + self.gen_repository = self.env["odoo.repository"].create( + { + "name": "new_repo", + "org_id": self.odoo_repository.org_id.id, + "repo_url": "http://example.net/new_repo", + "specific": False, + "to_scan": False, + } + ) + self.gen_repository.addons_path_ids = self.odoo_repository.addons_path_ids def _simulate_migration_scan(self, target_commit, report=None): """Helper method that pushes scanned migration data.""" @@ -166,3 +178,113 @@ def test_migration_scan_target_module_in_review_then_merged(self): self.assertEqual(self.module_branch.migration_ids.state, "fully_ported") self.assertFalse(self.module_branch.migration_ids.migration_scan) self.assertFalse(self.module_branch.migration_scan) + + def test_migration_scan_target_module_moved_to_standard(self): + """Module moved into a standard repository.""" + # Simulate a scan of a given migration path while the target module is + # not yet migrated/available in a repository + self.env["odoo.migration.path"].create( + { + "source_branch_id": self.branch.id, + "target_branch_id": self.branch2.id, + } + ) + self._simulate_migration_scan( + "target_commit1", report={"process": "migrate", "results": {}} + ) + self.assertTrue(self.module_branch.migration_ids) + mig = self.module_branch.migration_ids + self.assertFalse(mig.target_module_branch_id) + self.assertFalse(mig.migration_scan) + self.assertFalse(self.module_branch.migration_scan) + self.assertEqual(mig.state, "migrate") + # Then the module is discovered in a std repository + std_repo_branch = self._create_odoo_repository_branch( + self.std_repository, self.branch2 + ) + target_module_branch = self._create_odoo_module_branch( + self.module, + self.branch2, + specific=False, + is_standard=True, + repository_branch_id=std_repo_branch.id, + ) + self.assertEqual(mig.target_module_branch_id, target_module_branch) + self.assertTrue(mig.moved_to_standard) + self.assertFalse(mig.moved_to_oca) + self.assertFalse(mig.moved_to_generic) + self.assertEqual(mig.state, "moved_to_standard") + self.assertFalse(mig.migration_scan) + + def test_migration_scan_target_module_moved_to_oca(self): + """Module moved into an OCA repository.""" + # Simulate a scan of a given migration path while the target module is + # not yet migrated/available in a repository + self.env["odoo.migration.path"].create( + { + "source_branch_id": self.branch.id, + "target_branch_id": self.branch2.id, + } + ) + self._simulate_migration_scan( + "target_commit1", report={"process": "migrate", "results": {}} + ) + self.assertTrue(self.module_branch.migration_ids) + mig = self.module_branch.migration_ids + self.assertFalse(mig.target_module_branch_id) + self.assertFalse(mig.migration_scan) + self.assertFalse(self.module_branch.migration_scan) + self.assertEqual(mig.state, "migrate") + # Then the module is discovered in an OCA repository + oca_repo_branch = self._create_odoo_repository_branch( + self.oca_repository, self.branch2 + ) + target_module_branch = self._create_odoo_module_branch( + self.module, + self.branch2, + specific=False, + repository_branch_id=oca_repo_branch.id, + ) + self.assertEqual(mig.target_module_branch_id, target_module_branch) + self.assertFalse(mig.moved_to_standard) + self.assertTrue(mig.moved_to_oca) + self.assertFalse(mig.moved_to_generic) + self.assertEqual(mig.state, "moved_to_oca") + self.assertFalse(mig.migration_scan) + + def test_migration_scan_target_module_moved_to_generic(self): + """Specific module moved into a generic repository (that is not std or OCA).""" + self.odoo_repository.specific = True + # Simulate a scan of a given migration path while the target module is + # not yet migrated/available in a repository + self.env["odoo.migration.path"].create( + { + "source_branch_id": self.branch.id, + "target_branch_id": self.branch2.id, + } + ) + self._simulate_migration_scan( + "target_commit1", report={"process": "migrate", "results": {}} + ) + self.assertTrue(self.module_branch.migration_ids) + mig = self.module_branch.migration_ids + self.assertFalse(mig.target_module_branch_id) + self.assertFalse(mig.migration_scan) + self.assertFalse(self.module_branch.migration_scan) + self.assertEqual(mig.state, "migrate") + # Then the module is discovered in an OCA repository + gen_repo_branch = self._create_odoo_repository_branch( + self.gen_repository, self.branch2 + ) + target_module_branch = self._create_odoo_module_branch( + self.module, + self.branch2, + specific=False, + repository_branch_id=gen_repo_branch.id, + ) + self.assertEqual(mig.target_module_branch_id, target_module_branch) + self.assertFalse(mig.moved_to_standard) + self.assertFalse(mig.moved_to_oca) + self.assertTrue(mig.moved_to_generic) + self.assertEqual(mig.state, "moved_to_generic") + self.assertFalse(mig.migration_scan)