Syncing from upstream odoo/runbot (19.0-less-sources-exports-xdo)

runbot/__manifest__.py

@@ -6,7 +6,7 @@
     'author': "Odoo SA",
     'website': "http://runbot.odoo.com",
     'category': 'Website',
-    'version': '5.14',
+    'version': '5.16',
     'application': True,
     'depends': ['base', 'base_automation', 'website', 'auth_oauth'],
     'data': [
@@ -57,6 +57,7 @@
         'views/oauth_provider_views.xml',
         'views/repo_views.xml',
         'views/res_config_settings_views.xml',
+        'views/semgrep_rules.xml',
         'views/stat_views.xml',
         'views/upgrade.xml',
         'views/upgrade_matrix_views.xml',

runbot/common.py

@@ -24,6 +24,13 @@
 dest_reg = re.compile(r'^\d{5,}-.+$')
 
 
+try:
+    from odoo.addons.saas_worker.util import from_role
+except ImportError:
+    def from_role(*_, **__):
+        return lambda _: None
+
+
 def transactioncache(method):
     @functools.wraps(method)
     def wrapper(self, *args, **kwargs):

runbot/container.py

@@ -116,7 +116,7 @@ def _docker_build(build_dir, image_tag, pull=False):
     """Build the docker image
     :param build_dir: the build directory that contains Dockerfile.
     :param image_tag: name used to tag the resulting docker image
-    :return: tuple(success, msg) where success is a boolean and msg is the error message or None
+    :return: dict
     """
 
     with DockerManager(image_tag) as dm:
@@ -259,7 +259,10 @@ def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False
     else:
         run_cmd = cmd
     run_cmd = f'cd /data/build;touch start-{container_name};{run_cmd};cd /data/build;touch end-{container_name}'
-    _logger.info('Docker run command: %s', run_cmd)
+    run_cmd_repr = str(run_cmd)
+    if len(run_cmd_repr) > 250:
+        run_cmd_repr = run_cmd_repr[:250] + '...'
+    _logger.info('Docker run command: %s', run_cmd_repr)
     docker_clear_state(container_name, build_dir)  # ensure that no state are remaining
     build_dir = file_path(build_dir)
 

runbot/controllers/frontend.py

@@ -80,7 +80,7 @@ def _pending(self):
             '/runbot/<model("runbot.project"):project>',
             '/runbot/<model("runbot.project"):project>/search/<search>'], website=True, auth='public', type='http')
     def bundles(self, project=None, search='', refresh=False, limit=40, has_pr=None, **kwargs):
-        search = search if len(search) < 60 else search[:60]
+        search = search if len(search) < 60 else search[:200]
         env = request.env
         categories = env['runbot.category'].search([])
         projects = self.env['runbot.project'].search([('hidden', '=', False)])
@@ -119,13 +119,11 @@ def bundles(self, project=None, search='', refresh=False, limit=40, has_pr=None,
                 pr_numbers = []
                 for search_elem in search.split("|"):
                     if search_elem.isnumeric():
-                        pr_numbers.append(int(search_elem))
+                        search_domains.append([('branch_ids', 'any', [('name', '=', search_elem)])])
+                    if ':' in search_elem:
+                        search_domains.append([('branch_ids', 'any', [('pull_head_name', '=', search_elem)])])
                     operator = '=ilike' if '%' in search_elem else 'ilike'
                     search_domains.append([('name', operator, search_elem)])
-                if pr_numbers:
-                    res = request.env['runbot.branch'].search([('name', 'in', pr_numbers)])
-                    if res:
-                        search_domains.append([('id', 'in', res.mapped('bundle_id').ids)])
                 search_domain = Domain.OR(search_domains)
                 domain = Domain.AND([domain, search_domain])
 
@@ -166,7 +164,7 @@ def bundles(self, project=None, search='', refresh=False, limit=40, has_pr=None,
         '/runbot/bundle/<model("runbot.bundle"):bundle>/page/<int:page>',
         '/runbot/bundle/<string:bundle>',
         ], website=True, auth='public', type='http', sitemap=False)
-    def bundle(self, bundle=None, page=1, limit=50, **kwargs):
+    def bundle(self, bundle=None, page=1, limit=50, expand_custom=False, **kwargs):
         if isinstance(bundle, str):
             bundle = request.env['runbot.bundle'].search([('name', '=', bundle)], limit=1, order='id')
             if not bundle:
@@ -183,13 +181,16 @@ def bundle(self, bundle=None, page=1, limit=50, **kwargs):
         )
         batchs = request.env['runbot.batch'].search(domain, limit=limit, offset=pager.get('offset', 0), order='id desc')
 
+        # compute if we should display the new batch button
         context = {
             'bundle': bundle,
             'batchs': batchs,
             'pager': pager,
             'project': bundle.project_id,
             'title': 'Bundle %s' % bundle.name,
             'page_info_state': bundle.last_batch._get_global_result(),
+            'expand_custom': expand_custom,
+            'needs_update': bundle.last_batch and bundle.last_batch.sudo().needs_update(),
         }
 
         return request.render('runbot.bundle', context)
@@ -199,7 +200,7 @@ def bundle(self, bundle=None, page=1, limit=50, **kwargs):
         '/runbot/bundle/<model("runbot.bundle"):bundle>/force/<int:auto_rebase>',
     ], type='http', auth="user", methods=['GET', 'POST'], csrf=False)
     def force_bundle(self, bundle, auto_rebase=False, use_base_commits=False, **_post):
-        if not request.env.user.has_group('runbot.group_runbot_advanced_user') and ':' not in bundle.name:
+        if not request.env.user.has_group('runbot.group_runbot_advanced_user') and ':' not in bundle.name and not bundle.last_batch.needs_update():
             message = "Only users with a specific group can do that. Please contact runbot administrators"
             raise Forbidden(message)
         _logger.info('user %s forcing bundle %s', request.env.user.name, bundle.name)  # user must be able to read bundle
@@ -220,6 +221,12 @@ def batch(self, batch_id=None, **kwargs):
         }
         return request.render('runbot.batch', context)
 
+    @route(['/runbot/batch/<int:batch_id>/prioritize'], website=True, auth='user', type='http', sitemap=False)
+    def batch_priority(self, batch_id=None, **kwargs):
+        batch = request.env['runbot.batch'].browse(batch_id)
+        batch.sudo().priority_level = int(batch.create_date.timestamp() - 3600)
+        return werkzeug.utils.redirect('/runbot/batch/%s' % batch_id)
+
     @route(['/runbot/batch/slot/<model("runbot.batch.slot"):slot>/build'], auth='user', type='http')
     def slot_create_build(self, slot=None, **kwargs):
         build = slot.sudo()._create_missing_build()
@@ -316,7 +323,8 @@ def build(self, build_id, search=None, from_batch=None, **post):
     @route([
     '/runbot/build/search',
     ], website=True, auth='public', type='http', sitemap=False)
-    def builds(self, **kwargs):
+    def builds(self, limit=100, **kwargs):
+        limit = min(int(limit), 1000)
         domain = []
         for key in ('config_id', 'version_id', 'project_id', 'trigger_id', 'create_batch_id.bundle_id', 'create_batch_id'):  # allowed params
             value = kwargs.get(key)
@@ -330,10 +338,12 @@ def builds(self, **kwargs):
 
         for key in ('description',):
             if key in kwargs:
-                domain.append((f'{key}', 'ilike', kwargs.get(key)))
+                value = kwargs.get(key)
+                operator = 'ilike' if '%' in value else '='
+                domain.append((f'{key}', operator, value))
 
         context = {
-            'builds': request.env['runbot.build'].search(domain, limit=100),
+            'builds': request.env['runbot.build'].search(domain, limit=limit),
         }
 
         return request.render('runbot.build_search', context)
@@ -663,19 +673,40 @@ def parse_log(self, ir_log, **kwargs):
         request.env['runbot.build.error']._parse_logs(ir_log)
         return werkzeug.utils.redirect('/runbot/build/%s' % ir_log.build_id.id)
 
-    @route(['/runbot/bundle/toggle_no_build/<int:bundle_id>/<int:value>'], type='http', auth='user', sitemap=False)
-    def toggle_no_build(self, bundle_id, value, **kwargs):
-        if not request.env.user.has_group('base.group_user'):
-            return 'Forbidden'
-        bundle = request.env['runbot.bundle'].browse(bundle_id).exists()
-        if bundle.sticky or bundle.is_base:
-            return 'Forbidden'
-        if bundle.project_id.tmp_prefix and bundle.name.startswith(bundle.project_id.tmp_prefix):
-            return 'Forbidden'
-        bundle.sudo().no_build = bool(value)
-        _logger.info('Bundle %s no_build set to %s by %s', bundle.name, bool(value), request.env.user.name)
+    @route(['/runbot/bundle/<int:bundle_id>/triggers/<string:action>'], type='http', auth='user', sitemap=False)
+    def configure_bundle_triggers(self, bundle_id, action, expand_custom=False, **kwargs):
+        if not request.env.user.has_group('runbot.group_user'):
+            raise NotFound()
+
+        bundle = request.env['runbot.bundle'].browse(bundle_id)
+        if bundle.is_base or bundle.is_staging:
+            raise NotFound()
+        if action == 'disable_all':
+            bundle.sudo()._configure_custom_trigger_start_mode('disabled')
+        elif action == 'force_all':
+            bundle.sudo()._configure_custom_trigger_start_mode('force')
+        elif action == 'auto_all':
+            bundle.sudo()._configure_custom_trigger_start_mode('auto')
+        elif action == 'light_all':
+            bundle.sudo()._configure_custom_trigger_start_mode('light')
+        else:
+            raise NotFound()
+        if expand_custom:
+            return werkzeug.utils.redirect(f'/runbot/bundle/{bundle_id}?expand_custom=1')
         return werkzeug.utils.redirect(f'/runbot/bundle/{bundle_id}')
 
+    @route(['/runbot/trigger_custom/<int:trigger_custom_id>/set_mode/<string:mode>'], type='http', auth='user', sitemap=False)
+    def configure_custom_trigger(self, trigger_custom_id, mode, **kwargs):
+        if not request.env.user.has_group('runbot.group_user'):
+            raise NotFound()
+        trigger_custom = request.env['runbot.bundle.trigger.custom'].browse(trigger_custom_id)
+        bundle = trigger_custom.bundle_id
+        if bundle.is_base or bundle.is_staging:
+            raise NotFound()
+
+        trigger_custom.sudo().start_mode = mode
+        return werkzeug.utils.redirect(f'/runbot/bundle/{trigger_custom.bundle_id.id}?expand_custom=1')
+
     @route(['/runbot/trigger/report/<model("runbot.trigger"):trigger_id>'], type='http', auth='user', website=True, sitemap=False)
     def report_view(self, trigger_id=None, **kwargs):
         return request.render("runbot.trigger_report", {
@@ -852,21 +883,20 @@ def repos_heads(self, project_id=None, bundle_name=None, **kwargs):
         else:
             domain = Domain.AND([domain, [('sticky', '=', True)]])
         bundles = request.env['runbot.bundle'].search(domain, order='id desc, name')
-
-        last_batches_infos = {
-            bundle.name: {
+        last_batches_infos = dict()
+        for bundle in bundles:
+            batch = bundle.last_batch if bundle.last_batch.state != 'preparing' else bundle.last_done_batch
+            last_batches_infos[bundle.name] = {
                 "commits": [
                     {
                         "repo": commit_link.commit_id.repo_id.name,
                         "head": commit_link.commit_id.name,
                         "match_type": commit_link.match_type,
                     }
-                    for commit_link in bundle.last_batch.commit_link_ids
+                    for commit_link in batch.commit_link_ids
                 ],
-                "autotags": request.env["runbot.build.error"].sudo()._disabling_tags(build_id=bundle.last_batch.slot_ids.build_id[0]),
+                "autotags": request.env["runbot.build.error"].sudo()._disabling_tags(build_id=batch.slot_ids.build_id[0]),
             }
-            for bundle in bundles
-        }
         return request.make_json_response(last_batches_infos)
 
     @route([

runbot/controllers/hook.py

@@ -4,8 +4,9 @@
 import json
 import logging
 
-from odoo import http
+from odoo import http, fields
 from odoo.http import request
+from ..common import from_role
 
 _logger = logging.getLogger(__name__)
 
@@ -50,3 +51,18 @@ def hook(self, remote_id=None, **_post):
                 branch = request.env['runbot.branch'].sudo().search([('remote_id', '=', remote.id), ('name', '=', branch_ref)])
                 branch.alive = False
         return ""
+
+    @from_role('mergebot', signed=True)
+    @http.route(['/runbot/request_ci'], type='http', methods=["POST"], auth="public", website=True, csrf=False, sitemap=False)
+    def force_ci(self):
+        pull_request_names = request.get_json_data().get('pull_requests', [])
+        pull_domains = []
+        for pull_request_names in pull_request_names:
+            remote_short_name, name = pull_request_names.split('#')
+            owner, repo_name = remote_short_name.split('/')
+            pull_domains.append([('remote_id.owner', '=', owner), ('remote_id.repo_name', '=', repo_name), ('name', '=', name)])
+        pull_domains = fields.Domain.OR(pull_domains)
+        pull_requests = request.env['runbot.branch'].sudo().search([('is_pr', '=', True)] + pull_domains)
+        bundles = pull_requests.bundle_id
+        _logger.info('Received CI request for bundles: %s', bundles.mapped('name'))
+        bundles._force_ci()

runbot/data/dockerfile_data.xml

@@ -120,6 +120,7 @@
         <field name="name">Install branch debian/control with latest postgresql-client</field>
         <field name="values" eval="{'odoo_branch': 'master', 'os_release_name': '`lsb_release -s -c`'}"/>
         <field name="content"># This layer updates the repository list to get the latest postgresql-client, mainly needed if the host postgresql version is higher than the default version of the docker os
+# CACHE 60
 ADD https://raw.githubusercontent.com/odoo/odoo/{odoo_branch}/debian/control /tmp/control.txt
 RUN curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc -o /etc/apt/trusted.gpg.d/psql_client.asc \
     &amp;&amp; echo "deb http://apt.postgresql.org/pub/repos/apt/ {os_release_name}-pgdg main" &gt; /etc/apt/sources.list.d/pgclient.list \
@@ -136,7 +137,7 @@ RUN curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc -o /etc/apt/tru
         <field name="dockerfile_id" ref="runbot.docker_default"/>
         <field name="layer_type">template</field>
         <field name="name">Install chrome</field>
-        <field name="values" eval="{'chrome_version': '126.0.6478.182-1'}"/>
+        <field name="values" eval="{'chrome_version': '145.0.7632.116-1'}"/>
         <field name="content">RUN curl -sSL https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_{chrome_version}_amd64.deb -o /tmp/chrome.deb \
     &amp;&amp; apt-get update \
     &amp;&amp; apt-get -y install --no-install-recommends /tmp/chrome.deb \
@@ -195,7 +196,8 @@ ENV PIP_BREAK_SYSTEM_PACKAGES=1</field>
         <field name="layer_type">template</field>
         <field name="name">Install branch requirements</field>
         <field name="values" eval="{'odoo_branch': 'master'}"/>
-        <field name="content">ADD --chown={USERNAME} https://raw.githubusercontent.com/odoo/odoo/{odoo_branch}/requirements.txt /tmp/requirements.txt
+        <field name="content"># CACHE 60
+ADD --chown={USERNAME} https://raw.githubusercontent.com/odoo/odoo/{odoo_branch}/requirements.txt /tmp/requirements.txt
 RUN python3 -m pip install --no-cache-dir -r /tmp/requirements.txt</field>
     </record>
 

runbot/documentation/dynamic_config.md

@@ -75,7 +75,7 @@ The config steps are mainly defined by their `job_type`. The `name` key is also
 ```
 The `db_name` is optionnal, usually set to all as a convention on runbot for databases that contains *almost* all modules. If not defined the sanitized version of the name will be used.
 
-`install_modules` and `install_default_modules` behave the same way except that `install_modules` will consider that we start with no module (prepends `.*` filter) while `install_default_modules` will be based on the runbot default module list (all available modules minus the repo blacklist)
+`install_modules` and `install_default_modules` behave the same way except that `install_modules` will consider that we start with no module (prepends `-*` filter) while `install_default_modules` will be based on the runbot default module list (all available modules minus the repo blacklist)
 
 Both entries will use the value as a runbot module filter, and then passed as the -i, [see corresponding section](#module-selection) for more info.
 
@@ -344,21 +344,43 @@ Filters are a way to transform dynamic values before using them. They are define
 
 For example, to transform a module filter into test tags:
 
+#### filter_all_modules, make_module_test_tags
+
 ```json
     {"test_tags": "-at_install,{{test_module_filter|filter_all_modules|make_module_test_tags}}",
 ```
 
 In this example, the `filter_all_modules` filters will first transform the `test_module_filter` variable (which is a module filter) into a list of modules, and then the `make_module_test_tags` filters will transform this list of modules into test tags by prepending each module with a `/` to indicate that we want to run all tests from these modules.
 
-Note that `filter_all_modules` is actually equivalent to `filter_default_modules`, but prepending a `*` at the begining of the filter.
+#### filter_default_modules
+
+`filter_all_modules` is actually equivalent to `filter_default_modules`, but prepending a `*` at the begining of the filter. Without that a runbot defined filter is applied, returning a default list of modules per repo.
 
 `*,mail -> !web|filter_default_modules` is the same as `mail -> !web|filter_all_modules`
 
+
+#### prepend, append
 In some case we also want to combine the test-tags module with another tag or test method, this can be done using prepend and append
 
 `"{{-*,web*|filter_all_modules|make_module_test_tags|append('.test_method')}}`
 `{{-*,web*|filter_all_modules|make_module_test_tags|prepend('custom_tag')}}`
 
 
-It is also possible to filter modules based on the one modified in the current bundle.
+#### modified_modules
+
+It is possible to filter modules based on the one modified in the current bundle.
 `{{*|filter_all_modules|modified_modules}}"`
+
+#### select_existing_modules
+
+`select_existing_modules` is equivalent to `filter_default_modules` but with a -* at the beginning of the filter, meaning that we start with an empty selection and only add modules that are explicitly selected. 
+
+This is a solution to keep only existing modules from a specific list, when we are not sure modules exists:
+`{{*|filter_all_modules|modified_modules|prepend('test_')|select_existing_modules|make_module_test_tags}}`
+
+- `*|filter_all_modules` will select all existing modules
+- `|modified_modules` will only keep the modified ones
+- `prepend('test_')` will prepend test_ to have the test equivalent name of the modified modules (mail-> test_mail, base -> test_base)
+- `select_existing_modules` will only keep modules that exists (test_mail)
+- `make_module_test_tags` make the module test tags by prepending a / to each module.
+