Syncing from upstream odoo/runbot (19.0-show-cached-add-xdo)

runbot/__manifest__.py

@@ -6,7 +6,7 @@
     'author': "Odoo SA",
     'website': "http://runbot.odoo.com",
     'category': 'Website',
-    'version': '5.16',
+    'version': '5.17',
     'application': True,
     'depends': ['base', 'base_automation', 'website', 'auth_oauth'],
     'data': [
@@ -82,6 +82,7 @@
             'runbot/static/lib/fontawesome/css/font-awesome.css',
             'runbot/static/src/css/runbot.css',
 
+            'runbot/static/src/js/polyfill_command_api.js',
             'runbot/static/lib/jquery/jquery.js',
             'runbot/static/lib/bootstrap/js/bootstrap.bundle.js',
             'runbot/static/src/js/runbot.js',

runbot/common.py

@@ -24,6 +24,13 @@
 dest_reg = re.compile(r'^\d{5,}-.+$')
 
 
+try:
+    from odoo.addons.saas_worker.util import from_role
+except ImportError:
+    def from_role(*_, **__):
+        return lambda _: None
+
+
 def transactioncache(method):
     @functools.wraps(method)
     def wrapper(self, *args, **kwargs):
@@ -322,18 +329,43 @@ class TestTagsParser:
                                 (?:\[(.*)\])?               # parameters
                                 $''', re.VERBOSE)  # [-][tag][/module][:class][.method][[params]]
 
-    def __init__(self, test_tags):
-        parts = re.split(r',(?![^\[]*\])', test_tags)  # split on all comma not inside [] (not followed by ])
+    def __init__(self, test_tags, keep_escape=True):
+        parts = ['']
+        bracket_level = 0
+        escape_next = False
+        for char in test_tags:
+            if char == ',' and bracket_level == 0:
+                parts.append('')
+                continue
+
+            if char == '\\':
+                if not escape_next:
+                    escape_next = True
+                    if keep_escape:
+                        parts[-1] += '\\'  # not as the TagsSelector, we keep the escape character
+                    continue
+            elif char == '[':
+                if not escape_next:
+                    bracket_level += 1
+            elif char == ']':
+                if not escape_next:
+                    bracket_level -= 1
+            elif not keep_escape and escape_next:  # the previous \ was not escaping anything, put it back
+                parts[-1] += '\\'
+
+            escape_next = False
+            parts[-1] += char
+
         filter_specs = [t.strip() for t in parts if t.strip()]
+        self.filter_specs = filter_specs
         self.exclude = set()
         self.include = set()
         self.parameters = OrderedSet()
 
         for filter_spec in filter_specs:
             match = self.filter_spec_re.match(filter_spec)
             if not match:
-                _logger.error('Invalid tag %s', filter_spec)
-                continue
+                raise ValueError('Invalid tag %s' % filter_spec)
 
             sign, tag, file_path, module, klass, method, parameters = match.groups()
             is_include = sign != '-'
@@ -362,13 +394,18 @@ def __init__(self, test_tags):
 
     def test_tags_to_search_domain(self, exclude_error_id=None):
         search_domains = []
+        params_by_spec = dict(self.parameters)
         for include in self.include:
             _, test_module, test_class, test_method, file_path = include
             module_path = file_path or ((test_module or '') + '%')
             test_class = test_class or '%'
             test_method = test_method or '%'
             search_pattern = f'{module_path}:{test_class}.{test_method}'
             tag_domain = [('canonical_tags', 'like', f'{search_pattern}')]
+            params = params_by_spec.get(include)
+            if params:
+                _sign, parameters = params
+                tag_domain.append(('canonical_tags', 'like', f'%[{parameters}%]%'))
             if exclude_error_id:
                 tag_domain.append(('id', '!=', exclude_error_id))
             search_domains.append(tag_domain)

runbot/container.py

@@ -108,21 +108,22 @@ def get_config(self, starting_config=''):
         return res.read()
 
 
-def docker_build(build_dir, image_tag, pull=False):
-    return _docker_build(build_dir, image_tag, pull)
+def docker_build(build_dir, image_tag, pull=False, nocache=False):
+    return _docker_build(build_dir, image_tag, pull, nocache)
 
 
-def _docker_build(build_dir, image_tag, pull=False):
+def _docker_build(build_dir, image_tag, pull=False, nocache=False):
     """Build the docker image
     :param build_dir: the build directory that contains Dockerfile.
     :param image_tag: name used to tag the resulting docker image
+    :param nocache: bypass Docker layer cache when True
     :return: dict
     """
 
     with DockerManager(image_tag) as dm:
         last_step = None
         dm.result['success'] = False  # waiting for an image_id
-        for chunk in dm.consume(dm.docker_client.api.build(path=build_dir, tag=image_tag, rm=True, pull=pull)):
+        for chunk in dm.consume(dm.docker_client.api.build(path=build_dir, tag=image_tag, rm=True, pull=pull, nocache=nocache)):
             if 'stream' in chunk:
                 stream = chunk['stream']
                 if stream.startswith('Step '):
@@ -259,7 +260,10 @@ def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False
     else:
         run_cmd = cmd
     run_cmd = f'cd /data/build;touch start-{container_name};{run_cmd};cd /data/build;touch end-{container_name}'
-    _logger.info('Docker run command: %s', run_cmd)
+    run_cmd_repr = str(run_cmd)
+    if len(run_cmd_repr) > 250:
+        run_cmd_repr = run_cmd_repr[:250] + '...'
+    _logger.info('Docker run command: %s', run_cmd_repr)
     docker_clear_state(container_name, build_dir)  # ensure that no state are remaining
     build_dir = file_path(build_dir)
 

runbot/controllers/badge.py

@@ -44,13 +44,9 @@ def badge(self, name, repo_id=False, trigger_id=False, theme='default'):
         if not builds:
             state = 'testing'
         else:
-            result = builds._result_multi()
-            if result == 'ok':
+            state = 'failed'
+            if all(build.global_result == 'ok' for build in builds):
                 state = 'success'
-            elif result == 'warn':
-                state = 'warning'
-            else:
-                state = 'failed'
 
         etag = request.httprequest.headers.get('If-None-Match')
         retag = hashlib.md5(state.encode()).hexdigest()

runbot/controllers/frontend.py

@@ -80,7 +80,7 @@ def _pending(self):
             '/runbot/<model("runbot.project"):project>',
             '/runbot/<model("runbot.project"):project>/search/<search>'], website=True, auth='public', type='http')
     def bundles(self, project=None, search='', refresh=False, limit=40, has_pr=None, **kwargs):
-        search = search if len(search) < 60 else search[:60]
+        search = search if len(search) < 60 else search[:200]
         env = request.env
         categories = env['runbot.category'].search([])
         projects = self.env['runbot.project'].search([('hidden', '=', False)])
@@ -119,13 +119,11 @@ def bundles(self, project=None, search='', refresh=False, limit=40, has_pr=None,
                 pr_numbers = []
                 for search_elem in search.split("|"):
                     if search_elem.isnumeric():
-                        pr_numbers.append(int(search_elem))
+                        search_domains.append([('branch_ids', 'any', [('name', '=', search_elem)])])
+                    if ':' in search_elem:
+                        search_domains.append([('branch_ids', 'any', [('pull_head_name', '=', search_elem)])])
                     operator = '=ilike' if '%' in search_elem else 'ilike'
                     search_domains.append([('name', operator, search_elem)])
-                if pr_numbers:
-                    res = request.env['runbot.branch'].search([('name', 'in', pr_numbers)])
-                    if res:
-                        search_domains.append([('id', 'in', res.mapped('bundle_id').ids)])
                 search_domain = Domain.OR(search_domains)
                 domain = Domain.AND([domain, search_domain])
 
@@ -166,7 +164,7 @@ def bundles(self, project=None, search='', refresh=False, limit=40, has_pr=None,
         '/runbot/bundle/<model("runbot.bundle"):bundle>/page/<int:page>',
         '/runbot/bundle/<string:bundle>',
         ], website=True, auth='public', type='http', sitemap=False)
-    def bundle(self, bundle=None, page=1, limit=50, **kwargs):
+    def bundle(self, bundle=None, page=1, limit=50, expand_custom=False, **kwargs):
         if isinstance(bundle, str):
             bundle = request.env['runbot.bundle'].search([('name', '=', bundle)], limit=1, order='id')
             if not bundle:
@@ -183,13 +181,16 @@ def bundle(self, bundle=None, page=1, limit=50, **kwargs):
         )
         batchs = request.env['runbot.batch'].search(domain, limit=limit, offset=pager.get('offset', 0), order='id desc')
 
+        # compute if we should display the new batch button
         context = {
             'bundle': bundle,
             'batchs': batchs,
             'pager': pager,
             'project': bundle.project_id,
             'title': 'Bundle %s' % bundle.name,
             'page_info_state': bundle.last_batch._get_global_result(),
+            'expand_custom': expand_custom,
+            'needs_update': bundle.last_batch and bundle.last_batch.sudo().needs_update(),
         }
 
         return request.render('runbot.bundle', context)
@@ -199,7 +200,7 @@ def bundle(self, bundle=None, page=1, limit=50, **kwargs):
         '/runbot/bundle/<model("runbot.bundle"):bundle>/force/<int:auto_rebase>',
     ], type='http', auth="user", methods=['GET', 'POST'], csrf=False)
     def force_bundle(self, bundle, auto_rebase=False, use_base_commits=False, **_post):
-        if not request.env.user.has_group('runbot.group_runbot_advanced_user') and ':' not in bundle.name:
+        if not request.env.user.has_group('runbot.group_runbot_advanced_user') and ':' not in bundle.name and not bundle.last_batch.needs_update():
             message = "Only users with a specific group can do that. Please contact runbot administrators"
             raise Forbidden(message)
         _logger.info('user %s forcing bundle %s', request.env.user.name, bundle.name)  # user must be able to read bundle
@@ -275,6 +276,7 @@ def resend_status(self, status_id=None, **kwargs):
     ], type='http', auth="user", methods=['POST'], csrf=False)
     def build_operations(self, build_id, operation, **post):
         build = request.env['runbot.build'].sudo().browse(build_id)
+        build.check_access('read')
         if operation == 'rebuild':
             build = build._rebuild()
         elif operation == 'kill':
@@ -289,18 +291,17 @@ def build_operations(self, build_id, operation, **post):
         '/runbot/batch/<int:from_batch>/build/<int:build_id>',
     ], type='http', auth="public", website=True, sitemap=False)
     def build(self, build_id, search=None, from_batch=None, **post):
-        """Events/Logs"""
-
+        build = request.env['runbot.build'].browse(build_id)
         if from_batch:
             from_batch = request.env['runbot.batch'].browse(int(from_batch))
-            if build_id not in from_batch.with_context(active_test=False).slot_ids.build_id.ids:
+            if build.top_parent not in from_batch.with_context(active_test=False).slot_ids.build_id and build.create_batch_id != from_batch:
                 # the url may have been forged replacing the build id, redirect to hide the batch
                 return werkzeug.utils.redirect('/runbot/build/%s' % build_id)
 
             from_batch = from_batch.with_context(batch=from_batch)
         Build = request.env['runbot.build'].with_context(batch=from_batch)
 
-        build = Build.browse([build_id])[0]
+        build = Build.browse(build_id)
         if not build.exists():
             return request.not_found()
         siblings = (build.parent_id.children_ids if build.parent_id else from_batch.slot_ids.build_id if from_batch else build).sorted('id')
@@ -322,7 +323,8 @@ def build(self, build_id, search=None, from_batch=None, **post):
     @route([
     '/runbot/build/search',
     ], website=True, auth='public', type='http', sitemap=False)
-    def builds(self, **kwargs):
+    def builds(self, limit=100, **kwargs):
+        limit = min(int(limit), 1000)
         domain = []
         for key in ('config_id', 'version_id', 'project_id', 'trigger_id', 'create_batch_id.bundle_id', 'create_batch_id'):  # allowed params
             value = kwargs.get(key)
@@ -336,10 +338,12 @@ def builds(self, **kwargs):
 
         for key in ('description',):
             if key in kwargs:
-                domain.append((f'{key}', 'ilike', kwargs.get(key)))
+                value = kwargs.get(key)
+                operator = 'ilike' if '%' in value else '='
+                domain.append((f'{key}', operator, value))
 
         context = {
-            'builds': request.env['runbot.build'].search(domain, limit=100),
+            'builds': request.env['runbot.build'].search(domain, limit=limit),
         }
 
         return request.render('runbot.build_search', context)
@@ -450,8 +454,8 @@ def build_errors(self, sort=None, page=1, limit=20, **kwargs):
             'build_count asc': 'Number seen: Low to High',
             'responsible asc': 'Assignee: A - Z',
             'responsible desc': 'Assignee: Z - A',
-            'module_name asc': 'Module name: A - Z',
-            'module_name desc': 'Module name: Z -A',
+            'team_id asc': 'Team',
+            'name asc': 'Name',
         }
 
         sort_order = sort if sort in sort_order_choices else 'last_seen_date desc'
@@ -669,19 +673,40 @@ def parse_log(self, ir_log, **kwargs):
         request.env['runbot.build.error']._parse_logs(ir_log)
         return werkzeug.utils.redirect('/runbot/build/%s' % ir_log.build_id.id)
 
-    @route(['/runbot/bundle/toggle_no_build/<int:bundle_id>/<int:value>'], type='http', auth='user', sitemap=False)
-    def toggle_no_build(self, bundle_id, value, **kwargs):
-        if not request.env.user.has_group('base.group_user'):
-            return 'Forbidden'
-        bundle = request.env['runbot.bundle'].browse(bundle_id).exists()
-        if bundle.sticky or bundle.is_base:
-            return 'Forbidden'
-        if bundle.project_id.tmp_prefix and bundle.name.startswith(bundle.project_id.tmp_prefix):
-            return 'Forbidden'
-        bundle.sudo().no_build = bool(value)
-        _logger.info('Bundle %s no_build set to %s by %s', bundle.name, bool(value), request.env.user.name)
+    @route(['/runbot/bundle/<int:bundle_id>/triggers/<string:action>'], type='http', auth='user', sitemap=False)
+    def configure_bundle_triggers(self, bundle_id, action, expand_custom=False, **kwargs):
+        if not request.env.user.has_group('runbot.group_user'):
+            raise NotFound()
+
+        bundle = request.env['runbot.bundle'].browse(bundle_id)
+        if bundle.is_base or bundle.is_staging:
+            raise NotFound()
+        if action == 'disable_all':
+            bundle.sudo()._configure_custom_trigger_start_mode('disabled')
+        elif action == 'force_all':
+            bundle.sudo()._configure_custom_trigger_start_mode('force')
+        elif action == 'auto_all':
+            bundle.sudo()._configure_custom_trigger_start_mode('auto')
+        elif action == 'light_all':
+            bundle.sudo()._configure_custom_trigger_start_mode('light')
+        else:
+            raise NotFound()
+        if expand_custom:
+            return werkzeug.utils.redirect(f'/runbot/bundle/{bundle_id}?expand_custom=1')
         return werkzeug.utils.redirect(f'/runbot/bundle/{bundle_id}')
 
+    @route(['/runbot/trigger_custom/<int:trigger_custom_id>/set_mode/<string:mode>'], type='http', auth='user', sitemap=False)
+    def configure_custom_trigger(self, trigger_custom_id, mode, **kwargs):
+        if not request.env.user.has_group('runbot.group_user'):
+            raise NotFound()
+        trigger_custom = request.env['runbot.bundle.trigger.custom'].browse(trigger_custom_id)
+        bundle = trigger_custom.bundle_id
+        if bundle.is_base or bundle.is_staging:
+            raise NotFound()
+
+        trigger_custom.sudo().start_mode = mode
+        return werkzeug.utils.redirect(f'/runbot/bundle/{trigger_custom.bundle_id.id}?expand_custom=1')
+
     @route(['/runbot/trigger/report/<model("runbot.trigger"):trigger_id>'], type='http', auth='user', website=True, sitemap=False)
     def report_view(self, trigger_id=None, **kwargs):
         return request.render("runbot.trigger_report", {

runbot/controllers/hook.py

@@ -4,8 +4,9 @@
 import json
 import logging
 
-from odoo import http
+from odoo import http, fields
 from odoo.http import request
+from ..common import from_role
 
 _logger = logging.getLogger(__name__)
 
@@ -50,3 +51,18 @@ def hook(self, remote_id=None, **_post):
                 branch = request.env['runbot.branch'].sudo().search([('remote_id', '=', remote.id), ('name', '=', branch_ref)])
                 branch.alive = False
         return ""
+
+    @from_role('mergebot', signed=True)
+    @http.route(['/runbot/request_ci'], type='http', methods=["POST"], auth="public", website=True, csrf=False, sitemap=False)
+    def force_ci(self):
+        pull_request_names = request.get_json_data().get('pull_requests', [])
+        pull_domains = []
+        for pull_request_names in pull_request_names:
+            remote_short_name, name = pull_request_names.split('#')
+            owner, repo_name = remote_short_name.split('/')
+            pull_domains.append([('remote_id.owner', '=', owner), ('remote_id.repo_name', '=', repo_name), ('name', '=', name)])
+        pull_domains = fields.Domain.OR(pull_domains)
+        pull_requests = request.env['runbot.branch'].sudo().search([('is_pr', '=', True)] + pull_domains)
+        bundles = pull_requests.bundle_id
+        _logger.info('Received CI request for bundles: %s', bundles.mapped('name'))
+        bundles._force_ci()

runbot/data/dockerfile_data.xml

@@ -71,7 +71,7 @@
         <field name="dockerfile_id" ref="runbot.docker_default"/>
         <field name="layer_type">reference_layer</field>
         <field name="name">Install python debian packages</field>
-        <field name="packages">publicsuffix python3 flake8 python3-dbfread python3-dev python3-gevent python3-pip python3-setuptools python3-wheel python3-markdown python3-mock python3-phonenumbers python3-websocket python3-google-auth libpq-dev pylint python3-jwt python3-asn1crypto python3-html2text python3-suds python3-xmlsec python3-markdown2 python3-aiosmtpd</field>
+        <field name="packages">publicsuffix python3 flake8 python3-dbfread python3-dev python3-gevent python3-pip python3-setuptools python3-wheel python3-markdown python3-mock python3-phonenumbers python3-websocket python3-google-auth libpq-dev pylint python3-jwt python3-asn1crypto python3-html2text python3-suds python3-xmlsec python3-markdown2 python3-aiosmtpd python3-paramiko</field>
         <field name="reference_docker_layer_id" ref="runbot.docker_layer_debian_packages_template"/>
     </record>
 
@@ -137,7 +137,7 @@ RUN curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc -o /etc/apt/tru
         <field name="dockerfile_id" ref="runbot.docker_default"/>
         <field name="layer_type">template</field>
         <field name="name">Install chrome</field>
-        <field name="values" eval="{'chrome_version': '141.0.7390.54-1'}"/>
+        <field name="values" eval="{'chrome_version': '145.0.7632.116-1'}"/>
         <field name="content">RUN curl -sSL https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_{chrome_version}_amd64.deb -o /tmp/chrome.deb \
     &amp;&amp; apt-get update \
     &amp;&amp; apt-get -y install --no-install-recommends /tmp/chrome.deb \