Syncing from upstream odoo/runbot (19.0-build-errors-frontend-app)

runbot/__manifest__.py

@@ -6,7 +6,7 @@
     'author': "Odoo SA",
     'website': "http://runbot.odoo.com",
     'category': 'Website',
-    'version': '5.16',
+    'version': '5.17',
     'application': True,
     'depends': ['base', 'base_automation', 'website', 'auth_oauth'],
     'data': [
@@ -80,10 +80,14 @@
             'web/static/lib/odoo_ui_icons/style.css',
             'runbot/static/lib/bootstrap/css/bootstrap.css',
             'runbot/static/lib/fontawesome/css/font-awesome.css',
+            'runbot/static/src/css/table_group.css',
             'runbot/static/src/css/runbot.css',
 
+            'runbot/static/src/js/polyfill_command_api.js',
             'runbot/static/lib/jquery/jquery.js',
             'runbot/static/lib/bootstrap/js/bootstrap.bundle.js',
+            'runbot/static/src/js/table_filter.js',
+            'runbot/static/src/js/table_group.js',
             'runbot/static/src/js/runbot.js',
         ],
     },

runbot/common.py

@@ -19,6 +19,8 @@
 from odoo.fields import Domain
 from odoo.tools.misc import DEFAULT_SERVER_DATETIME_FORMAT, file_open, html_escape, OrderedSet
 
+DEFAULT_MAX_FILE_SIZE = 500 * 1024 * 1024
+
 _logger = logging.getLogger(__name__)
 
 dest_reg = re.compile(r'^\d{5,}-.+$')
@@ -329,18 +331,43 @@ class TestTagsParser:
                                 (?:\[(.*)\])?               # parameters
                                 $''', re.VERBOSE)  # [-][tag][/module][:class][.method][[params]]
 
-    def __init__(self, test_tags):
-        parts = re.split(r',(?![^\[]*\])', test_tags)  # split on all comma not inside [] (not followed by ])
+    def __init__(self, test_tags, keep_escape=True):
+        parts = ['']
+        bracket_level = 0
+        escape_next = False
+        for char in test_tags:
+            if char == ',' and bracket_level == 0:
+                parts.append('')
+                continue
+
+            if char == '\\':
+                if not escape_next:
+                    escape_next = True
+                    if keep_escape:
+                        parts[-1] += '\\'  # not as the TagsSelector, we keep the escape character
+                    continue
+            elif char == '[':
+                if not escape_next:
+                    bracket_level += 1
+            elif char == ']':
+                if not escape_next:
+                    bracket_level -= 1
+            elif not keep_escape and escape_next:  # the previous \ was not escaping anything, put it back
+                parts[-1] += '\\'
+
+            escape_next = False
+            parts[-1] += char
+
         filter_specs = [t.strip() for t in parts if t.strip()]
+        self.filter_specs = filter_specs
         self.exclude = set()
         self.include = set()
         self.parameters = OrderedSet()
 
         for filter_spec in filter_specs:
             match = self.filter_spec_re.match(filter_spec)
             if not match:
-                _logger.error('Invalid tag %s', filter_spec)
-                continue
+                raise ValueError('Invalid tag %s' % filter_spec)
 
             sign, tag, file_path, module, klass, method, parameters = match.groups()
             is_include = sign != '-'
@@ -369,13 +396,18 @@ def __init__(self, test_tags):
 
     def test_tags_to_search_domain(self, exclude_error_id=None):
         search_domains = []
+        params_by_spec = dict(self.parameters)
         for include in self.include:
             _, test_module, test_class, test_method, file_path = include
             module_path = file_path or ((test_module or '') + '%')
             test_class = test_class or '%'
             test_method = test_method or '%'
             search_pattern = f'{module_path}:{test_class}.{test_method}'
             tag_domain = [('canonical_tags', 'like', f'{search_pattern}')]
+            params = params_by_spec.get(include)
+            if params:
+                _sign, parameters = params
+                tag_domain.append(('canonical_tags', 'like', f'%[{parameters}%]%'))
             if exclude_error_id:
                 tag_domain.append(('id', '!=', exclude_error_id))
             search_domains.append(tag_domain)

runbot/container.py

@@ -108,21 +108,22 @@ def get_config(self, starting_config=''):
         return res.read()
 
 
-def docker_build(build_dir, image_tag, pull=False):
-    return _docker_build(build_dir, image_tag, pull)
+def docker_build(build_dir, image_tag, pull=False, nocache=False):
+    return _docker_build(build_dir, image_tag, pull, nocache)
 
 
-def _docker_build(build_dir, image_tag, pull=False):
+def _docker_build(build_dir, image_tag, pull=False, nocache=False):
     """Build the docker image
     :param build_dir: the build directory that contains Dockerfile.
     :param image_tag: name used to tag the resulting docker image
+    :param nocache: bypass Docker layer cache when True
     :return: dict
     """
 
     with DockerManager(image_tag) as dm:
         last_step = None
         dm.result['success'] = False  # waiting for an image_id
-        for chunk in dm.consume(dm.docker_client.api.build(path=build_dir, tag=image_tag, rm=True, pull=pull)):
+        for chunk in dm.consume(dm.docker_client.api.build(path=build_dir, tag=image_tag, rm=True, pull=pull, nocache=nocache)):
             if 'stream' in chunk:
                 stream = chunk['stream']
                 if stream.startswith('Step '):
@@ -259,7 +260,10 @@ def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False
     else:
         run_cmd = cmd
     run_cmd = f'cd /data/build;touch start-{container_name};{run_cmd};cd /data/build;touch end-{container_name}'
-    _logger.info('Docker run command: %s', run_cmd)
+    run_cmd_repr = str(run_cmd)
+    if len(run_cmd_repr) > 250:
+        run_cmd_repr = run_cmd_repr[:250] + '...'
+    _logger.info('Docker run command: %s', run_cmd_repr)
     docker_clear_state(container_name, build_dir)  # ensure that no state are remaining
     build_dir = file_path(build_dir)
 

runbot/controllers/badge.py

@@ -44,13 +44,9 @@ def badge(self, name, repo_id=False, trigger_id=False, theme='default'):
         if not builds:
             state = 'testing'
         else:
-            result = builds._result_multi()
-            if result == 'ok':
+            state = 'failed'
+            if all(build.global_result == 'ok' for build in builds):
                 state = 'success'
-            elif result == 'warn':
-                state = 'warning'
-            else:
-                state = 'failed'
 
         etag = request.httprequest.headers.get('If-None-Match')
         retag = hashlib.md5(state.encode()).hexdigest()

runbot/controllers/frontend.py

@@ -276,6 +276,7 @@ def resend_status(self, status_id=None, **kwargs):
     ], type='http', auth="user", methods=['POST'], csrf=False)
     def build_operations(self, build_id, operation, **post):
         build = request.env['runbot.build'].sudo().browse(build_id)
+        build.check_access('read')
         if operation == 'rebuild':
             build = build._rebuild()
         elif operation == 'kill':
@@ -290,18 +291,17 @@ def build_operations(self, build_id, operation, **post):
         '/runbot/batch/<int:from_batch>/build/<int:build_id>',
     ], type='http', auth="public", website=True, sitemap=False)
     def build(self, build_id, search=None, from_batch=None, **post):
-        """Events/Logs"""
-
+        build = request.env['runbot.build'].browse(build_id)
         if from_batch:
             from_batch = request.env['runbot.batch'].browse(int(from_batch))
-            if build_id not in from_batch.with_context(active_test=False).slot_ids.build_id.ids:
+            if build.top_parent not in from_batch.with_context(active_test=False).slot_ids.build_id and build.create_batch_id != from_batch:
                 # the url may have been forged replacing the build id, redirect to hide the batch
                 return werkzeug.utils.redirect('/runbot/build/%s' % build_id)
 
             from_batch = from_batch.with_context(batch=from_batch)
         Build = request.env['runbot.build'].with_context(batch=from_batch)
 
-        build = Build.browse([build_id])[0]
+        build = Build.browse(build_id)
         if not build.exists():
             return request.not_found()
         siblings = (build.parent_id.children_ids if build.parent_id else from_batch.slot_ids.build_id if from_batch else build).sorted('id')
@@ -323,7 +323,8 @@ def build(self, build_id, search=None, from_batch=None, **post):
     @route([
     '/runbot/build/search',
     ], website=True, auth='public', type='http', sitemap=False)
-    def builds(self, **kwargs):
+    def builds(self, limit=100, **kwargs):
+        limit = min(int(limit), 1000)
         domain = []
         for key in ('config_id', 'version_id', 'project_id', 'trigger_id', 'create_batch_id.bundle_id', 'create_batch_id'):  # allowed params
             value = kwargs.get(key)
@@ -337,10 +338,12 @@ def builds(self, **kwargs):
 
         for key in ('description',):
             if key in kwargs:
-                domain.append((f'{key}', 'ilike', kwargs.get(key)))
+                value = kwargs.get(key)
+                operator = 'ilike' if '%' in value else '='
+                domain.append((f'{key}', operator, value))
 
         context = {
-            'builds': request.env['runbot.build'].search(domain, limit=100),
+            'builds': request.env['runbot.build'].search(domain, limit=limit),
         }
 
         return request.render('runbot.build_search', context)
@@ -451,8 +454,8 @@ def build_errors(self, sort=None, page=1, limit=20, **kwargs):
             'build_count asc': 'Number seen: Low to High',
             'responsible asc': 'Assignee: A - Z',
             'responsible desc': 'Assignee: Z - A',
-            'module_name asc': 'Module name: A - Z',
-            'module_name desc': 'Module name: Z -A',
+            'team_id asc': 'Team',
+            'name asc': 'Name',
         }
 
         sort_order = sort if sort in sort_order_choices else 'last_seen_date desc'
@@ -478,6 +481,7 @@ def build_errors(self, sort=None, page=1, limit=20, **kwargs):
             'build_errors': build_errors,
             'title': 'Build Errors',
             'sort_order_choices': sort_order_choices,
+            'sort_order': sort_order,
             'page': page,
             'pager': pager,
         }
@@ -667,7 +671,7 @@ def access_running(self, build_id, db_suffix=None, **kwargs):
 
     @route(['/runbot/parse_log/<model("ir.logging"):ir_log>'], type='http', auth='user', sitemap=False)
     def parse_log(self, ir_log, **kwargs):
-        request.env['runbot.build.error']._parse_logs(ir_log)
+        request.env['runbot.build.error']._parse_logs(ir_log, update_tags=True)
         return werkzeug.utils.redirect('/runbot/build/%s' % ir_log.build_id.id)
 
     @route(['/runbot/bundle/<int:bundle_id>/triggers/<string:action>'], type='http', auth='user', sitemap=False)
@@ -905,19 +909,18 @@ def bundles_by_tag(self, bundle_tag_id=None, project=None, **kwargs):
         if not project and projects:
             project = projects[0]
         bundles_by_team = defaultdict(list)
-        nb_bundles = 0
         nb_bundles_done = 0
-        for bundle in self.env['runbot.bundle'].search([('tag_ids', 'in', bundle_tag_id.id)]):
+        bundles = self.env['runbot.bundle'].search([('tag_ids', 'in', bundle_tag_id.id)])
+        for bundle in bundles:
             bundles_by_team[bundle.team_id.name or 'No Team Defined'].append(bundle)
-            nb_bundles += 1
             bundle_prs = bundle.branch_ids.filtered(lambda rec: rec.is_pr)
             if any(bundle_prs) and not any(bundle_prs.mapped('alive')):
                 nb_bundles_done += 1
 
         qctx = {
             'tag': bundle_tag_id,
+            'bundles': bundles,
             'bundles_by_team': bundles_by_team,
-            'nb_bundles': nb_bundles,
             'nb_bundles_done': nb_bundles_done,
         }
         return request.render('runbot.bundles_by_tag', qctx)

runbot/data/build_parse.xml

@@ -6,7 +6,7 @@
         <field name="type">ir.actions.server</field>
         <field name="state">code</field>
         <field name="code">
-            action = records._parse_logs()
+            action = records._parse_logs(update_tags=True)
         </field>
     </record>
 </odoo>

runbot/data/dockerfile_data.xml

@@ -71,7 +71,7 @@
         <field name="dockerfile_id" ref="runbot.docker_default"/>
         <field name="layer_type">reference_layer</field>
         <field name="name">Install python debian packages</field>
-        <field name="packages">publicsuffix python3 flake8 python3-dbfread python3-dev python3-gevent python3-pip python3-setuptools python3-wheel python3-markdown python3-mock python3-phonenumbers python3-websocket python3-google-auth libpq-dev pylint python3-jwt python3-asn1crypto python3-html2text python3-suds python3-xmlsec python3-markdown2 python3-aiosmtpd</field>
+        <field name="packages">publicsuffix python3 flake8 python3-dbfread python3-dev python3-gevent python3-pip python3-setuptools python3-wheel python3-markdown python3-mock python3-phonenumbers python3-websocket python3-google-auth libpq-dev pylint python3-jwt python3-asn1crypto python3-html2text python3-suds python3-xmlsec python3-markdown2 python3-aiosmtpd python3-paramiko</field>
         <field name="reference_docker_layer_id" ref="runbot.docker_layer_debian_packages_template"/>
     </record>
 

runbot/migrations/19.0.5.17/pre-migration.py

@@ -0,0 +1,3 @@
+def migrate(cr, version):
+    cr.execute("""UPDATE runbot_build set local_result = 'killed' where local_result = 'manually_killed'""")
+    cr.execute("""UPDATE runbot_build set global_result = 'killed' where global_result = 'manually_killed'""")

runbot/models/batch.py

@@ -15,6 +15,7 @@ class Batch(models.Model):
 
     last_update = fields.Datetime('Last ref update')
     bundle_id = fields.Many2one('runbot.bundle', required=True, index=True, ondelete='cascade')
+    build_all = fields.Boolean('Force all triggers')
     commit_link_ids = fields.Many2many('runbot.commit.link')
     commit_ids = fields.Many2many('runbot.commit', compute='_compute_commit_ids')
     slot_ids = fields.One2many('runbot.batch.slot', 'batch_id')
@@ -153,7 +154,7 @@ def _create_build(self, params, slot):
 
         build = self.env['runbot.build'].search(domain, limit=1, order='id desc')
         link_type = 'matched'
-        killed_states = ('skipped', 'killed', 'manually_killed')
+        killed_states = ('skipped', 'killed')
         if build and build.local_result not in killed_states and build.global_result not in killed_states:
             if build.killable:
                 build.killable = False
@@ -185,8 +186,16 @@ def _prepare(self, auto_rebase=False, use_base_commits=False):
         _logger.info('Preparing batch %s', self.id)
 
         priority_offset = self.bundle_id.priority_offset
-        if not priority_offset and self.bundle_id.branch_ids.forwardport_of_id and self.bundle_id.last_batchs == self:  # this is the only batch of a forwardported pr.
-            priority_offset = - 3600 * 5
+        if not priority_offset and self.bundle_id.branch_ids.forwardport_of_id:
+            if self.bundle_id.last_batchs == self:  # this is the first batch of a forwardported pr.
+                priority_offset = - 3600 * 5
+            if len(self.bundle_id.last_batchs) <= 2:
+                # for normal pr, mergebot will request all ci on r+ if needed, for forward port, we need to ensure they are all created or the chain could be blocked
+                # In some rare cases the branch can create a batch and the pr a second one (github issues)
+                # in this case, forwardport_of_id is falsy for the fisrt, and self.bundle_id.last_batchs == self is falsy for the second
+                # we still want to build all for the second one, relaxing the condition to len(self.bundle_id.last_batchs) <= 2
+                # It also means that fixing a conflict or an error will build all on the first push, but not on second (looks almost like a feature)
+                self.build_all = True
         self.priority_level = int(self.create_date.timestamp() - priority_offset)
         if use_base_commits:
             self._warning('This batch will use base commits instead of bundle commits')
@@ -383,7 +392,7 @@ def _fill_missing(branch_commits, match_type):
                 continue
             # in any case, search for an existing build
             config = trigger.config_id
-            if not trigger_custom and trigger.light_config_id and not bundle.build_all and not bundle.is_staging and not bundle.is_base:
+            if not trigger_custom and trigger.light_config_id and not bundle.build_all and not self.build_all and not bundle.is_staging and not bundle.is_base:
                 if (project.use_light_default
                     or
                     project.use_light_draft and any(branch.draft for branch in self.bundle_id.branch_ids)
@@ -455,7 +464,10 @@ def _start_builds(self):
         is_dev = not bundle.is_staging and not bundle.is_base
         for trigger in self.slot_ids.trigger_id:
             enable_on_bundle = (trigger.on_staging and bundle.is_staging) or (trigger.on_base and bundle.is_base) or (trigger.on_dev and is_dev)
-            if ((trigger.repo_ids & bundle_repos) or bundle.build_all or bundle.sticky) and enable_on_bundle:
+            common_repo = (trigger.repo_ids & bundle_repos)
+            if self.build_all and not common_repo:
+                common_repo = (trigger.dependency_ids & bundle_repos)
+            if (common_repo or bundle.build_all or bundle.sticky) and enable_on_bundle:
                 should_start_triggers_ids.add(trigger.id)
 
         disabled_triggers = self.bundle_id.all_trigger_custom_ids.filtered(lambda tc: tc.start_mode == 'disabled').trigger_id