diff --git a/runbot/__manifest__.py b/runbot/__manifest__.py index 9a8491ab8..b3589f8dc 100644 --- a/runbot/__manifest__.py +++ b/runbot/__manifest__.py @@ -6,7 +6,7 @@ 'author': "Odoo SA", 'website': "http://runbot.odoo.com", 'category': 'Website', - 'version': '5.15', + 'version': '5.17', 'application': True, 'depends': ['base', 'base_automation', 'website', 'auth_oauth'], 'data': [ @@ -82,6 +82,7 @@ 'runbot/static/lib/fontawesome/css/font-awesome.css', 'runbot/static/src/css/runbot.css', + 'runbot/static/src/js/polyfill_command_api.js', 'runbot/static/lib/jquery/jquery.js', 'runbot/static/lib/bootstrap/js/bootstrap.bundle.js', 'runbot/static/src/js/runbot.js', diff --git a/runbot/common.py b/runbot/common.py index de9f5b333..0740ce89e 100644 --- a/runbot/common.py +++ b/runbot/common.py @@ -19,11 +19,20 @@ from odoo.fields import Domain from odoo.tools.misc import DEFAULT_SERVER_DATETIME_FORMAT, file_open, html_escape, OrderedSet +DEFAULT_MAX_FILE_SIZE = 500 * 1024 * 1024 + _logger = logging.getLogger(__name__) dest_reg = re.compile(r'^\d{5,}-.+$') +try: + from odoo.addons.saas_worker.util import from_role +except ImportError: + def from_role(*_, **__): + return lambda _: None + + def transactioncache(method): @functools.wraps(method) def wrapper(self, *args, **kwargs): @@ -322,9 +331,35 @@ class TestTagsParser: (?:\[(.*)\])? # parameters $''', re.VERBOSE) # [-][tag][/module][:class][.method][[params]] - def __init__(self, test_tags): - parts = re.split(r',(?![^\[]*\])', test_tags) # split on all comma not inside [] (not followed by ]) + def __init__(self, test_tags, keep_escape=True): + parts = [''] + bracket_level = 0 + escape_next = False + for char in test_tags: + if char == ',' and bracket_level == 0: + parts.append('') + continue + + if char == '\\': + if not escape_next: + escape_next = True + if keep_escape: + parts[-1] += '\\' # not as the TagsSelector, we keep the escape character + continue + elif char == '[': + if not escape_next: + bracket_level += 1 + elif char == ']': + if not escape_next: + bracket_level -= 1 + elif not keep_escape and escape_next: # the previous \ was not escaping anything, put it back + parts[-1] += '\\' + + escape_next = False + parts[-1] += char + filter_specs = [t.strip() for t in parts if t.strip()] + self.filter_specs = filter_specs self.exclude = set() self.include = set() self.parameters = OrderedSet() @@ -332,8 +367,7 @@ def __init__(self, test_tags): for filter_spec in filter_specs: match = self.filter_spec_re.match(filter_spec) if not match: - _logger.error('Invalid tag %s', filter_spec) - continue + raise ValueError('Invalid tag %s' % filter_spec) sign, tag, file_path, module, klass, method, parameters = match.groups() is_include = sign != '-' @@ -362,6 +396,7 @@ def __init__(self, test_tags): def test_tags_to_search_domain(self, exclude_error_id=None): search_domains = [] + params_by_spec = dict(self.parameters) for include in self.include: _, test_module, test_class, test_method, file_path = include module_path = file_path or ((test_module or '') + '%') @@ -369,6 +404,10 @@ def test_tags_to_search_domain(self, exclude_error_id=None): test_method = test_method or '%' search_pattern = f'{module_path}:{test_class}.{test_method}' tag_domain = [('canonical_tags', 'like', f'{search_pattern}')] + params = params_by_spec.get(include) + if params: + _sign, parameters = params + tag_domain.append(('canonical_tags', 'like', f'%[{parameters}%]%')) if exclude_error_id: tag_domain.append(('id', '!=', exclude_error_id)) search_domains.append(tag_domain) diff --git a/runbot/container.py b/runbot/container.py index 679ea35f1..2fe340d9d 100644 --- a/runbot/container.py +++ b/runbot/container.py @@ -108,21 +108,22 @@ def get_config(self, starting_config=''): return res.read() -def docker_build(build_dir, image_tag, pull=False): - return _docker_build(build_dir, image_tag, pull) +def docker_build(build_dir, image_tag, pull=False, nocache=False): + return _docker_build(build_dir, image_tag, pull, nocache) -def _docker_build(build_dir, image_tag, pull=False): +def _docker_build(build_dir, image_tag, pull=False, nocache=False): """Build the docker image :param build_dir: the build directory that contains Dockerfile. :param image_tag: name used to tag the resulting docker image + :param nocache: bypass Docker layer cache when True :return: dict """ with DockerManager(image_tag) as dm: last_step = None dm.result['success'] = False # waiting for an image_id - for chunk in dm.consume(dm.docker_client.api.build(path=build_dir, tag=image_tag, rm=True, pull=pull)): + for chunk in dm.consume(dm.docker_client.api.build(path=build_dir, tag=image_tag, rm=True, pull=pull, nocache=nocache)): if 'stream' in chunk: stream = chunk['stream'] if stream.startswith('Step '): @@ -259,7 +260,10 @@ def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False else: run_cmd = cmd run_cmd = f'cd /data/build;touch start-{container_name};{run_cmd};cd /data/build;touch end-{container_name}' - _logger.info('Docker run command: %s', run_cmd) + run_cmd_repr = str(run_cmd) + if len(run_cmd_repr) > 250: + run_cmd_repr = run_cmd_repr[:250] + '...' + _logger.info('Docker run command: %s', run_cmd_repr) docker_clear_state(container_name, build_dir) # ensure that no state are remaining build_dir = file_path(build_dir) diff --git a/runbot/controllers/badge.py b/runbot/controllers/badge.py index fa6e031ee..662163682 100644 --- a/runbot/controllers/badge.py +++ b/runbot/controllers/badge.py @@ -44,13 +44,9 @@ def badge(self, name, repo_id=False, trigger_id=False, theme='default'): if not builds: state = 'testing' else: - result = builds._result_multi() - if result == 'ok': + state = 'failed' + if all(build.global_result == 'ok' for build in builds): state = 'success' - elif result == 'warn': - state = 'warning' - else: - state = 'failed' etag = request.httprequest.headers.get('If-None-Match') retag = hashlib.md5(state.encode()).hexdigest() diff --git a/runbot/controllers/frontend.py b/runbot/controllers/frontend.py index 1512a7a3c..181a2c184 100644 --- a/runbot/controllers/frontend.py +++ b/runbot/controllers/frontend.py @@ -80,7 +80,7 @@ def _pending(self): '/runbot/', '/runbot//search/'], website=True, auth='public', type='http') def bundles(self, project=None, search='', refresh=False, limit=40, has_pr=None, **kwargs): - search = search if len(search) < 60 else search[:60] + search = search if len(search) < 60 else search[:200] env = request.env categories = env['runbot.category'].search([]) projects = self.env['runbot.project'].search([('hidden', '=', False)]) @@ -119,13 +119,11 @@ def bundles(self, project=None, search='', refresh=False, limit=40, has_pr=None, pr_numbers = [] for search_elem in search.split("|"): if search_elem.isnumeric(): - pr_numbers.append(int(search_elem)) + search_domains.append([('branch_ids', 'any', [('name', '=', search_elem)])]) + if ':' in search_elem: + search_domains.append([('branch_ids', 'any', [('pull_head_name', '=', search_elem)])]) operator = '=ilike' if '%' in search_elem else 'ilike' search_domains.append([('name', operator, search_elem)]) - if pr_numbers: - res = request.env['runbot.branch'].search([('name', 'in', pr_numbers)]) - if res: - search_domains.append([('id', 'in', res.mapped('bundle_id').ids)]) search_domain = Domain.OR(search_domains) domain = Domain.AND([domain, search_domain]) @@ -166,7 +164,7 @@ def bundles(self, project=None, search='', refresh=False, limit=40, has_pr=None, '/runbot/bundle//page/', '/runbot/bundle/', ], website=True, auth='public', type='http', sitemap=False) - def bundle(self, bundle=None, page=1, limit=50, **kwargs): + def bundle(self, bundle=None, page=1, limit=50, expand_custom=False, **kwargs): if isinstance(bundle, str): bundle = request.env['runbot.bundle'].search([('name', '=', bundle)], limit=1, order='id') if not bundle: @@ -183,6 +181,7 @@ def bundle(self, bundle=None, page=1, limit=50, **kwargs): ) batchs = request.env['runbot.batch'].search(domain, limit=limit, offset=pager.get('offset', 0), order='id desc') + # compute if we should display the new batch button context = { 'bundle': bundle, 'batchs': batchs, @@ -190,6 +189,8 @@ def bundle(self, bundle=None, page=1, limit=50, **kwargs): 'project': bundle.project_id, 'title': 'Bundle %s' % bundle.name, 'page_info_state': bundle.last_batch._get_global_result(), + 'expand_custom': expand_custom, + 'needs_update': bundle.last_batch and bundle.last_batch.sudo().needs_update(), } return request.render('runbot.bundle', context) @@ -199,7 +200,7 @@ def bundle(self, bundle=None, page=1, limit=50, **kwargs): '/runbot/bundle//force/', ], type='http', auth="user", methods=['GET', 'POST'], csrf=False) def force_bundle(self, bundle, auto_rebase=False, use_base_commits=False, **_post): - if not request.env.user.has_group('runbot.group_runbot_advanced_user') and ':' not in bundle.name: + if not request.env.user.has_group('runbot.group_runbot_advanced_user') and ':' not in bundle.name and not bundle.last_batch.needs_update(): message = "Only users with a specific group can do that. Please contact runbot administrators" raise Forbidden(message) _logger.info('user %s forcing bundle %s', request.env.user.name, bundle.name) # user must be able to read bundle @@ -220,6 +221,12 @@ def batch(self, batch_id=None, **kwargs): } return request.render('runbot.batch', context) + @route(['/runbot/batch//prioritize'], website=True, auth='user', type='http', sitemap=False) + def batch_priority(self, batch_id=None, **kwargs): + batch = request.env['runbot.batch'].browse(batch_id) + batch.sudo().priority_level = int(batch.create_date.timestamp() - 3600) + return werkzeug.utils.redirect('/runbot/batch/%s' % batch_id) + @route(['/runbot/batch/slot//build'], auth='user', type='http') def slot_create_build(self, slot=None, **kwargs): build = slot.sudo()._create_missing_build() @@ -269,6 +276,7 @@ def resend_status(self, status_id=None, **kwargs): ], type='http', auth="user", methods=['POST'], csrf=False) def build_operations(self, build_id, operation, **post): build = request.env['runbot.build'].sudo().browse(build_id) + build.check_access('read') if operation == 'rebuild': build = build._rebuild() elif operation == 'kill': @@ -283,18 +291,17 @@ def build_operations(self, build_id, operation, **post): '/runbot/batch//build/', ], type='http', auth="public", website=True, sitemap=False) def build(self, build_id, search=None, from_batch=None, **post): - """Events/Logs""" - + build = request.env['runbot.build'].browse(build_id) if from_batch: from_batch = request.env['runbot.batch'].browse(int(from_batch)) - if build_id not in from_batch.with_context(active_test=False).slot_ids.build_id.ids: + if build.top_parent not in from_batch.with_context(active_test=False).slot_ids.build_id and build.create_batch_id != from_batch: # the url may have been forged replacing the build id, redirect to hide the batch return werkzeug.utils.redirect('/runbot/build/%s' % build_id) from_batch = from_batch.with_context(batch=from_batch) Build = request.env['runbot.build'].with_context(batch=from_batch) - build = Build.browse([build_id])[0] + build = Build.browse(build_id) if not build.exists(): return request.not_found() siblings = (build.parent_id.children_ids if build.parent_id else from_batch.slot_ids.build_id if from_batch else build).sorted('id') @@ -316,7 +323,8 @@ def build(self, build_id, search=None, from_batch=None, **post): @route([ '/runbot/build/search', ], website=True, auth='public', type='http', sitemap=False) - def builds(self, **kwargs): + def builds(self, limit=100, **kwargs): + limit = min(int(limit), 1000) domain = [] for key in ('config_id', 'version_id', 'project_id', 'trigger_id', 'create_batch_id.bundle_id', 'create_batch_id'): # allowed params value = kwargs.get(key) @@ -330,10 +338,12 @@ def builds(self, **kwargs): for key in ('description',): if key in kwargs: - domain.append((f'{key}', 'ilike', kwargs.get(key))) + value = kwargs.get(key) + operator = 'ilike' if '%' in value else '=' + domain.append((f'{key}', operator, value)) context = { - 'builds': request.env['runbot.build'].search(domain, limit=100), + 'builds': request.env['runbot.build'].search(domain, limit=limit), } return request.render('runbot.build_search', context) @@ -444,8 +454,8 @@ def build_errors(self, sort=None, page=1, limit=20, **kwargs): 'build_count asc': 'Number seen: Low to High', 'responsible asc': 'Assignee: A - Z', 'responsible desc': 'Assignee: Z - A', - 'module_name asc': 'Module name: A - Z', - 'module_name desc': 'Module name: Z -A', + 'team_id asc': 'Team', + 'name asc': 'Name', } sort_order = sort if sort in sort_order_choices else 'last_seen_date desc' @@ -660,22 +670,43 @@ def access_running(self, build_id, db_suffix=None, **kwargs): @route(['/runbot/parse_log/'], type='http', auth='user', sitemap=False) def parse_log(self, ir_log, **kwargs): - request.env['runbot.build.error']._parse_logs(ir_log) + request.env['runbot.build.error']._parse_logs(ir_log, update_tags=True) return werkzeug.utils.redirect('/runbot/build/%s' % ir_log.build_id.id) - @route(['/runbot/bundle/toggle_no_build//'], type='http', auth='user', sitemap=False) - def toggle_no_build(self, bundle_id, value, **kwargs): - if not request.env.user.has_group('base.group_user'): - return 'Forbidden' - bundle = request.env['runbot.bundle'].browse(bundle_id).exists() - if bundle.sticky or bundle.is_base: - return 'Forbidden' - if bundle.project_id.tmp_prefix and bundle.name.startswith(bundle.project_id.tmp_prefix): - return 'Forbidden' - bundle.sudo().no_build = bool(value) - _logger.info('Bundle %s no_build set to %s by %s', bundle.name, bool(value), request.env.user.name) + @route(['/runbot/bundle//triggers/'], type='http', auth='user', sitemap=False) + def configure_bundle_triggers(self, bundle_id, action, expand_custom=False, **kwargs): + if not request.env.user.has_group('runbot.group_user'): + raise NotFound() + + bundle = request.env['runbot.bundle'].browse(bundle_id) + if bundle.is_base or bundle.is_staging: + raise NotFound() + if action == 'disable_all': + bundle.sudo()._configure_custom_trigger_start_mode('disabled') + elif action == 'force_all': + bundle.sudo()._configure_custom_trigger_start_mode('force') + elif action == 'auto_all': + bundle.sudo()._configure_custom_trigger_start_mode('auto') + elif action == 'light_all': + bundle.sudo()._configure_custom_trigger_start_mode('light') + else: + raise NotFound() + if expand_custom: + return werkzeug.utils.redirect(f'/runbot/bundle/{bundle_id}?expand_custom=1') return werkzeug.utils.redirect(f'/runbot/bundle/{bundle_id}') + @route(['/runbot/trigger_custom//set_mode/'], type='http', auth='user', sitemap=False) + def configure_custom_trigger(self, trigger_custom_id, mode, **kwargs): + if not request.env.user.has_group('runbot.group_user'): + raise NotFound() + trigger_custom = request.env['runbot.bundle.trigger.custom'].browse(trigger_custom_id) + bundle = trigger_custom.bundle_id + if bundle.is_base or bundle.is_staging: + raise NotFound() + + trigger_custom.sudo().start_mode = mode + return werkzeug.utils.redirect(f'/runbot/bundle/{trigger_custom.bundle_id.id}?expand_custom=1') + @route(['/runbot/trigger/report/'], type='http', auth='user', website=True, sitemap=False) def report_view(self, trigger_id=None, **kwargs): return request.render("runbot.trigger_report", { diff --git a/runbot/controllers/hook.py b/runbot/controllers/hook.py index d46c7f699..4bd36a20d 100644 --- a/runbot/controllers/hook.py +++ b/runbot/controllers/hook.py @@ -4,8 +4,9 @@ import json import logging -from odoo import http +from odoo import http, fields from odoo.http import request +from ..common import from_role _logger = logging.getLogger(__name__) @@ -50,3 +51,18 @@ def hook(self, remote_id=None, **_post): branch = request.env['runbot.branch'].sudo().search([('remote_id', '=', remote.id), ('name', '=', branch_ref)]) branch.alive = False return "" + + @from_role('mergebot', signed=True) + @http.route(['/runbot/request_ci'], type='http', methods=["POST"], auth="public", website=True, csrf=False, sitemap=False) + def force_ci(self): + pull_request_names = request.get_json_data().get('pull_requests', []) + pull_domains = [] + for pull_request_names in pull_request_names: + remote_short_name, name = pull_request_names.split('#') + owner, repo_name = remote_short_name.split('/') + pull_domains.append([('remote_id.owner', '=', owner), ('remote_id.repo_name', '=', repo_name), ('name', '=', name)]) + pull_domains = fields.Domain.OR(pull_domains) + pull_requests = request.env['runbot.branch'].sudo().search([('is_pr', '=', True)] + pull_domains) + bundles = pull_requests.bundle_id + _logger.info('Received CI request for bundles: %s', bundles.mapped('name')) + bundles._force_ci() diff --git a/runbot/data/build_parse.xml b/runbot/data/build_parse.xml index a164c660d..dd2279edb 100644 --- a/runbot/data/build_parse.xml +++ b/runbot/data/build_parse.xml @@ -6,7 +6,7 @@ ir.actions.server code - action = records._parse_logs() + action = records._parse_logs(update_tags=True) diff --git a/runbot/data/dockerfile_data.xml b/runbot/data/dockerfile_data.xml index 9d2601f97..720a70bf6 100644 --- a/runbot/data/dockerfile_data.xml +++ b/runbot/data/dockerfile_data.xml @@ -71,7 +71,7 @@ reference_layer Install python debian packages - publicsuffix python3 flake8 python3-dbfread python3-dev python3-gevent python3-pip python3-setuptools python3-wheel python3-markdown python3-mock python3-phonenumbers python3-websocket python3-google-auth libpq-dev pylint python3-jwt python3-asn1crypto python3-html2text python3-suds python3-xmlsec python3-markdown2 python3-aiosmtpd + publicsuffix python3 flake8 python3-dbfread python3-dev python3-gevent python3-pip python3-setuptools python3-wheel python3-markdown python3-mock python3-phonenumbers python3-websocket python3-google-auth libpq-dev pylint python3-jwt python3-asn1crypto python3-html2text python3-suds python3-xmlsec python3-markdown2 python3-aiosmtpd python3-paramiko @@ -120,6 +120,7 @@ Install branch debian/control with latest postgresql-client # This layer updates the repository list to get the latest postgresql-client, mainly needed if the host postgresql version is higher than the default version of the docker os +# CACHE 60 ADD https://raw.githubusercontent.com/odoo/odoo/{odoo_branch}/debian/control /tmp/control.txt RUN curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc -o /etc/apt/trusted.gpg.d/psql_client.asc \ && echo "deb http://apt.postgresql.org/pub/repos/apt/ {os_release_name}-pgdg main" > /etc/apt/sources.list.d/pgclient.list \ @@ -136,7 +137,7 @@ RUN curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc -o /etc/apt/tru template Install chrome - + RUN curl -sSL https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_{chrome_version}_amd64.deb -o /tmp/chrome.deb \ && apt-get update \ && apt-get -y install --no-install-recommends /tmp/chrome.deb \ @@ -195,7 +196,8 @@ ENV PIP_BREAK_SYSTEM_PACKAGES=1 template Install branch requirements - ADD --chown={USERNAME} https://raw.githubusercontent.com/odoo/odoo/{odoo_branch}/requirements.txt /tmp/requirements.txt + # CACHE 60 +ADD --chown={USERNAME} https://raw.githubusercontent.com/odoo/odoo/{odoo_branch}/requirements.txt /tmp/requirements.txt RUN python3 -m pip install --no-cache-dir -r /tmp/requirements.txt diff --git a/runbot/documentation/dynamic_config.md b/runbot/documentation/dynamic_config.md index 1d647758d..9282bd816 100644 --- a/runbot/documentation/dynamic_config.md +++ b/runbot/documentation/dynamic_config.md @@ -344,21 +344,43 @@ Filters are a way to transform dynamic values before using them. They are define For example, to transform a module filter into test tags: +#### filter_all_modules, make_module_test_tags + ```json {"test_tags": "-at_install,{{test_module_filter|filter_all_modules|make_module_test_tags}}", ``` In this example, the `filter_all_modules` filters will first transform the `test_module_filter` variable (which is a module filter) into a list of modules, and then the `make_module_test_tags` filters will transform this list of modules into test tags by prepending each module with a `/` to indicate that we want to run all tests from these modules. -Note that `filter_all_modules` is actually equivalent to `filter_default_modules`, but prepending a `*` at the begining of the filter. +#### filter_default_modules + +`filter_all_modules` is actually equivalent to `filter_default_modules`, but prepending a `*` at the begining of the filter. Without that a runbot defined filter is applied, returning a default list of modules per repo. `*,mail -> !web|filter_default_modules` is the same as `mail -> !web|filter_all_modules` + +#### prepend, append In some case we also want to combine the test-tags module with another tag or test method, this can be done using prepend and append `"{{-*,web*|filter_all_modules|make_module_test_tags|append('.test_method')}}` `{{-*,web*|filter_all_modules|make_module_test_tags|prepend('custom_tag')}}` -It is also possible to filter modules based on the one modified in the current bundle. +#### modified_modules + +It is possible to filter modules based on the one modified in the current bundle. `{{*|filter_all_modules|modified_modules}}"` + +#### select_existing_modules + +`select_existing_modules` is equivalent to `filter_default_modules` but with a -* at the beginning of the filter, meaning that we start with an empty selection and only add modules that are explicitly selected. + +This is a solution to keep only existing modules from a specific list, when we are not sure modules exists: +`{{*|filter_all_modules|modified_modules|prepend('test_')|select_existing_modules|make_module_test_tags}}` + +- `*|filter_all_modules` will select all existing modules +- `|modified_modules` will only keep the modified ones +- `prepend('test_')` will prepend test_ to have the test equivalent name of the modified modules (mail-> test_mail, base -> test_base) +- `select_existing_modules` will only keep modules that exists (test_mail) +- `make_module_test_tags` make the module test tags by prepending a / to each module. + diff --git a/runbot/migrations/19.0.5.16/pre-migration.py b/runbot/migrations/19.0.5.16/pre-migration.py new file mode 100644 index 000000000..bffcd9582 --- /dev/null +++ b/runbot/migrations/19.0.5.16/pre-migration.py @@ -0,0 +1,8 @@ +import logging + +_logger = logging.getLogger(__name__) + + +def migrate(cr, version): + cr.execute("""ALTER TABLE runbot_batch ADD COLUMN priority_level integer""") + cr.execute("""ALTER TABLE runbot_build ADD COLUMN priority_level integer""") diff --git a/runbot/migrations/19.0.5.17/pre-migration.py b/runbot/migrations/19.0.5.17/pre-migration.py new file mode 100644 index 000000000..c21faba28 --- /dev/null +++ b/runbot/migrations/19.0.5.17/pre-migration.py @@ -0,0 +1,3 @@ +def migrate(cr, version): + cr.execute("""UPDATE runbot_build set local_result = 'killed' where local_result = 'manually_killed'""") + cr.execute("""UPDATE runbot_build set global_result = 'killed' where global_result = 'manually_killed'""") diff --git a/runbot/models/batch.py b/runbot/models/batch.py index e044662f8..5f03d4973 100644 --- a/runbot/models/batch.py +++ b/runbot/models/batch.py @@ -15,11 +15,13 @@ class Batch(models.Model): last_update = fields.Datetime('Last ref update') bundle_id = fields.Many2one('runbot.bundle', required=True, index=True, ondelete='cascade') + build_all = fields.Boolean('Force all triggers') commit_link_ids = fields.Many2many('runbot.commit.link') commit_ids = fields.Many2many('runbot.commit', compute='_compute_commit_ids') slot_ids = fields.One2many('runbot.batch.slot', 'batch_id') all_build_ids = fields.Many2many('runbot.build', compute='_compute_all_build_ids', help="Recursive builds") state = fields.Selection([('preparing', 'Preparing'), ('ready', 'Ready'), ('done', 'Done'), ('skipped', 'Skipped')]) + priority_level = fields.Integer("Priority level", help="Priority level of the batch, determined from the create date and the bundle priority offset. The lower, the higher priority.") hidden = fields.Boolean('Hidden', default=False) age = fields.Integer(compute='_compute_age', string='Build age') category_id = fields.Many2one('runbot.category', index=True, default=lambda self: self.env.ref('runbot.default_category', raise_if_not_found=False)) @@ -152,7 +154,7 @@ def _create_build(self, params, slot): build = self.env['runbot.build'].search(domain, limit=1, order='id desc') link_type = 'matched' - killed_states = ('skipped', 'killed', 'manually_killed') + killed_states = ('skipped', 'killed') if build and build.local_result not in killed_states and build.global_result not in killed_states: if build.killable: build.killable = False @@ -163,7 +165,7 @@ def _create_build(self, params, slot): build_type = 'normal' if self.category_id != self.env.ref('runbot.default_category'): build_type = 'scheduled' - elif self.bundle_id.priority: + elif self.bundle_id.priority or params.trigger_id.use_extra_slot: build_type = 'priority' build = self.env['runbot.build'].create({ @@ -182,6 +184,19 @@ def _create_build(self, params, slot): def _prepare(self, auto_rebase=False, use_base_commits=False): _logger.info('Preparing batch %s', self.id) + + priority_offset = self.bundle_id.priority_offset + if not priority_offset and self.bundle_id.branch_ids.forwardport_of_id: + if self.bundle_id.last_batchs == self: # this is the first batch of a forwardported pr. + priority_offset = - 3600 * 5 + if len(self.bundle_id.last_batchs) <= 2: + # for normal pr, mergebot will request all ci on r+ if needed, for forward port, we need to ensure they are all created or the chain could be blocked + # In some rare cases the branch can create a batch and the pr a second one (github issues) + # in this case, forwardport_of_id is falsy for the fisrt, and self.bundle_id.last_batchs == self is falsy for the second + # we still want to build all for the second one, relaxing the condition to len(self.bundle_id.last_batchs) <= 2 + # It also means that fixing a conflict or an error will build all on the first push, but not on second (looks almost like a feature) + self.build_all = True + self.priority_level = int(self.create_date.timestamp() - priority_offset) if use_base_commits: self._warning('This batch will use base commits instead of bundle commits') if not self.bundle_id.base_id: @@ -376,7 +391,22 @@ def _fill_missing(branch_commits, match_type): self._warning('Missing commit for repo %s for trigger %s', (trigger_repos & missing_repos).mapped('name'), trigger.name) continue # in any case, search for an existing build - config = trigger_custom.config_id or trigger.config_id + config = trigger.config_id + if not trigger_custom and trigger.light_config_id and not bundle.build_all and not self.build_all and not bundle.is_staging and not bundle.is_base: + if (project.use_light_default + or + project.use_light_draft and any(branch.draft for branch in self.bundle_id.branch_ids) + or + project.use_light_no_pr and not any(branch.is_pr for branch in self.bundle_id.branch_ids) + ): + config = trigger.light_config_id + + if trigger_custom.config_id: + config = trigger_custom.config_id + elif trigger_custom.start_mode == 'light' and trigger.light_config_id: + config = trigger.light_config_id + + extra_params = trigger_custom.extra_params or '' config_data = dict(trigger.config_data or {}) | dict(trigger_custom.config_data or {}) trigger_commit_link_by_repos = commit_link_by_repos @@ -396,7 +426,7 @@ def _fill_missing(branch_commits, match_type): 'modules': bundle.modules, 'dockerfile_id': dockerfile_id, 'create_batch_id': self.id, - 'used_custom_trigger': bool(trigger_custom), + 'used_custom_trigger': bool(trigger_custom.config_id or trigger_custom.extra_params or trigger_custom.config_data or trigger_custom.use_base_commits), } params = self.env['runbot.build.params'].create(params_value) @@ -434,16 +464,22 @@ def _start_builds(self): is_dev = not bundle.is_staging and not bundle.is_base for trigger in self.slot_ids.trigger_id: enable_on_bundle = (trigger.on_staging and bundle.is_staging) or (trigger.on_base and bundle.is_base) or (trigger.on_dev and is_dev) - if ((trigger.repo_ids & bundle_repos) or bundle.build_all or bundle.sticky) and enable_on_bundle: + common_repo = (trigger.repo_ids & bundle_repos) + if self.build_all and not common_repo: + common_repo = (trigger.dependency_ids & bundle_repos) + if (common_repo or bundle.build_all or bundle.sticky) and enable_on_bundle: should_start_triggers_ids.add(trigger.id) + disabled_triggers = self.bundle_id.all_trigger_custom_ids.filtered(lambda tc: tc.start_mode == 'disabled').trigger_id for slot in self.slot_ids: if slot.build_id: continue trigger = slot.trigger_id - if trigger.starts_after_ids - success_trigger: # some required triggers are missing - continue trigger_custom = trigger_customs.get(trigger, self.env['runbot.bundle.trigger.custom']) + missing_triggers = trigger.starts_after_ids - success_trigger + if missing_triggers: + if not trigger_custom or (missing_triggers - disabled_triggers): + continue force_trigger = trigger_custom and trigger_custom.start_mode == 'force' skip_trigger = (trigger_custom and trigger_custom.start_mode == 'disabled') or trigger.manual should_start = slot.trigger_id.id in should_start_triggers_ids @@ -508,6 +544,23 @@ def _log(self, message, *args, level='INFO'): 'level': level, }) + def needs_update(self): + bundle = self.bundle_id + custom_trigger_per_trigger = {ct.trigger_id: ct for ct in bundle.trigger_custom_ids} + for slot in self.slot_ids: + trigger = slot.trigger_id + custom_trigger = custom_trigger_per_trigger.get(trigger) + if not custom_trigger: + continue + expected_config = trigger.config_id + if custom_trigger.config_id: + expected_config = custom_trigger.config_id + elif trigger.light_config_id and custom_trigger.start_mode == 'light': + expected_config = trigger.light_config_id + if slot.params_id.config_id != expected_config: + return True + return False + class BatchLog(models.Model): _name = 'runbot.batch.log' _description = 'Batch log' diff --git a/runbot/models/branch.py b/runbot/models/branch.py index f2cf6edf0..8deb9ec5c 100644 --- a/runbot/models/branch.py +++ b/runbot/models/branch.py @@ -101,7 +101,8 @@ def _compute_reference_name(self): # branch.reference_name = '%s~%s' % (branch.pull_head_name, branch.name) else: reference_name = branch.name - forced_version = branch.remote_id.repo_id.single_version # we don't add a depend on repo.single_version to avoid mass recompute of existing branches + repo = branch.remote_id.repo_id + forced_version = repo.enforce_version and repo.single_version # we don't add a depend on repo.single_version to avoid mass recompute of existing branches if forced_version and not (reference_name.startswith(f'{forced_version.name}-') or reference_name == forced_version.name): reference_name = f'{forced_version.name}---{reference_name}' branch.reference_name = reference_name diff --git a/runbot/models/build.py b/runbot/models/build.py index 854e38bd6..a822f69c7 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +import ast import datetime import getpass import hashlib @@ -10,27 +10,42 @@ import shutil import time import uuid - from collections import defaultdict -from dateutil import parser from pathlib import Path + +from dateutil import parser +from markupsafe import Markup from psycopg2 import sql from psycopg2.extensions import TransactionRollbackError -from ..common import dt2time, now, grep, local_pgadmin_cursor, dest_reg, os, list_local_dbs, pseudo_markdown, RunbotException, findall, sanitize, markdown_escape, tail -from ..container import docker_stop, docker_state, Command, docker_run, docker_pull -from ..fields import JsonDictField - -from odoo import models, fields, api - +from odoo import api, fields, models from odoo.exceptions import ValidationError -from odoo.tools import file_open, file_path +from odoo.tools import file_open, file_path, html_escape from odoo.tools.safe_eval import safe_eval +from ..common import ( + RunbotException, + dest_reg, + dt2time, + findall, + grep, + list_local_dbs, + local_pgadmin_cursor, + markdown_escape, + now, + os, + pseudo_markdown, + sanitize, + tail, + transactioncache, + DEFAULT_MAX_FILE_SIZE, +) +from ..container import Command, docker_pull, docker_run, docker_state, docker_stop +from ..fields import JsonDictField _logger = logging.getLogger(__name__) -result_order = ['ok', 'warn', 'ko', 'skipped', 'killed', 'manually_killed'] +result_order = ['ok', 'warn', 'ko', 'skipped', 'killed', 'manually_killed'] # TODO remove manually_killed state_order = ['pending', 'testing', 'waiting', 'running', 'done'] COPY_WHITELIST = [ @@ -61,7 +76,6 @@ def remove_readonly(func, path_str, exinfo): def make_selection(array): return [(elem, elem.replace('_', ' ').capitalize()) if isinstance(elem, str) else elem for elem in array] - class BuildParameters(models.Model): _name = 'runbot.build.params' _description = "Build parameters" @@ -105,8 +119,18 @@ class BuildParameters(models.Model): # @api.depends('version_id', 'project_id', 'extra_params', 'config_id', 'config_data', 'modules', 'commit_link_ids', 'builds_reference_ids') def _compute_fingerprint(self): + def get_commit_links_ident(commit_link): + commit_idents = [] + for c in commit_link.commit_id: + commit_ident = c.tree_hash or c.name + if c.rebase_on_id: + commit_ident += (c.rebase_on_id.tree_hash or c.rebase_on_id.name) + # in a ideal world, we would be able to determine what the real threehash would be + commit_idents.append(commit_ident) + return sorted(commit_idents) + for param in self: - commit_ident = sorted([c.tree_hash or '' for c in param.commit_link_ids.commit_id]) + commit_ident = get_commit_links_ident(param.commit_link_ids) if param.trigger_id.batch_dependent: commit_ident = sorted(param.commit_link_ids.commit_id.ids) cleaned_vals = { @@ -125,7 +149,7 @@ def _compute_fingerprint(self): } if param.upgrade_to_build_id: cleaned_vals['upgrade_to_build_dockerfile_id'] = param.upgrade_to_build_id.params_id.dockerfile_id.id - cleaned_vals['upgrade_to_build_commits'] = sorted([c.tree_hash or c.id for c in param.upgrade_to_build_id.params_id.commit_link_ids.commit_id]) + cleaned_vals['upgrade_to_build_commits'] = get_commit_links_ident(param.upgrade_to_build_id.params_id.commit_link_ids) if param.upgrade_from_build_id: cleaned_vals['upgrade_from_build_id'] = param.upgrade_from_build_id.id if param.trigger_id.batch_dependent: @@ -266,6 +290,7 @@ class BuildResult(models.Model): create_batch_id = fields.Many2one('runbot.batch', related='params_id.create_batch_id', store=True, index=True) create_bundle_id = fields.Many2one('runbot.bundle', related='params_id.create_batch_id.bundle_id', index=True) dynamic_config = JsonDictField('Dynamic Config', related='params_id.dynamic_config') + priority_level = fields.Integer('Priority', related='create_batch_id.priority_level', store=True, index=True) # state machine global_state = fields.Selection(make_selection(state_order), string='Status', compute='_compute_global_state', store=True, recursive=True) @@ -274,6 +299,8 @@ class BuildResult(models.Model): local_result = fields.Selection(make_selection(result_order), string='Build Result', default='ok') requested_action = fields.Selection([('wake_up', 'To wake up'), ('deathrow', 'To kill')], string='Action requested', index=True) + to_kill = fields.Boolean('To kill', compute='_compute_to_kill') + message_ids = fields.One2many('runbot.host.message', 'build_id', string='Messages') # web infos host = fields.Char('Host name') host_id = fields.Many2one('runbot.host', string="Host", compute='_compute_host_id') @@ -291,12 +318,13 @@ class BuildResult(models.Model): active_step = fields.Many2one('runbot.build.config.step', 'Active step') job = fields.Char('Active step display name', compute='_compute_job') dynamic_active_step_index = fields.Integer('Dynamic active step index') - + cpu_limit = fields.Integer('CPU limit for the current running docker') job_start = fields.Datetime('Job start') job_end = fields.Datetime('Job end') build_start = fields.Datetime('Build start') build_end = fields.Datetime('Build end') docker_start = fields.Datetime('Docker start') + docker_time = fields.Integer('Docker time', default=0, help='Accumulated time spent in Docker containers') job_time = fields.Integer(compute='_compute_job_time', string='Job time') build_time = fields.Integer(compute='_compute_build_time', string='Build time') wait_time = fields.Integer(compute='_compute_wait_time', string='Wait time') @@ -325,6 +353,7 @@ class BuildResult(models.Model): ancestors = fields.Many2many('runbot.build', compute='_compute_ancestors') # should we add a has children stored boolean? children_ids = fields.One2many('runbot.build', 'parent_id') + all_children_ids = fields.One2many('runbot.build', compute='_compute_all_children_ids') # config of top_build is inherithed from params, but subbuild will have different configs @@ -333,7 +362,7 @@ class BuildResult(models.Model): build_url = fields.Char('Build url', compute='_compute_build_url', store=False) build_error_link_ids = fields.One2many('runbot.build.error.link', 'build_id') build_error_ids = fields.Many2many('runbot.build.error', compute='_compute_build_error_ids', string='Errors') - keep_running = fields.Boolean('Keep running', help='Keep running', index=True) + gc_running_date = fields.Date('GC Running Date', help='Running build cannot be killed before this date', index='btree_not_null') log_counter = fields.Integer('Log Lines counter', default=100) slot_ids = fields.One2many('runbot.batch.slot', 'build_id') @@ -371,6 +400,11 @@ def _compute_global_state(self): else: record.global_state = record.local_state + @api.depends('message_ids') + def _compute_to_kill(self): + for record in self: + record.to_kill = any(message.message == 'kill' for message in record.message_ids) + @api.depends('gc_delay', 'job_end') def _compute_gc_date(self): icp = self.env['ir.config_parameter'].sudo() @@ -395,6 +429,10 @@ def _compute_ancestors(self): for build in self: build.ancestors = self.browse([int(b) for b in build.parent_path.split('/') if b]) + def _compute_all_children_ids(self): + for build in self: + build.all_children_ids = self.search([('parent_path', '=like', build.parent_path + '%')]) + def _get_youngest_state(self, states): index = min([self._get_state_score(state) for state in states]) return state_order[index] @@ -512,6 +550,7 @@ def _add_child(self, param_values, orphan=False, description=False, additionnal_ 'params_id': self.params_id.copy(param_values).id, 'parent_id': self.id, 'build_type': self.build_type, + 'priority_level': self.priority_level, 'description': description, 'orphan_result': orphan, 'keep_host': self.keep_host, @@ -519,17 +558,6 @@ def _add_child(self, param_values, orphan=False, description=False, additionnal_ **build_values, }) - def _result_multi(self): - if all(build.global_result == 'ok' or not build.global_result for build in self): - return 'ok' - if any(build.global_result in ('skipped', 'killed', 'manually_killed') for build in self): - return 'killed' - if any(build.global_result == 'ko' for build in self): - return 'ko' - if any(build.global_result == 'warning' for build in self): - return 'warning' - return 'ko' # ? - @api.depends('params_id.version_id.name') def _compute_dest(self): for build in self: @@ -607,7 +635,7 @@ def _rebuild(self, message=None): # TODO don't rebuild if there is a more recent build for this params? values = { 'params_id': self.params_id.id, - 'build_type': 'rebuild', + 'build_type': 'rebuild' if self.build_type == "normal" else self.build_type, } if self.keep_host: values['host'] = self.host @@ -802,11 +830,13 @@ def _init_pendings(self): def _process_requested_actions(self): self.ensure_one() build = self + # TODO remove, replaced by queue if build.requested_action == 'deathrow': result = None if build.local_state != 'running' and build.global_result not in ('warn', 'ko'): - result = 'manually_killed' + result = 'killed' build._kill(result=result) + build.requested_action = False return if build.requested_action == 'wake_up': @@ -857,7 +887,8 @@ def _schedule(self): else: _docker_state = docker_state(build._get_docker_name(), build._path()) if _docker_state == 'RUNNING': - timeout = min(build.active_step.cpu_limit, int(icp.get_param('runbot.runbot_timeout', default=10000))) + build_limit = build.cpu_limit or build.active_step.cpu_limit + timeout = min(build_limit, int(icp.get_param('runbot.runbot_timeout', default=10000))) if build.local_state != 'running' and build.job_time > timeout: build.active_step._make_stats(build) build._log('_schedule', '%s time exceeded (%ss)' % (build.active_step._get_display_name(self) if build.active_step else "?", build.job_time)) @@ -879,7 +910,9 @@ def _schedule(self): if self.env['runbot.host']._fetch_local_logs(build_ids=build.ids): return True # avoid to make results with remaining logs # No job running, make result and select next job - + if build.docker_start: + docker_duration = int(time.time() - dt2time(build.docker_start)) + build.docker_time += docker_duration build.job_end = now() build.docker_start = False # make result of previous job @@ -1009,7 +1042,9 @@ def _docker_run(self, step, cmd=None, ro_volumes=None, env_variables=None, **kwa containers_memory_limit = self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_containers_memory', 0) if containers_memory_limit and 'memory' not in kwargs: - kwargs['memory'] = int(float(containers_memory_limit) * 1024 ** 3) + memory_limit_factor = float(self.params_id.config_data.get('memory_limit_factor', 1)) + containers_memory_limit = int(float(containers_memory_limit) * 1024 ** 3) * memory_limit_factor + kwargs['memory'] = containers_memory_limit self.docker_start = now() if self.job_start: @@ -1020,6 +1055,8 @@ def _docker_run(self, step, cmd=None, ro_volumes=None, env_variables=None, **kwa starting_config = self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_default_odoorc') if isinstance(cmd, Command): rc_content = cmd.get_config(starting_config=starting_config) + if step.check_exit_status: + cmd.finals = [['echo', r'$?', '>', f'/data/build/logs/{step.sanitized_name(self)}_exit_status.txt']] + cmd.finals else: rc_content = starting_config self._write_file('.odoorc', rc_content) @@ -1033,6 +1070,7 @@ def _docker_run(self, step, cmd=None, ro_volumes=None, env_variables=None, **kwa self.env.flush_all() env_variables = env_variables or [] env_variables.append('ODOO_RUNBOT=1') + self.cpu_limit = kwargs.get('cpu_limit') def start_docker(): docker_run( cmd=cmd, @@ -1079,25 +1117,28 @@ def _checkout(self): return exports + def _list_available_modules(self): + for commit in self.env.context.get('defined_commit_ids') or self.params_id.commit_ids: + for (addons_path, module, manifest_file_name) in commit._list_available_modules(): + yield commit, addons_path, module, manifest_file_name + def _get_available_modules(self): all_modules = dict() available_modules = defaultdict(list) # repo_modules = [] - for commit in self.env.context.get('defined_commit_ids') or self.params_id.commit_ids: - for (addons_path, module, manifest_file_name) in commit._get_available_modules(): - if module in all_modules: - self._log( - 'Building environment', - '%s is a duplicated modules (found in "%s", already defined in %s)' % ( - module, - commit._source_path(addons_path, module, manifest_file_name), - all_modules[module]._source_path(addons_path, module, manifest_file_name)), - level='WARNING', - ) - else: - available_modules[commit.repo_id].append(module) - all_modules[module] = commit - # return repo_modules, available_modules + for commit, addons_path, module, manifest_file_name in self._list_available_modules(): + if module in all_modules: + self._log( + 'Building environment', + '%s is a duplicated modules (found in "%s", already defined in %s)' % ( + module, + commit._source_path(addons_path, module, manifest_file_name), + all_modules[module]._source_path(addons_path, module, manifest_file_name)), + level='WARNING', + ) + else: + available_modules[commit.repo_id].append(module) + all_modules[module] = commit return available_modules def _get_modules_to_test(self, modules_patterns=''): @@ -1108,6 +1149,49 @@ def _get_modules_to_test(self, modules_patterns=''): modules_patterns = (modules_patterns or '').split(',') return trigger._filter_modules_to_test(modules, params_patterns + modules_patterns) # we may switch params_patterns and modules_patterns order + @transactioncache + def _dependency_graph(self): + dependency_graph = defaultdict(set) + dependant_graph = defaultdict(set) + for commit in self.env.context.get('defined_commit_ids') or self.params_id.commit_ids: + file_paths = [] + modules = [] + for (addons_path, module, manifest_file_name) in commit._list_available_modules(): + file_paths.append(os.path.join(addons_path, module, manifest_file_name)) + modules.append(module) + contents = commit._git_show_files(file_paths) + for module, manifest in zip(modules, contents): + manifest_content = ast.literal_eval(manifest) + depends = manifest_content.get('depends', []) + if not depends and module != 'base': + depends = ['base'] + for dep in depends: + dependency_graph[module].add(dep) + dependant_graph[dep].add(module) + return dependency_graph, dependant_graph + + def search_modules_graph(self, modules, graph, depth=None): + def search(modules, depth=None, visited=None): + visited = visited or set() + modules = set(modules) - visited + visited |= modules + dependencies = set(modules) + if depth == 0 or not modules: + return dependencies + for module in modules: + dependencies |= search(graph[module], depth - 1 if depth is not None else None, visited) + return dependencies + return sorted(search(modules, depth)) + + def _get_modules_dependencies(self, modules, depth=None): + self.ensure_one() + dependency_graph, _ = self._dependency_graph() + return self.search_modules_graph(modules, dependency_graph, depth) + + def _get_dependant_modules(self, modules, depth=None): + _, dependant_graph = self._dependency_graph() + return self.search_modules_graph(modules, dependant_graph, depth) + def _local_pg_dropdb(self, dbname): msg = '' try: @@ -1162,7 +1246,7 @@ def truncate(message, maxlenght=300000): 'line': '0', }) - def _kill(self, result=None): + def _kill(self, result='killed'): host_name = self.env['runbot.host']._get_current_name() self.ensure_one() build = self @@ -1170,30 +1254,38 @@ def _kill(self, result=None): return build._log('kill', 'Kill build %s' % build.dest) docker_stop(build._get_docker_name(), build._path()) - v = {'local_state': 'done', 'requested_action': False, 'active_step': False, 'job_end': now()} + build.local_state = 'done' + build.active_step = False + build.job_end = now() if not build.build_end: - v['build_end'] = now() + build.build_end = now() if result: - v['local_result'] = result - build.write(v) - - def _ask_kill(self, lock=True, message=None): - # if build remains in same bundle, it's ok like that - # if build can be cross bundle, need to check number of ref to build - if lock: - self.env.cr.execute("""SELECT id FROM runbot_build WHERE parent_path like %s FOR UPDATE""", ['%s%%' % self.parent_path]) + build.local_result = result + + def _ask_kill(self, message=None): self.ensure_one() user = self.env.user uid = user.id build = self message = message or 'Killing build %s, requested by %s (user #%s)' % (build.dest, user.name, uid) build._log('_ask_kill', message) - if build.local_state == 'pending': - build._skip() - elif build.local_state in ['testing', 'running']: - build.requested_action = 'deathrow' - for child in build.children_ids: - child._ask_kill(lock=False) + + self.env.cr.execute("""SELECT id, local_state FROM runbot_build WHERE parent_path like %s""", ['%s%%' % self.parent_path]) + builds = self.browse([b[0] for b in self.env.cr.fetchall()]) + pending = builds.filtered(lambda b: b.local_state == 'pending') + killable = builds.filtered(lambda b: b.local_state in ('running', 'testing')) + if pending: + pending.local_state = 'done' + pending.local_result = 'killed' + pending.flush_recordset() # faster concurrent error or lock row + + values = [{ + 'host_id': b.host_id.id, + 'build_id': b.id, + 'message': 'kill', + } for b in killable] + + self.env['runbot.host.message'].sudo().create(values) def _wake_up(self): user = self.env.user @@ -1230,19 +1322,24 @@ def _modified_files(self, commit_link_links=None): commit_link_links = self.params_id.commit_link_ids for commit_link in commit_link_links: commit = commit_link.commit_id - modified = commit.repo_id._git(['diff', '--name-only', '%s..%s' % (commit_link.merge_base_commit_id.name, commit.name)]) + commit._fetch() + modified = commit.repo_id._git(['diff', '--name-only', '%s..%s' % (commit_link.merge_base_commit_id.tree_hash, commit.tree_hash)]) if modified: files = [os.sep.join([self._docker_source_folder(commit), file]) for file in modified.split('\n') if file] modified_files[commit_link] = files return modified_files - def _modified_modules(self, commit_link_links=None): + def _modified_modules(self, commit_link_links=None, defaults=None): modified_files = self._modified_files(commit_link_links) modified_modules = set() for commit_link, files in modified_files.items(): commit = commit_link.commit_id for file in files: - modified_modules.add(commit.repo_id._get_module(file)) + module = commit.repo_id._get_module(file) + if module: + modified_modules.add(module) + elif defaults: + modified_modules |= set(defaults) return modified_modules def _get_upgrade_path(self): @@ -1396,12 +1493,12 @@ def _get_py_version(self): return '3' return '' - def _parse_logs(self): + def _parse_logs(self, update_tags=False): """ Parse build logs to classify errors """ # only parse logs from builds in error and not already scanned builds_to_scan = self.filtered(lambda b: b.local_result in ('ko', 'killed', 'warn') and not b.build_error_link_ids) ir_logs = builds_to_scan.log_ids.filtered(lambda l: l.level in ('ERROR', 'WARNING', 'CRITICAL')) - return self.env['runbot.build.error']._parse_logs(ir_logs) + return self.env['runbot.build.error']._parse_logs(ir_logs, update_tags=update_tags) def _is_file(self, file, mode='r'): file_path = self._path(file) @@ -1409,6 +1506,10 @@ def _is_file(self, file, mode='r'): def _read_file(self, file, mode='r'): file_path = self._path(file) + max_log_file_size = int(self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_max_log_size', DEFAULT_MAX_FILE_SIZE)) + if os.path.getsize(file_path) > max_log_file_size: + self._log('readfile', f"File size exceeds {max_log_file_size} limit", level="ERROR") + return False try: with file_open(file_path, mode) as f: return f.read() @@ -1443,10 +1544,62 @@ def _get_color_class(self): if self.global_result == 'ok': return 'success' - if self.global_result in ('skipped', 'killed', 'manually_killed'): + if self.global_result in ('skipped', 'killed'): return 'secondary' return 'default' + def _get_file_url(self, path, line=None): + repo_name = path.replace('/data/build/', '').split('/')[0] + for commit_link in self.params_id.commit_link_ids: + if commit_link.commit_id.repo_id.name == repo_name: + repo_base_url = commit_link.branch_id.remote_id.base_url + commit_hash = commit_link.commit_id.name + path = path.replace('/data/build/%s/' % repo_name, '') + url = f'https://{repo_base_url}/blob/{commit_hash}/{path}' + if line: + url = f'{url}#L{line}' + return url + return '' + + def _format_message(self, log): + text = log.message + if not "\n" in text and 'in: /data/build/' in text: + parts = text.split('in: /data/build/') + text = parts[0] + url = f'http://{self.host}/runbot/static/build/{self.dest}/{parts[-1]}' + template = Markup('%s') + return template % (url, text) + text = text.strip('\n') + text = html_escape(text) + + def get_link(match): + path = match.group(1) + line = match.group(2) + url = self._get_file_url(path, line) + if url: + if line: + return Markup('%s", line %s') % (url, path, line) + return Markup('%s') % (url, path) + return match.group(0) + regex = r''' + (/data/build/[\w\-\./]+\.(?:py|xml|js|css)) # Path in /data/build ending with a common extension + (?: + \&\#34;,\sline\s(\d+) # Optional line number (escaped quote) + )? + ''' + text = Markup(re.sub(regex, get_link, text, flags=re.VERBOSE)) + + return text + + def _log_details(self, log): + title = f"Logger: {log.name}\nFunc: {log.func}" + test_data = log.metadata.dict.get('test') + if test_data: + title += '\n' + for test_line in test_data: + title += f'\n{test_line}: {test_data[test_line]}' + return title + def _github_status(self): """Notify github of failed/successful builds""" for build in self: @@ -1458,14 +1611,18 @@ def _github_status(self): build.parent_id._github_status() else: trigger = build.params_id.trigger_id - if not trigger.ci_context: + ci_context = trigger.ci_context + if not ci_context: continue desc = trigger.ci_description or " (runtime %ss)" % (build.job_time,) if build.params_id.used_custom_trigger: - state = 'error' + ci_context += " (custom)" desc = "This build used custom config. Remove custom trigger to restore default ci" - elif build.global_result in ('ko', 'warn'): + if build.params_id.config_id == build.trigger_id.light_config_id: + ci_context += " (light)" + desc = "This build used a light config. Enable default build configuration to restore default ci" + if build.global_result in ('ko', 'warn'): state = 'error' elif build.global_state in ('pending', 'testing'): state = 'pending' @@ -1518,7 +1675,7 @@ def _github_status(self): else: target_url = f"{self.get_base_url()}/runbot/build/{build.id}" - commit._github_status(build, trigger.ci_context, state, target_url, desc, ci_strategy=trigger.ci_strategy) + commit._github_status(build, ci_context, state, target_url, desc, ci_strategy=trigger.ci_strategy) def _parse_config(self): return set(findall(self._server("tools/config.py"), r'--[\w-]+', )) diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py index c7bc622a0..af90073a8 100644 --- a/runbot/models/build_config.py +++ b/runbot/models/build_config.py @@ -1,35 +1,47 @@ import base64 +import fnmatch import glob import json import logging -import fnmatch -import psutil import re import shlex import time -from unidiff import PatchSet -from ..common import now, grep, time2str, rfind, s2human, os, RunbotException, ReProxy, markdown_escape -from ..container import docker_get_gateway_ip, Command -from odoo import models, fields, api, tools -from odoo.exceptions import UserError, ValidationError -from odoo.tools.misc import file_open -from odoo.tools.safe_eval import safe_eval, test_python_expr, _SAFE_OPCODES, to_opcodes -# adding some additionnal optcode to safe_eval. This is not 100% needed and won't be done in standard but will help -# to simplify some python step by wraping the content in a function to allow return statement and get closer to other -# steps +import psutil +import requests +from unidiff import VERSION, PatchSet, patch +from odoo import api, fields, models, tools +from odoo.exceptions import UserError, ValidationError +from odoo.tools.safe_eval import _SAFE_OPCODES, safe_eval, test_python_expr, to_opcodes + +from ..common import ( + ReProxy, + RunbotException, + TestTagsParser, + grep, + markdown_escape, + now, + os, + rfind, + s2human, + time2str, + DEFAULT_MAX_FILE_SIZE, +) +from ..container import Command, docker_get_gateway_ip # There is an issue in unidiff 0.7.3 fixed in 0.7.4 # https://github.com/matiasb/python-unidiff/commit/a3faffc54e5aacaee3ded4565c534482d5cc3465 # Since the unidiff packaged version in noble is 0.7.3 # patching it looks like the easiest solution -from unidiff import patch, VERSION if VERSION == '0.7.3': patch.RE_DIFF_GIT_DELETED_FILE = re.compile(r'^deleted file mode \d+$') patch.RE_DIFF_GIT_NEW_FILE = re.compile(r'^new file mode \d+$') +# adding some additionnal optcode to safe_eval. This is not 100% needed and won't be done in standard but will help +# to simplify some python step by wraping the content in a function to allow return statement and get closer to other +# steps _SAFE_OPCODES |= set(to_opcodes(['LOAD_DEREF', 'STORE_DEREF', 'LOAD_CLOSURE', 'MAKE_CELL', 'COPY_FREE_VARS'])) @@ -46,28 +58,41 @@ def filter_all_modules(selector, build, dynamic_vars): return filter_default_modules(selector, build, dynamic_vars) +def get_dependencies(modules, build, dynamic_vars, depth=None): + depth = int(depth) if depth else None + modules = modules.split(',') + dependant = set(build._get_modules_dependencies(modules, depth)) - set(modules) + return ','.join(sorted(dependant)) + + +def get_dependant(modules, build, dynamic_vars, depth=None): + depth = int(depth) if depth else None + modules = modules.split(',') + dependant = set(build._get_dependant_modules(modules, depth)) - set(modules) + return ','.join(sorted(dependant)) + + def filter_default_modules(selector, build, dynamic_vars): - build._checkout() # we need to ensure source are exported before _get_modules_to_test modules = build._get_modules_to_test(selector) return ','.join(modules) -def keep_modified_modules(modules, build, dynamic_vars): +def select_existing_modules(selector, build, dynamic_vars): + selector = f'-*,{selector}' + return filter_default_modules(selector, build, dynamic_vars) + + +def keep_modified_modules(modules, build, dynamic_vars, *defaults): if build.params_id.config_data.get('skip_modified_modules_filter', False): return modules - modified_modules = build._modified_modules() + if defaults: + defaults = [d[1:-1] if re.match(r'^[\'"].*[\'"]$', d) else d for d in defaults] + modified_modules = build._modified_modules(defaults=defaults) modules = modules.split(',') filtered_modules = [module for module in modules if module in modified_modules] return ','.join(filtered_modules) -def keep_modified_modules_or_base(modules, build, dynamic_vars): - bundle = build.params_id.create_batch_id.bundle_id - if bundle.is_base or bundle.is_staging: - return modules - return keep_modified_modules(modules, build, dynamic_vars) - - def make_module_test_tags(modules, build, dynamic_vars): return ','.join([f'/{module}' for module in modules.split(',')]) @@ -88,6 +113,17 @@ def append_string(modules, build, dynamic_vars, element): return ','.join([f'{module}{element}' for module in modules.split(',')]) +def union(modules, build, dynamic_vars, element): + if re.match(r'^[\'"].*[\'"]$', element): + element = element[1:-1] + else: + element = dynamic_vars.get(element, element) + element = element.strip() + modules = set(modules.split(',')) if modules else set() + new_modules = set(element.split(',')) if element else set() + return ','.join(sorted(modules | new_modules)) + + class Config(models.Model): _name = 'runbot.build.config' _description = "Build config" @@ -222,11 +258,15 @@ def wrapper(value, path): return wrapper def VARS(vars, path): - if not isinstance(vars, dict): - raise ValidationError(f'{path} ({vars}) should be a dict') - for key, val in vars.items(): - TECHNICAL_NAME(key, f'{path}.{key}') - STR(val, f'{path}.{key}') + if isinstance(vars, list): + for item in vars: + VARS(item, path) + else: + if not isinstance(vars, dict): + raise ValidationError(f'{path} ({vars}) should be a dict') + for key, val in vars.items(): + TECHNICAL_NAME(key, f'{path}.{key}') + STR(val, f'{path}.{key}') NAME = str_checker(r'^[\w \-]+$') STR = str_checker(r'.*') @@ -241,6 +281,7 @@ def VARS(vars, path): 'vars': OPTIONAL(VARS), 'steps': REQUIRED(LIST(STEP)), 'description': OPTIONAL(DYNAMIC_VALUE), + 'log': OPTIONAL(DYNAMIC_VALUE), } valid_steps['odoo'] = { 'name': REQUIRED(NAME), @@ -251,9 +292,11 @@ def VARS(vars, path): 'test_tags': OPTIONAL(DYNAMIC_VALUE), 'demo_mode': OPTIONAL(IN(['default', 'with_demo', 'without_demo'])), 'enable_auto_tags': OPTIONAL(BOOL), + 'extra_params': OPTIONAL(DYNAMIC_VALUE), 'cpu_limit': OPTIONAL(INT), 'export_database': OPTIONAL(BOOL), 'make_stats': OPTIONAL(BOOL), + 'log': OPTIONAL(DYNAMIC_VALUE), } valid_steps['create_build'] = { 'name': REQUIRED(NAME), @@ -262,6 +305,8 @@ def VARS(vars, path): 'for_each_vars': OPTIONAL(LIST(VARS)), 'for_each_module': OPTIONAL(DYNAMIC_VALUE), 'max_builds': OPTIONAL(INT), + 'if': OPTIONAL(DYNAMIC_VALUE), + 'log': OPTIONAL(DYNAMIC_VALUE), } valid_steps['restore'] = { 'name': REQUIRED(NAME), @@ -271,6 +316,7 @@ def VARS(vars, path): 'trigger_id': OPTIONAL(INT), 'use_current_batch': OPTIONAL(BOOL), 'zip_url': OPTIONAL(STR), + 'log': OPTIONAL(DYNAMIC_VALUE), } valid_steps['command'] = { 'name': REQUIRED(NAME), @@ -283,6 +329,7 @@ def VARS(vars, path): 'check_logs': OPTIONAL(LIST(STR)), 'expected_logs': OPTIONAL(LIST(STR)), 'make_stats': OPTIONAL(BOOL), + 'log': OPTIONAL(DYNAMIC_VALUE), } validate(config_schema, config, 'config') @@ -388,10 +435,14 @@ class ConfigStep(models.Model): cpu_limit = fields.Integer('Cpu limit', default=3600, tracking=True) container_cpus = fields.Integer('Allowed CPUs', help='Allowed container CPUs. Fallback on config parameter if 0.', default=0, tracking=True) coverage = fields.Boolean('Coverage', default=False, tracking=True) + coverage_branch = fields.Boolean('Coverage branch', default=False, tracking=True) + coverage_concurrency = fields.Boolean('Coverage concurrency', default=False, tracking=True) + coverage_test_context = fields.Boolean('Coverage test context', default=False, tracking=True) + coverage_make_report = fields.Boolean('Make coverage report', default=False, tracking=True) paths_to_omit = fields.Char('Paths to omit from coverage', tracking=True) flamegraph = fields.Boolean('Allow Flamegraph', default=False, tracking=True) test_enable = fields.Boolean('Test enable', default=True, tracking=True) - test_tags = fields.Char('Test tags', help="comma separated list of test tags", tracking=True) + test_tags = fields.Char('Test tags', help="new line (or comma) separated list of test tags", tracking=True) enable_auto_tags = fields.Boolean('Allow auto tag', default=True, tracking=True) sub_command = fields.Char('Subcommand', tracking=True) extra_params = fields.Char('Extra cmd args', tracking=True) @@ -429,6 +480,9 @@ class ConfigStep(models.Model): restore_download_db_suffix = fields.Char('Download db suffix') restore_rename_db_suffix = fields.Char('Rename db suffix') + # TODO change the default to True once we are sure that it works as expected + check_exit_status = fields.Boolean('Check exit status', default=False, help='Check exit status of the main command') + semgrep_category = fields.Many2one('runbot.checker_category', string='Semgrep Category', tracking=True) custom_link = fields.Char('Custom link for semgrep codes', tracking=True) disable_nosem = fields.Boolean('Disable nosem', default=False, tracking=True) @@ -546,6 +600,10 @@ def _run_step(self, build, **kwargs): max_timeout = int(self.env['ir.config_parameter'].get_param('runbot.runbot_timeout', default=10000)) docker_params['cpu_limit'] = min(self.cpu_limit, max_timeout) + config_data = {**kwargs.get('config_data', {}), **build.params_id.config_data} + if docker_params['cpu_limit'] and config_data.get('cpu_limit_factor'): + docker_params['cpu_limit'] = int(docker_params['cpu_limit'] * min(float(config_data['cpu_limit_factor']), 3)) + container_cpus = float(self.container_cpus or self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_containers_cpus', 0)) if 'cpus' not in docker_params and container_cpus: logical_cpu_count = psutil.cpu_count(logical=True) @@ -582,6 +640,9 @@ def _run_create_build(self, build, config_data=None, max_build=200): build._log('create_build', 'created with config %s' % config_name, log_type='subbuild', path=str(child.id)) def _make_python_ctx(self, build): + def log(*args, **kwargs): + args = [str(arg) for arg in args] + build._log(self.name, *args, **kwargs) return { 'datetime': tools.safe_eval.datetime, 'dateutil': tools.safe_eval.dateutil, @@ -602,6 +663,9 @@ def _make_python_ctx(self, build): 'json_loads': json.loads, 'PatchSet': PatchSet, 'markdown_escape': markdown_escape, + 'TestTagsParser': TestTagsParser, + 'requests': requests.Session(), + 'log': log, } def _run_python(self, build, force=False): @@ -688,7 +752,6 @@ def _run_run_odoo(self, build, force=False): return dict(cmd=cmd, exposed_ports=[build_port, build_port + 1], ro_volumes=exports, env_variables=env_variables, cpu_limit=None, network_enabled=True) def _run_install_odoo(self, build, config_data=None): - if config_data: config_data = {**config_data, **build.params_id.config_data} else: @@ -699,10 +762,14 @@ def _run_install_odoo(self, build, config_data=None): mods = ",".join(modules_to_install) python_params = [] py_version = build._get_py_version() - if self.coverage: + if self.coverage or config_data.get('coverage'): build.coverage = True - coverage_extra_params = self._coverage_params(build, modules_to_install) - python_params = ['-m', 'coverage', 'run', '--branch', '--source', '/data/build'] + coverage_extra_params + python_params = ['-m', 'coverage', 'run', '--source', '/data/build'] + if config_data.get('coverage_branch', self.coverage_branch): + python_params += ['--branch'] + if config_data.get('coverage_concurrency', self.coverage_concurrency): + python_params += ['--concurrency=thread'] + python_params += self._coverage_params(build, config_data) elif self.flamegraph: python_params = ['-m', 'flamegraph', '-o', self._perfs_data_path(build)] cmd = build._cmd(python_params, py_version, sub_command=self.sub_command, enable_log_db=self.enable_log_db) @@ -724,8 +791,8 @@ def _run_install_odoo(self, build, config_data=None): elif demo_mode == 'without_demo' and demo_installed_by_default: cmd.append('--without-demo=true') + extra_params = config_data.get('extra_params', build.params_id.extra_params or self.extra_params or '') # list module to install - extra_params = build.params_id.extra_params or self.extra_params or '' if mods and '-i' not in extra_params: cmd += ['-i', mods] config_path = build._server("tools/config.py") @@ -739,7 +806,7 @@ def _run_install_odoo(self, build, config_data=None): test_tags_in_extra = '--test-tags' in extra_params if (test_enable or test_tags) and "--test-tags" in available_options and not test_tags_in_extra: - test_tags = [t.strip() for t in (test_tags or '').split(',')] + test_tags = [t.strip() for t in TestTagsParser(test_tags or '').filter_specs] if enable_auto_tags and not config_data.get('disable_auto_tags', False): if grep(config_path, "[/module][:class]"): auto_tags = self.env['runbot.build.error']._disabling_tags(build) @@ -771,7 +838,7 @@ def _run_install_odoo(self, build, config_data=None): if extra_params: cmd.extend(shlex.split(extra_params)) - cmd.finals.extend(self._post_install_commands(build, modules_to_install, py_version)) # coverage post, extra-checks, ... + cmd.finals.extend(self._post_install_commands(build, config_data, py_version)) # coverage post, extra-checks, ... if config_data.get('export_database', True): self._add_zip_generation(build, cmd, db_name) @@ -783,10 +850,12 @@ def _run_install_odoo(self, build, config_data=None): if config_env_variables := config_data.get('env_variables', False): env_variables += config_env_variables.split(';') - cpu_limit = None + if config_data.get('coverage_test_context', self.coverage_test_context): + env_variables.append("COVERAGE_DYNAMIC_CONTEXT=test_function") + + cpu_limit = self.cpu_limit if config_data.get('cpu_limit'): cpu_limit = min(self.cpu_limit, int(config_data['cpu_limit'])) - return dict(cmd=cmd, ro_volumes=exports, cpu_limit=cpu_limit, env_variables=env_variables) def _add_zip_generation(self, build, cmd, db_name): @@ -992,7 +1061,9 @@ def get_reference_builds_for_versions(versions): ) if self.allow_similar_build_quick_result: - existing_done_build = next((build for build in child.params_id.build_ids.sorted('id') if build.global_state == 'done' and build.local_result not in ('skipped', 'killed', 'manually_killed')), None) + existing_done_build = next((build for build in child.params_id.build_ids.sorted('id') if build.global_state == 'done' and build.global_result == 'ok'), None) + if not existing_done_build: + existing_done_build = next((build for build in child.params_id.build_ids.sorted('id') if build.global_state == 'done' and build.local_result not in ('skipped', 'killed')), None) if existing_done_build: child._log('', 'A similar [build](%s) has been found, marking as done directly', existing_done_build.build_url, log_type='markdown') child.local_state = 'done' @@ -1152,11 +1223,11 @@ def _log_end(self, build): log_type = 'markdown' build._log('', message, *args, log_type=log_type) - if self.coverage: - xml_url = '%scoverage.xml' % build._http_log_url() - html_url = 'http://%s/runbot/static/build/%s/coverage/index.html' % (build.host, build.dest) - message = 'Coverage report: [xml @icon-download](%s), [html @icon-eye](%s)' - build._log('end_job', message, xml_url, html_url, log_type='markdown') + if self.coverage and self.coverage_make_report: + json_url = f'http://{build.host}/runbot/static/build/{build.dest}/logs/coverage/coverage.json' + html_url = f'http://{build.host}/runbot/static/build/{build.dest}/logs/coverage/' + message = 'Coverage report: [json @icon-download](%s), [html @icon-eye](%s)' + build._log('end_job', message, json_url, html_url, log_type='markdown') if self.flamegraph: dat_url = '%sflame_%s.%s' % (build._http_log_url(), self.sanitized_name(build), 'log.gz') @@ -1164,34 +1235,27 @@ def _log_end(self, build): message = 'Flamegraph report: [data @icon-download](%s), [svg @icon-eye](%s)' build._log('end_job', message, dat_url, svg_url, log_type='markdown') - def _post_install_commands(self, build, modules_to_install, py_version=None): + def _post_install_commands(self, build, config_data, py_version): cmds = [] - if self.coverage: - py_version = py_version if py_version is not None else build._get_py_version() - # prepare coverage result - cov_path = build._path('coverage') - os.makedirs(cov_path, exist_ok=True) - cmds.append(['python%s' % py_version, "-m", "coverage", "html", "-d", "/data/build/coverage", "--ignore-errors"]) - cmds.append(['python%s' % py_version, "-m", "coverage", "xml", "-o", "/data/build/logs/coverage.xml", "--ignore-errors"]) + if config_data.get('coverage_make_report', (self.coverage and self.coverage_make_report)): + cmds.append(['python%s' % py_version, "-m", "coverage", "html", "-d", "/data/build/logs/coverage", "--ignore-errors"]) + cmds.append(['python%s' % py_version, "-m", "coverage", "json", "-o", "/data/build/logs/coverage.json", "--ignore-errors"]) + if config_data.get('coverage', self.coverage): + cmds.append(['mv', "/data/build/.coverage", f"/data/build/logs/coverage.{build.id}.{int(time.time())}"]) return cmds def _perfs_data_path(self, build, ext='log'): return '/data/build/logs/flame_%s.%s' % (self.sanitized_name(build), ext) - def _coverage_params(self, build, modules_to_install): + def _coverage_params(self, build, config_data): pattern_to_omit = set() if self.paths_to_omit: - pattern_to_omit = set(self.paths_to_omit.split(',')) - for commit in build.params_id.commit_ids: - docker_source_folder = build._docker_source_folder(commit) - for manifest_file in commit.repo_id.manifest_files.split(','): - pattern_to_omit.add('*%s' % manifest_file) - for (addons_path, module, _) in commit._get_available_modules(): - if module not in modules_to_install: - # we want to omit docker_source_folder/[addons/path/]module/* - module_path_in_docker = os.sep.join([docker_source_folder, addons_path, module]) - pattern_to_omit.add('%s/*' % (module_path_in_docker)) - return ['--omit', ','.join(sorted(pattern_to_omit))] + pattern_to_omit |= set(self.paths_to_omit.split(',')) + if config_data.get('paths_to_omit'): + pattern_to_omit |= set(config_data.get('paths_to_omit').split(',')) + if pattern_to_omit: + return ['--omit', ','.join(sorted(pattern_to_omit))] + return [] def _make_results(self, build): # TODO fixme config data are not the same as the run part in dynamic steps @@ -1222,6 +1286,10 @@ def _make_results(self, build): if log_time: build.job_end = log_time + if self.check_exit_status and (exit_status := self._get_exit_status(build)) != 0: + build._log('_make_results', f'Main command exited with status code {exit_status}', level='ERROR') + build.local_result = 'ko' + if check_logs or expected_logs: self._make_custom_result(build, check_logs, expected_logs) elif active_job_type == 'python': @@ -1230,8 +1298,6 @@ def _make_results(self, build): elif self.test_enable or self.test_tags: self._make_odoo_results(build) elif active_job_type == 'install_odoo': - if self.coverage: - build.write(self._make_coverage_results(build)) if not self.sub_command: self._make_odoo_results(build) elif active_job_type == 'test_upgrade': @@ -1250,23 +1316,6 @@ def _make_python_results(self, build): raise RunbotException('python_result_code must set return_value to a dict values on build') build.write(return_value) # old style support - def _make_coverage_results(self, build): - build_values = {} - build._log('coverage_result', 'Start getting coverage result') - cov_path = build._path('coverage/index.html') - if os.path.exists(cov_path): - with file_open(cov_path, 'r') as f: - data = f.read() - covgrep = re.search(r'pc_cov.>(?P\d+)%', data) - build_values['coverage_result'] = covgrep and covgrep.group('coverage') or False - if build_values['coverage_result']: - build._log('coverage_result', 'Coverage result: %s' % build_values['coverage_result']) - else: - build._log('coverage_result', 'Coverage result not found', level='WARNING') - else: - build._log('coverage_result', 'Coverage file not found', level='WARNING') - return build_values - def _make_upgrade_results(self, build): build._log('upgrade', 'Getting results for build %s' % build.dest) @@ -1298,6 +1347,10 @@ def _check_log(self, build): if not os.path.isfile(log_path): build._log('_make_tests_results', "Log file not found at the end of test job", level="ERROR") return 'ko' + max_log_file_size = int(self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_max_log_size', DEFAULT_MAX_FILE_SIZE)) + if os.path.getsize(log_path) > max_log_file_size: + build._log('_make_tests_results', f"Log file exceeds {max_log_file_size} limit", level="ERROR") + return 'ko' return 'ok' def _check_module_loaded(self, build): @@ -1380,6 +1433,21 @@ def _get_checkers_result(self, build, checkers): return result return 'ok' + def _get_exit_status(self, build): + exit_status_filename = f'{self.sanitized_name(build)}_exit_status.txt' + if not os.path.exists(build._path(exit_status_filename)): + build._log('_make_tests_results', f'Exit status file "{exit_status_filename}" not found', level="ERROR") + return 1 + res = build._read_file(exit_status_filename) + if res: + try: + return int(res.strip('\n')) + except ValueError: + build._log('_make_tests_results', f'Status file "{exit_status_filename}" does not contain an integer', level="ERROR") + return -242 + build._log('_make_tests_results', f'Exception or file empty while reading status file "{exit_status_filename}"', level="ERROR") + return -241 + def _make_custom_result(self, build, enabled_checkers=None, expected_logs=None): build._log('run', 'Getting results for build %s' % build.dest) if build.local_result != 'ko': @@ -1499,35 +1567,70 @@ def _run_dynamic(self, build): raise RunbotException('Too many ancestors builds, possible cyclic dynamic build creation') if build.parent_id and build.dynamic_config == build.parent_id.dynamic_config: raise RunbotException('A child build cannot load the same dynamic config if parent, recursion detected') + + config_vars_list = build.dynamic_config.get('vars', {}) + if not isinstance(config_vars_list, list): + config_vars_list = [config_vars_list] + raw_vars = {} + for config_vars in config_vars_list: + raw_vars.update(config_vars) + + raw_vars.update(build.params_id.config_data.get('dynamic_vars', {})) + dynamic_vars = {} + # dynamic_vars can either be raw value like 'account', value to evaluate lazily in anothed dynamic value like 'account->!mail' + # or dynamic value that we want to evaluate early like '{{*|filter_all_modules|modified_modules}}' (between {{}}) + # this loop will evalute the third category + # this alows to evaluate only once an expression that could be expensive to use it in multiple dynamic values + # this also allow to clarify the config by chaining vars definition + # TODO check ordering + for key, value in raw_vars.items(): + dynamic_vars[key] = self._parse_dynamic_entry(value, build, dynamic_vars=dynamic_vars) + current_step = self._get_dynamic_step(build) if not current_step: build._log('Dynamic Step', 'No dynamic config or steps found, skipping', level="WARNING") return + if current_step.get('log'): + text = self._parse_dynamic_entry(current_step['log'], build, dynamic_vars=dynamic_vars) + build._log('_run_dynamic', text) if current_step['job_type'] == 'create_build': for_each_vars_list = current_step.get('for_each_vars', [{}]) if 'for_each_module' in current_step: modules_vars = [] for for_each_vars in for_each_vars_list: - modules_entry = self._parse_dynamic_entry(current_step['for_each_module'], build, additional_dynamic_vars=for_each_vars) + modules_entry = self._parse_dynamic_entry(current_step['for_each_module'], build, dynamic_vars={**dynamic_vars, **for_each_vars}) modules = [m.strip() for m in modules_entry.split(',') if m.strip()] for module in modules: module_vars = {**for_each_vars, 'module': module} modules_vars.append(module_vars) for_each_vars_list = modules_vars - parent_vars = {**build.dynamic_config.get('vars', {}), **build.params_id.config_data.get('dynamic_vars', {})} + child_data_list = [] for child_index, child in enumerate(current_step.get('children', [])): child_vars = child.get('vars', {}) for for_each_vars in for_each_vars_list: config_name = child.get('name', build.params_id.config_id.name) - dynamic_vars = {**parent_vars, **child_vars, **for_each_vars} + raw_dynamic_vars = {**dynamic_vars, **for_each_vars, **child_vars} + child_dynamic_vars = {} + # evaluate for_each_vars + for key, value in raw_dynamic_vars.items(): + child_dynamic_vars[key] = self._parse_dynamic_entry(value, build, dynamic_vars=child_dynamic_vars) + if 'if' in current_step: + condition = self._parse_dynamic_entry(current_step['if'], build, dynamic_vars=child_dynamic_vars) + if not condition: + continue if 'description' in child: - description = self._parse_dynamic_entry(child['description'], build, additional_dynamic_vars=dynamic_vars) + description = self._parse_dynamic_entry(child['description'], build, dynamic_vars=child_dynamic_vars) # note: we mainly need to provide additional_dynamic_vars because the child is not created yet at this point else: description = config_name + # filter vars not prefixed with _ to simplify child values + if child.get('log'): + text = self._parse_dynamic_entry(child['log'], build, dynamic_vars=child_dynamic_vars) + build._log('_run_dynamic', text) + public_child_dynamic_vars = {key: value for key, value in child_dynamic_vars.items() if not key.startswith('_')} child_data = { - 'config_data': {**build.params_id.config_data.dict, "dynamic_vars": dynamic_vars}, + 'config_data': {**build.params_id.config_data.dict, "dynamic_vars": public_child_dynamic_vars}, 'config_id': build.params_id.config_id.id, 'dynamic_active_step_index': 0, 'dynamic_config_position': f'{build.params_id.dynamic_config_position or ""}/{build.dynamic_active_step_index}.{child_index}', @@ -1558,12 +1661,18 @@ def _run_dynamic(self, build): install_modules_pattern = current_step.get('install_modules', '') if install_modules_pattern.split(',', 1)[0] not in ('*', '-*'): install_modules_pattern = '-*,' + install_modules_pattern - config_data['install_module_pattern'] = self._parse_dynamic_entry(install_modules_pattern, build) + config_data['install_module_pattern'] = self._parse_dynamic_entry(install_modules_pattern, build, dynamic_vars) if 'test_tags' in current_step: - config_data['test_tags'] = self._parse_dynamic_entry(current_step.get('test_tags'), build) + config_data['test_tags'] = self._parse_dynamic_entry(current_step.get('test_tags'), build, dynamic_vars) config_data['test_enable'] = bool(current_step.get('test_enable') or current_step.get('test_tags')) + if 'extra_params' in current_step: + config_data['extra_params'] = self._parse_dynamic_entry(current_step.get('extra_params'), build, dynamic_vars) + + if 'cpu_limit' in current_step: + config_data['cpu_limit'] = int(current_step.get('cpu_limit')) + for key in ('screencast', 'demo_mode', 'enable_auto_tags'): if key in current_step: value = current_step[key] @@ -1584,6 +1693,7 @@ def _run_dynamic(self, build): 'addons_path': ",".join(build._get_addons_path()), 'exports': ",".join(exports.keys()), 'exports_paths': ",".join(exports.values()), + **dynamic_vars, } command = [shlex.quote(self._parse_dynamic_entry(part, build, values)) for part in command] pres = [] @@ -1605,22 +1715,23 @@ def _get_dynamic_db_suffix(self, step): db_suffix = re.sub(r'[^a-z0-9_\-]', '_', db_suffix.lower()) return db_suffix - def _parse_dynamic_entry(self, entry, build, additional_dynamic_vars=None): + def _parse_dynamic_entry(self, entry, build, dynamic_vars): """ transforms a module/test-tags entry dynamically """ - dynamic_config = build.dynamic_config - expression_filters = { 'filter_all_modules': filter_all_modules, 'filter_default_modules': filter_default_modules, 'make_module_test_tags': make_module_test_tags, + 'select_existing_modules': select_existing_modules, + 'get_dependencies': get_dependencies, + 'get_dependant': get_dependant, 'prepend': prepend_string, 'append': append_string, 'modified_modules': keep_modified_modules, - 'modified_modules_or_base': keep_modified_modules_or_base, + 'union': union, } - dynamic_vars = {**dynamic_config.get('vars', {}), **build.params_id.config_data.get('dynamic_vars', {}), **(additional_dynamic_vars or {})} + dynamic_vars = dynamic_vars or {} def parse_expression(match): # inspired by jinja but with limited features diff --git a/runbot/models/build_error.py b/runbot/models/build_error.py index b165914d6..2d07236fd 100644 --- a/runbot/models/build_error.py +++ b/runbot/models/build_error.py @@ -9,16 +9,15 @@ from dateutil import rrule from dateutil.relativedelta import relativedelta from markupsafe import Markup - from werkzeug.urls import url_join from odoo import api, fields, models from odoo.exceptions import AccessError, UserError, ValidationError -from odoo.tools import SQL, lazy, ormcache from odoo.fields import Domain +from odoo.tools import SQL, lazy, ormcache +from ..common import TestTagsParser, transactioncache from ..fields import JsonDictField -from ..common import transactioncache, TestTagsParser _logger = logging.getLogger(__name__) @@ -215,7 +214,6 @@ class BuildError(models.Model): _inherit = ('mail.thread', 'mail.activity.mixin', 'runbot.build.error.seen.mixin') _mail_post_access = 'read' - name = fields.Char("Name") active = fields.Boolean('Open (not fixed)', default=True, tracking=True) description = fields.Text("Description", store=True, compute='_compute_description') @@ -242,9 +240,10 @@ class BuildError(models.Model): breaking_bundle_id = fields.Many2one('runbot.bundle', 'Breaking bundle', tracking=True, help="Bundle that introduced the error", related='breaking_pr_id.bundle_id') breaking_bundle_url = fields.Char('Breaking bundle url', related='breaking_bundle_id.frontend_url') breaking_pr_date = fields.Datetime('Breaking date', related="breaking_pr_id.close_date", help="Date of the merge of the first pr") + duplicate_breaking_pr_count = fields.Integer('Same Breaking PR', compute='_compute_duplicate_breaking_pr_count', help='Other errors with same breaking PR') - test_tags = fields.Char(string='Test tags', help="Comma separated list of test_tags to use to reproduce/remove this error", tracking=True) - canonical_tags = fields.Char('Canonical tag', compute='_compute_canonical_tags', store=True) + test_tags = fields.Text(string='Test tags', help="Comma separated list of test_tags to use to reproduce/remove this error", tracking=True) + canonical_tags = fields.Text('Canonical tag', compute='_compute_canonical_tags', store=True) tags_match_count = fields.Integer('Nb errors matching the test_tags', compute='_compute_tags_match_count') tags_min_version_excluded_id = fields.Many2one('runbot.version', 'Tag min version (excluded)') tags_min_version_id = fields.Many2one('runbot.version', 'Tags Min version', compute="_compute_tags_min_version_id", inverse="_inverse_tags_min_version_id", help="Minimal version where the test tags will be applied.", tracking=True) @@ -289,7 +288,7 @@ def _inverse_tags_min_version_id(self): def _compute_canonical_tags(self): for record in self: canonical_tags = sorted(set(record.error_content_ids.filtered('canonical_tag').mapped('canonical_tag'))) - record.canonical_tags = ','.join(canonical_tags) + record.canonical_tags = '\n'.join(canonical_tags) @api.depends('tags_min_version_id') def _compute_tags_min_version_id(self): @@ -322,6 +321,25 @@ def _compute_fixing_bundle_id(self): for record in self: record.fixing_bundle_id = record.fixing_pr_id.bundle_id if record.fixing_pr_id else False + @api.depends('breaking_pr_id') + def _compute_duplicate_breaking_pr_count(self): + breaking_counts = self.env["runbot.build.error"]._read_group( + domain=[ + ("breaking_pr_id", "in", self.breaking_pr_id.ids), + ("active", "=", True), + ], + groupby=["breaking_pr_id"], + aggregates=["id:count"], + having=[('id:count', '>', 1)], + ) + + count_by_pr = {pr_count[0]: pr_count[1] for pr_count in breaking_counts} + + for record in self: + # remove 1 to not count the current error + record.duplicate_breaking_pr_count = count_by_pr.get(record.breaking_pr_id, 1) - 1 + + @api.depends('error_content_ids.version_ids') def _compute_version_ids(self): for record in self: @@ -431,10 +449,10 @@ def _compute_unique_qualifiers(self): @api.depends('common_qualifiers') def _compute_similar_ids(self): for record in self: - if record.common_qualifiers: + if record.common_qualifiers and (record.id or record.id.origin): query = SQL( r"""SELECT id FROM runbot_build_error WHERE id != %s AND common_qualifiers @> %s""", - record.id, + record.id or record.id.origin, json.dumps(record.common_qualifiers.dict), ) self.env.cr.execute(query) @@ -445,10 +463,10 @@ def _compute_similar_ids(self): @api.depends('common_qualifiers') def _compute_similar_content_ids(self): for record in self: - if record.common_qualifiers: + if record.common_qualifiers and (record.id or record.id.origin): query = SQL( r"""SELECT id FROM runbot_build_error_content WHERE error_id != %s AND qualifiers @> %s""", - record.id, + record.id or record.id.origin, json.dumps(record.common_qualifiers.dict), ) self.env.cr.execute(query) @@ -459,10 +477,10 @@ def _compute_similar_content_ids(self): @api.depends('common_qualifiers') def _compute_analogous_ids(self): for record in self: - if record.common_qualifiers: + if record.common_qualifiers and (record.id or record.id.origin): query = SQL( r"""SELECT id FROM runbot_build_error WHERE id != %s AND unique_qualifiers @> %s""", - record.id, + record.id or record.id.origin, json.dumps(record.unique_qualifiers.dict), ) self.env.cr.execute(query) @@ -473,10 +491,10 @@ def _compute_analogous_ids(self): @api.depends('common_qualifiers') def _compute_analogous_content_ids(self): for record in self: - if record.common_qualifiers: + if record.common_qualifiers and (record.id or record.id.origin): query = SQL( r"""SELECT id FROM runbot_build_error_content WHERE error_id != %s AND qualifiers @> %s""", - record.id, + record.id or record.id.origin, json.dumps(record.unique_qualifiers.dict), ) self.env.cr.execute(query) @@ -489,16 +507,20 @@ def _compute_tags_match_count(self): for record in self: record.tags_match_count = 0 if record.test_tags: - tags_parser = TestTagsParser(record.test_tags) - search_domain = tags_parser.test_tags_to_search_domain(exclude_error_id=record.id) - if search_domain: - record.tags_match_count = self.env['runbot.build.error'].with_context(active_test=True).search_count(search_domain) + try: + tags_parser = TestTagsParser(record.test_tags.replace('\n', ',')) + search_domain = tags_parser.test_tags_to_search_domain(exclude_error_id=record.id or record.id.origin) + if search_domain: + record.tags_match_count = self.env['runbot.build.error'].with_context(active_test=True).search_count(search_domain) + except Exception as e: # noqa: BLE001 + record.tags_match_count = -1 + _logger.warning("Error while computing tags_match_count for error %s with test_tags %s: %s", record.id, record.test_tags, e) def action_view_impacted_by_tag(self): self.ensure_one() if not self.test_tags: return - tags_parser = TestTagsParser(self.test_tags) + tags_parser = TestTagsParser(self.test_tags.replace('\n', ',')) return { 'type': 'ir.actions.act_window', 'views': [(False, 'list'), (False, 'form')], @@ -511,8 +533,15 @@ def action_view_impacted_by_tag(self): @api.constrains('test_tags') def _check_test_tags(self): for build_error in self: - if build_error.test_tags and '-' in build_error.test_tags: - raise ValidationError('Build error test_tags should not be negated') + if build_error.test_tags: + try: + test_tags = build_error.test_tags.replace('\n', ',') + tags_parser = TestTagsParser(test_tags) + tags_parser = TestTagsParser(test_tags, keep_escape=False) + except Exception as e: # noqa: BLE001 + raise ValidationError(f'Invalid test_tags format: {e}') + if tags_parser.exclude or any(params[0] == '-' for p, params in tags_parser.parameters): + raise ValidationError('Build error test_tags should not be negated') @api.onchange('test_tags') def _onchange_test_tags(self): @@ -520,6 +549,17 @@ def _onchange_test_tags(self): self.tags_min_version_id = min(self.version_ids, key=lambda rec: rec.number) self.tags_max_version_id = max(self.version_ids, key=lambda rec: rec.number) + def _update_version_tags(self): + for error in self: + if not (error.test_tags and error.version_ids): + continue + new_min = min(error.version_ids, key=lambda rec: rec.number) + new_max = max(error.version_ids, key=lambda rec: rec.number) + if not error.tags_min_version_id or new_min.number < error.tags_min_version_id.number: + error.tags_min_version_id = new_min + if not error.tags_max_version_id or new_max.number > error.tags_max_version_id.number: + error.tags_max_version_id = new_max + @api.onchange('customer') def _onchange_customer(self): if not self.responsible: @@ -587,12 +627,12 @@ def _merge(self, others): error.sudo().test_tags = previous_error.test_tags previous_error.sudo().test_tags = False elif self.env.su: - test_tags = error.test_tags.split(',') - previous_error - for tag in previous_error.test_tags.split(','): + test_tags = TestTagsParser(error.test_tags.replace('\n', ',')).filter_specs + previous_error_tags = TestTagsParser(previous_error.test_tags.replace('\n', ',')).filter_specs + for tag in previous_error_tags: if tag not in test_tags: test_tags.append(tag) - error.test_tags = ','.join(test_tags) + error.test_tags = '\n'.join(test_tags) previous_error.test_tags = False for field in fields_to_merge + fields_to_copy: if previous_error[field]: @@ -623,7 +663,16 @@ def filter_tags(e): return True test_tag_list = self.search([('test_tags', '!=', False)]).filtered(filter_tags).mapped('test_tags') - return [test_tag for error_tags in test_tag_list for test_tag in (error_tags).split(',')] + parsed_test_tags = [] + for error_tags in test_tag_list: + try: + # we cannot rely only on '\n' since old test-tags or user defined ones could be comma separated + error_tags = error_tags.replace('\n', ',') + tags_parser = TestTagsParser(error_tags) + parsed_test_tags.extend(tags_parser.filter_specs) + except Exception as e: # noqa: BLE001 + _logger.warning('Error while parsing test_tags for error with id %s: %s', self.id, e) + return parsed_test_tags @api.model def _disabling_tags(self, build_id=False): @@ -701,6 +750,19 @@ def action_view_analogous_qualified_contents(self): 'name': 'Similary Qualified Contents' } + def action_view_duplicate_breaking_pr(self): + self.ensure_one() + return { + 'type': 'ir.actions.act_window', + 'res_model': 'runbot.build.error', + 'domain': [ + ('breaking_pr_id', '=', self.breaking_pr_id.id), + ('active', '=', True), + ], + 'view_mode': 'list,form', + 'name': 'Errors with same breaking PR', + } + @api.depends('manual_team_id', 'auto_team_id') def _compute_team_id(self): for error in self: @@ -738,7 +800,7 @@ def action_copy_canonical_tag(self): record._onchange_test_tags() @api.model - def _parse_logs(self, ir_logs): + def _parse_logs(self, ir_logs, update_tags=False): if not ir_logs: return None regexes = self.env['runbot.error.regex'].search([]) @@ -794,6 +856,11 @@ def _parse_logs(self, ir_logs): 'log_date': rec.create_date, }) + if update_tags: + errors_to_update = build_error_contents.error_id.filtered('test_tags') + for error in errors_to_update: + error._update_version_tags() + if build_error_contents: window_action = { "type": "ir.actions.act_window", @@ -854,7 +921,7 @@ class BuildErrorContent(models.Model): breaking_pr_id = fields.Many2one(related='error_id.breaking_pr_id') fixing_pr_alive = fields.Boolean(related='error_id.fixing_pr_alive') fixing_pr_url = fields.Char(related='error_id.fixing_pr_url') - test_tags = fields.Char(related='error_id.test_tags') + test_tags = fields.Text(related='error_id.test_tags') tags_min_version_id = fields.Many2one(related='error_id.tags_min_version_id') tags_max_version_id = fields.Many2one(related='error_id.tags_max_version_id') @@ -992,10 +1059,10 @@ def _compute_error_display_id(self): def _compute_similar_ids(self): """error contents having the exactly the same qualifiers""" for record in self: - if record.qualifiers: + if record.qualifiers and (record.id or record.id.origin): query = SQL( r"""SELECT id FROM runbot_build_error_content WHERE id != %s AND qualifiers @> %s AND qualifiers <@ %s""", - record.id, + record.id or record.id.origin, json.dumps(record.qualifiers.dict), json.dumps(record.qualifiers.dict), ) diff --git a/runbot/models/build_stat_regex.py b/runbot/models/build_stat_regex.py index e845cbad7..9b6b8747c 100644 --- a/runbot/models/build_stat_regex.py +++ b/runbot/models/build_stat_regex.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import logging -from ..common import os +from ..common import os, DEFAULT_MAX_FILE_SIZE import re from odoo import models, fields, api @@ -53,6 +53,10 @@ def _find_in_file(self, file_path): """ if not os.path.exists(file_path): return {} + max_log_file_size = int(self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_max_log_size', DEFAULT_MAX_FILE_SIZE)) + if os.path.getsize(file_path) > max_log_file_size: + _logger.warning("Log file '%s' exceeds %s limit", file_path, max_log_file_size) + return {} stats_matches = {} with file_open(file_path, "r") as log_file: data = log_file.read() diff --git a/runbot/models/bundle.py b/runbot/models/bundle.py index 1c54d9d53..3e8073e89 100644 --- a/runbot/models/bundle.py +++ b/runbot/models/bundle.py @@ -57,6 +57,8 @@ class Bundle(models.Model): tag_ids = fields.Many2many('runbot.bundle.tag', string='Tags') team_id = fields.Many2one('runbot.team', compute='_compute_team_id', store=True, readonly=False) + priority_offset = fields.Integer("Priority offset", help="Offset in seconds to remove from the create date of a batch to define priority, positive value means higher priority, negative value means lower priority.") + def _compute_frontend_url(self): for bundle in self: bundle.frontend_url = f'/runbot/bundle/{bundle.id}' @@ -275,7 +277,7 @@ def _consistency_warning(self): warnings.append(('info', 'PR %s targeting a non base branch: %s' % (branch.dname, branch.target_branch_name))) else: warnings.append(('warning' if branch.alive else 'info', 'PR %s targeting wrong version: %s (expecting %s)' % (branch.dname, branch.target_branch_name, self.base_id.name))) - elif not branch.is_pr and not branch.name.startswith(self.base_id.name) and not self.defined_base_id: + elif not branch.is_pr and not branch.name.startswith(self.base_id.name) and not self.defined_base_id and branch.remote_id.repo_id.enforce_version: warnings.append(('warning', 'Branch %s not starting with version name (%s)' % (branch.dname, self.base_id.name))) return warnings @@ -320,7 +322,12 @@ def action_generate_custom_trigger_restore_action(self): return self._generate_custom_trigger_action(context) def action_disable_all_triggers(self): - triggers_to_disable = ( + self._configure_custom_trigger_start_mode('disable') + + def _configure_custom_trigger_start_mode(self, mode): + self.ensure_one() + + triggers_to_create = ( self.env["runbot.trigger"] .search([ ("id", "not in", self.trigger_custom_ids.trigger_id.ids), @@ -333,13 +340,26 @@ def action_disable_all_triggers(self): ) ) vals = [] - for trigger in triggers_to_disable: - vals.append({ - 'bundle_id': self.id, - 'trigger_id': trigger.id, - 'start_mode': 'disabled', - }) + bundle_repos = self.branch_ids.remote_id.repo_id + for trigger in triggers_to_create: + if trigger.repo_ids & bundle_repos or trigger.dependency_ids & bundle_repos: + vals.append({ + 'bundle_id': self.id, + 'trigger_id': trigger.id, + }) self.env['runbot.bundle.trigger.custom'].create(vals) + for custom_trigger in self.trigger_custom_ids: + trigger_mode = mode + if mode == 'light' and not custom_trigger.trigger_id.light_config_id: + trigger_mode = 'auto' + custom_trigger.start_mode = trigger_mode + + def _force_ci(self): + for bundle in self: + bundle._configure_custom_trigger_start_mode('force') + # we need to create a new batch in case some of the triggers were in minimal mode + batch = bundle._force() or bundle.last_batch + batch._log("Batch was requested for ci") class BundleTag(models.Model): diff --git a/runbot/models/commit.py b/runbot/models/commit.py index 255bb44f3..b0aba479a 100644 --- a/runbot/models/commit.py +++ b/runbot/models/commit.py @@ -3,7 +3,6 @@ import subprocess from ..common import os, RunbotException, make_github_session, transactioncache -import glob import shutil from odoo import models, fields, api @@ -66,22 +65,14 @@ def _rebase_on(self, commit): return self return self._get(self.name, self.repo_id.id, self.read()[0], commit.id) - def _get_available_modules(self): - for manifest_file_name in self.repo_id.manifest_files.split(','): # '__manifest__.py' '__openerp__.py' - for addons_path in (self.repo_id.addons_paths or '').split(','): # '' 'addons' 'odoo/addons' - sep = os.path.join(addons_path, '*') - for manifest_path in glob.glob(self._source_path(sep, manifest_file_name)): - module = os.path.basename(os.path.dirname(manifest_path)) - yield (addons_path, module, manifest_file_name) - def _list_files(self, patterns): #example: git ls-files --with-tree=abcf390f90dbdd39fd61abc53f8516e7278e0931 ':(glob)addons/*/*.py' ':(glob)odoo/addons/*/*.py' # note that glob is needed to avoid the star matching ** self.ensure_one() - return self.repo_id._git(['ls-files', '--with-tree', self.name, *patterns]).split('\n') + self._fetch() + return self.repo_id._git(['ls-files', '--with-tree', self.tree_hash, *patterns]).split('\n') def _list_available_modules(self): - # beta version, may replace _get_available_modules latter addons_paths = (self.repo_id.addons_paths or '').split(',') patterns = [] for manifest_file_name in self.repo_id.manifest_files.split(','): # '__manifest__.py' '__openerp__.py' @@ -98,18 +89,24 @@ def _list_available_modules(self): module, manifest_file_name = elems yield (addons_path, module, manifest_file_name) + @transactioncache # hack to avoid to fetch two time the same commit inside the same transaction + def _fetch(self): + try: + self.repo_id._fetch(self.name) + except RunbotException: + self.repo_id._fetch(self.tree_hash) def _export(self, build): """Export a git repo into a sources""" # TODO add automated tests self.ensure_one() - self.repo_id._fetch(self.tree_hash) + self._fetch() if not self.env['runbot.commit.export'].search([('build_id', '=', build.id), ('commit_id', '=', self.id)]): self.env['runbot.commit.export'].create({'commit_id': self.id, 'build_id': build.id}) export_path = self._source_path() if os.path.isdir(export_path): - _logger.info('git export: exporting to %s (already exists)', export_path) + _logger.debug('git export: exporting to %s (already exists)', export_path) return export_path _logger.info('git export: exporting to %s (new)', export_path) @@ -166,12 +163,43 @@ def _read_source(self, file, mode='r'): @transactioncache def _git_show_file(self, file): + return self._git_show_files([file])[0] + + def _git_show_files(self, files): self.ensure_one() + if not files: + return [] + self.repo_id._fetch(self.name) + + queries = "\n".join([f"{self.name}:{f}" for f in files]) + "\n" + try: - return self.repo_id._git(['show', '%s:%s' % (self.name, file)]) + buffer = self.repo_id._git( + ['cat-file', '--batch'], + input_data=queries, + raw=True, + ) except subprocess.CalledProcessError: - return False + return [False] * len(files) + + results = [] + offset = 0 + buffer_len = len(buffer) + while offset < buffer_len: + newline_idx = buffer.find(b'\n', offset) + if newline_idx == -1: + break + header = buffer[offset:newline_idx].decode('utf-8') + offset = newline_idx + 1 + try: + size_in_bytes = int(header.rsplit(' ', 1)[-1]) + except ValueError: # most likely missing + results.append(False) + continue + results.append(buffer[offset : offset + size_in_bytes].decode('utf-8', errors='replace')) + offset += size_in_bytes + 1 + return results def _source_path(self, *paths): if not self.tree_hash: diff --git a/runbot/models/custom_trigger.py b/runbot/models/custom_trigger.py index 53cc74281..a31d7fa55 100644 --- a/runbot/models/custom_trigger.py +++ b/runbot/models/custom_trigger.py @@ -8,7 +8,7 @@ class BundleTriggerCustomization(models.Model): _description = 'Custom trigger' trigger_id = fields.Many2one('runbot.trigger') - start_mode = fields.Selection([('disabled', 'Disabled'), ('auto', 'Auto'), ('force', 'Force')], required=True, default='auto') + start_mode = fields.Selection([('disabled', 'Disabled'), ('auto', 'Auto'), ('light', 'Light'), ('force', 'Force')], required=True, default='auto') use_base_commits = fields.Boolean("Use base commits", help="Allow to test a trigger without the branch changes", default=False) bundle_id = fields.Many2one('runbot.bundle') config_id = fields.Many2one('runbot.build.config') diff --git a/runbot/models/docker.py b/runbot/models/docker.py index 547a3c6e0..d7115e3d5 100644 --- a/runbot/models/docker.py +++ b/runbot/models/docker.py @@ -145,6 +145,7 @@ class Dockerfile(models.Model): dockerfile = fields.Text(compute='_compute_dockerfile', recursive=True, tracking=True) in_error = fields.Boolean('In error', help='The last build failed.', default=False) to_build = fields.Boolean('To Build', help='Build Dockerfile. Check this when the Dockerfile is ready.', default=True) + nocache = fields.Boolean('No Cache', help='Force a full rebuild on next build, bypassing the Docker layer cache. Automatically reset to False after the build.', copy=False) always_pull = fields.Boolean('Always pull', help='Always Pull on the hosts, not only at the use time', default=False, tracking=True, copy=False) version_ids = fields.One2many('runbot.version', 'dockerfile_id', string='Versions') description = fields.Text('Description') @@ -351,6 +352,7 @@ def _get_cached_content(self, docker_build_path): destination = add_match.group('destination') # Use the destination name as hardlink name to avoid rebuild if file content is the same but not the url hardlink_name = re.sub(r'[^a-zA-Z0-9]', '_', destination) + lines[i] = f'# CACHED {lines[i + 1]}' lines[i + 1] = f'COPY {hardlink_name} {destination}' cache_file_path = cache_dir / filename if not cache_file_path.exists() or time.time() - cache_file_path.lstat().st_mtime > cache_duration: @@ -383,7 +385,7 @@ def _build(self, host=None): content = self._get_cached_content(docker_build_path) with open(self.env['runbot.runbot']._path('docker', tag_dir, 'Dockerfile'), 'w', encoding="utf-8") as Dockerfile: Dockerfile.write(content) - result = docker_build(docker_build_path, self.image_future_tag, self.pull_on_build) + result = docker_build(docker_build_path, self.image_future_tag, self.pull_on_build, self.nocache) duration = result['duration'] msg = result['msg'] success = image_id = result.get('image_id') @@ -427,6 +429,9 @@ def clean_output(output): message = f'Build failure, check results for more info ({result.summary})' self.message_post(body=message) _logger.error(message) + + if self.nocache: + self.nocache = False return image_id diff --git a/runbot/models/host.py b/runbot/models/host.py index 7dea8fba1..6b9055ef5 100644 --- a/runbot/models/host.py +++ b/runbot/models/host.py @@ -341,7 +341,7 @@ def _get_builds(self, domain, order=None): return self.env['runbot.build'].search(self._get_build_domain(domain), order=order) def _process_messages(self): - self.host_message_ids._process() + return self.host_message_ids._process() class MessageQueue(models.Model): @@ -351,14 +351,25 @@ class MessageQueue(models.Model): _log_access = False create_date = fields.Datetime('Create date', default=fields.Datetime.now) - host_id = fields.Many2one('runbot.host', required=True, ondelete='cascade') - build_id = fields.Many2one('runbot.build') + host_id = fields.Many2one('runbot.host', required=True, ondelete='cascade', index=True) + build_id = fields.Many2one('runbot.build', index=True) message = fields.Char('Message') def _process(self): records = self + processed = False # todo consume messages here if records: + processed = True for record in records: - self.env['runbot.runbot']._warning(f'Host {record.host_id.name} got an unexpected message {record.message}') + if record.message == 'kill': + if record.build_id: + build = record.build_id + result = None + if build.local_state != 'running' and build.global_result not in ('warn', 'ko'): + result = 'killed' + build._kill(result=result) + else: + self.env['runbot.runbot']._warning(f'Host {record.host_id.name} got an unexpected message {record.message}') self.unlink() + return processed diff --git a/runbot/models/ir_qweb.py b/runbot/models/ir_qweb.py index e9fa061ce..c8ea0eb37 100644 --- a/runbot/models/ir_qweb.py +++ b/runbot/models/ir_qweb.py @@ -1,5 +1,5 @@ -from ..common import s2human, s2human_long, precise_s2human -from odoo import models +from ..common import s2human, s2human_long, precise_s2human, transactioncache +from odoo import models, tools from odoo.http import request from odoo.addons.website.controllers.main import QueryURL @@ -12,3 +12,10 @@ def _prepare_frontend_environment(self, values): values['s2human_long'] = s2human_long values['precise_s2human'] = precise_s2human return response + + @tools.conditional( + 'xml' in tools.config['dev_mode'], + transactioncache, + ) # replace ormcache by transaction cache to avoid reading the same template multiple times in the same requests. Context is ignored but should be the same for each call in the same request + def _generate_code_cached(self, ref: int): + return super()._generate_code_cached(ref) diff --git a/runbot/models/project.py b/runbot/models/project.py index f60844bfb..7032754c4 100644 --- a/runbot/models/project.py +++ b/runbot/models/project.py @@ -25,6 +25,9 @@ class Project(models.Model): active = fields.Boolean("Active", default=True) process_delay = fields.Integer('Process delay', default=60, required=True, help="Delay between a push and a batch starting its process.") next_freeze_tag_id = fields.Many2one('runbot.bundle.tag', string="Next freeze tag") + use_light_default = fields.Boolean('Use light config by default', help="Use the light config when possible for all triggers") + use_light_draft = fields.Boolean('Use light config for draft PRs', help="Use the light config when possible for bundle having draft pr") + use_light_no_pr = fields.Boolean('Use light config when no PR', help="Use the light config when possible for all bundles not having any pr") @api.constrains('process_delay') def _constraint_process_delay(self): diff --git a/runbot/models/repo.py b/runbot/models/repo.py index b3ac2872d..32c12acb6 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -6,6 +6,7 @@ import re import subprocess import time +import psutil import requests import markupsafe import shlex @@ -55,6 +56,7 @@ class Trigger(models.Model): project_id = fields.Many2one('runbot.project', string="Project id", required=True) repo_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_triggers', string="Triggers", domain="[('project_id', '=', project_id)]") dependency_ids = fields.Many2many('runbot.repo', relation='runbot_trigger_dependencies', string="Dependencies") + use_extra_slot = fields.Boolean('Use extra slot', help="If checked, builds from this trigger can use an extra slot on the builders (for light and fast triggers)") starts_before_ids = fields.Many2many( 'runbot.trigger', @@ -72,6 +74,7 @@ class Trigger(models.Model): ) module_filters = fields.One2many('runbot.module.filter', 'trigger_id', string="Module filters", help='Will be combined with repo module filters when used with this trigger') config_id = fields.Many2one('runbot.build.config', string="Config", required=True) + light_config_id = fields.Many2one('runbot.build.config', string="Light config", help="Alternative config to use when light mode is enabled") config_data = JsonDictField('Config Data') network_enabled = fields.Boolean('Network Enabled') batch_dependent = fields.Boolean('Batch Dependent', help="Force adding batch in build parameters to make it unique and give access to bundle") @@ -262,6 +265,11 @@ def _compute_remote_name(self): for remote in self: remote.remote_name = sanitize(remote.short_name) + def _get_fetch_url(self): + if not self.name.startswith('https://') or not self.token: + return self.name + return self.name.replace('https://', 'https://%s@' % self.token, 1) + def create(self, values_list): remote = super().create(values_list) if not remote.repo_id.main_remote_id: @@ -428,8 +436,10 @@ class Repo(models.Model): get_ref_time = fields.Float('Last refs db update', compute='_compute_get_ref_time') trigger_ids = fields.Many2many('runbot.trigger', relation='runbot_trigger_triggers', readonly=True) single_version = fields.Many2one('runbot.version', "Single version", help="Limit the repo to a single version for non versionned repo") + enforce_version = fields.Boolean('Force version', help="Force all bundle containing branch from this repo to be prefixed with the correct version", default=True) forbidden_regex = fields.Char('Forbidden regex', help="Regex that forid bundle creation if branch name is matching", tracking=True) invalid_branch_message = fields.Char('Forbidden branch message', tracking=True) + allow_slashes = fields.Boolean('Allow slashes in branch names', help="Allow branches with slashes in their name (e.g. odoo/tests/my_branch). If unchecked, only one level of branches is allowed (e.g. odoo/my_branch)", default=True) def _compute_get_ref_time(self): self.env.cr.execute(""" @@ -499,25 +509,56 @@ def _get_git_command(self, cmd, errors='strict'): cmd = ['git', '-C', self.path] + config_args + cmd return cmd - def _git(self, cmd, errors='strict', quiet=False): + def _git(self, cmd, errors='strict', quiet=False, input_data=None, raw=False): cmd = self._get_git_command(cmd, errors) - if not quiet: - _logger.info("git command: %s", shlex.join(cmd)) - return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode(errors=errors) + if input_data is not None and isinstance(input_data, str): + input_data = input_data.encode('utf-8') + + fetch_timeout = int(self.env['ir.config_parameter'].get_param('runbot.runbot_fetch_timeout', default=30)) + for i in range(3): # retry in case of timeout + if not quiet: + _logger.info("git command: %s", shlex.join(cmd)) + + killed_fetch = False + stdin = subprocess.PIPE if input_data is not None else None + process = subprocess.Popen(cmd, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + while True: + try: + output, _ = process.communicate(input=input_data, timeout=fetch_timeout) + break + except subprocess.TimeoutExpired: + try: + parent = psutil.Process(process.pid) + for child in parent.children(recursive=True): + if 'git-upload-pack' in str(child.cmdline()) and child.status() == psutil.STATUS_SLEEPING: + child.kill() + _logger.info("Killed sleeping git subprocess (pid: %s)", child.pid) + killed_fetch = True + fetch_timeout *= 10 # increase the timeout for the next try + except psutil.NoSuchProcess: + pass + + if not killed_fetch: + break + if process.returncode: + raise subprocess.CalledProcessError(process.returncode, cmd, output=output) + + if raw: + return output + return output.decode(errors=errors) def _fetch(self, sha): + self._git_init() if not self._hash_exists(sha): - self._update(force=True) + for remote in self.remote_ids: + try: + self._git(['fetch', remote.remote_name, sha]) + break + except subprocess.CalledProcessError: + pass if not self._hash_exists(sha): - for remote in self.remote_ids: - try: - self._git(['fetch', remote.remote_name, sha]) - _logger.info('Success fetching specific head %s on %s', sha, remote) - break - except subprocess.CalledProcessError: - pass - if not self._hash_exists(sha): - raise RunbotException("Commit %s is unreachable. Did you force push the branch?" % sha) + raise RunbotException("Commit %s is unreachable, most likely because it is not attached to any branch anymore" % sha) def _hash_exists(self, commit_hash): """ Verify that a commit hash exists in the repo """ @@ -554,7 +595,10 @@ def _get_refs(self, max_age=30, ignore=None): self._set_ref_time(get_ref_time) fields = ['refname', 'objectname', 'committerdate:unix', 'authorname', 'authoremail', 'subject', 'committername', 'committeremail', 'tree'] fmt = "%00".join(["%(" + field + ")" for field in fields]) - cmd = ['for-each-ref', '--format', fmt, '--sort=-committerdate', 'refs/*/heads/*'] + refs_desc = 'refs/*/heads/*' + if self.allow_slashes: + refs_desc = 'refs/*/heads/**' + cmd = ['for-each-ref', '--format', fmt, '--sort=-committerdate', refs_desc] if any(remote.fetch_pull for remote in self.remote_ids): cmd.append('refs/*/pull/*') git_refs = self._git(cmd) @@ -581,7 +625,7 @@ def _find_or_create_branches(self, refs): """ # FIXME WIP - names = [r[0].split('/')[-1] for r in refs] + names = [r[0].split('/', 3)[-1] for r in refs] branches = self.env['runbot.branch'].search([('name', 'in', names), ('remote_id', 'in', self.remote_ids.ids)]) ref_branches = {branch._ref(): branch for branch in branches} new_branch_values = [] @@ -590,7 +634,7 @@ def _find_or_create_branches(self, refs): # format example: # refs/ruodoo-dev/heads/12.0-must-fail # refs/ruodoo/pull/1 - _, remote_name, branch_type, name = ref_name.split('/') + _, remote_name, branch_type, name = ref_name.split('/', 3) remote_id = self.remote_ids.filtered(lambda r: r.remote_name == remote_name).id if not remote_id: _logger.warning('Remote %s not found', remote_name) @@ -670,6 +714,9 @@ def _update_batches(self, force=False, ignore=None): def _update_git_config(self): """ Update repo git config file """ + if not self: + return + _logger.info('Updating git config for %s repos', len(self)) for repo in self: if repo.mode == 'disabled': _logger.info(f'skipping disabled repo {repo.name}') @@ -683,6 +730,7 @@ def _update_git_config(self): _logger.info('Config updated for repo %s' % repo.name) else: _logger.info('Repo not cloned, skiping config update for %s' % repo.name) + return max(self.mapped('write_date')) def _git_init(self): """ Clone the remote repo if needed """ diff --git a/runbot/models/runbot.py b/runbot/models/runbot.py index f750ef8fc..195efe4a7 100644 --- a/runbot/models/runbot.py +++ b/runbot/models/runbot.py @@ -53,10 +53,10 @@ def _scheduler(self, host): processed += 1 build._process_requested_actions() self._commit() + if host._process_messages(): + self._commit() host._process_logs() self._commit() - host._process_messages() - self._commit() for build in host._get_builds([('local_state', 'in', ['testing', 'running'])]) | self._get_builds_to_init(host): build = build.browse(build.id) # remove preftech ids, manage build one by one result = build._schedule() @@ -104,7 +104,7 @@ def _get_builds_to_init(self, host): def _gc_running(self, host): running_max = host._get_running_max() Build = self.env['runbot.build'] - cannot_be_killed_ids = host._get_builds([('keep_running', '=', True)]).ids + cannot_be_killed_ids = host._get_builds([('gc_running_date', '>', fields.Date.today())]).ids sticky_bundles = self.env['runbot.bundle'].search([('sticky', '=', True), ('project_id.keep_sticky_running', '=', True)]) cannot_be_killed_ids += [ build.id @@ -113,14 +113,14 @@ def _gc_running(self, host): ][:running_max] build_ids = host._get_builds([('local_state', '=', 'running'), ('id', 'not in', cannot_be_killed_ids)], order='job_start desc').ids for build in Build.browse(build_ids)[running_max:]: - build._kill() + build._kill(None) def _gc_testing(self, host): """garbage collect builds that could be killed""" # decide if we need room Build = self.env['runbot.build'] domain_host = host._get_build_domain() - testing_builds = Build.search(domain_host + [('local_state', 'in', ['testing', 'pending']), ('requested_action', '!=', 'deathrow')]) + testing_builds = Build.search(domain_host + [('local_state', 'in', ['testing', 'pending']), ('message_ids', '=', False)]) used_slots = len(testing_builds) available_slots = host.nb_worker - used_slots nb_pending = Build.search_count([('local_state', '=', 'pending'), ('host', '=', False)]) @@ -138,7 +138,7 @@ def _allocate_builds(self, host, nb_slots, domain=None): if domain: non_allocated_domain = Domain.AND([non_allocated_domain, domain]) query = self.env['runbot.build']._search(non_allocated_domain) - query.order = 'runbot_build.create_batch_id' + query.order = 'runbot_build.priority_level' self.env.execute_query(SQL("""UPDATE runbot_build SET diff --git a/runbot/security/ir.model.access.csv b/runbot/security/ir.model.access.csv index 6c233c654..f0ca84182 100644 --- a/runbot/security/ir.model.access.csv +++ b/runbot/security/ir.model.access.csv @@ -68,6 +68,9 @@ access_runbot_error_regex_manager,runbot_error_regex_manager,runbot.model_runbot access_runbot_host_public,runbot_host_public,runbot.model_runbot_host,runbot.base_runbot_model_access,1,0,0,0 access_runbot_host_manager,runbot_host_manager,runbot.model_runbot_host,runbot.group_runbot_admin,1,1,1,1 +access_runbot_host_message_public,runbot_host_message_public,runbot.model_runbot_host_message,runbot.base_runbot_model_access,1,0,0,0 +access_runbot_host_message_admin,runbot_host_message_admin,runbot.model_runbot_host_message,runbot.group_runbot_admin,1,1,1,1 + access_runbot_repo_hooktime,runbot_repo_hooktime,runbot.model_runbot_repo_hooktime,group_user,1,0,0,0 access_runbot_repo_referencetime,runbot_repo_referencetime,runbot.model_runbot_repo_reftime,group_user,1,0,0,0 access_runbot_build_stat_admin,runbot_build_stat_admin,runbot.model_runbot_build_stat,runbot.group_runbot_admin,1,1,1,1 @@ -106,7 +109,6 @@ access_runbot_bundle_public,access_runbot_bundle_public,runbot.model_runbot_bund access_runbot_bundle_runbot_bundle_manager,access_runbot_bundle_runbot_manager,runbot.model_runbot_bundle,runbot.group_runbot_bundle_manager,1,1,0,0 access_runbot_bundle_runbot_admin,access_runbot_bundle_runbot_admin,runbot.model_runbot_bundle,runbot.group_runbot_admin,1,1,1,1 - access_runbot_batch_public,access_runbot_batch_public,runbot.model_runbot_batch,runbot.base_runbot_model_access,1,0,0,0 access_runbot_batch_runbot_admin,access_runbot_batch_runbot_admin,runbot.model_runbot_batch,runbot.group_runbot_admin,1,1,1,1 diff --git a/runbot/static/src/css/runbot.css b/runbot/static/src/css/runbot.css index 46415f421..39df923c8 100644 --- a/runbot/static/src/css/runbot.css +++ b/runbot/static/src/css/runbot.css @@ -1,5 +1,9 @@ :root { --gray: #6c757d; /* used for batch limitation */ + --btn-default-color: var(--bs-body-color); + --btn-default-bg: var(--bs-body-bg); + --btn-default-border: #ccc; + --active-project-color: #777; } /* @@ -13,7 +17,8 @@ --bs-info-bg-subtle: #d9edf7; --bs-info-rgb: 23, 162, 184; } -:root[data-bs-theme=red404] { + +:root[data-bs-theme=red404] { --bs-success-bg-subtle: #cdffb9; --bs-danger-bg-subtle: #e67ecf; --bs-warning-bg-subtle: #fae9b1; @@ -21,6 +26,11 @@ --bs-info-rgb: 23, 162, 184; } +:root[data-bs-theme=dark] { + --btn-default-border: #333; + --active-project-color: #CCC; +} + [data-bs-theme=legacy] .text-bg-info { color: #fff !important; /* black by default, changes from previous version, color forced to fit with --bs-info-rgb*/ } @@ -59,37 +69,28 @@ --bs-btn-disabled-border-color: #b90e6c; } - -:root { - --alternative:#ccc; - --btn-default-color: var(--bs-body-color); - --btn-default-border:#ccc; - --bs-default-rgb: var(--bs-body-color-rgb); - --active-project-color: #777; - -} - -:root[data-bs-theme=dark] { - --btn-default-border:#333; - --btn-default-color: var(--bs-body-color); - --active-project-color: #CCC; -} - .btn-default { --bs-btn-color: var(--btn-default-color); - --bs-btn-bg: var(--bs-body-bg); + --bs-btn-bg: var(--btn-default-bg); --bs-btn-border-color: var(--btn-default-border); --bs-btn-hover-color: var(--btn-default-color); - --bs-btn-hover-bg: var(--btn-default-border); - --bs-btn-hover-border-color: var(--btn-default-border); + --bs-btn-hover-bg: color-mix(in lab, var(--btn-default-bg), black 15%); + --bs-btn-hover-border-color: color-mix(in lab, var(--btn-default-border), black 10%); --bs-btn-focus-shadow-rgb: 60, 153, 110; --bs-btn-active-color: var(--btn-default-color); - --bs-btn-active-bg: var(--bs-body-bg); - --bs-btn-active-border-color: var(--bs-body-bg); + --bs-btn-active-bg: color-mix(in lab, var(--btn-default-bg), black 20%); + --bs-btn-active-border-color: color-mix(in lab, var(--btn-default-border), black 15%); --bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125); --bs-btn-disabled-color: var(--btn-default-color); - --bs-btn-disabled-bg: var(--bs-body-bg); - --bs-btn-disabled-border-color: var(--btn-default-border);; + --bs-btn-disabled-bg: var(--btn-default-bg); + --bs-btn-disabled-border-color: var(--btn-default-border); +} + +[data-bs-theme=dark] .btn-default { + --bs-btn-hover-bg: color-mix(in lab, var(--btn-default-bg), white 15%); + --bs-btn-hover-border-color: color-mix(in lab, var(--btn-default-border), white 10%); + --bs-btn-active-bg: color-mix(in lab, var(--btn-default-bg), white 20%); + --bs-btn-active-border-color: color-mix(in lab, var(--btn-default-border), white 15%); } .btn-info { @@ -152,6 +153,22 @@ table { font-size: 0.875rem; } +dialog.modal { + --bs-modal-zindex: auto; + margin: 0; + padding: 0; + border: none; + background-color: transparent; + + &::backdrop { + background-color: rgba(0, 0, 0, 0.5); + } + + &[open] { + display: block; + } +} + .fa { line-height: inherit; /* reset fa icon line height to body height*/ } @@ -184,7 +201,7 @@ a.slots_infos:hover { } .separator { - border-top: 2px solid #666; + border-top: 0.2em solid #666; } body, .table { @@ -426,3 +443,31 @@ code { .hide-success tr.bg-success-subtle { display: none; } + +.pre { + display: block; + font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; + white-space: pre; + overflow: auto; + font-size: 0.875em; + margin:0; + padding:0; + margin-top: 0.2em; + border: none; + +} + +.subtle_link { + color: var(--bs-body-color); + text-decoration: underline; +} + +.table-condensed .log-server td, .table-condensed .log-details td { + padding-top: 0; + padding-bottom: 0; + border: none; +} + +.log-details td { + padding-left: 20px; +} diff --git a/runbot/static/src/js/fields/fields.js b/runbot/static/src/js/fields/fields.js index 2568a9e2b..9d36fb7ff 100644 --- a/runbot/static/src/js/fields/fields.js +++ b/runbot/static/src/js/fields/fields.js @@ -1,44 +1,41 @@ -/** @odoo-module **/ - import { TextField } from "@web/views/fields/text/text_field"; import { CharField } from "@web/views/fields/char/char_field"; import { Many2OneField } from "@web/views/fields/many2one/many2one_field"; -import { _lt } from "@web/core/l10n/translation"; import { formatDateTime } from "@web/core/l10n/dates"; import { registry } from "@web/core/registry"; import { useInputField } from "@web/views/fields/input_field_hook"; -import { useRef, xml, Component, markup} from "@odoo/owl"; +import { useRef, xml, Component, markup } from "@odoo/owl"; import { useAutoresize } from "@web/core/utils/autoresize"; import { getFormattedValue } from "@web/views/utils"; import { UrlField } from "@web/views/fields/url/url_field"; -import { X2ManyField , x2ManyField} from "@web/views/fields/x2many/x2many_field"; +import { X2ManyField , x2ManyField } from "@web/views/fields/x2many/x2many_field"; import { BooleanToggleField } from "@web/views/fields/boolean_toggle/boolean_toggle_field"; - // https://stackoverflow.com/questions/4810841/pretty-print-json-using-javascript function colorizeJson(json) { - json = json.replace(/&/g, '&').replace(//g, '>'); - return json.replace(/("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?)/g, function (match) { - var cls = ''; + json = json.replace(/&/g, "&").replace(//g, ">"); + return json.replace(/("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+-]?\d+)?)/g, function (match) { + let cls = ""; if (/^"/.test(match)) { if (/:$/.test(match)) { - cls = 'o_runbot_json_key'; + cls = "o_runbot_json_key"; } else { - cls = 'o_runbot_json_value'; + cls = "o_runbot_json_value"; } } - return '' + match + ''; + return `${match}`; }); } function stringify(obj) { - return JSON.stringify(obj, null, '\t'); - } + return JSON.stringify(obj, null, "\t"); +} + export class JsonField extends TextField { static template = xml` - + @@ -54,6 +51,7 @@ export class JsonField extends TextField { `; + setup() { this.divRef = useRef("div"); this.textareaRef = useRef("textarea"); @@ -86,9 +84,7 @@ export class FrontendUrl extends Component { static template = xml`
`; - static components = { Many2OneField }; - static props = { ...Many2OneField.props, linkField: { type: String, optional: true }, @@ -96,21 +92,21 @@ export class FrontendUrl extends Component { get displayValue() { if (this.props.record.data[this.props.name].isLuxonDateTime){ - return formatDateTime(this.props.record.data[this.props.name]) + return formatDateTime(this.props.record.data[this.props.name]); } else { - return this.props.record.data[this.props.name] ? getFormattedValue(this.props.record, this.props.name) : '' + return this.props.record.data[this.props.name] ? getFormattedValue(this.props.record, this.props.name) : ""; } } get route() { - return this._route(this.props.linkField || this.props.name) + return this._route(this.props.linkField || this.props.name); } _route(fieldName) { const model = this.props.record.fields[fieldName].relation || "runbot.unknown"; const { id } = this.props.record.data[fieldName]; - if (model.startsWith('runbot.')){ - return '/runbot/' + model.split('.')[1] + '/' + id; + if (model.startsWith("runbot.")){ + return `/runbot/${model.split(".")[1]}/${id}`; } else { return false; } @@ -127,22 +123,20 @@ registry.category("fields").add("frontend_url", { }, }); - export class FieldCharFrontendUrl extends Component { - static template = xml` -
-
-
-
`; - - static components = { CharField } +
+
+
+
+ `; + static components = { CharField }; get route() { const model = this.props.record.resModel; const id = this.props.record.resId; - if (model.startsWith('runbot.')) { - return '/runbot/' + model.split('.')[1] + '/' + id; + if (model.startsWith("runbot.")) { + return `/runbot/${model.split(".")[1]}/${id}`; } else { return false; } @@ -154,36 +148,33 @@ registry.category("fields").add("char_frontend_url", { component: FieldCharFrontendUrl, }); - // Pull Request URL Widget const pullRequestRegex = /\/([a-zA-Z-_]+\/[a-zA-Z-_]+)\/pull\/(\d+)/; class PullRequestUrlField extends UrlField { static template = xml` `; - static components = { UrlField } + static components = { UrlField }; + get fieldProps() { - const props = {...this.props }; - const parts = pullRequestRegex.exec(this.props.record.data[props.name]) + const props = { ...this.props }; + const parts = pullRequestRegex.exec(this.props.record.data[props.name]); if (parts) { props.text = `${parts[1]}#${parts[2]}`; } - return props + return props; } } PullRequestUrlField.supportedTypes = ["char"]; - registry.category("fields").add("pull_request_url", { supportedTypes: ["char"], component: PullRequestUrlField, }); - export class Matrixx2ManyField extends X2ManyField { - static template = 'runbot.Matrixx2ManyField'; - + static template = "runbot.Matrixx2ManyField"; static components = { BooleanToggleField }; getEntry(from, to) { @@ -200,11 +191,11 @@ export class Matrixx2ManyField extends X2ManyField { return [...new Set(versions)].sort().reverse(); } } + export const matrixx2ManyField = { ...x2ManyField, component: Matrixx2ManyField, useSubView: false, }; - registry.category("fields").add("version_matrix", matrixx2ManyField); diff --git a/runbot/static/src/js/fields/history_graph.js b/runbot/static/src/js/fields/history_graph.js index e285d963c..6f6a3370c 100644 --- a/runbot/static/src/js/fields/history_graph.js +++ b/runbot/static/src/js/fields/history_graph.js @@ -1,5 +1,3 @@ -/** @odoo-module **/ -import { _lt } from "@web/core/l10n/translation"; import { registry } from "@web/core/registry"; import { useRef, xml, Component, useEffect } from "@odoo/owl"; @@ -9,21 +7,20 @@ export class HistoryGraph extends Component { `; + setup() { this.canvasRef = useRef("canvas"); useEffect(() => this.renderErrorGraph()); } renderErrorGraph(activeCell) { - const data = this.props.record.data[this.props.name] || {}; const errorId = data.error_id; const projectId = data.project_id; const categoryId = data.category_id; const breaking_pr_close_dates = data.breaking_pr_close_dates; const fixing_pr_close_dates = data.fixing_pr_close_dates; - - const canvas = this.canvasRef.el + const canvas = this.canvasRef.el; const ctx = canvas.getContext("2d"); const maxValue = data.max_count; const canvasBorder = 1; @@ -37,26 +34,25 @@ export class HistoryGraph extends Component { canvas.width = canvasWidth; canvas.height = canvasHeight; - function getColor(value, opacity) { if (value >= 10) { return `rgba(255, 0, 0, ${opacity})`; // red } else if (value >= 5) { return `rgba(255, 165, 0, ${opacity})`; // orange } - return `rgba(0, 170, 0, ${opacity})` // green + return `rgba(0, 170, 0, ${opacity})`; // green } ctx.clearRect(0, 0, canvasWidth, canvasHeight); ctx.fillStyle = "#EEE"; ctx.fillRect(0, 0, canvasWidth, canvasHeight); ctx.strokeStyle = "#333"; - ctx.lineWidth = canvasBorder * 2; // * 2 to account for each side, not only inner width - ctx.strokeRect(0, 0, canvasWidth, canvasHeight,); + ctx.lineWidth = canvasBorder * 2; // * 2 to account for each side, not only inner width + ctx.strokeRect(0, 0, canvasWidth, canvasHeight); data.date_labels.forEach((dateLabel, idx) => { data.version_labels.forEach((versionLabel, idy) => { - let version_id = data.versions_ids[idy] + const version_id = data.versions_ids[idy]; let value = data.daily_version_freq[idx][idy] || 0; let cellColor = "white"; let cellOpacity = 0; @@ -70,13 +66,13 @@ export class HistoryGraph extends Component { ctx.fillStyle = cellColor; ctx.fillRect(posX, posY, cellWidth, cellHeight); + if (activeCell && activeCell.col === idx && activeCell.row === idy) { ctx.strokeStyle = "black"; ctx.lineWidth = 2; ctx.strokeRect(posX, posY, cellWidth, cellHeight); } - if (fixing_pr_close_dates[version_id] == dateLabel) { ctx.fillStyle = "black"; ctx.font = "12px Arial"; @@ -87,13 +83,11 @@ export class HistoryGraph extends Component { ctx.font = "12px Arial"; ctx.fillText("✗", posX + cellWidth / 2 - 4, posY + cellHeight / 2 + 4); } - - }); }); if (mouseActions) { canvas.onmousemove = (event) => { - let tooltip = canvas.parentElement.querySelector('.history-graph-tooltip'); + let tooltip = canvas.parentElement.querySelector(".history-graph-tooltip"); if (tooltip) { tooltip.remove(); } @@ -101,16 +95,16 @@ export class HistoryGraph extends Component { const { col, row, value, dateLabel, versionLabel } = this.getCellFromEvent(event); if ( col >= 0 && row >= 0) { - tooltip = document.createElement('div'); - tooltip.className = 'history-graph-tooltip'; - tooltip.style.position = 'absolute'; + tooltip = document.createElement("div"); + tooltip.className = "history-graph-tooltip"; + tooltip.style.position = "absolute"; tooltip.style.left = `${canvas.offsetLeft}px`; tooltip.style.top = `${canvas.offsetTop + canvas.height}px`; - tooltip.style.background = '#fff'; - tooltip.style.border = '1px solid #333'; - tooltip.style.padding = '4px 8px'; - tooltip.style.fontSize = '12px'; - tooltip.style.pointerEvents = 'none'; + tooltip.style.background = "#fff"; + tooltip.style.border = "1px solid #333"; + tooltip.style.padding = "4px 8px"; + tooltip.style.fontSize = "12px"; + tooltip.style.pointerEvents = "none"; tooltip.style.zIndex = 1000; tooltip.innerHTML = ` Date: ${dateLabel} @@ -125,23 +119,23 @@ export class HistoryGraph extends Component { }; canvas.onmouseleave = () => { - const tooltip = canvas.parentElement.querySelector('.history-graph-tooltip'); + const tooltip = canvas.parentElement.querySelector(".history-graph-tooltip"); if (tooltip) { tooltip.remove(); - this.renderErrorGraph() + this.renderErrorGraph(); } }; canvas.onclick = (event) => { - const { col, row, value, dateLabel, versionLabel } = this.getCellFromEvent(event); + const { col, row, dateLabel } = this.getCellFromEvent(event); if (col >= 0 && row >= 0) { const url = `/runbot/batches/${projectId}/${categoryId}/${dateLabel}/${errorId}`; - window.open(url, '_blank'); + window.open(url, "_blank"); } - } + }; } - } + getCellFromEvent(event) { const data = this.props.record.data[this.props.name] || {}; const rect = this.canvasRef.el.getBoundingClientRect(); @@ -149,13 +143,13 @@ export class HistoryGraph extends Component { const y = event.clientY - rect.top - 1; // Adjust for canvas border const col = Math.floor(x / this.props.cellSize); const row = Math.floor(y / this.props.cellSize); - if ( col >= 0 && col < data.date_labels.length && row >= 0 && row < data.version_labels.length) { + if ( col >= 0 && col < data.date_labels.length && row >= 0 && row < data.version_labels.length) { const value = data.daily_version_freq[col][row] || 0; const dateLabel = data.date_labels[col]; const versionLabel = data.version_labels[row]; return { col, row, value, dateLabel, versionLabel }; } else { - return { col: -1, row: -1, value: 0, dateLabel: '', versionLabel: '' }; + return { col: -1, row: -1, value: 0, dateLabel: "", versionLabel: "" }; } } } diff --git a/runbot/static/src/js/fields/tracking_value.js b/runbot/static/src/js/fields/tracking_value.js index 14058b1cd..b08647d1b 100644 --- a/runbot/static/src/js/fields/tracking_value.js +++ b/runbot/static/src/js/fields/tracking_value.js @@ -1,4 +1,3 @@ -/** @odoo-module **/ import { patch } from "@web/core/utils/patch"; import { Message } from "@mail/core/common/message"; @@ -7,22 +6,27 @@ patch(Message.prototype, { super.setup(...arguments); this.kept = false; }, + isMultiline(trackingValue) { const oldValue = trackingValue.oldValue; const newValue = trackingValue.newValue; - return ((oldValue && typeof oldValue=== 'string' && oldValue.includes('\n')) && (newValue && typeof oldValue=== 'string' && newValue.includes('\n'))) + return ((oldValue && typeof oldValue=== "string" && oldValue.includes("\n")) && (newValue && typeof oldValue=== "string" && newValue.includes("\n"))); }, + formatTracking(trackingFieldInfo, trackingValue) { - return super.formatTracking(trackingFieldInfo, trackingValue) + return super.formatTracking(trackingFieldInfo, trackingValue); }, + toggleKept() { this.kept = !this.kept; }, + copyToClipboard(trackingValue) { return function () { navigator.clipboard.writeText(trackingValue); }; }, + lines(trackingValue) { const oldValue = trackingValue.oldValue; const newValue = trackingValue.newValue; @@ -30,45 +34,47 @@ patch(Message.prototype, { const lines = this.prepareForRendering(diff); return lines; }, + makeDiff(text1, text2) { - var dmp = new diff_match_patch(); - var a = dmp.diff_linesToChars_(text1, text2); - var lineText1 = a.chars1; - var lineText2 = a.chars2; - var lineArray = a.lineArray; - var diffs = dmp.diff_main(lineText1, lineText2, false); + const dmp = new diff_match_patch(); + const a = dmp.diff_linesToChars_(text1, text2); + const lineText1 = a.chars1; + const lineText2 = a.chars2; + const lineArray = a.lineArray; + const diffs = dmp.diff_main(lineText1, lineText2, false); dmp.diff_charsToLines_(diffs, lineArray); dmp.diff_cleanupSemantic(diffs); return diffs; }, + prepareForRendering(diffs) { - var lines = []; - var pre_line_counter = 0 - var post_line_counter = 0 - for (var x = 0; x < diffs.length; x++) { - var diff_type = diffs[x][0]; - var data = diffs[x][1]; - var data_lines = data.split('\n'); - for (var line_index in data_lines) { - var line = data_lines[line_index]; - line = line.replace(/&/g, '&'); - line = line.replace(//g, '>'); + const lines = []; + let pre_line_counter = 0; + let post_line_counter = 0; + for (let x = 0; x < diffs.length; x++) { + const diff_type = diffs[x][0]; + const data = diffs[x][1]; + const data_lines = data.split("\n"); + for (const line_index in data_lines) { + let line = data_lines[line_index]; + line = line.replace(/&/g, "&"); + line = line.replace(//g, ">"); //text = text.replace(/\n/g, '
'); //text = text.replace(/ /g, '  '); if (diff_type == -1) { - lines.push({type:'removed', pre_line_counter: pre_line_counter, post_line_counter: '-', line: line}) - pre_line_counter += 1 + lines.push({ type: "removed", pre_line_counter: pre_line_counter, post_line_counter: "-", line: line }); + pre_line_counter += 1; } else if (diff_type == 0) { - lines.push({type:'kept', pre_line_counter: '', post_line_counter: post_line_counter, line: line}) - pre_line_counter += 1 - post_line_counter +=1 + lines.push({ type: "kept", pre_line_counter: "", post_line_counter: post_line_counter, line: line }); + pre_line_counter += 1; + post_line_counter += 1; } else if (diff_type == 1) { - lines.push({type:'added', pre_line_counter: '+', post_line_counter: post_line_counter, line: line}) - post_line_counter +=1 + lines.push({ type: "added", pre_line_counter: "+", post_line_counter: post_line_counter, line: line }); + post_line_counter += 1; } } } return lines; - }, + }, }); diff --git a/runbot/static/src/js/polyfill_command_api.js b/runbot/static/src/js/polyfill_command_api.js new file mode 100644 index 000000000..701498808 --- /dev/null +++ b/runbot/static/src/js/polyfill_command_api.js @@ -0,0 +1,28 @@ +// @odoo-module ignore +(function () { + if ( + typeof HTMLButtonElement !== "undefined" && + "command" in HTMLButtonElement.prototype && + // eslint-disable-next-line no-undef + "source" in ((CommandEvent || {}).prototype || {}) + ) { + return; + } + const SUPPORTED_COMMANDS = { + "show-modal": "showModal", + "close": "close", + }; + document.addEventListener("click", (ev) => { + const commandEl = ev.target.closest("[commandfor]"); + if (!commandEl) { + return; + } + const forTarget = document.getElementById(commandEl.getAttribute("commandfor")); + const command = commandEl.getAttribute("command"); + if (command in SUPPORTED_COMMANDS) { + forTarget[SUPPORTED_COMMANDS[command]](); + } else { + throw new Error(`UnsupportedCommand: ${command} is not a supported command.`); + } + }); +})(); diff --git a/runbot/static/src/js/runbot.js b/runbot/static/src/js/runbot.js index 58a902976..76b2a129b 100644 --- a/runbot/static/src/js/runbot.js +++ b/runbot/static/src/js/runbot.js @@ -38,3 +38,19 @@ function copyToClipboard(text) { } navigator.clipboard.writeText(text); } + +document.addEventListener('DOMContentLoaded', function() { + const collapseElement = document.getElementById('customTriggers'); + if (collapseElement) { + collapseElement.addEventListener('show.bs.collapse', function () { + const url = new URL(window.location); + url.searchParams.set('expand_custom', '1'); + window.history.replaceState({}, '', url); + }); + collapseElement.addEventListener('hide.bs.collapse', function () { + const url = new URL(window.location); + url.searchParams.delete('expand_custom'); + window.history.replaceState({}, '', url); + }); + } +}); diff --git a/runbot/static/src/js/views/form_controller.js b/runbot/static/src/js/views/form_controller.js index f7a1ea72c..4fc4d83a2 100644 --- a/runbot/static/src/js/views/form_controller.js +++ b/runbot/static/src/js/views/form_controller.js @@ -1,19 +1,18 @@ -/** @odoo-module **/ - -import { FormController } from '@web/views/form/form_controller'; -import { patch } from '@web/core/utils/patch'; +import { FormController } from "@web/views/form/form_controller"; +import { patch } from "@web/core/utils/patch"; patch(FormController.prototype, { // Prevent saving on tab switching beforeVisibilityChange: () => {}, + // Prevent closing page with dirty fields async beforeUnload(ev) { if (await this.model.root.isDirty()) { ev.preventDefault(); - ev.returnValue = 'Unsaved changes'; + ev.returnValue = "Unsaved changes"; } else { super.beforeUnload(ev); } - } -}) + }, +}); diff --git a/runbot/templates/batch.xml b/runbot/templates/batch.xml index 54a25393e..0896e8a1b 100644 --- a/runbot/templates/batch.xml +++ b/runbot/templates/batch.xml @@ -13,11 +13,28 @@ &emsp; + t-attf-href="/odoo/batch/{{batch.id}}" class="btn btn-default btn-sm" target="_blank" title="View Batch in Backend"> + + Priority + + + High + + + + Low + + + Normal + + Set to high + + + Category diff --git a/runbot/templates/branch.xml b/runbot/templates/branch.xml index 78e0948e0..611f06e77 100644 --- a/runbot/templates/branch.xml +++ b/runbot/templates/branch.xml @@ -12,7 +12,7 @@
+ t-attf-href="/odoo/branch/{{branch.id}}" target="_blank" title="View Branch in Backend"/>
diff --git a/runbot/templates/build.xml b/runbot/templates/build.xml index d9428a0d3..9f57f309c 100644 --- a/runbot/templates/build.xml +++ b/runbot/templates/build.xml @@ -44,7 +44,7 @@
  • - +
    • @@ -68,7 +68,7 @@
    -
    +
    This build is referenced in bundles
    @@ -114,6 +114,9 @@
    + + Mode: Light Configure
    +     Version: @@ -222,7 +225,7 @@ - + Build @@ -234,7 +237,7 @@ with config - ... + ... @@ -265,25 +268,27 @@
    - + - - - + - + - - + @@ -291,54 +296,22 @@ - - + diff --git a/runbot/templates/build_error.xml b/runbot/templates/build_error.xml index e64fc4706..87ad07804 100644 --- a/runbot/templates/build_error.xml +++ b/runbot/templates/build_error.xml @@ -36,7 +36,7 @@
    + t-attf-href="/odoo/error/{{build_error.id}}" target="_blank" title="View in Backend"> @@ -113,7 +113,7 @@

    Team + t-attf-href="/odoo/team/{{team.id}}" target="_blank" title="View in Backend">

    diff --git a/runbot/templates/build_stats.xml b/runbot/templates/build_stats.xml index 8f36b6850..7721b8490 100644 --- a/runbot/templates/build_stats.xml +++ b/runbot/templates/build_stats.xml @@ -4,7 +4,7 @@ diff --git a/runbot/templates/utils.xml b/runbot/templates/utils.xml index 3a7d06da1..1b9554322 100644 --- a/runbot/templates/utils.xml +++ b/runbot/templates/utils.xml @@ -15,7 +15,7 @@ - + @@ -31,7 +31,7 @@ - + @@ -67,11 +67,11 @@
  • - -
  • - + +
  • +
  • @@ -105,7 +105,7 @@
    • -
    -
    - -
    -
    - - -
    -
    - -
    - - - - -
    -
    - -
    - - - - -
    -
    - -
    - -
    - -
    -
    - - - - -

    -
    - -
    - -
    +
    +
    + + + + +
    + +

    + @@ -273,6 +283,7 @@ +     Light @@ -329,7 +340,7 @@ default - + killed @@ -364,12 +375,12 @@ Database selector - + Rebuild - + Kill @@ -414,8 +425,7 @@ Find similar builds - + View in backend diff --git a/runbot/tests/common.py b/runbot/tests/common.py index 9f6ba86c8..81cc8e784 100644 --- a/runbot/tests/common.py +++ b/runbot/tests/common.py @@ -12,7 +12,7 @@ class RunbotCase(TransactionCase): - def mock_git_helper(self, repo, cmd): + def mock_git_helper(self, repo, cmd, input_data=None, raw=False): """Helper that returns a mock for repo._git()""" if cmd[:2] == ['show', '-s'] or cmd[:3] == ['show', '--pretty="%H -- %s"', '-s']: return 'commit message for %s' % cmd[-1] @@ -82,7 +82,9 @@ def setUp(self): self.repo_odoo: [ ('odoo/addons', 'base', '__manifest__.py'), ('odoo/addons', 'test_lint', '__manifest__.py'), + ('addons', 'account', '__manifest__.py'), ('addons', 'mail', '__manifest__.py'), + ('addons', 'test_mail', '__manifest__.py'), ('addons', 'web', '__manifest__.py'), ('addons', 'crm', '__manifest__.py'), ('addons', 'project', '__manifest__.py'), @@ -194,8 +196,8 @@ def setUp(self): self.docker_run_calls = [] self.diff = '' - def mock_git(repo, cmd, quiet=False): - return self.mock_git_helper(repo, cmd) + def mock_git(repo, cmd, quiet=False, input_data=None, raw=False): + return self.mock_git_helper(repo, cmd, input_data=input_data, raw=raw) self.start_patcher('git_patcher', 'odoo.addons.runbot.models.repo.Repo._git', new=mock_git) self.start_patcher('hostname_patcher', 'odoo.addons.runbot.common.socket.gethostname', 'host.runbot.com') @@ -232,10 +234,10 @@ def mock_git(repo, cmd, quiet=False): self.start_patcher('_write_file', 'odoo.addons.runbot.models.build.BuildResult._write_file', None) self.start_patcher('_parse_config', 'odoo.addons.runbot.models.build.BuildResult._parse_config', {'--test-enable', '--test-tags', '--with-demo'}) - def get_available_modules(self_commit): + def _list_available_modules(self_commit): return self.addons_per_repo.get(self_commit.repo_id, []) - self.start_patcher('_get_available_modules', 'odoo.addons.runbot.models.commit.Commit._get_available_modules', new=get_available_modules) + self.start_patcher('_list_available_modules', 'odoo.addons.runbot.models.commit.Commit._list_available_modules', new=_list_available_modules) def no_commit(*_args, **_kwargs): _logger.info('Skipping commit') diff --git a/runbot/tests/test_build.py b/runbot/tests/test_build.py index e55aab30a..03e0cd831 100644 --- a/runbot/tests/test_build.py +++ b/runbot/tests/test_build.py @@ -7,7 +7,6 @@ from odoo.tests import tagged from odoo.exceptions import UserError, ValidationError from .common import RunbotCase, RunbotCaseMinimalSetup -from unittest.mock import MagicMock def rev_parse(repo, branch_name): @@ -705,6 +704,32 @@ def test_build_cmd_faketime(self): second_child_cmd = second_child._cmd(py_version=3) self.assertIn('faketime "2024-02-04 04:42 UTC" python3 odoo/server.py', str(second_child_cmd)) + def test_format_message(self): + def get_log(message): + return self.env['ir.logging'].create({ + 'build_id': build.id, + 'type': 'server', + 'message': message, + 'level': 'INFO', + 'name': 'odoo.addons.web.tests.test_web', + 'path': '', + 'func': '', + 'line': '', + }) + build = self.Build.create({ + 'params_id': self.server_params.id, + 'description': 'A nice **description** with a link to odoo.com', + }) + self.assertEqual( + build._format_message(get_log('File "/data/build/odoo/addons/web/tests/test_web.py", line 42, in test_web')), + 'File "/data/build/odoo/addons/web/tests/test_web.py", line 42, in test_web' + ) + + self.assertEqual( + build._format_message(get_log('File "/data/build/odoo/addons/web/tests/test_web.py", in test_web')), + 'File "/data/build/odoo/addons/web/tests/test_web.py", in test_web' + ) + class TestGc(RunbotCaseMinimalSetup): def test_repo_gc_testing(self): @@ -744,7 +769,6 @@ def test_repo_gc_testing(self): bundle_b = self.env['runbot.bundle'].search([('name', '=', branch_b_name)]) bundle_b.last_batch._process() - build_b = bundle_b.last_batch.slot_ids[0].build_id # the two builds are starting tests on two different hosts @@ -754,8 +778,8 @@ def test_repo_gc_testing(self): # no room needed, verify that nobody got killed self.Runbot._gc_testing(host) - self.assertFalse(build_a.requested_action) - self.assertFalse(build_b.requested_action) + self.assertFalse(build_a.to_kill) + self.assertFalse(build_b.to_kill) # a new commit is pushed on branch_a self.push_commit(self.remote_odoo_dev, branch_a_name, 'new subject', sha='d0cad0ca') @@ -775,10 +799,10 @@ def test_repo_gc_testing(self): # no room needed, verify that nobody got killed self.Runbot._gc_testing(host) - self.assertFalse(build_a.requested_action) - self.assertFalse(build_b.requested_action) - self.assertFalse(build_a_last.requested_action) - self.assertFalse(children_b.requested_action) + self.assertFalse(build_a.to_kill) + self.assertFalse(build_b.to_kill) + self.assertFalse(build_a_last.to_kill) + self.assertFalse(children_b.to_kill) # now children_b starts on runbot_xxx children_b.write({'local_state': 'testing', 'host': host.name}) @@ -789,10 +813,10 @@ def test_repo_gc_testing(self): self.Runbot._gc_testing(host) # the killable build should have been marked to be killed - self.assertEqual(build_a.requested_action, 'deathrow') - self.assertFalse(build_b.requested_action) - self.assertFalse(build_a_last.requested_action) - self.assertFalse(children_b.requested_action) + self.assertTrue(build_a.to_kill) + self.assertFalse(build_b.to_kill) + self.assertFalse(build_a_last.to_kill) + self.assertFalse(children_b.to_kill) class TestGithubStatus(RunbotCase): diff --git a/runbot/tests/test_build_config_step.py b/runbot/tests/test_build_config_step.py index 063c318ec..a91044f0a 100644 --- a/runbot/tests/test_build_config_step.py +++ b/runbot/tests/test_build_config_step.py @@ -439,25 +439,58 @@ def setUp(self): }).id, 'local_result': 'ok', }) + self.module_dependencies = { + "test_mail": ["mail"], + "mail": ["web"], + "account": ["web"], + "crm": ["web"], + "project": ["web"], + "test_l10n": ["l10n_be", "l10n_in"], + "l10n_be": ["account"], + "l10n_in": ["account"], + "web_enterprise": ["web"], + } + + def mock_git_helper(self, repo, cmd, input_data=None, raw=False): + def make_catfile_output(commit, content): + content_bytes = content.encode('utf-8') + header = f"{commit} blob {len(content_bytes)}\n".encode() + result = header + content_bytes + b"\n" + return result + + if cmd == ['cat-file', '--batch']: + if repo == self.repo_odoo and input_data == 'dfdfcfcf0000ffffffffffffffffffffffffffff:odoo/tests/.runbot/parallel_testing.json\n': + return make_catfile_output('dfdfcfcf0000ffffffffffffffffffffffffffff', self.config_file) + if repo == self.repo_odoo and input_data == 'dfdfcfcf0000ffffffffffffffffffffffffffff:odoo/tests/.runbot/l10n_standalone_testing.json\n': + return make_catfile_output('dfdfcfcf0000ffffffffffffffffffffffffffff', self.l10n_standalone_testing_file) + + if "__manifest__.py" in input_data: + modules_info = [ + (line, line.split(':')[-1].split('/')[-2]) + for line in input_data.splitlines() + if line.endswith('__manifest__.py') + ] + result = b"" + for original_query, module in modules_info: + content = '''{'name': '%s', 'depends': %s}''' % (module, self.module_dependencies.get(module, [])) + result += make_catfile_output(original_query.split(':')[0], content) + return result - def mock_git_helper(self, repo, cmd): - if repo == self.repo_odoo and cmd == ['show', 'dfdfcfcf0000ffffffffffffffffffffffffffff:odoo/tests/.runbot/parallel_testing.json']: - return self.config_file - elif repo == self.repo_odoo and cmd == ['show', 'dfdfcfcf0000ffffffffffffffffffffffffffff:odoo/tests/.runbot/l10n_standalone_testing.json']: - return self.l10n_standalone_testing_file - elif 'show' in cmd: + if cmd == ['cat-file', '--batch']: raise subprocess.CalledProcessError(cmd, 128) - return super().mock_git_helper(repo, cmd) + elif 'diff' in cmd: + return 'odoo/addons/crm/some/file.py\nodoo/addons/project/some/file.py' + return super().mock_git_helper(repo, cmd, input_data, raw) def test_module_filters(self): - self.assertEqual(self.build._get_modules_to_test('-> !mail'), ['base', 'crm', 'documents']) - self.assertEqual(self.build._get_modules_to_test('mail -> !web'), ['mail', 'project', 'test_l10n', 'test_lint']) + self.assertEqual(self.build._get_modules_to_test('-> !mail'), ['account', 'base', 'crm', 'documents']) + self.assertEqual(self.build._get_modules_to_test('mail -> !web'), ['mail', 'project', 'test_l10n', 'test_lint', 'test_mail']) self.assertEqual(self.build._get_modules_to_test('web -> web'), ['web']) self.assertEqual(self.build._get_modules_to_test('!web ->'), ['web_enterprise']) - self.assertEqual(self.build._get_modules_to_test('-> !mail, -crm'), ['base', 'documents']) - self.assertEqual(self.build._get_modules_to_test('mail -> !web, !project'), ['mail', 'test_l10n', 'test_lint']) - self.assertEqual(self.build._get_modules_to_test('-*,odoo/*'), ['base', 'crm', 'hw_drivers', 'mail', 'project', 'test_l10n', 'test_lint', 'web']) - self.assertEqual(self.build._get_modules_to_test('-*,odoo/test_*'), ['test_l10n', 'test_lint']) + self.assertEqual(self.build._get_modules_to_test('-> !mail, -crm'), ['account', 'base', 'documents']) + self.assertEqual(self.build._get_modules_to_test('mail -> !web, !project'), ['mail', 'test_l10n', 'test_lint', 'test_mail']) + self.assertEqual(self.build._get_modules_to_test('-*,odoo/*'), ['account', 'base', 'crm', 'hw_drivers', 'mail', 'project', 'test_l10n', 'test_lint', 'test_mail', 'web']) + self.assertEqual(self.build._get_modules_to_test('-*,odoo/test_*'), ['test_l10n', 'test_lint', 'test_mail']) self.assertEqual(self.build._get_modules_to_test('-*,enterprise/*'), ['documents', 'l10n_be', 'l10n_in', 'web_enterprise']) self.assertEqual(self.build._get_modules_to_test('-*,web*'), ['web', 'web_enterprise']) self.assertEqual(self.build._get_modules_to_test('-*,web*,-enterprise/web*'), ['web']) @@ -467,6 +500,35 @@ def test_config_extension(self): self.assertEqual(json.loads(self.config.default_dynamic_config)['vars']['module_filter'], '*,-hw_*') self.assertEqual(self.build.dynamic_config['vars']['module_filter'], '*,-hw_*,-l10n_*') + def test_parse_dynamic_entry(self): + Step = self.env['runbot.build.config.step'] + + def check_parse(entry, expected): + res = Step._parse_dynamic_entry(entry, self.build, {'key': 'value', 'test_method': '.test_method'}) + self.assertEqual(res, expected) + check_parse('{{-test_*|filter_all_modules}}', 'account,base,crm,documents,hw_drivers,l10n_be,l10n_in,mail,project,web,web_enterprise') + check_parse('{{-*,web*|filter_all_modules}}', 'web,web_enterprise') + check_parse('{{-*,web*|filter_all_modules|make_module_test_tags}}', '/web,/web_enterprise') + check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|prepend("some_tag")}}', 'some_tag/web,some_tag/web_enterprise') + check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|prepend(key)}}', 'value/web,value/web_enterprise') + check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|append(".test_method")}}', '/web.test_method,/web_enterprise.test_method') + check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|append(test_method)}}', '/web.test_method,/web_enterprise.test_method') + + self.patch(type(self.build), '_modified_modules', lambda cl, defaults=None: {'crm'}) + + check_parse('{{*|filter_all_modules|modified_modules}}', 'crm') + + def test_modules_dependencies(self): + self.assertEqual(self.build._get_modules_dependencies(['test_mail'], 1), ['mail', 'test_mail']) + self.assertEqual(self.build._get_modules_dependencies(['test_mail']), ['base', 'mail', 'test_mail', 'web']) + self.assertEqual(self.build._get_modules_dependencies(['test_l10n']), ['account', 'base', 'l10n_be', 'l10n_in', 'test_l10n', 'web']) + self.assertEqual(self.build._get_modules_dependencies(['test_mail', 'test_l10n']), ['account', 'base', 'l10n_be', 'l10n_in', 'mail', 'test_l10n', 'test_mail', 'web']) + self.assertEqual(self.build._get_modules_dependencies(['test_mail', 'test_l10n'], 1), ['l10n_be', 'l10n_in', 'mail', 'test_l10n', 'test_mail']) + + self.assertEqual(self.build._get_dependant_modules(['account'], 1), ['account', 'l10n_be', 'l10n_in']) + self.assertEqual(self.build._get_dependant_modules(['account']), ['account', 'l10n_be', 'l10n_in', 'test_l10n']) + self.assertEqual(self.build._get_dependant_modules(['base']), ['account', 'base', 'crm', 'documents', 'hw_drivers', 'l10n_be', 'l10n_in', 'mail', 'project', 'test_l10n', 'test_lint', 'test_mail', 'web', 'web_enterprise']) + def check_server_cmd(self, cmd, install, test_enable, test_tags, db=None): self.assertIn('odoo/server.py', cmd) if install: @@ -537,7 +599,7 @@ def test_dynamic_step_parallel_testing(self): cmd = self.docker_run_calls[0][0] odoo_cmd = cmd.cmd self.check_server_cmd(odoo_cmd, - install=['base', 'crm', 'documents', 'mail', 'project', 'test_l10n', 'test_lint', 'web', 'web_enterprise'], + install=['account', 'base', 'crm', 'documents', 'mail', 'project', 'test_l10n', 'test_lint', 'test_mail', 'web', 'web_enterprise'], test_enable=False, test_tags=None, db=f'{build.dest}-all', @@ -572,7 +634,7 @@ def test_dynamic_step_parallel_testing(self): cmd = self.docker_run_calls[0][0] odoo_cmd = cmd.cmd self.check_server_cmd(odoo_cmd, - install=['base', 'crm', 'documents', 'mail', 'project', 'test_l10n', 'test_lint', 'web', 'web_enterprise'], + install=['account', 'base', 'crm', 'documents', 'mail', 'project', 'test_l10n', 'test_lint', 'test_mail', 'web', 'web_enterprise'], test_enable=True, test_tags='-post_install,-/test_lint', ) @@ -589,8 +651,8 @@ def test_dynamic_step_parallel_testing(self): ) for post_install, expected_tags in [ - (post_install_1, '-at_install,/base,/crm,/documents,/hw_drivers,/l10n_be,/l10n_in'), # we need the blacklisted modules here - (post_install_2, '-at_install,/mail,/project,/test_l10n,/test_lint'), + (post_install_1, '-at_install,/account,/base,/crm,/documents,/hw_drivers,/l10n_be,/l10n_in'), # we need the blacklisted modules here + (post_install_2, '-at_install,/mail,/project,/test_l10n,/test_lint,/test_mail'), (post_install_3, '-at_install,/web'), (post_install_4, '-at_install,/web_enterprise'), ]: @@ -662,11 +724,14 @@ def test_dynamic_step_l10n_standalone(self): ) # 0.2 run standalone l10n script + self.config_step.check_exit_status = True self.docker_run_calls = [] build._schedule()() self.assertEqual(len(self.docker_run_calls), 1, "One docker run should have been called for install_all step") cmd = self.docker_run_calls[0][0] - self.assertEqual(cmd.build(), f'odoo/odoo/tests/test_module_operations.py -d {build.dest}-l10n --data-dir /data/build/datadir/ --addons-path odoo/addons,odoo/core/addons,enterprise --standalone all_l10n') + expected_cmd = f'odoo/odoo/tests/test_module_operations.py -d {build.dest}-l10n --data-dir /data/build/datadir/ --addons-path odoo/addons,odoo/core/addons,enterprise --standalone all_l10n' + expected_cmd += r' ; echo $? > /data/build/logs/running_standalone_exit_status.txt' + self.assertEqual(cmd.build(), expected_cmd) # 0.3. create post install builds build._schedule() @@ -694,7 +759,7 @@ def test_dynamic_step_l10n_standalone(self): (post_install_1, '-external,-external_l10n,post_install_l10n/l10n_hr_payroll_be,post_install_l10n/l10n_hr_payroll_in'), # we need the blacklisted modules here (post_install_2, '-external,-external_l10n,post_install_l10n/l10n_edi_be,post_install_l10n/l10n_edi_in'), (post_install_3, '-external,-external_l10n,post_install_l10n/l10n_reports_be,post_install_l10n/l10n_reports_in'), - (post_install_4, Like('-external,-external_l10n,post_install_l10n/base,post_install_l10n/crm,...')), + (post_install_4, Like('-external,-external_l10n,post_install_l10n/account,post_install_l10n/base,post_install_l10n/crm,...')), ]: with self.subTest(post_install=expected_tags): # 4.1 post install restore @@ -738,6 +803,7 @@ def test_foreach_module(self): self.config.step_ids[0]._run_dynamic(self.build) self.assertEqual(self.build.children_ids.mapped('description'), [ + 'Post install tests for **account**', 'Post install tests for **base**', 'Post install tests for **crm**', 'Post install tests for **documents**', @@ -767,31 +833,154 @@ def test_foreach_modified_module(self): }] }''' - self.patch(type(self.build), '_modified_modules', lambda cl: {'crm'}) + self.patch(type(self.build), '_modified_modules', lambda cl, defaults=None: {'crm'}) self.config.default_dynamic_config = dynamic_config self.config.step_ids[0]._run_dynamic(self.build) self.assertEqual(self.build.children_ids.mapped('description'), - [ - 'Post install tests for **crm**', + [ + 'Post install tests for **crm**', ]) - def test_parse_dynamic_entry(self): - Step = self.env['runbot.build.config.step'] + def test_modified_existing_module(self): + dynamic_config = '''{ + "vars": { + "modified_modules": "{{*|filter_all_modules|modified_modules}}", + "test_modules": "{{modified_modules|prepend('test_')|select_existing_modules}}", + "modules_to_test": "{{modified_modules|union(test_modules)}}" + }, + "name": "Foreach module testing", + "steps": [{ + "name": "Create module builds", + "job_type": "create_build", + "children": [{ + "name": "Test single module", + "description": "Post install tests for **{{modules_to_test}}**", + "steps": [{ + "name": "Start single module test", + "job_type": "odoo", + "install_modules": "{{modules_to_test}}", + "test_tags": "{{modules_to_test|make_module_test_tags}}" + }] + }] + }] + }''' - def check_parse(entry, expected): - res = Step._parse_dynamic_entry(entry, self.build, {'key': 'value', 'test_method': '.test_method'}) - self.assertEqual(res, expected) - check_parse('{{-test_*|filter_all_modules}}', 'base,crm,documents,hw_drivers,l10n_be,l10n_in,mail,project,web,web_enterprise') - check_parse('{{-*,web*|filter_all_modules}}', 'web,web_enterprise') - check_parse('{{-*,web*|filter_all_modules|make_module_test_tags}}', '/web,/web_enterprise') - check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|prepend("some_tag")}}', 'some_tag/web,some_tag/web_enterprise') - check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|prepend(key)}}', 'value/web,value/web_enterprise') - check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|append(".test_method")}}', '/web.test_method,/web_enterprise.test_method') - check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|append(test_method)}}', '/web.test_method,/web_enterprise.test_method') + self.patch(type(self.build), '_modified_modules', lambda cl, defaults=None: {'crm', 'mail'}) + self.config.default_dynamic_config = dynamic_config + self.config.step_ids[0]._run_dynamic(self.build) + self.assertEqual(self.build.children_ids.mapped('description'), + [ + 'Post install tests for **crm,mail,test_mail**', + ]) + child_dynamic_vars = self.build.children_ids.params_id.config_data['dynamic_vars'] + self.assertEqual(child_dynamic_vars, { + 'modified_modules': 'crm,mail', + 'test_modules': 'test_mail', + 'modules_to_test': 'crm,mail,test_mail', + }) + + def test_modified_existing_module_parallel(self): + dynamic_config = '''{ + "vars": { + "modified_modules": "{{*|filter_all_modules|modified_modules}}", + "modules_to_test": "{{modified_modules|prepend('test_')|select_existing_modules|union(modified_modules)}}" + }, + "name": "Parallel split modified", + "steps": [{ + "name": "Create module builds", + "job_type": "create_build", + "for_each_vars": [{ + "test_module_filter": "{{modules_to_test}},->!mail" + }, + { + "test_module_filter": "{{modules_to_test}},mail->!website" + }, + { + "test_module_filter": "{{modules_to_test}},website->" + } + ], + "if": "{{child_modules_to_test}}", + "children": [{ + "vars": { + "child_modules_to_test": "{{test_module_filter|select_existing_modules}}" + }, + "name": "Test single module", + "description": "Post install tests for **{{child_modules_to_test}}**", + "steps": [{ + "name": "Start single module test", + "job_type": "odoo", + "install_modules": "{{child_modules_to_test}}", + "test_tags": "{{child_modules_to_test|make_module_test_tags}}" + }] + }] + }] + }''' - self.patch(type(self.build), '_modified_modules', lambda cl: {'crm'}) + self.patch(type(self.build), '_modified_modules', lambda cl, defaults=None: {'crm', 'mail'}) + self.config.default_dynamic_config = dynamic_config + self.config.step_ids[0]._run_dynamic(self.build) + self.assertEqual(self.build.children_ids.mapped('description'), + [ + 'Post install tests for **crm**', + 'Post install tests for **mail,test_mail**', + ]) - check_parse('{{*|filter_all_modules|modified_modules}}', 'crm') + self.assertEqual(self.build.children_ids[0].params_id.config_data['dynamic_vars']['child_modules_to_test'], 'crm') + self.assertEqual(self.build.children_ids[1].params_id.config_data['dynamic_vars']['child_modules_to_test'], 'mail,test_mail') + + def test_modified_existing_module_parallel_relations(self): + dynamic_config = '''{ + "vars": [ + {"module_filter": "*,-hw_*,-*l10n_*,-theme_*,-account_bacs,-account_reports_cash_basis,-auth_ldap,-base_gengo,-document_ftp,-iot_drivers,-note_pad,-odoo_referral,-odoo_referral_portal,-pad,-pad_project,-pos_blackbox_be,-pos_cache,-pos_six,-social_demo,-website_gengo,-website_instantclick,test_l10n_be_hr_payroll_account,test_l10n_us_hr_payroll_account"}, + {"_modified_modules": "{{module_filter|filter_all_modules|modified_modules}}"}, + {"_modules_dependencies": "{{_modified_modules|get_dependencies(1)}}"}, + {"_dependant_modules": "{{_modified_modules|get_dependant(1)}}"}, + {"_test_modules": "{{_modified_modules|prepend('test_')|select_existing_modules}}"}, + {"_modules_to_test": "{{_modified_modules|union(_test_modules)|union(_dependant_modules)|union(_modules_dependencies)}}"} + ], + "name": "Parallel split modified", + "steps": [{ + "name": "Create module builds", + "job_type": "create_build", + "for_each_vars": [{ + "_test_module_filter": "{{_modules_to_test}},->!mail" + }, + { + "_test_module_filter": "{{_modules_to_test}},mail->!website" + }, + { + "_test_module_filter": "{{_modules_to_test}},website->" + } + ], + "if": "{{child_modules_to_test}}", + "log": "Modified modules: {{_modified_modules}}\\nDepenencies: {{_modules_dependencies}}\\nDependant: {{_dependant_modules}}\\nTest modules: {{_test_modules}}", + "children": [{ + "vars": { + "child_modules_to_test": "{{_test_module_filter|select_existing_modules}}" + }, + "name": "Test single module", + "description": "Post install tests for **{{child_modules_to_test}}**", + "steps": [{ + "name": "Start single module test", + "job_type": "odoo", + "install_modules": "{{child_modules_to_test}}", + "test_tags": "{{child_modules_to_test|make_module_test_tags}}" + }] + }] + }] + }''' + + self.patch(type(self.build), '_modified_modules', lambda cl, defaults=None: {'crm', 'mail'}) + self.config.default_dynamic_config = dynamic_config + self.config.step_ids[0]._run_dynamic(self.build) + self.assertEqual(self.build.children_ids.mapped('description'), + [ + 'Post install tests for **crm**', + 'Post install tests for **mail,test_mail,web**', + ]) + self.assertEqual(self.build.children_ids[0].params_id.config_data['dynamic_vars']['child_modules_to_test'], 'crm') + self.assertEqual(self.build.children_ids[1].params_id.config_data['dynamic_vars']['child_modules_to_test'], 'mail,test_mail,web') + self.assertEqual(list(self.build.children_ids[0].params_id.config_data['dynamic_vars'].keys()), ['module_filter', 'child_modules_to_test']) class TestBuildConfigStep(TestBuildConfigStepCommon): @@ -864,12 +1053,13 @@ def test_coverage(self, mock_checkout): 'name': 'coverage', 'job_type': 'install_odoo', 'coverage': True, + 'coverage_make_report': True }) cmd = config_step._run_install_odoo(self.parent_build)['cmd'] - self.assertEqual(cmd.cmd[:10], ['python3', '-m', 'coverage', 'run', '--branch', '--source', '/data/build', '--omit', '*__manifest__.py,odoo/addons/hw_drivers/*', 'odoo/server.py']) - self.assertIn(['python3', '-m', 'coverage', 'html', '-d', '/data/build/coverage', '--ignore-errors'], cmd.finals) - self.assertIn(['python3', '-m', 'coverage', 'xml', '-o', '/data/build/logs/coverage.xml', '--ignore-errors'], cmd.finals) + self.assertEqual(cmd.cmd[:7], ['python3', '-m', 'coverage', 'run', '--source', '/data/build', 'odoo/server.py']) + self.assertIn(['python3', '-m', 'coverage', 'html', '-d', '/data/build/logs/coverage', '--ignore-errors'], cmd.finals) + self.assertIn(['python3', '-m', 'coverage', 'json', '-o', '/data/build/logs/coverage.json', '--ignore-errors'], cmd.finals) @patch('odoo.addons.runbot.models.build.BuildResult._checkout') @@ -1185,22 +1375,15 @@ def test_network_can_be_enable(self, mock_checkout): 'job_type': 'install_odoo', }) - # by default, network is disabled - def first_docker_run(cmd, log_path, *args, **kwargs): - self.assertFalse(kwargs['network_enabled']) - - self.docker_run_patch = first_docker_run config_step._run_step(self.parent_build)() - - def second_docker_run(cmd, log_path, *args, **kwargs): - self.assertTrue(kwargs['network_enabled']) - - self.docker_run_patch = second_docker_run + self.assertFalse(self.docker_run_calls[0][3]['network_enabled']) parent_build_params = self.parent_build.params_id.copy({'config_data': {'network_enabled': True}}) parent_build = self.parent_build.copy({'params_id': parent_build_params.id}) config_step._run_step(parent_build)() + self.assertTrue(self.docker_run_calls[1][3]['network_enabled']) + @patch('odoo.addons.runbot.models.build.BuildResult._checkout') def test_run_python_networkcan_be_enabled(self, mock_checkout): @@ -1241,6 +1424,21 @@ def test_install_custom_parametric_tags(self, mock_checkout, parse_config): tags = self.get_test_tags(params) self.assertEqual(tags, '"-at_install,/web/foo.py:WebSuite.test_unit_desktop[@bar/test with spaces],-/web/bar.py[@snafu/other test with spaces]"') + @patch('odoo.addons.runbot.models.build.BuildResult._checkout') + def test_memory_limit(self, mock_checkout): + """ test that network can be disabled with config_data """ + config_step = self.ConfigStep.create({ + 'name': 'default', + 'job_type': 'install_odoo', + }) + self.env['ir.config_parameter'].sudo().set_param('runbot.runbot_containers_memory', 10) + config_step._run_step(self.parent_build)() + self.assertEqual(self.docker_run_calls[0][3]['memory'], 10 * 1024 ** 3) + + parent_build_params = self.parent_build.params_id.copy({'config_data': {'memory_limit_factor': 0.9}}) + parent_build = self.parent_build.copy({'params_id': parent_build_params.id}) + config_step._run_step(parent_build)() + self.assertEqual(self.docker_run_calls[1][3]['memory'], 9 * 1024 ** 3) class TestMakeResult(RunbotCase): def setUp(self): @@ -1253,6 +1451,7 @@ def _log(build, func, message, level='INFO', log_type='runbot', path='runbot'): self.logs.append((level, message)) self.start_patcher('log_patcher', 'odoo.addons.runbot.models.build.BuildResult._log', new=_log) + self.start_patcher('get_size', 'odoo.addons.runbot.models.build_config.os.path.getsize', return_value=100) self.build = self.Build.create({ 'params_id': self.base_params.id, @@ -1474,3 +1673,112 @@ def make_warn(build): mock_make_odoo_results.side_effect = make_warn config_step._make_results(build) self.assertEqual(build.local_result, 'warn') + + def test_check_exit_status_ok(self): + self.config_step.write({'check_exit_status': True}) + file_content = """ +Loading stuff +odoo.stuff.modules.loading: Modules loaded. +Some post install stuff +Initiating shutdown + """ + exit_status_content = "0\n" + + def mock_open_files(filename, mode='r', *args, **kwargs): + if filename.endswith('_exit_status.txt'): + return mock_open(read_data=exit_status_content)() + return mock_open(read_data=file_content)() + + with patch('builtins.open', mock_open_files): + with patch('os.path.exists', return_value=True): + self.config_step._make_results(self.build) + + self.assertEqual(self.build.local_result, 'ok') + + def test_check_exit_status_ko(self): + self.config_step.write({'check_exit_status': True}) + file_content = """ +Loading stuff +odoo.stuff.modules.loading: Modules loaded. +Some post install stuff +Initiating shutdown + """ + exit_status_content = "1\n" + + def mock_open_files(filename, mode='r', *args, **kwargs): + if filename.endswith('_exit_status.txt'): + return mock_open(read_data=exit_status_content)() + return mock_open(read_data=file_content)() + + with patch('builtins.open', mock_open_files): + with patch('os.path.exists', return_value=True): + self.config_step._make_results(self.build) + + self.assertEqual(self.build.local_result, 'ko') + self.assertIn(('ERROR', 'Main command exited with status code 1'), self.logs) + + def test_check_exit_status_file_empty(self): + self.config_step.write({'check_exit_status': True}) + file_content = """ +Loading stuff +odoo.stuff.modules.loading: Modules loaded. +Some post install stuff +Initiating shutdown + """ + exit_status_content = "" + + def mock_open_files(filename, mode='r', *args, **kwargs): + if filename.endswith('_exit_status.txt'): + return mock_open(read_data=exit_status_content)() + return mock_open(read_data=file_content)() + + with patch('builtins.open', mock_open_files): + with patch('os.path.exists', return_value=True): + self.config_step._make_results(self.build) + + self.assertEqual(self.build.local_result, 'ko') + self.assertIn(('ERROR', 'Exception or file empty while reading status file "all_exit_status.txt"'), self.logs) + self.assertIn(('ERROR', 'Main command exited with status code -241'), self.logs) + + def test_check_exit_status_file_non_integer(self): + self.config_step.write({'check_exit_status': True}) + file_content = """ +Loading stuff +odoo.stuff.modules.loading: Modules loaded. +Some post install stuff +Initiating shutdown + """ + exit_status_content = "d0d0caca" + + def mock_open_files(filename, mode='r', *args, **kwargs): + if filename.endswith('_exit_status.txt'): + return mock_open(read_data=exit_status_content)() + return mock_open(read_data=file_content)() + + with patch('builtins.open', mock_open_files): + with patch('os.path.exists', return_value=True): + self.config_step._make_results(self.build) + + self.assertEqual(self.build.local_result, 'ko') + self.assertIn(('ERROR', 'Status file "all_exit_status.txt" does not contain an integer'), self.logs) + self.assertIn(('ERROR', 'Main command exited with status code -242'), self.logs) + + def test_check_exit_status_file_not_found(self): + config_step = self.ConfigStep.create({ + 'name': 'test', + 'job_type': 'install_odoo', + 'check_exit_status': True, + }) + self.patchers['file_exist'].return_value = False + config_step._make_results(self.build) + self.assertEqual(self.build.local_result, 'ko') + self.assertIn(('ERROR', 'Exit status file "test_exit_status.txt" not found'), self.logs) + + def test_make_result_large_file(self): + custom_limit = 1024 + self.env['ir.config_parameter'].sudo().set_param('runbot.runbot_max_log_size', custom_limit) + self.patchers['get_size'].return_value = custom_limit + 1 + self.config_step._make_results(self.build) + self.assertEqual(str(self.build.job_end), '1970-01-01 02:00:00') + self.assertIn(('ERROR', 'Log file exceeds %s limit' % custom_limit), self.logs) + self.assertEqual(self.build.local_result, 'ko') diff --git a/runbot/tests/test_build_error.py b/runbot/tests/test_build_error.py index 24e9ab6a2..8d4a27ec6 100644 --- a/runbot/tests/test_build_error.py +++ b/runbot/tests/test_build_error.py @@ -219,6 +219,158 @@ def test_merge_pr_ids(self): self.assertEqual(error_a.fixing_pr_id, self.dev_pr) self.assertEqual(error_a.breaking_pr_id, self.dev_pr) + def test_duplicate_breaking_pr(self): + pr_branch = self.Branch.create({ + 'name': '242', + 'is_pr': True, + 'alive': True, + 'remote_id': self.remote_odoo.id, + 'target_branch_name': self.branch_odoo.name, + 'pull_head_name': f'{self.remote_odoo.owner}:{self.dev_branch.name}', + }) + + error_a = self.BuildError.create({'content': 'error A', 'breaking_pr_id': pr_branch.id}) + error_b = self.BuildError.create({'content': 'error B', 'breaking_pr_id': pr_branch.id}) + error_c = self.BuildError.create({'content': 'error C', 'breaking_pr_id': pr_branch.id}) + error_d = self.BuildError.create({'content': 'error D'}) + + # Test compute count (excludes self) + self.assertEqual(error_a.duplicate_breaking_pr_count, 2) + self.assertEqual(error_b.duplicate_breaking_pr_count, 2) + self.assertEqual(error_c.duplicate_breaking_pr_count, 2) + self.assertEqual(error_d.duplicate_breaking_pr_count, 0) + + # Test action returns all errors including self + action = error_a.action_view_duplicate_breaking_pr() + self.assertEqual(action['type'], 'ir.actions.act_window') + self.assertEqual(action['res_model'], 'runbot.build.error') + errors_in_action = self.BuildError.search(action['domain']) + self.assertIn(error_a, errors_in_action) + self.assertEqual(len(errors_in_action), 3) + + # Test count update on deactivation + error_b.active = False + error_a.invalidate_recordset(['duplicate_breaking_pr_count']) + self.assertEqual(error_a.duplicate_breaking_pr_count, 1) + + # Test count update on breaking PR removal + error_a.breaking_pr_id = False + self.assertEqual(error_a.duplicate_breaking_pr_count, 0) + + def test_onchange_test_tags(self): + error = self.BuildError.create({}) + error_content = self.BuildErrorContent.create({'content': 'very bad trip', 'error_id': error.id}) + build_13 = self.create_test_build({'local_result': 'ko'}) + self.BuildErrorLink.create({ + 'build_id': build_13.id, + 'error_content_id': error_content.id, + }) + self.assertEqual(error.tags_min_version_id.id, False) + self.assertEqual(error.tags_max_version_id.id, False) + + error.test_tags = '/account' + error._onchange_test_tags() + self.assertEqual(error.tags_min_version_id, self.version_13) + self.assertEqual(error.tags_max_version_id, self.version_13) + + version_14 = self.Version._get('14.0') + params_14 = self.create_params({'version_id': version_14.id}) + build_14 = self.create_test_build({'local_result': 'ko', 'params_id': params_14.id}) + self.BuildErrorLink.create({ + 'build_id': build_14.id, + 'error_content_id': error_content.id, + }) + error._onchange_test_tags() + self.assertEqual(error.tags_min_version_id, self.version_13) + self.assertEqual(error.tags_max_version_id, version_14) + + def test_parse_logs_updates_version_bounds(self): + build_13 = self.create_test_build({'local_result': 'ko', 'local_state': 'done'}) + log_data = { + 'name': 'test-parse-logs', + 'type': 'server', + 'path': 'runbot', + 'level': 'ERROR', + 'func': 'test_trip', + 'line': 242, + 'message': 'Error Very Bad Trip', + 'build_id': build_13.id, + } + log_13 = self.IrLog.create(log_data) + + action = self.BuildError._parse_logs(log_13, update_tags=True) + self.assertEqual(action.get('type'), 'ir.actions.act_window') + error_content = self.BuildErrorContent.browse(action.get('res_id')) + error = error_content.error_id + error.test_tags = '/account' + error._update_version_tags() + self.assertEqual(error.tags_min_version_id, self.version_13) + self.assertEqual(error.tags_max_version_id, self.version_13) + + # Scanning a newer build extends the max without updating the min + version_14 = self.Version._get('14.0') + params_14 = self.create_params({'version_id': version_14.id}) + build_14 = self.create_test_build({'local_result': 'ko', 'local_state': 'done', 'params_id': params_14.id}) + log_data.update({'build_id': build_14.id}) + log_14 = self.IrLog.create(log_data) + self.BuildError._parse_logs(log_14, update_tags=True) + self.assertIn(build_14, error.build_ids) + self.assertEqual(error.tags_min_version_id, self.version_13) + self.assertEqual(error.tags_max_version_id, version_14) + + def test_parse_logs_no_update_without_flag(self): + build_13 = self.create_test_build({'local_result': 'ko', 'local_state': 'done'}) + log_13 = self.IrLog.create({ + 'name': 'test-parse-logs', + 'type': 'server', + 'path': 'runbot', + 'level': 'ERROR', + 'func': 'test_trip', + 'line': 242, + 'message': 'Error Very Bad Trip', + 'build_id': build_13.id, + }) + action = self.BuildError._parse_logs(log_13) + self.assertEqual(action.get('type'), 'ir.actions.act_window') + error_content = self.BuildErrorContent.browse(action.get('res_id')) + error = error_content.error_id + error.test_tags = '/discuss' + self.assertFalse(error.tags_min_version_id) + self.assertFalse(error.tags_max_version_id) + + def test_update_version_tags_no_update_inside_bounds(self): + version_14 = self.Version._get('14.0') + version_16 = self.Version._get('16.0') + params_14 = self.create_params({'version_id': version_14.id}) + params_16 = self.create_params({'version_id': version_16.id}) + build_14 = self.create_test_build({'local_result': 'ko', 'params_id': params_14.id}) + build_16 = self.create_test_build({'local_result': 'ko', 'params_id': params_16.id}) + + error = self.BuildError.create({}) + error_content = self.BuildErrorContent.create({'content': 'inside bounds test', 'error_id': error.id}) + link_14 = self.BuildErrorLink.create({'build_id': build_14.id, 'error_content_id': error_content.id}) + link_16 = self.BuildErrorLink.create({'build_id': build_16.id, 'error_content_id': error_content.id}) + + error.test_tags = '/account' + error._update_version_tags() + self.assertEqual(error.tags_min_version_id, version_14) + self.assertEqual(error.tags_max_version_id, version_16) + + link_14.unlink() + self.assertNotIn(version_14, error.version_ids) + # There is only version 16.0 on the error now but if we update the tags, it should leave the min at 14.0 + error._update_version_tags() + self.assertEqual(error.tags_min_version_id, version_14) + self.assertEqual(error.tags_max_version_id, version_16) + + link_14 = self.BuildErrorLink.create({'build_id': build_14.id, 'error_content_id': error_content.id}) + link_16.unlink() + self.assertNotIn(version_16, error.version_ids) + # Now there is only version 14.0 on the error but if we update the tags, it should leave the max at 16.0 + error._update_version_tags() + self.assertEqual(error.tags_min_version_id, version_14) + self.assertEqual(error.tags_max_version_id, version_16) + def test_relink_contents(self): build_a = self.create_test_build({'local_result': 'ko', 'local_state': 'done'}) error_content_a = self.BuildErrorContent.create({'content': 'foo bar'}) @@ -266,7 +418,7 @@ def test_relink_simple(self): build_b = self.create_test_build({'local_result': 'ko', 'local_state': 'done'}) error_content_b = self.BuildErrorContent.create({'content': 'foo bar'}) error_b = error_content_b.error_id - error_b.test_tags = 'footag' + error_b.test_tags = 'footag[@test, comma]\nfootag2[@test, comma],footag3[@test, comma]' self.BuildErrorLink.create({'build_id': build_b.id, 'error_content_id': error_content_b.id}) self.assertEqual(self.BuildErrorContent.search([('fingerprint', '=', error_content_a.fingerprint)]), error_content_a | error_content_b) @@ -275,7 +427,7 @@ def test_relink_simple(self): self.assertFalse(error_b.error_content_ids) self.assertTrue(error_a.active, 'The merged error without test tags should have been deactivated') - self.assertEqual(error_a.test_tags, 'footag', 'Tags should have been transfered from b to a') + self.assertEqual(error_a.test_tags, 'footag[@test, comma]\nfootag2[@test, comma],footag3[@test, comma]', 'Tags should have been transfered from b to a') self.assertFalse(error_b.active, 'The merged error with test tags should remain active') self.assertIn(build_a, error_content_a.build_ids) self.assertIn(build_b, error_content_a.build_ids) @@ -284,9 +436,9 @@ def test_relink_simple(self): tagged_error_content = self.BuildErrorContent.create({'content': 'foo bar'}) tagged_error = tagged_error_content.error_id - tagged_error.test_tags = 'bartag' + tagged_error.test_tags = 'bartag[@test, comma]\nbartag2[@test, comma],bartag3[@test, comma]' (error_content_a | tagged_error_content)._relink() - self.assertEqual(error_a.test_tags, 'footag,bartag') + self.assertEqual(error_a.test_tags, 'footag[@test, comma]\nfootag2[@test, comma]\nfootag3[@test, comma]\nbartag[@test, comma]\nbartag2[@test, comma]\nbartag3[@test, comma]') self.assertTrue(error_a.active) self.assertFalse(tagged_error.active) @@ -869,7 +1021,38 @@ def test_error_content_multiple_canonical_tags(self): self.assertEqual(error_content_2.canonical_tag, '/web/tests/test_file.py:TestUi.TestUi') self.assertNotEqual(error_content_1, error_content_2) self.assertEqual(error_content_1.error_id, error_content_2.error_id) - self.assertEqual(error_content_1.error_id.canonical_tags, '/base/tests/test_file.py:TestUi.TestUi,/web/tests/test_file.py:TestUi.TestUi') + self.assertEqual(error_content_1.error_id.canonical_tags, '/base/tests/test_file.py:TestUi.TestUi\n/web/tests/test_file.py:TestUi.TestUi') + error_content_1.error_id.test_tags = error_content_1.error_id.canonical_tags + self.assertEqual(error_content_1.error_id._disabling_tags(), [ + '-/base/tests/test_file.py:TestUi.TestUi', + '-/web/tests/test_file.py:TestUi.TestUi', + ]) + error_content_3 = self.env['runbot.build.error.content'].create({ + 'content': 'Tour foobar_tour failed at step step_14 in mode mode', + 'metadata': {'test': {'canonical_tag': '/web/tests/test_file.py:TestJs.test_unit_desktop[@web/some , comma\\[\\] and brackets]'}}, + }) + self.assertEqual(error_content_3.canonical_tag, '/web/tests/test_file.py:TestJs.test_unit_desktop[@web/some , comma\\[\\] and brackets]') + self.assertNotEqual(error_content_1, error_content_2) + self.assertEqual(error_content_1.error_id, error_content_2.error_id) + self.assertEqual(error_content_1.error_id.canonical_tags, """/base/tests/test_file.py:TestUi.TestUi\n/web/tests/test_file.py:TestJs.test_unit_desktop[@web/some , comma\\[\\] and brackets]\n/web/tests/test_file.py:TestUi.TestUi""") + error_content_1.error_id.test_tags = error_content_1.error_id.canonical_tags + self.assertEqual(error_content_1.error_id._disabling_tags(), [ + '-/base/tests/test_file.py:TestUi.TestUi', + '-/web/tests/test_file.py:TestJs.test_unit_desktop[@web/some , comma\\[\\] and brackets]', + '-/web/tests/test_file.py:TestUi.TestUi', + ], "disabling tags must keep the escaping and correct format") + error_content_1.error_id.test_tags = error_content_1.error_id.test_tags.replace('\n', ',') + self.assertEqual(error_content_1.error_id._disabling_tags(), [ + '-/base/tests/test_file.py:TestUi.TestUi', + '-/web/tests/test_file.py:TestJs.test_unit_desktop[@web/some , comma\\[\\] and brackets]', + '-/web/tests/test_file.py:TestUi.TestUi', + ], "_disabling_tags shoudl also work fine with comma separted tags") + error_content_1.error_id.test_tags = error_content_1.error_id.test_tags.replace('\\', '') + self.assertEqual(error_content_1.error_id._disabling_tags(), [ + '-/base/tests/test_file.py:TestUi.TestUi', + '-/web/tests/test_file.py:TestJs.test_unit_desktop[@web/some , comma[] and brackets]', + '-/web/tests/test_file.py:TestUi.TestUi', + ]) class TestCodeOwner(RunbotCase): diff --git a/runbot/tests/test_build_stat.py b/runbot/tests/test_build_stat.py index 6f8282585..5a5e616a7 100644 --- a/runbot/tests/test_build_stat.py +++ b/runbot/tests/test_build_stat.py @@ -9,6 +9,7 @@ class TestBuildStatRegex(RunbotCase): def setUp(self): super(TestBuildStatRegex, self).setUp() + self.start_patcher('get_size', 'odoo.addons.runbot.models.build_config.os.path.getsize', return_value=100) self.StatRegex = self.env["runbot.build.stat.regex"] self.ConfigStep = self.env["runbot.build.config.step"] self.BuildStat = self.env["runbot.build.stat"] diff --git a/runbot/tests/test_dockerfile.py b/runbot/tests/test_dockerfile.py index cb7539bc2..b04ea6fb2 100644 --- a/runbot/tests/test_dockerfile.py +++ b/runbot/tests/test_dockerfile.py @@ -44,7 +44,7 @@ def test_docker_default(self): && rm -rf /var/lib/apt/lists/* RUN set -x ; \ apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends publicsuffix python3 flake8 python3-dbfread python3-dev python3-gevent python3-pip python3-setuptools python3-wheel python3-markdown python3-mock python3-phonenumbers python3-websocket python3-google-auth libpq-dev pylint python3-jwt python3-asn1crypto python3-html2text python3-suds python3-xmlsec python3-markdown2 python3-aiosmtpd \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends publicsuffix python3 flake8 python3-dbfread python3-dev python3-gevent python3-pip python3-setuptools python3-wheel python3-markdown python3-mock python3-phonenumbers python3-websocket python3-google-auth libpq-dev pylint python3-jwt python3-asn1crypto python3-html2text python3-suds python3-xmlsec python3-markdown2 python3-aiosmtpd python3-paramiko \ && rm -rf /var/lib/apt/lists/* RUN curl -sSL https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox_0.12.6.1-2.jammy_amd64.deb -o /tmp/wkhtml.deb \ && apt-get update \ @@ -204,7 +204,7 @@ def test_dockerfile_get_cached_content(self): expected_content = """# CacheAddTest FROM ubuntu:noble -# CACHE 60 +# CACHED ADD https://nowhere.example.org/nothing.txt /data/nothing.txt COPY _data_nothing_txt /data/nothing.txt @@ -270,7 +270,7 @@ def test_dockerfile_build_with_cached_content(self): expected_content = """# Cache Test FROM ubuntu:noble -# CACHE 60 +# CACHED ADD https://nowhere.example.org/nothing.txt /data/nothing.txt COPY _data_nothing_txt /data/nothing.txt diff --git a/runbot/tests/test_repo.py b/runbot/tests/test_repo.py index 81c940ab0..d4c40fcae 100644 --- a/runbot/tests/test_repo.py +++ b/runbot/tests/test_repo.py @@ -372,7 +372,7 @@ def setUp(self): self.fetch_count = 0 self.force_failure = False - def mock_git_helper(self, repo, cmd): + def mock_git_helper(self, repo, cmd, input_data=None, raw=False): self.assertIn('fetch', cmd) self.fetch_count += 1 if self.fetch_count < 3 or self.force_failure: @@ -383,26 +383,28 @@ def mock_git_helper(self, repo, cmd): class TestIdentityFile(RunbotCase): - def check_output_helper(self): - """Helper that returns a mock for repo._git()""" - def mock_check_output(cmd, *args, **kwargs): - expected_option = r'-c core.sshCommand=ssh -i \/.+\/\.ssh\/fake_identity' - git_cmd = ' '.join(cmd) - self.assertTrue(re.search(expected_option, git_cmd), '%s did not match %s' % (git_cmd, expected_option)) - return Mock() + def popen_helper(self): + """Helper that returns a mock for repo._git()""" + def mock_popen(cmd, *args, **kwargs): + expected_option = r'-c core.sshCommand=ssh -i \/.+\/\.ssh\/fake_identity' + git_cmd = ' '.join(cmd) + self.assertTrue(re.search(expected_option, git_cmd), '%s did not match %s' % (git_cmd, expected_option)) + popen_mock = Mock() + attrs = {"communicate.return_value": (b"", b"")} + popen_mock.configure_mock(**attrs) + return popen_mock - return mock_check_output + return mock_popen - def test_identity_file(self): - """test that the identity file is used in git command""" + def test_identity_file(self): + """test that the identity file is used in git command""" - self.patcher_objects['git_patcher'].stop() - self.start_patcher('check_output_patcher', 'odoo.addons.runbot.models.repo.subprocess.check_output', new=self.check_output_helper()) + self.patcher_objects['git_patcher'].stop() + self.start_patcher('popen_patcher', 'odoo.addons.runbot.models.repo.subprocess.Popen', new=self.popen_helper()) + self.repo_odoo.identity_file = 'fake_identity' - self.repo_odoo.identity_file = 'fake_identity' - - with mute_logger("odoo.addons.runbot.models.repo"): - self.repo_odoo._update_fetch_cmd() + with mute_logger("odoo.addons.runbot.models.repo"): + self.repo_odoo._update_fetch_cmd() class TestRepoScheduler(RunbotCase): @@ -457,7 +459,7 @@ def setUp(self): super().setUp() self.test_refs = [] - def mock_git_helper(self, repo, cmd): + def mock_git_helper(self, repo, cmd, input_data=None, raw=False): self.assertIn('for-each-ref', cmd) self.assertIn('refs/*/pull/*', cmd) return '\n'.join(['\x00'.join(ref_data) for ref_data in self.test_refs]) diff --git a/runbot/tests/test_upgrade.py b/runbot/tests/test_upgrade.py index dbd7bb48d..e52ef5a8d 100644 --- a/runbot/tests/test_upgrade.py +++ b/runbot/tests/test_upgrade.py @@ -19,6 +19,7 @@ def setUp(self): def upgrade_flow_setup(self): self.start_patcher('find_patcher', 'odoo.addons.runbot.common.find', 0) + self.start_patcher('get_size', 'odoo.addons.runbot.models.build_config.os.path.getsize', return_value=100) self.additionnal_setup() self.master_bundle = self.branch_odoo.bundle_id diff --git a/runbot/views/branch_views.xml b/runbot/views/branch_views.xml index 943ef6e30..3791f562e 100644 --- a/runbot/views/branch_views.xml +++ b/runbot/views/branch_views.xml @@ -46,6 +46,7 @@ Branches runbot.branch list,form + branch diff --git a/runbot/views/build_error_views.xml b/runbot/views/build_error_views.xml index 1f92e0619..e521b5eb4 100644 --- a/runbot/views/build_error_views.xml +++ b/runbot/views/build_error_views.xml @@ -10,6 +10,9 @@ +
    @@ -439,6 +442,7 @@ Errors regex runbot.error.regex list,form + error-regex runbot.error.qualify.regex.list @@ -514,6 +518,7 @@ Build Errors Qualifying Regexes runbot.error.qualify.regex list,form + error-qualify diff --git a/runbot/views/build_views.xml b/runbot/views/build_views.xml index 722201d09..71a1b49e5 100644 --- a/runbot/views/build_views.xml +++ b/runbot/views/build_views.xml @@ -12,6 +12,8 @@ + + @@ -63,7 +65,7 @@ - + @@ -80,9 +82,11 @@ - + + + @@ -148,12 +152,14 @@ ir.actions.act_window runbot.build list,form,graph,pivot + build Builds Params ir.actions.act_window runbot.build.params list,form + build-param - \ No newline at end of file + diff --git a/runbot/views/bundle_views.xml b/runbot/views/bundle_views.xml index d3e004185..ff76a3fb4 100644 --- a/runbot/views/bundle_views.xml +++ b/runbot/views/bundle_views.xml @@ -18,6 +18,9 @@ + + + @@ -56,6 +59,7 @@ + @@ -259,36 +263,42 @@ ir.actions.act_window runbot.bundle.trigger.custom list,form + custom-trigger Bundles ir.actions.act_window runbot.bundle list,form + bundle Projects ir.actions.act_window runbot.project list,form + project Versions ir.actions.act_window runbot.version list,form + version Batches ir.actions.act_window runbot.batch list,form + batch Bundle Tags ir.actions.act_window runbot.bundle.tag list,form + bundle-tag diff --git a/runbot/views/codeowner_views.xml b/runbot/views/codeowner_views.xml index afb81aaed..e7ccdfacf 100644 --- a/runbot/views/codeowner_views.xml +++ b/runbot/views/codeowner_views.xml @@ -39,6 +39,7 @@ Codeowner runbot.codeowner list,form + codeowner diff --git a/runbot/views/commit_views.xml b/runbot/views/commit_views.xml index 8bc09c10c..fed406ef9 100644 --- a/runbot/views/commit_views.xml +++ b/runbot/views/commit_views.xml @@ -59,6 +59,7 @@ Commits runbot.commit list,form + commit @@ -93,6 +94,7 @@ Commit Links runbot.commit.link list + commit-link diff --git a/runbot/views/config_views.xml b/runbot/views/config_views.xml index a2b8a0bbe..11d52ac7c 100644 --- a/runbot/views/config_views.xml +++ b/runbot/views/config_views.xml @@ -76,6 +76,10 @@ + + + + @@ -207,12 +211,14 @@ Build Configs runbot.build.config list,form + config Build Config Steps runbot.build.config.step list,form + config-step diff --git a/runbot/views/dashboard_views.xml b/runbot/views/dashboard_views.xml index fa0f4ad8e..3a127a766 100644 --- a/runbot/views/dashboard_views.xml +++ b/runbot/views/dashboard_views.xml @@ -215,30 +215,35 @@ Runbot Dashboards Tiles runbot.dashboard.tile list,form + dashboard-tile Runbot teams runbot.team list,form + team Runbot Dashboards runbot.dashboard list,form + dashboard Runbot modules runbot.module list,form + module Runbot modules ownership runbot.module.ownership list,form + module-ownership diff --git a/runbot/views/dockerfile_views.xml b/runbot/views/dockerfile_views.xml index 79e9f58af..27d45ac5e 100644 --- a/runbot/views/dockerfile_views.xml +++ b/runbot/views/dockerfile_views.xml @@ -23,6 +23,7 @@ + @@ -107,6 +108,8 @@ + + @@ -268,12 +271,14 @@ Docker build results runbot.docker_build_result list,form + docker-build Docker Layers runbot.docker_layer list,form + docker-layer diff --git a/runbot/views/host_views.xml b/runbot/views/host_views.xml index f22f34e2b..726aeff48 100644 --- a/runbot/views/host_views.xml +++ b/runbot/views/host_views.xml @@ -72,6 +72,7 @@ Host runbot.host list,form + host diff --git a/runbot/views/repo_views.xml b/runbot/views/repo_views.xml index 02d7e44e6..41b620f6e 100644 --- a/runbot/views/repo_views.xml +++ b/runbot/views/repo_views.xml @@ -17,10 +17,12 @@ + + @@ -198,6 +200,7 @@ + @@ -264,30 +267,35 @@ Repositories runbot.repo list,form + repository Triggers runbot.trigger list,form + trigger Triggers dependency runbot.trigger.dependency list,form + trigger-dependency Remotes runbot.remote list,form + remote Trigger Categories runbot.category list,form + trigger-category diff --git a/runbot/views/semgrep_rules.xml b/runbot/views/semgrep_rules.xml index 21cef13ac..a325ea0ed 100644 --- a/runbot/views/semgrep_rules.xml +++ b/runbot/views/semgrep_rules.xml @@ -79,12 +79,14 @@ Semgrep Rules runbot.semgrep_rule list,form + semgrep-rule Checker Categories runbot.checker_category list,form + checker-category diff --git a/runbot/views/stat_views.xml b/runbot/views/stat_views.xml index 6da213540..1131525d3 100644 --- a/runbot/views/stat_views.xml +++ b/runbot/views/stat_views.xml @@ -37,6 +37,7 @@ Stat regex runbot.build.stat.regex list,form + stat-regex diff --git a/runbot/views/upgrade.xml b/runbot/views/upgrade.xml index 2338aa16e..680b978da 100644 --- a/runbot/views/upgrade.xml +++ b/runbot/views/upgrade.xml @@ -86,12 +86,14 @@ Upgrade Exceptions runbot.upgrade.exception list,form + upgrade-exception Upgrade Regexes runbot.upgrade.regex list,form + upgrade-regex diff --git a/runbot/views/upgrade_matrix_views.xml b/runbot/views/upgrade_matrix_views.xml index 594127b71..a692ea132 100644 --- a/runbot/views/upgrade_matrix_views.xml +++ b/runbot/views/upgrade_matrix_views.xml @@ -71,6 +71,7 @@ Upgrade matrix runbot.upgrade.matrix list,form + upgrade-matrix diff --git a/runbot/views/warning_views.xml b/runbot/views/warning_views.xml index be9a0ba0c..edccdf541 100644 --- a/runbot/views/warning_views.xml +++ b/runbot/views/warning_views.xml @@ -16,6 +16,7 @@ Warnings runbot.warning list + warning diff --git a/runbot_builder/builder.py b/runbot_builder/builder.py index 90637ee64..5ca51d0c8 100755 --- a/runbot_builder/builder.py +++ b/runbot_builder/builder.py @@ -13,6 +13,7 @@ class BuilderClient(RunbotClient): def on_start(self): + self.last_update = datetime(1970, 1, 1) self.last_docker_updates = None if self.host.is_builder: builds_path = self.env['runbot.runbot']._path('build') @@ -30,6 +31,8 @@ def loop_turn(self): self.host._docker_update_images() self.env.cr.commit() if self.host.is_builder: + self.last_update = self.env['runbot.repo'].search([('write_date', '>', self.last_update)])._update_git_config() + self.env.cr.commit() if self.count == 1: # cleanup at second iteration self.env['runbot.runbot']._source_cleanup() self.env.cr.commit() @@ -39,8 +42,6 @@ def loop_turn(self): self.env.cr.commit() self.host._set_psql_conn_count() self.env.cr.commit() - self.env['runbot.repo']._update_git_config() - self.env.cr.commit() self.git_gc() self.env.cr.commit() return self.env['runbot.runbot']._scheduler_loop_turn(self.host) diff --git a/runbot_builder/leader.py b/runbot_builder/leader.py index c39cb8ea5..106d83eb1 100755 --- a/runbot_builder/leader.py +++ b/runbot_builder/leader.py @@ -1,23 +1,27 @@ #!/usr/bin/python3 from tools import RunbotClient, run import logging -import time + +from datetime import datetime + _logger = logging.getLogger(__name__) -class LeaderClient(RunbotClient): # Conductor, Director, Main, Maestro, Lead +class LeaderClient(RunbotClient): def __init__(self, env): self.pull_info_failures = {} + self.last_update = datetime(1970, 1, 1) super().__init__(env) def loop_turn(self): if not self.host.is_leader: _logger.warning('Leader client is not a leader host, skipping loop_turn') return 10 + + self.last_update = self.env['runbot.repo'].search([('write_date', '>', self.last_update)])._update_git_config() + self.env.cr.commit() if self.count == 0: - self.env['runbot.repo']._update_git_config() - self.env.cr.commit() self.git_gc() self.env.cr.commit()
    DateDate (UTC) LevelType Message
    - -
    - + - + + + + + + + + - - - - Build # - - - - - - - - - - - - - : - + + + + Build # + + + - - - - - - - - - - - - - - - - - - - - - -
    -     + + +     		- + @@ -348,17 +321,16 @@ -
    +
    - This error is already . - - - - - () + + This error is already . + + + + () +