diff --git a/runbot/__manifest__.py b/runbot/__manifest__.py index 38db70949..b3589f8dc 100644 --- a/runbot/__manifest__.py +++ b/runbot/__manifest__.py @@ -6,7 +6,7 @@ 'author': "Odoo SA", 'website': "http://runbot.odoo.com", 'category': 'Website', - 'version': '5.16', + 'version': '5.17', 'application': True, 'depends': ['base', 'base_automation', 'website', 'auth_oauth'], 'data': [ @@ -82,6 +82,7 @@ 'runbot/static/lib/fontawesome/css/font-awesome.css', 'runbot/static/src/css/runbot.css', + 'runbot/static/src/js/polyfill_command_api.js', 'runbot/static/lib/jquery/jquery.js', 'runbot/static/lib/bootstrap/js/bootstrap.bundle.js', 'runbot/static/src/js/runbot.js', diff --git a/runbot/common.py b/runbot/common.py index de9f5b333..fe34c2d44 100644 --- a/runbot/common.py +++ b/runbot/common.py @@ -24,6 +24,13 @@ dest_reg = re.compile(r'^\d{5,}-.+$') +try: + from odoo.addons.saas_worker.util import from_role +except ImportError: + def from_role(*_, **__): + return lambda _: None + + def transactioncache(method): @functools.wraps(method) def wrapper(self, *args, **kwargs): @@ -322,9 +329,35 @@ class TestTagsParser: (?:\[(.*)\])? # parameters $''', re.VERBOSE) # [-][tag][/module][:class][.method][[params]] - def __init__(self, test_tags): - parts = re.split(r',(?![^\[]*\])', test_tags) # split on all comma not inside [] (not followed by ]) + def __init__(self, test_tags, keep_escape=True): + parts = [''] + bracket_level = 0 + escape_next = False + for char in test_tags: + if char == ',' and bracket_level == 0: + parts.append('') + continue + + if char == '\\': + if not escape_next: + escape_next = True + if keep_escape: + parts[-1] += '\\' # not as the TagsSelector, we keep the escape character + continue + elif char == '[': + if not escape_next: + bracket_level += 1 + elif char == ']': + if not escape_next: + bracket_level -= 1 + elif not keep_escape and escape_next: # the previous \ was not escaping anything, put it back + parts[-1] += '\\' + + escape_next = False + parts[-1] += char + filter_specs = [t.strip() for t in parts if t.strip()] + self.filter_specs = filter_specs self.exclude = set() self.include = set() self.parameters = OrderedSet() @@ -332,8 +365,7 @@ def __init__(self, test_tags): for filter_spec in filter_specs: match = self.filter_spec_re.match(filter_spec) if not match: - _logger.error('Invalid tag %s', filter_spec) - continue + raise ValueError('Invalid tag %s' % filter_spec) sign, tag, file_path, module, klass, method, parameters = match.groups() is_include = sign != '-' @@ -362,6 +394,7 @@ def __init__(self, test_tags): def test_tags_to_search_domain(self, exclude_error_id=None): search_domains = [] + params_by_spec = dict(self.parameters) for include in self.include: _, test_module, test_class, test_method, file_path = include module_path = file_path or ((test_module or '') + '%') @@ -369,6 +402,10 @@ def test_tags_to_search_domain(self, exclude_error_id=None): test_method = test_method or '%' search_pattern = f'{module_path}:{test_class}.{test_method}' tag_domain = [('canonical_tags', 'like', f'{search_pattern}')] + params = params_by_spec.get(include) + if params: + _sign, parameters = params + tag_domain.append(('canonical_tags', 'like', f'%[{parameters}%]%')) if exclude_error_id: tag_domain.append(('id', '!=', exclude_error_id)) search_domains.append(tag_domain) diff --git a/runbot/container.py b/runbot/container.py index 679ea35f1..2fe340d9d 100644 --- a/runbot/container.py +++ b/runbot/container.py @@ -108,21 +108,22 @@ def get_config(self, starting_config=''): return res.read() -def docker_build(build_dir, image_tag, pull=False): - return _docker_build(build_dir, image_tag, pull) +def docker_build(build_dir, image_tag, pull=False, nocache=False): + return _docker_build(build_dir, image_tag, pull, nocache) -def _docker_build(build_dir, image_tag, pull=False): +def _docker_build(build_dir, image_tag, pull=False, nocache=False): """Build the docker image :param build_dir: the build directory that contains Dockerfile. :param image_tag: name used to tag the resulting docker image + :param nocache: bypass Docker layer cache when True :return: dict """ with DockerManager(image_tag) as dm: last_step = None dm.result['success'] = False # waiting for an image_id - for chunk in dm.consume(dm.docker_client.api.build(path=build_dir, tag=image_tag, rm=True, pull=pull)): + for chunk in dm.consume(dm.docker_client.api.build(path=build_dir, tag=image_tag, rm=True, pull=pull, nocache=nocache)): if 'stream' in chunk: stream = chunk['stream'] if stream.startswith('Step '): @@ -259,7 +260,10 @@ def _docker_run(cmd=False, log_path=False, build_dir=False, container_name=False else: run_cmd = cmd run_cmd = f'cd /data/build;touch start-{container_name};{run_cmd};cd /data/build;touch end-{container_name}' - _logger.info('Docker run command: %s', run_cmd) + run_cmd_repr = str(run_cmd) + if len(run_cmd_repr) > 250: + run_cmd_repr = run_cmd_repr[:250] + '...' + _logger.info('Docker run command: %s', run_cmd_repr) docker_clear_state(container_name, build_dir) # ensure that no state are remaining build_dir = file_path(build_dir) diff --git a/runbot/controllers/badge.py b/runbot/controllers/badge.py index fa6e031ee..662163682 100644 --- a/runbot/controllers/badge.py +++ b/runbot/controllers/badge.py @@ -44,13 +44,9 @@ def badge(self, name, repo_id=False, trigger_id=False, theme='default'): if not builds: state = 'testing' else: - result = builds._result_multi() - if result == 'ok': + state = 'failed' + if all(build.global_result == 'ok' for build in builds): state = 'success' - elif result == 'warn': - state = 'warning' - else: - state = 'failed' etag = request.httprequest.headers.get('If-None-Match') retag = hashlib.md5(state.encode()).hexdigest() diff --git a/runbot/controllers/frontend.py b/runbot/controllers/frontend.py index 1598fda9d..6735fe586 100644 --- a/runbot/controllers/frontend.py +++ b/runbot/controllers/frontend.py @@ -80,7 +80,7 @@ def _pending(self): '/runbot/', '/runbot//search/'], website=True, auth='public', type='http') def bundles(self, project=None, search='', refresh=False, limit=40, has_pr=None, **kwargs): - search = search if len(search) < 60 else search[:60] + search = search if len(search) < 60 else search[:200] env = request.env categories = env['runbot.category'].search([]) projects = self.env['runbot.project'].search([('hidden', '=', False)]) @@ -119,13 +119,11 @@ def bundles(self, project=None, search='', refresh=False, limit=40, has_pr=None, pr_numbers = [] for search_elem in search.split("|"): if search_elem.isnumeric(): - pr_numbers.append(int(search_elem)) + search_domains.append([('branch_ids', 'any', [('name', '=', search_elem)])]) + if ':' in search_elem: + search_domains.append([('branch_ids', 'any', [('pull_head_name', '=', search_elem)])]) operator = '=ilike' if '%' in search_elem else 'ilike' search_domains.append([('name', operator, search_elem)]) - if pr_numbers: - res = request.env['runbot.branch'].search([('name', 'in', pr_numbers)]) - if res: - search_domains.append([('id', 'in', res.mapped('bundle_id').ids)]) search_domain = Domain.OR(search_domains) domain = Domain.AND([domain, search_domain]) @@ -166,7 +164,7 @@ def bundles(self, project=None, search='', refresh=False, limit=40, has_pr=None, '/runbot/bundle//page/', '/runbot/bundle/', ], website=True, auth='public', type='http', sitemap=False) - def bundle(self, bundle=None, page=1, limit=50, **kwargs): + def bundle(self, bundle=None, page=1, limit=50, expand_custom=False, **kwargs): if isinstance(bundle, str): bundle = request.env['runbot.bundle'].search([('name', '=', bundle)], limit=1, order='id') if not bundle: @@ -183,6 +181,7 @@ def bundle(self, bundle=None, page=1, limit=50, **kwargs): ) batchs = request.env['runbot.batch'].search(domain, limit=limit, offset=pager.get('offset', 0), order='id desc') + # compute if we should display the new batch button context = { 'bundle': bundle, 'batchs': batchs, @@ -190,6 +189,8 @@ def bundle(self, bundle=None, page=1, limit=50, **kwargs): 'project': bundle.project_id, 'title': 'Bundle %s' % bundle.name, 'page_info_state': bundle.last_batch._get_global_result(), + 'expand_custom': expand_custom, + 'needs_update': bundle.last_batch and bundle.last_batch.sudo().needs_update(), } return request.render('runbot.bundle', context) @@ -199,7 +200,7 @@ def bundle(self, bundle=None, page=1, limit=50, **kwargs): '/runbot/bundle//force/', ], type='http', auth="user", methods=['GET', 'POST'], csrf=False) def force_bundle(self, bundle, auto_rebase=False, use_base_commits=False, **_post): - if not request.env.user.has_group('runbot.group_runbot_advanced_user') and ':' not in bundle.name: + if not request.env.user.has_group('runbot.group_runbot_advanced_user') and ':' not in bundle.name and not bundle.last_batch.needs_update(): message = "Only users with a specific group can do that. Please contact runbot administrators" raise Forbidden(message) _logger.info('user %s forcing bundle %s', request.env.user.name, bundle.name) # user must be able to read bundle @@ -275,6 +276,7 @@ def resend_status(self, status_id=None, **kwargs): ], type='http', auth="user", methods=['POST'], csrf=False) def build_operations(self, build_id, operation, **post): build = request.env['runbot.build'].sudo().browse(build_id) + build.check_access('read') if operation == 'rebuild': build = build._rebuild() elif operation == 'kill': @@ -289,18 +291,17 @@ def build_operations(self, build_id, operation, **post): '/runbot/batch//build/', ], type='http', auth="public", website=True, sitemap=False) def build(self, build_id, search=None, from_batch=None, **post): - """Events/Logs""" - + build = request.env['runbot.build'].browse(build_id) if from_batch: from_batch = request.env['runbot.batch'].browse(int(from_batch)) - if build_id not in from_batch.with_context(active_test=False).slot_ids.build_id.ids: + if build.top_parent not in from_batch.with_context(active_test=False).slot_ids.build_id and build.create_batch_id != from_batch: # the url may have been forged replacing the build id, redirect to hide the batch return werkzeug.utils.redirect('/runbot/build/%s' % build_id) from_batch = from_batch.with_context(batch=from_batch) Build = request.env['runbot.build'].with_context(batch=from_batch) - build = Build.browse([build_id])[0] + build = Build.browse(build_id) if not build.exists(): return request.not_found() siblings = (build.parent_id.children_ids if build.parent_id else from_batch.slot_ids.build_id if from_batch else build).sorted('id') @@ -322,7 +323,8 @@ def build(self, build_id, search=None, from_batch=None, **post): @route([ '/runbot/build/search', ], website=True, auth='public', type='http', sitemap=False) - def builds(self, **kwargs): + def builds(self, limit=100, **kwargs): + limit = min(int(limit), 1000) domain = [] for key in ('config_id', 'version_id', 'project_id', 'trigger_id', 'create_batch_id.bundle_id', 'create_batch_id'): # allowed params value = kwargs.get(key) @@ -336,10 +338,12 @@ def builds(self, **kwargs): for key in ('description',): if key in kwargs: - domain.append((f'{key}', 'ilike', kwargs.get(key))) + value = kwargs.get(key) + operator = 'ilike' if '%' in value else '=' + domain.append((f'{key}', operator, value)) context = { - 'builds': request.env['runbot.build'].search(domain, limit=100), + 'builds': request.env['runbot.build'].search(domain, limit=limit), } return request.render('runbot.build_search', context) @@ -450,8 +454,8 @@ def build_errors(self, sort=None, page=1, limit=20, **kwargs): 'build_count asc': 'Number seen: Low to High', 'responsible asc': 'Assignee: A - Z', 'responsible desc': 'Assignee: Z - A', - 'module_name asc': 'Module name: A - Z', - 'module_name desc': 'Module name: Z -A', + 'team_id asc': 'Team', + 'name asc': 'Name', } sort_order = sort if sort in sort_order_choices else 'last_seen_date desc' @@ -669,19 +673,40 @@ def parse_log(self, ir_log, **kwargs): request.env['runbot.build.error']._parse_logs(ir_log) return werkzeug.utils.redirect('/runbot/build/%s' % ir_log.build_id.id) - @route(['/runbot/bundle/toggle_no_build//'], type='http', auth='user', sitemap=False) - def toggle_no_build(self, bundle_id, value, **kwargs): - if not request.env.user.has_group('base.group_user'): - return 'Forbidden' - bundle = request.env['runbot.bundle'].browse(bundle_id).exists() - if bundle.sticky or bundle.is_base: - return 'Forbidden' - if bundle.project_id.tmp_prefix and bundle.name.startswith(bundle.project_id.tmp_prefix): - return 'Forbidden' - bundle.sudo().no_build = bool(value) - _logger.info('Bundle %s no_build set to %s by %s', bundle.name, bool(value), request.env.user.name) + @route(['/runbot/bundle//triggers/'], type='http', auth='user', sitemap=False) + def configure_bundle_triggers(self, bundle_id, action, expand_custom=False, **kwargs): + if not request.env.user.has_group('runbot.group_user'): + raise NotFound() + + bundle = request.env['runbot.bundle'].browse(bundle_id) + if bundle.is_base or bundle.is_staging: + raise NotFound() + if action == 'disable_all': + bundle.sudo()._configure_custom_trigger_start_mode('disabled') + elif action == 'force_all': + bundle.sudo()._configure_custom_trigger_start_mode('force') + elif action == 'auto_all': + bundle.sudo()._configure_custom_trigger_start_mode('auto') + elif action == 'light_all': + bundle.sudo()._configure_custom_trigger_start_mode('light') + else: + raise NotFound() + if expand_custom: + return werkzeug.utils.redirect(f'/runbot/bundle/{bundle_id}?expand_custom=1') return werkzeug.utils.redirect(f'/runbot/bundle/{bundle_id}') + @route(['/runbot/trigger_custom//set_mode/'], type='http', auth='user', sitemap=False) + def configure_custom_trigger(self, trigger_custom_id, mode, **kwargs): + if not request.env.user.has_group('runbot.group_user'): + raise NotFound() + trigger_custom = request.env['runbot.bundle.trigger.custom'].browse(trigger_custom_id) + bundle = trigger_custom.bundle_id + if bundle.is_base or bundle.is_staging: + raise NotFound() + + trigger_custom.sudo().start_mode = mode + return werkzeug.utils.redirect(f'/runbot/bundle/{trigger_custom.bundle_id.id}?expand_custom=1') + @route(['/runbot/trigger/report/'], type='http', auth='user', website=True, sitemap=False) def report_view(self, trigger_id=None, **kwargs): return request.render("runbot.trigger_report", { diff --git a/runbot/controllers/hook.py b/runbot/controllers/hook.py index d46c7f699..4bd36a20d 100644 --- a/runbot/controllers/hook.py +++ b/runbot/controllers/hook.py @@ -4,8 +4,9 @@ import json import logging -from odoo import http +from odoo import http, fields from odoo.http import request +from ..common import from_role _logger = logging.getLogger(__name__) @@ -50,3 +51,18 @@ def hook(self, remote_id=None, **_post): branch = request.env['runbot.branch'].sudo().search([('remote_id', '=', remote.id), ('name', '=', branch_ref)]) branch.alive = False return "" + + @from_role('mergebot', signed=True) + @http.route(['/runbot/request_ci'], type='http', methods=["POST"], auth="public", website=True, csrf=False, sitemap=False) + def force_ci(self): + pull_request_names = request.get_json_data().get('pull_requests', []) + pull_domains = [] + for pull_request_names in pull_request_names: + remote_short_name, name = pull_request_names.split('#') + owner, repo_name = remote_short_name.split('/') + pull_domains.append([('remote_id.owner', '=', owner), ('remote_id.repo_name', '=', repo_name), ('name', '=', name)]) + pull_domains = fields.Domain.OR(pull_domains) + pull_requests = request.env['runbot.branch'].sudo().search([('is_pr', '=', True)] + pull_domains) + bundles = pull_requests.bundle_id + _logger.info('Received CI request for bundles: %s', bundles.mapped('name')) + bundles._force_ci() diff --git a/runbot/data/dockerfile_data.xml b/runbot/data/dockerfile_data.xml index 806d9d8fc..720a70bf6 100644 --- a/runbot/data/dockerfile_data.xml +++ b/runbot/data/dockerfile_data.xml @@ -71,7 +71,7 @@ reference_layer Install python debian packages - publicsuffix python3 flake8 python3-dbfread python3-dev python3-gevent python3-pip python3-setuptools python3-wheel python3-markdown python3-mock python3-phonenumbers python3-websocket python3-google-auth libpq-dev pylint python3-jwt python3-asn1crypto python3-html2text python3-suds python3-xmlsec python3-markdown2 python3-aiosmtpd + publicsuffix python3 flake8 python3-dbfread python3-dev python3-gevent python3-pip python3-setuptools python3-wheel python3-markdown python3-mock python3-phonenumbers python3-websocket python3-google-auth libpq-dev pylint python3-jwt python3-asn1crypto python3-html2text python3-suds python3-xmlsec python3-markdown2 python3-aiosmtpd python3-paramiko @@ -137,7 +137,7 @@ RUN curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc -o /etc/apt/tru template Install chrome - + RUN curl -sSL https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_{chrome_version}_amd64.deb -o /tmp/chrome.deb \ && apt-get update \ && apt-get -y install --no-install-recommends /tmp/chrome.deb \ diff --git a/runbot/documentation/dynamic_config.md b/runbot/documentation/dynamic_config.md index 1d647758d..9282bd816 100644 --- a/runbot/documentation/dynamic_config.md +++ b/runbot/documentation/dynamic_config.md @@ -344,21 +344,43 @@ Filters are a way to transform dynamic values before using them. They are define For example, to transform a module filter into test tags: +#### filter_all_modules, make_module_test_tags + ```json {"test_tags": "-at_install,{{test_module_filter|filter_all_modules|make_module_test_tags}}", ``` In this example, the `filter_all_modules` filters will first transform the `test_module_filter` variable (which is a module filter) into a list of modules, and then the `make_module_test_tags` filters will transform this list of modules into test tags by prepending each module with a `/` to indicate that we want to run all tests from these modules. -Note that `filter_all_modules` is actually equivalent to `filter_default_modules`, but prepending a `*` at the begining of the filter. +#### filter_default_modules + +`filter_all_modules` is actually equivalent to `filter_default_modules`, but prepending a `*` at the begining of the filter. Without that a runbot defined filter is applied, returning a default list of modules per repo. `*,mail -> !web|filter_default_modules` is the same as `mail -> !web|filter_all_modules` + +#### prepend, append In some case we also want to combine the test-tags module with another tag or test method, this can be done using prepend and append `"{{-*,web*|filter_all_modules|make_module_test_tags|append('.test_method')}}` `{{-*,web*|filter_all_modules|make_module_test_tags|prepend('custom_tag')}}` -It is also possible to filter modules based on the one modified in the current bundle. +#### modified_modules + +It is possible to filter modules based on the one modified in the current bundle. `{{*|filter_all_modules|modified_modules}}"` + +#### select_existing_modules + +`select_existing_modules` is equivalent to `filter_default_modules` but with a -* at the beginning of the filter, meaning that we start with an empty selection and only add modules that are explicitly selected. + +This is a solution to keep only existing modules from a specific list, when we are not sure modules exists: +`{{*|filter_all_modules|modified_modules|prepend('test_')|select_existing_modules|make_module_test_tags}}` + +- `*|filter_all_modules` will select all existing modules +- `|modified_modules` will only keep the modified ones +- `prepend('test_')` will prepend test_ to have the test equivalent name of the modified modules (mail-> test_mail, base -> test_base) +- `select_existing_modules` will only keep modules that exists (test_mail) +- `make_module_test_tags` make the module test tags by prepending a / to each module. + diff --git a/runbot/migrations/19.0.5.17/pre-migration.py b/runbot/migrations/19.0.5.17/pre-migration.py new file mode 100644 index 000000000..c21faba28 --- /dev/null +++ b/runbot/migrations/19.0.5.17/pre-migration.py @@ -0,0 +1,3 @@ +def migrate(cr, version): + cr.execute("""UPDATE runbot_build set local_result = 'killed' where local_result = 'manually_killed'""") + cr.execute("""UPDATE runbot_build set global_result = 'killed' where global_result = 'manually_killed'""") diff --git a/runbot/models/batch.py b/runbot/models/batch.py index 84e2bb707..5f03d4973 100644 --- a/runbot/models/batch.py +++ b/runbot/models/batch.py @@ -15,6 +15,7 @@ class Batch(models.Model): last_update = fields.Datetime('Last ref update') bundle_id = fields.Many2one('runbot.bundle', required=True, index=True, ondelete='cascade') + build_all = fields.Boolean('Force all triggers') commit_link_ids = fields.Many2many('runbot.commit.link') commit_ids = fields.Many2many('runbot.commit', compute='_compute_commit_ids') slot_ids = fields.One2many('runbot.batch.slot', 'batch_id') @@ -153,7 +154,7 @@ def _create_build(self, params, slot): build = self.env['runbot.build'].search(domain, limit=1, order='id desc') link_type = 'matched' - killed_states = ('skipped', 'killed', 'manually_killed') + killed_states = ('skipped', 'killed') if build and build.local_result not in killed_states and build.global_result not in killed_states: if build.killable: build.killable = False @@ -185,8 +186,16 @@ def _prepare(self, auto_rebase=False, use_base_commits=False): _logger.info('Preparing batch %s', self.id) priority_offset = self.bundle_id.priority_offset - if not priority_offset and self.bundle_id.branch_ids.forwardport_of_id and self.bundle_id.last_batchs == self: # this is the only batch of a forwardported pr. - priority_offset = - 3600 * 5 + if not priority_offset and self.bundle_id.branch_ids.forwardport_of_id: + if self.bundle_id.last_batchs == self: # this is the first batch of a forwardported pr. + priority_offset = - 3600 * 5 + if len(self.bundle_id.last_batchs) <= 2: + # for normal pr, mergebot will request all ci on r+ if needed, for forward port, we need to ensure they are all created or the chain could be blocked + # In some rare cases the branch can create a batch and the pr a second one (github issues) + # in this case, forwardport_of_id is falsy for the fisrt, and self.bundle_id.last_batchs == self is falsy for the second + # we still want to build all for the second one, relaxing the condition to len(self.bundle_id.last_batchs) <= 2 + # It also means that fixing a conflict or an error will build all on the first push, but not on second (looks almost like a feature) + self.build_all = True self.priority_level = int(self.create_date.timestamp() - priority_offset) if use_base_commits: self._warning('This batch will use base commits instead of bundle commits') @@ -382,7 +391,22 @@ def _fill_missing(branch_commits, match_type): self._warning('Missing commit for repo %s for trigger %s', (trigger_repos & missing_repos).mapped('name'), trigger.name) continue # in any case, search for an existing build - config = trigger_custom.config_id or trigger.config_id + config = trigger.config_id + if not trigger_custom and trigger.light_config_id and not bundle.build_all and not self.build_all and not bundle.is_staging and not bundle.is_base: + if (project.use_light_default + or + project.use_light_draft and any(branch.draft for branch in self.bundle_id.branch_ids) + or + project.use_light_no_pr and not any(branch.is_pr for branch in self.bundle_id.branch_ids) + ): + config = trigger.light_config_id + + if trigger_custom.config_id: + config = trigger_custom.config_id + elif trigger_custom.start_mode == 'light' and trigger.light_config_id: + config = trigger.light_config_id + + extra_params = trigger_custom.extra_params or '' config_data = dict(trigger.config_data or {}) | dict(trigger_custom.config_data or {}) trigger_commit_link_by_repos = commit_link_by_repos @@ -402,7 +426,7 @@ def _fill_missing(branch_commits, match_type): 'modules': bundle.modules, 'dockerfile_id': dockerfile_id, 'create_batch_id': self.id, - 'used_custom_trigger': bool(trigger_custom), + 'used_custom_trigger': bool(trigger_custom.config_id or trigger_custom.extra_params or trigger_custom.config_data or trigger_custom.use_base_commits), } params = self.env['runbot.build.params'].create(params_value) @@ -440,16 +464,22 @@ def _start_builds(self): is_dev = not bundle.is_staging and not bundle.is_base for trigger in self.slot_ids.trigger_id: enable_on_bundle = (trigger.on_staging and bundle.is_staging) or (trigger.on_base and bundle.is_base) or (trigger.on_dev and is_dev) - if ((trigger.repo_ids & bundle_repos) or bundle.build_all or bundle.sticky) and enable_on_bundle: + common_repo = (trigger.repo_ids & bundle_repos) + if self.build_all and not common_repo: + common_repo = (trigger.dependency_ids & bundle_repos) + if (common_repo or bundle.build_all or bundle.sticky) and enable_on_bundle: should_start_triggers_ids.add(trigger.id) + disabled_triggers = self.bundle_id.all_trigger_custom_ids.filtered(lambda tc: tc.start_mode == 'disabled').trigger_id for slot in self.slot_ids: if slot.build_id: continue trigger = slot.trigger_id - if trigger.starts_after_ids - success_trigger: # some required triggers are missing - continue trigger_custom = trigger_customs.get(trigger, self.env['runbot.bundle.trigger.custom']) + missing_triggers = trigger.starts_after_ids - success_trigger + if missing_triggers: + if not trigger_custom or (missing_triggers - disabled_triggers): + continue force_trigger = trigger_custom and trigger_custom.start_mode == 'force' skip_trigger = (trigger_custom and trigger_custom.start_mode == 'disabled') or trigger.manual should_start = slot.trigger_id.id in should_start_triggers_ids @@ -514,6 +544,23 @@ def _log(self, message, *args, level='INFO'): 'level': level, }) + def needs_update(self): + bundle = self.bundle_id + custom_trigger_per_trigger = {ct.trigger_id: ct for ct in bundle.trigger_custom_ids} + for slot in self.slot_ids: + trigger = slot.trigger_id + custom_trigger = custom_trigger_per_trigger.get(trigger) + if not custom_trigger: + continue + expected_config = trigger.config_id + if custom_trigger.config_id: + expected_config = custom_trigger.config_id + elif trigger.light_config_id and custom_trigger.start_mode == 'light': + expected_config = trigger.light_config_id + if slot.params_id.config_id != expected_config: + return True + return False + class BatchLog(models.Model): _name = 'runbot.batch.log' _description = 'Batch log' diff --git a/runbot/models/branch.py b/runbot/models/branch.py index f2cf6edf0..8deb9ec5c 100644 --- a/runbot/models/branch.py +++ b/runbot/models/branch.py @@ -101,7 +101,8 @@ def _compute_reference_name(self): # branch.reference_name = '%s~%s' % (branch.pull_head_name, branch.name) else: reference_name = branch.name - forced_version = branch.remote_id.repo_id.single_version # we don't add a depend on repo.single_version to avoid mass recompute of existing branches + repo = branch.remote_id.repo_id + forced_version = repo.enforce_version and repo.single_version # we don't add a depend on repo.single_version to avoid mass recompute of existing branches if forced_version and not (reference_name.startswith(f'{forced_version.name}-') or reference_name == forced_version.name): reference_name = f'{forced_version.name}---{reference_name}' branch.reference_name = reference_name diff --git a/runbot/models/build.py b/runbot/models/build.py index 65c884ab8..8ba62d047 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +import ast import datetime import getpass import hashlib @@ -10,27 +10,41 @@ import shutil import time import uuid - from collections import defaultdict -from dateutil import parser from pathlib import Path + +from dateutil import parser +from markupsafe import Markup from psycopg2 import sql from psycopg2.extensions import TransactionRollbackError -from ..common import dt2time, now, grep, local_pgadmin_cursor, dest_reg, os, list_local_dbs, pseudo_markdown, RunbotException, findall, sanitize, markdown_escape, tail -from ..container import docker_stop, docker_state, Command, docker_run, docker_pull -from ..fields import JsonDictField - -from odoo import models, fields, api - +from odoo import api, fields, models from odoo.exceptions import ValidationError -from odoo.tools import file_open, file_path +from odoo.tools import file_open, file_path, html_escape from odoo.tools.safe_eval import safe_eval +from ..common import ( + RunbotException, + dest_reg, + dt2time, + findall, + grep, + list_local_dbs, + local_pgadmin_cursor, + markdown_escape, + now, + os, + pseudo_markdown, + sanitize, + tail, + transactioncache, +) +from ..container import Command, docker_pull, docker_run, docker_state, docker_stop +from ..fields import JsonDictField _logger = logging.getLogger(__name__) -result_order = ['ok', 'warn', 'ko', 'skipped', 'killed', 'manually_killed'] +result_order = ['ok', 'warn', 'ko', 'skipped', 'killed', 'manually_killed'] # TODO remove manually_killed state_order = ['pending', 'testing', 'waiting', 'running', 'done'] COPY_WHITELIST = [ @@ -61,7 +75,6 @@ def remove_readonly(func, path_str, exinfo): def make_selection(array): return [(elem, elem.replace('_', ' ').capitalize()) if isinstance(elem, str) else elem for elem in array] - class BuildParameters(models.Model): _name = 'runbot.build.params' _description = "Build parameters" @@ -285,6 +298,8 @@ class BuildResult(models.Model): local_result = fields.Selection(make_selection(result_order), string='Build Result', default='ok') requested_action = fields.Selection([('wake_up', 'To wake up'), ('deathrow', 'To kill')], string='Action requested', index=True) + to_kill = fields.Boolean('To kill', compute='_compute_to_kill') + message_ids = fields.One2many('runbot.host.message', 'build_id', string='Messages') # web infos host = fields.Char('Host name') host_id = fields.Many2one('runbot.host', string="Host", compute='_compute_host_id') @@ -302,12 +317,13 @@ class BuildResult(models.Model): active_step = fields.Many2one('runbot.build.config.step', 'Active step') job = fields.Char('Active step display name', compute='_compute_job') dynamic_active_step_index = fields.Integer('Dynamic active step index') - + cpu_limit = fields.Integer('CPU limit for the current running docker') job_start = fields.Datetime('Job start') job_end = fields.Datetime('Job end') build_start = fields.Datetime('Build start') build_end = fields.Datetime('Build end') docker_start = fields.Datetime('Docker start') + docker_time = fields.Integer('Docker time', default=0, help='Accumulated time spent in Docker containers') job_time = fields.Integer(compute='_compute_job_time', string='Job time') build_time = fields.Integer(compute='_compute_build_time', string='Build time') wait_time = fields.Integer(compute='_compute_wait_time', string='Wait time') @@ -336,6 +352,7 @@ class BuildResult(models.Model): ancestors = fields.Many2many('runbot.build', compute='_compute_ancestors') # should we add a has children stored boolean? children_ids = fields.One2many('runbot.build', 'parent_id') + all_children_ids = fields.One2many('runbot.build', compute='_compute_all_children_ids') # config of top_build is inherithed from params, but subbuild will have different configs @@ -344,7 +361,7 @@ class BuildResult(models.Model): build_url = fields.Char('Build url', compute='_compute_build_url', store=False) build_error_link_ids = fields.One2many('runbot.build.error.link', 'build_id') build_error_ids = fields.Many2many('runbot.build.error', compute='_compute_build_error_ids', string='Errors') - keep_running = fields.Boolean('Keep running', help='Keep running', index=True) + gc_running_date = fields.Date('GC Running Date', help='Running build cannot be killed before this date', index='btree_not_null') log_counter = fields.Integer('Log Lines counter', default=100) slot_ids = fields.One2many('runbot.batch.slot', 'build_id') @@ -382,6 +399,11 @@ def _compute_global_state(self): else: record.global_state = record.local_state + @api.depends('message_ids') + def _compute_to_kill(self): + for record in self: + record.to_kill = any(message.message == 'kill' for message in record.message_ids) + @api.depends('gc_delay', 'job_end') def _compute_gc_date(self): icp = self.env['ir.config_parameter'].sudo() @@ -406,6 +428,10 @@ def _compute_ancestors(self): for build in self: build.ancestors = self.browse([int(b) for b in build.parent_path.split('/') if b]) + def _compute_all_children_ids(self): + for build in self: + build.all_children_ids = self.search([('parent_path', '=like', build.parent_path + '%')]) + def _get_youngest_state(self, states): index = min([self._get_state_score(state) for state in states]) return state_order[index] @@ -531,17 +557,6 @@ def _add_child(self, param_values, orphan=False, description=False, additionnal_ **build_values, }) - def _result_multi(self): - if all(build.global_result == 'ok' or not build.global_result for build in self): - return 'ok' - if any(build.global_result in ('skipped', 'killed', 'manually_killed') for build in self): - return 'killed' - if any(build.global_result == 'ko' for build in self): - return 'ko' - if any(build.global_result == 'warning' for build in self): - return 'warning' - return 'ko' # ? - @api.depends('params_id.version_id.name') def _compute_dest(self): for build in self: @@ -619,7 +634,7 @@ def _rebuild(self, message=None): # TODO don't rebuild if there is a more recent build for this params? values = { 'params_id': self.params_id.id, - 'build_type': 'rebuild', + 'build_type': 'rebuild' if self.build_type == "normal" else self.build_type, } if self.keep_host: values['host'] = self.host @@ -814,11 +829,13 @@ def _init_pendings(self): def _process_requested_actions(self): self.ensure_one() build = self + # TODO remove, replaced by queue if build.requested_action == 'deathrow': result = None if build.local_state != 'running' and build.global_result not in ('warn', 'ko'): - result = 'manually_killed' + result = 'killed' build._kill(result=result) + build.requested_action = False return if build.requested_action == 'wake_up': @@ -869,7 +886,8 @@ def _schedule(self): else: _docker_state = docker_state(build._get_docker_name(), build._path()) if _docker_state == 'RUNNING': - timeout = min(build.active_step.cpu_limit, int(icp.get_param('runbot.runbot_timeout', default=10000))) + build_limit = build.cpu_limit or build.active_step.cpu_limit + timeout = min(build_limit, int(icp.get_param('runbot.runbot_timeout', default=10000))) if build.local_state != 'running' and build.job_time > timeout: build.active_step._make_stats(build) build._log('_schedule', '%s time exceeded (%ss)' % (build.active_step._get_display_name(self) if build.active_step else "?", build.job_time)) @@ -891,7 +909,9 @@ def _schedule(self): if self.env['runbot.host']._fetch_local_logs(build_ids=build.ids): return True # avoid to make results with remaining logs # No job running, make result and select next job - + if build.docker_start: + docker_duration = int(time.time() - dt2time(build.docker_start)) + build.docker_time += docker_duration build.job_end = now() build.docker_start = False # make result of previous job @@ -1021,7 +1041,9 @@ def _docker_run(self, step, cmd=None, ro_volumes=None, env_variables=None, **kwa containers_memory_limit = self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_containers_memory', 0) if containers_memory_limit and 'memory' not in kwargs: - kwargs['memory'] = int(float(containers_memory_limit) * 1024 ** 3) + memory_limit_factor = float(self.params_id.config_data.get('memory_limit_factor', 1)) + containers_memory_limit = int(float(containers_memory_limit) * 1024 ** 3) * memory_limit_factor + kwargs['memory'] = containers_memory_limit self.docker_start = now() if self.job_start: @@ -1032,6 +1054,8 @@ def _docker_run(self, step, cmd=None, ro_volumes=None, env_variables=None, **kwa starting_config = self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_default_odoorc') if isinstance(cmd, Command): rc_content = cmd.get_config(starting_config=starting_config) + if step.check_exit_status: + cmd.finals = [['echo', r'$?', '>', f'/data/build/logs/{step.sanitized_name(self)}_exit_status.txt']] + cmd.finals else: rc_content = starting_config self._write_file('.odoorc', rc_content) @@ -1045,6 +1069,7 @@ def _docker_run(self, step, cmd=None, ro_volumes=None, env_variables=None, **kwa self.env.flush_all() env_variables = env_variables or [] env_variables.append('ODOO_RUNBOT=1') + self.cpu_limit = kwargs.get('cpu_limit') def start_docker(): docker_run( cmd=cmd, @@ -1091,25 +1116,28 @@ def _checkout(self): return exports + def _list_available_modules(self): + for commit in self.env.context.get('defined_commit_ids') or self.params_id.commit_ids: + for (addons_path, module, manifest_file_name) in commit._list_available_modules(): + yield commit, addons_path, module, manifest_file_name + def _get_available_modules(self): all_modules = dict() available_modules = defaultdict(list) # repo_modules = [] - for commit in self.env.context.get('defined_commit_ids') or self.params_id.commit_ids: - for (addons_path, module, manifest_file_name) in commit._get_available_modules(): - if module in all_modules: - self._log( - 'Building environment', - '%s is a duplicated modules (found in "%s", already defined in %s)' % ( - module, - commit._source_path(addons_path, module, manifest_file_name), - all_modules[module]._source_path(addons_path, module, manifest_file_name)), - level='WARNING', - ) - else: - available_modules[commit.repo_id].append(module) - all_modules[module] = commit - # return repo_modules, available_modules + for commit, addons_path, module, manifest_file_name in self._list_available_modules(): + if module in all_modules: + self._log( + 'Building environment', + '%s is a duplicated modules (found in "%s", already defined in %s)' % ( + module, + commit._source_path(addons_path, module, manifest_file_name), + all_modules[module]._source_path(addons_path, module, manifest_file_name)), + level='WARNING', + ) + else: + available_modules[commit.repo_id].append(module) + all_modules[module] = commit return available_modules def _get_modules_to_test(self, modules_patterns=''): @@ -1120,6 +1148,49 @@ def _get_modules_to_test(self, modules_patterns=''): modules_patterns = (modules_patterns or '').split(',') return trigger._filter_modules_to_test(modules, params_patterns + modules_patterns) # we may switch params_patterns and modules_patterns order + @transactioncache + def _dependency_graph(self): + dependency_graph = defaultdict(set) + dependant_graph = defaultdict(set) + for commit in self.env.context.get('defined_commit_ids') or self.params_id.commit_ids: + file_paths = [] + modules = [] + for (addons_path, module, manifest_file_name) in commit._list_available_modules(): + file_paths.append(os.path.join(addons_path, module, manifest_file_name)) + modules.append(module) + contents = commit._git_show_files(file_paths) + for module, manifest in zip(modules, contents): + manifest_content = ast.literal_eval(manifest) + depends = manifest_content.get('depends', []) + if not depends and module != 'base': + depends = ['base'] + for dep in depends: + dependency_graph[module].add(dep) + dependant_graph[dep].add(module) + return dependency_graph, dependant_graph + + def search_modules_graph(self, modules, graph, depth=None): + def search(modules, depth=None, visited=None): + visited = visited or set() + modules = set(modules) - visited + visited |= modules + dependencies = set(modules) + if depth == 0 or not modules: + return dependencies + for module in modules: + dependencies |= search(graph[module], depth - 1 if depth is not None else None, visited) + return dependencies + return sorted(search(modules, depth)) + + def _get_modules_dependencies(self, modules, depth=None): + self.ensure_one() + dependency_graph, _ = self._dependency_graph() + return self.search_modules_graph(modules, dependency_graph, depth) + + def _get_dependant_modules(self, modules, depth=None): + _, dependant_graph = self._dependency_graph() + return self.search_modules_graph(modules, dependant_graph, depth) + def _local_pg_dropdb(self, dbname): msg = '' try: @@ -1174,7 +1245,7 @@ def truncate(message, maxlenght=300000): 'line': '0', }) - def _kill(self, result=None): + def _kill(self, result='killed'): host_name = self.env['runbot.host']._get_current_name() self.ensure_one() build = self @@ -1182,30 +1253,38 @@ def _kill(self, result=None): return build._log('kill', 'Kill build %s' % build.dest) docker_stop(build._get_docker_name(), build._path()) - v = {'local_state': 'done', 'requested_action': False, 'active_step': False, 'job_end': now()} + build.local_state = 'done' + build.active_step = False + build.job_end = now() if not build.build_end: - v['build_end'] = now() + build.build_end = now() if result: - v['local_result'] = result - build.write(v) - - def _ask_kill(self, lock=True, message=None): - # if build remains in same bundle, it's ok like that - # if build can be cross bundle, need to check number of ref to build - if lock: - self.env.cr.execute("""SELECT id FROM runbot_build WHERE parent_path like %s FOR UPDATE""", ['%s%%' % self.parent_path]) + build.local_result = result + + def _ask_kill(self, message=None): self.ensure_one() user = self.env.user uid = user.id build = self message = message or 'Killing build %s, requested by %s (user #%s)' % (build.dest, user.name, uid) build._log('_ask_kill', message) - if build.local_state == 'pending': - build._skip() - elif build.local_state in ['testing', 'running']: - build.requested_action = 'deathrow' - for child in build.children_ids: - child._ask_kill(lock=False) + + self.env.cr.execute("""SELECT id, local_state FROM runbot_build WHERE parent_path like %s""", ['%s%%' % self.parent_path]) + builds = self.browse([b[0] for b in self.env.cr.fetchall()]) + pending = builds.filtered(lambda b: b.local_state == 'pending') + killable = builds.filtered(lambda b: b.local_state in ('running', 'testing')) + if pending: + pending.local_state = 'done' + pending.local_result = 'killed' + pending.flush_recordset() # faster concurrent error or lock row + + values = [{ + 'host_id': b.host_id.id, + 'build_id': b.id, + 'message': 'kill', + } for b in killable] + + self.env['runbot.host.message'].sudo().create(values) def _wake_up(self): user = self.env.user @@ -1242,19 +1321,24 @@ def _modified_files(self, commit_link_links=None): commit_link_links = self.params_id.commit_link_ids for commit_link in commit_link_links: commit = commit_link.commit_id - modified = commit.repo_id._git(['diff', '--name-only', '%s..%s' % (commit_link.merge_base_commit_id.name, commit.name)]) + commit._fetch() + modified = commit.repo_id._git(['diff', '--name-only', '%s..%s' % (commit_link.merge_base_commit_id.tree_hash, commit.tree_hash)]) if modified: files = [os.sep.join([self._docker_source_folder(commit), file]) for file in modified.split('\n') if file] modified_files[commit_link] = files return modified_files - def _modified_modules(self, commit_link_links=None): + def _modified_modules(self, commit_link_links=None, defaults=None): modified_files = self._modified_files(commit_link_links) modified_modules = set() for commit_link, files in modified_files.items(): commit = commit_link.commit_id for file in files: - modified_modules.add(commit.repo_id._get_module(file)) + module = commit.repo_id._get_module(file) + if module: + modified_modules.add(module) + elif defaults: + modified_modules |= set(defaults) return modified_modules def _get_upgrade_path(self): @@ -1455,10 +1539,62 @@ def _get_color_class(self): if self.global_result == 'ok': return 'success' - if self.global_result in ('skipped', 'killed', 'manually_killed'): + if self.global_result in ('skipped', 'killed'): return 'secondary' return 'default' + def _get_file_url(self, path, line=None): + repo_name = path.replace('/data/build/', '').split('/')[0] + for commit_link in self.params_id.commit_link_ids: + if commit_link.commit_id.repo_id.name == repo_name: + repo_base_url = commit_link.branch_id.remote_id.base_url + commit_hash = commit_link.commit_id.name + path = path.replace('/data/build/%s/' % repo_name, '') + url = f'https://{repo_base_url}/blob/{commit_hash}/{path}' + if line: + url = f'{url}#L{line}' + return url + return '' + + def _format_message(self, log): + text = log.message + if not "\n" in text and 'in: /data/build/' in text: + parts = text.split('in: /data/build/') + text = parts[0] + url = f'http://{self.host}/runbot/static/build/{self.dest}/{parts[-1]}' + template = Markup('%s') + return template % (url, text) + text = text.strip('\n') + text = html_escape(text) + + def get_link(match): + path = match.group(1) + line = match.group(2) + url = self._get_file_url(path, line) + if url: + if line: + return Markup('%s", line %s') % (url, path, line) + return Markup('%s') % (url, path) + return match.group(0) + regex = r''' + (/data/build/[\w\-\./]+\.(?:py|xml|js|css)) # Path in /data/build ending with a common extension + (?: + \&\#34;,\sline\s(\d+) # Optional line number (escaped quote) + )? + ''' + text = Markup(re.sub(regex, get_link, text, flags=re.VERBOSE)) + + return text + + def _log_details(self, log): + title = f"Logger: {log.name}\nFunc: {log.func}" + test_data = log.metadata.dict.get('test') + if test_data: + title += '\n' + for test_line in test_data: + title += f'\n{test_line}: {test_data[test_line]}' + return title + def _github_status(self): """Notify github of failed/successful builds""" for build in self: @@ -1470,14 +1606,18 @@ def _github_status(self): build.parent_id._github_status() else: trigger = build.params_id.trigger_id - if not trigger.ci_context: + ci_context = trigger.ci_context + if not ci_context: continue desc = trigger.ci_description or " (runtime %ss)" % (build.job_time,) if build.params_id.used_custom_trigger: - state = 'error' + ci_context += " (custom)" desc = "This build used custom config. Remove custom trigger to restore default ci" - elif build.global_result in ('ko', 'warn'): + if build.params_id.config_id == build.trigger_id.light_config_id: + ci_context += " (light)" + desc = "This build used a light config. Enable default build configuration to restore default ci" + if build.global_result in ('ko', 'warn'): state = 'error' elif build.global_state in ('pending', 'testing'): state = 'pending' @@ -1530,7 +1670,7 @@ def _github_status(self): else: target_url = f"{self.get_base_url()}/runbot/build/{build.id}" - commit._github_status(build, trigger.ci_context, state, target_url, desc, ci_strategy=trigger.ci_strategy) + commit._github_status(build, ci_context, state, target_url, desc, ci_strategy=trigger.ci_strategy) def _parse_config(self): return set(findall(self._server("tools/config.py"), r'--[\w-]+', )) diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py index c7bc622a0..500252ddf 100644 --- a/runbot/models/build_config.py +++ b/runbot/models/build_config.py @@ -1,35 +1,46 @@ import base64 +import fnmatch import glob import json import logging -import fnmatch -import psutil import re import shlex import time -from unidiff import PatchSet -from ..common import now, grep, time2str, rfind, s2human, os, RunbotException, ReProxy, markdown_escape -from ..container import docker_get_gateway_ip, Command -from odoo import models, fields, api, tools -from odoo.exceptions import UserError, ValidationError -from odoo.tools.misc import file_open -from odoo.tools.safe_eval import safe_eval, test_python_expr, _SAFE_OPCODES, to_opcodes -# adding some additionnal optcode to safe_eval. This is not 100% needed and won't be done in standard but will help -# to simplify some python step by wraping the content in a function to allow return statement and get closer to other -# steps +import psutil +import requests +from unidiff import VERSION, PatchSet, patch +from odoo import api, fields, models, tools +from odoo.exceptions import UserError, ValidationError +from odoo.tools.safe_eval import _SAFE_OPCODES, safe_eval, test_python_expr, to_opcodes + +from ..common import ( + ReProxy, + RunbotException, + TestTagsParser, + grep, + markdown_escape, + now, + os, + rfind, + s2human, + time2str, +) +from ..container import Command, docker_get_gateway_ip # There is an issue in unidiff 0.7.3 fixed in 0.7.4 # https://github.com/matiasb/python-unidiff/commit/a3faffc54e5aacaee3ded4565c534482d5cc3465 # Since the unidiff packaged version in noble is 0.7.3 # patching it looks like the easiest solution -from unidiff import patch, VERSION if VERSION == '0.7.3': patch.RE_DIFF_GIT_DELETED_FILE = re.compile(r'^deleted file mode \d+$') patch.RE_DIFF_GIT_NEW_FILE = re.compile(r'^new file mode \d+$') +# adding some additionnal optcode to safe_eval. This is not 100% needed and won't be done in standard but will help +# to simplify some python step by wraping the content in a function to allow return statement and get closer to other +# steps _SAFE_OPCODES |= set(to_opcodes(['LOAD_DEREF', 'STORE_DEREF', 'LOAD_CLOSURE', 'MAKE_CELL', 'COPY_FREE_VARS'])) @@ -46,28 +57,41 @@ def filter_all_modules(selector, build, dynamic_vars): return filter_default_modules(selector, build, dynamic_vars) +def get_dependencies(modules, build, dynamic_vars, depth=None): + depth = int(depth) if depth else None + modules = modules.split(',') + dependant = set(build._get_modules_dependencies(modules, depth)) - set(modules) + return ','.join(sorted(dependant)) + + +def get_dependant(modules, build, dynamic_vars, depth=None): + depth = int(depth) if depth else None + modules = modules.split(',') + dependant = set(build._get_dependant_modules(modules, depth)) - set(modules) + return ','.join(sorted(dependant)) + + def filter_default_modules(selector, build, dynamic_vars): - build._checkout() # we need to ensure source are exported before _get_modules_to_test modules = build._get_modules_to_test(selector) return ','.join(modules) -def keep_modified_modules(modules, build, dynamic_vars): +def select_existing_modules(selector, build, dynamic_vars): + selector = f'-*,{selector}' + return filter_default_modules(selector, build, dynamic_vars) + + +def keep_modified_modules(modules, build, dynamic_vars, *defaults): if build.params_id.config_data.get('skip_modified_modules_filter', False): return modules - modified_modules = build._modified_modules() + if defaults: + defaults = [d[1:-1] if re.match(r'^[\'"].*[\'"]$', d) else d for d in defaults] + modified_modules = build._modified_modules(defaults=defaults) modules = modules.split(',') filtered_modules = [module for module in modules if module in modified_modules] return ','.join(filtered_modules) -def keep_modified_modules_or_base(modules, build, dynamic_vars): - bundle = build.params_id.create_batch_id.bundle_id - if bundle.is_base or bundle.is_staging: - return modules - return keep_modified_modules(modules, build, dynamic_vars) - - def make_module_test_tags(modules, build, dynamic_vars): return ','.join([f'/{module}' for module in modules.split(',')]) @@ -88,6 +112,17 @@ def append_string(modules, build, dynamic_vars, element): return ','.join([f'{module}{element}' for module in modules.split(',')]) +def union(modules, build, dynamic_vars, element): + if re.match(r'^[\'"].*[\'"]$', element): + element = element[1:-1] + else: + element = dynamic_vars.get(element, element) + element = element.strip() + modules = set(modules.split(',')) if modules else set() + new_modules = set(element.split(',')) if element else set() + return ','.join(sorted(modules | new_modules)) + + class Config(models.Model): _name = 'runbot.build.config' _description = "Build config" @@ -222,11 +257,15 @@ def wrapper(value, path): return wrapper def VARS(vars, path): - if not isinstance(vars, dict): - raise ValidationError(f'{path} ({vars}) should be a dict') - for key, val in vars.items(): - TECHNICAL_NAME(key, f'{path}.{key}') - STR(val, f'{path}.{key}') + if isinstance(vars, list): + for item in vars: + VARS(item, path) + else: + if not isinstance(vars, dict): + raise ValidationError(f'{path} ({vars}) should be a dict') + for key, val in vars.items(): + TECHNICAL_NAME(key, f'{path}.{key}') + STR(val, f'{path}.{key}') NAME = str_checker(r'^[\w \-]+$') STR = str_checker(r'.*') @@ -241,6 +280,7 @@ def VARS(vars, path): 'vars': OPTIONAL(VARS), 'steps': REQUIRED(LIST(STEP)), 'description': OPTIONAL(DYNAMIC_VALUE), + 'log': OPTIONAL(DYNAMIC_VALUE), } valid_steps['odoo'] = { 'name': REQUIRED(NAME), @@ -251,9 +291,11 @@ def VARS(vars, path): 'test_tags': OPTIONAL(DYNAMIC_VALUE), 'demo_mode': OPTIONAL(IN(['default', 'with_demo', 'without_demo'])), 'enable_auto_tags': OPTIONAL(BOOL), + 'extra_params': OPTIONAL(DYNAMIC_VALUE), 'cpu_limit': OPTIONAL(INT), 'export_database': OPTIONAL(BOOL), 'make_stats': OPTIONAL(BOOL), + 'log': OPTIONAL(DYNAMIC_VALUE), } valid_steps['create_build'] = { 'name': REQUIRED(NAME), @@ -262,6 +304,8 @@ def VARS(vars, path): 'for_each_vars': OPTIONAL(LIST(VARS)), 'for_each_module': OPTIONAL(DYNAMIC_VALUE), 'max_builds': OPTIONAL(INT), + 'if': OPTIONAL(DYNAMIC_VALUE), + 'log': OPTIONAL(DYNAMIC_VALUE), } valid_steps['restore'] = { 'name': REQUIRED(NAME), @@ -271,6 +315,7 @@ def VARS(vars, path): 'trigger_id': OPTIONAL(INT), 'use_current_batch': OPTIONAL(BOOL), 'zip_url': OPTIONAL(STR), + 'log': OPTIONAL(DYNAMIC_VALUE), } valid_steps['command'] = { 'name': REQUIRED(NAME), @@ -283,6 +328,7 @@ def VARS(vars, path): 'check_logs': OPTIONAL(LIST(STR)), 'expected_logs': OPTIONAL(LIST(STR)), 'make_stats': OPTIONAL(BOOL), + 'log': OPTIONAL(DYNAMIC_VALUE), } validate(config_schema, config, 'config') @@ -388,10 +434,14 @@ class ConfigStep(models.Model): cpu_limit = fields.Integer('Cpu limit', default=3600, tracking=True) container_cpus = fields.Integer('Allowed CPUs', help='Allowed container CPUs. Fallback on config parameter if 0.', default=0, tracking=True) coverage = fields.Boolean('Coverage', default=False, tracking=True) + coverage_branch = fields.Boolean('Coverage branch', default=False, tracking=True) + coverage_concurrency = fields.Boolean('Coverage concurrency', default=False, tracking=True) + coverage_test_context = fields.Boolean('Coverage test context', default=False, tracking=True) + coverage_make_report = fields.Boolean('Make coverage report', default=False, tracking=True) paths_to_omit = fields.Char('Paths to omit from coverage', tracking=True) flamegraph = fields.Boolean('Allow Flamegraph', default=False, tracking=True) test_enable = fields.Boolean('Test enable', default=True, tracking=True) - test_tags = fields.Char('Test tags', help="comma separated list of test tags", tracking=True) + test_tags = fields.Char('Test tags', help="new line (or comma) separated list of test tags", tracking=True) enable_auto_tags = fields.Boolean('Allow auto tag', default=True, tracking=True) sub_command = fields.Char('Subcommand', tracking=True) extra_params = fields.Char('Extra cmd args', tracking=True) @@ -429,6 +479,9 @@ class ConfigStep(models.Model): restore_download_db_suffix = fields.Char('Download db suffix') restore_rename_db_suffix = fields.Char('Rename db suffix') + # TODO change the default to True once we are sure that it works as expected + check_exit_status = fields.Boolean('Check exit status', default=False, help='Check exit status of the main command') + semgrep_category = fields.Many2one('runbot.checker_category', string='Semgrep Category', tracking=True) custom_link = fields.Char('Custom link for semgrep codes', tracking=True) disable_nosem = fields.Boolean('Disable nosem', default=False, tracking=True) @@ -546,6 +599,10 @@ def _run_step(self, build, **kwargs): max_timeout = int(self.env['ir.config_parameter'].get_param('runbot.runbot_timeout', default=10000)) docker_params['cpu_limit'] = min(self.cpu_limit, max_timeout) + config_data = {**kwargs.get('config_data', {}), **build.params_id.config_data} + if docker_params['cpu_limit'] and config_data.get('cpu_limit_factor'): + docker_params['cpu_limit'] = int(docker_params['cpu_limit'] * min(float(config_data['cpu_limit_factor']), 3)) + container_cpus = float(self.container_cpus or self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_containers_cpus', 0)) if 'cpus' not in docker_params and container_cpus: logical_cpu_count = psutil.cpu_count(logical=True) @@ -582,6 +639,9 @@ def _run_create_build(self, build, config_data=None, max_build=200): build._log('create_build', 'created with config %s' % config_name, log_type='subbuild', path=str(child.id)) def _make_python_ctx(self, build): + def log(*args, **kwargs): + args = [str(arg) for arg in args] + build._log(self.name, *args, **kwargs) return { 'datetime': tools.safe_eval.datetime, 'dateutil': tools.safe_eval.dateutil, @@ -602,6 +662,9 @@ def _make_python_ctx(self, build): 'json_loads': json.loads, 'PatchSet': PatchSet, 'markdown_escape': markdown_escape, + 'TestTagsParser': TestTagsParser, + 'requests': requests.Session(), + 'log': log, } def _run_python(self, build, force=False): @@ -688,7 +751,6 @@ def _run_run_odoo(self, build, force=False): return dict(cmd=cmd, exposed_ports=[build_port, build_port + 1], ro_volumes=exports, env_variables=env_variables, cpu_limit=None, network_enabled=True) def _run_install_odoo(self, build, config_data=None): - if config_data: config_data = {**config_data, **build.params_id.config_data} else: @@ -699,10 +761,14 @@ def _run_install_odoo(self, build, config_data=None): mods = ",".join(modules_to_install) python_params = [] py_version = build._get_py_version() - if self.coverage: + if self.coverage or config_data.get('coverage'): build.coverage = True - coverage_extra_params = self._coverage_params(build, modules_to_install) - python_params = ['-m', 'coverage', 'run', '--branch', '--source', '/data/build'] + coverage_extra_params + python_params = ['-m', 'coverage', 'run', '--source', '/data/build'] + if config_data.get('coverage_branch', self.coverage_branch): + python_params += ['--branch'] + if config_data.get('coverage_concurrency', self.coverage_concurrency): + python_params += ['--concurrency=thread'] + python_params += self._coverage_params(build, config_data) elif self.flamegraph: python_params = ['-m', 'flamegraph', '-o', self._perfs_data_path(build)] cmd = build._cmd(python_params, py_version, sub_command=self.sub_command, enable_log_db=self.enable_log_db) @@ -724,8 +790,8 @@ def _run_install_odoo(self, build, config_data=None): elif demo_mode == 'without_demo' and demo_installed_by_default: cmd.append('--without-demo=true') + extra_params = config_data.get('extra_params', build.params_id.extra_params or self.extra_params or '') # list module to install - extra_params = build.params_id.extra_params or self.extra_params or '' if mods and '-i' not in extra_params: cmd += ['-i', mods] config_path = build._server("tools/config.py") @@ -739,7 +805,7 @@ def _run_install_odoo(self, build, config_data=None): test_tags_in_extra = '--test-tags' in extra_params if (test_enable or test_tags) and "--test-tags" in available_options and not test_tags_in_extra: - test_tags = [t.strip() for t in (test_tags or '').split(',')] + test_tags = [t.strip() for t in TestTagsParser(test_tags or '').filter_specs] if enable_auto_tags and not config_data.get('disable_auto_tags', False): if grep(config_path, "[/module][:class]"): auto_tags = self.env['runbot.build.error']._disabling_tags(build) @@ -771,7 +837,7 @@ def _run_install_odoo(self, build, config_data=None): if extra_params: cmd.extend(shlex.split(extra_params)) - cmd.finals.extend(self._post_install_commands(build, modules_to_install, py_version)) # coverage post, extra-checks, ... + cmd.finals.extend(self._post_install_commands(build, config_data, py_version)) # coverage post, extra-checks, ... if config_data.get('export_database', True): self._add_zip_generation(build, cmd, db_name) @@ -783,10 +849,12 @@ def _run_install_odoo(self, build, config_data=None): if config_env_variables := config_data.get('env_variables', False): env_variables += config_env_variables.split(';') - cpu_limit = None + if config_data.get('coverage_test_context', self.coverage_test_context): + env_variables.append("COVERAGE_DYNAMIC_CONTEXT=test_function") + + cpu_limit = self.cpu_limit if config_data.get('cpu_limit'): cpu_limit = min(self.cpu_limit, int(config_data['cpu_limit'])) - return dict(cmd=cmd, ro_volumes=exports, cpu_limit=cpu_limit, env_variables=env_variables) def _add_zip_generation(self, build, cmd, db_name): @@ -992,7 +1060,9 @@ def get_reference_builds_for_versions(versions): ) if self.allow_similar_build_quick_result: - existing_done_build = next((build for build in child.params_id.build_ids.sorted('id') if build.global_state == 'done' and build.local_result not in ('skipped', 'killed', 'manually_killed')), None) + existing_done_build = next((build for build in child.params_id.build_ids.sorted('id') if build.global_state == 'done' and build.global_result == 'ok'), None) + if not existing_done_build: + existing_done_build = next((build for build in child.params_id.build_ids.sorted('id') if build.global_state == 'done' and build.local_result not in ('skipped', 'killed')), None) if existing_done_build: child._log('', 'A similar [build](%s) has been found, marking as done directly', existing_done_build.build_url, log_type='markdown') child.local_state = 'done' @@ -1152,11 +1222,11 @@ def _log_end(self, build): log_type = 'markdown' build._log('', message, *args, log_type=log_type) - if self.coverage: - xml_url = '%scoverage.xml' % build._http_log_url() - html_url = 'http://%s/runbot/static/build/%s/coverage/index.html' % (build.host, build.dest) - message = 'Coverage report: [xml @icon-download](%s), [html @icon-eye](%s)' - build._log('end_job', message, xml_url, html_url, log_type='markdown') + if self.coverage and self.coverage_make_report: + json_url = f'http://{build.host}/runbot/static/build/{build.dest}/logs/coverage/coverage.json' + html_url = f'http://{build.host}/runbot/static/build/{build.dest}/logs/coverage/' + message = 'Coverage report: [json @icon-download](%s), [html @icon-eye](%s)' + build._log('end_job', message, json_url, html_url, log_type='markdown') if self.flamegraph: dat_url = '%sflame_%s.%s' % (build._http_log_url(), self.sanitized_name(build), 'log.gz') @@ -1164,34 +1234,27 @@ def _log_end(self, build): message = 'Flamegraph report: [data @icon-download](%s), [svg @icon-eye](%s)' build._log('end_job', message, dat_url, svg_url, log_type='markdown') - def _post_install_commands(self, build, modules_to_install, py_version=None): + def _post_install_commands(self, build, config_data, py_version): cmds = [] - if self.coverage: - py_version = py_version if py_version is not None else build._get_py_version() - # prepare coverage result - cov_path = build._path('coverage') - os.makedirs(cov_path, exist_ok=True) - cmds.append(['python%s' % py_version, "-m", "coverage", "html", "-d", "/data/build/coverage", "--ignore-errors"]) - cmds.append(['python%s' % py_version, "-m", "coverage", "xml", "-o", "/data/build/logs/coverage.xml", "--ignore-errors"]) + if config_data.get('coverage_make_report', (self.coverage and self.coverage_make_report)): + cmds.append(['python%s' % py_version, "-m", "coverage", "html", "-d", "/data/build/logs/coverage", "--ignore-errors"]) + cmds.append(['python%s' % py_version, "-m", "coverage", "json", "-o", "/data/build/logs/coverage.json", "--ignore-errors"]) + if config_data.get('coverage', self.coverage): + cmds.append(['mv', "/data/build/.coverage", f"/data/build/logs/coverage.{build.id}.{int(time.time())}"]) return cmds def _perfs_data_path(self, build, ext='log'): return '/data/build/logs/flame_%s.%s' % (self.sanitized_name(build), ext) - def _coverage_params(self, build, modules_to_install): + def _coverage_params(self, build, config_data): pattern_to_omit = set() if self.paths_to_omit: - pattern_to_omit = set(self.paths_to_omit.split(',')) - for commit in build.params_id.commit_ids: - docker_source_folder = build._docker_source_folder(commit) - for manifest_file in commit.repo_id.manifest_files.split(','): - pattern_to_omit.add('*%s' % manifest_file) - for (addons_path, module, _) in commit._get_available_modules(): - if module not in modules_to_install: - # we want to omit docker_source_folder/[addons/path/]module/* - module_path_in_docker = os.sep.join([docker_source_folder, addons_path, module]) - pattern_to_omit.add('%s/*' % (module_path_in_docker)) - return ['--omit', ','.join(sorted(pattern_to_omit))] + pattern_to_omit |= set(self.paths_to_omit.split(',')) + if config_data.get('paths_to_omit'): + pattern_to_omit |= set(config_data.get('paths_to_omit').split(',')) + if pattern_to_omit: + return ['--omit', ','.join(sorted(pattern_to_omit))] + return [] def _make_results(self, build): # TODO fixme config data are not the same as the run part in dynamic steps @@ -1222,6 +1285,10 @@ def _make_results(self, build): if log_time: build.job_end = log_time + if self.check_exit_status and (exit_status := self._get_exit_status(build)) != 0: + build._log('_make_results', f'Main command exited with status code {exit_status}', level='ERROR') + build.local_result = 'ko' + if check_logs or expected_logs: self._make_custom_result(build, check_logs, expected_logs) elif active_job_type == 'python': @@ -1230,8 +1297,6 @@ def _make_results(self, build): elif self.test_enable or self.test_tags: self._make_odoo_results(build) elif active_job_type == 'install_odoo': - if self.coverage: - build.write(self._make_coverage_results(build)) if not self.sub_command: self._make_odoo_results(build) elif active_job_type == 'test_upgrade': @@ -1250,23 +1315,6 @@ def _make_python_results(self, build): raise RunbotException('python_result_code must set return_value to a dict values on build') build.write(return_value) # old style support - def _make_coverage_results(self, build): - build_values = {} - build._log('coverage_result', 'Start getting coverage result') - cov_path = build._path('coverage/index.html') - if os.path.exists(cov_path): - with file_open(cov_path, 'r') as f: - data = f.read() - covgrep = re.search(r'pc_cov.>(?P\d+)%', data) - build_values['coverage_result'] = covgrep and covgrep.group('coverage') or False - if build_values['coverage_result']: - build._log('coverage_result', 'Coverage result: %s' % build_values['coverage_result']) - else: - build._log('coverage_result', 'Coverage result not found', level='WARNING') - else: - build._log('coverage_result', 'Coverage file not found', level='WARNING') - return build_values - def _make_upgrade_results(self, build): build._log('upgrade', 'Getting results for build %s' % build.dest) @@ -1380,6 +1428,21 @@ def _get_checkers_result(self, build, checkers): return result return 'ok' + def _get_exit_status(self, build): + exit_status_filename = f'{self.sanitized_name(build)}_exit_status.txt' + if not os.path.exists(build._path(exit_status_filename)): + build._log('_make_tests_results', f'Exit status file "{exit_status_filename}" not found', level="ERROR") + return 1 + res = build._read_file(exit_status_filename) + if res: + try: + return int(res.strip('\n')) + except ValueError: + build._log('_make_tests_results', f'Status file "{exit_status_filename}" does not contain an integer', level="ERROR") + return -242 + build._log('_make_tests_results', f'Exception or file empty while reading status file "{exit_status_filename}"', level="ERROR") + return -241 + def _make_custom_result(self, build, enabled_checkers=None, expected_logs=None): build._log('run', 'Getting results for build %s' % build.dest) if build.local_result != 'ko': @@ -1499,35 +1562,70 @@ def _run_dynamic(self, build): raise RunbotException('Too many ancestors builds, possible cyclic dynamic build creation') if build.parent_id and build.dynamic_config == build.parent_id.dynamic_config: raise RunbotException('A child build cannot load the same dynamic config if parent, recursion detected') + + config_vars_list = build.dynamic_config.get('vars', {}) + if not isinstance(config_vars_list, list): + config_vars_list = [config_vars_list] + raw_vars = {} + for config_vars in config_vars_list: + raw_vars.update(config_vars) + + raw_vars.update(build.params_id.config_data.get('dynamic_vars', {})) + dynamic_vars = {} + # dynamic_vars can either be raw value like 'account', value to evaluate lazily in anothed dynamic value like 'account->!mail' + # or dynamic value that we want to evaluate early like '{{*|filter_all_modules|modified_modules}}' (between {{}}) + # this loop will evalute the third category + # this alows to evaluate only once an expression that could be expensive to use it in multiple dynamic values + # this also allow to clarify the config by chaining vars definition + # TODO check ordering + for key, value in raw_vars.items(): + dynamic_vars[key] = self._parse_dynamic_entry(value, build, dynamic_vars=dynamic_vars) + current_step = self._get_dynamic_step(build) if not current_step: build._log('Dynamic Step', 'No dynamic config or steps found, skipping', level="WARNING") return + if current_step.get('log'): + text = self._parse_dynamic_entry(current_step['log'], build, dynamic_vars=dynamic_vars) + build._log('_run_dynamic', text) if current_step['job_type'] == 'create_build': for_each_vars_list = current_step.get('for_each_vars', [{}]) if 'for_each_module' in current_step: modules_vars = [] for for_each_vars in for_each_vars_list: - modules_entry = self._parse_dynamic_entry(current_step['for_each_module'], build, additional_dynamic_vars=for_each_vars) + modules_entry = self._parse_dynamic_entry(current_step['for_each_module'], build, dynamic_vars={**dynamic_vars, **for_each_vars}) modules = [m.strip() for m in modules_entry.split(',') if m.strip()] for module in modules: module_vars = {**for_each_vars, 'module': module} modules_vars.append(module_vars) for_each_vars_list = modules_vars - parent_vars = {**build.dynamic_config.get('vars', {}), **build.params_id.config_data.get('dynamic_vars', {})} + child_data_list = [] for child_index, child in enumerate(current_step.get('children', [])): child_vars = child.get('vars', {}) for for_each_vars in for_each_vars_list: config_name = child.get('name', build.params_id.config_id.name) - dynamic_vars = {**parent_vars, **child_vars, **for_each_vars} + raw_dynamic_vars = {**dynamic_vars, **for_each_vars, **child_vars} + child_dynamic_vars = {} + # evaluate for_each_vars + for key, value in raw_dynamic_vars.items(): + child_dynamic_vars[key] = self._parse_dynamic_entry(value, build, dynamic_vars=child_dynamic_vars) + if 'if' in current_step: + condition = self._parse_dynamic_entry(current_step['if'], build, dynamic_vars=child_dynamic_vars) + if not condition: + continue if 'description' in child: - description = self._parse_dynamic_entry(child['description'], build, additional_dynamic_vars=dynamic_vars) + description = self._parse_dynamic_entry(child['description'], build, dynamic_vars=child_dynamic_vars) # note: we mainly need to provide additional_dynamic_vars because the child is not created yet at this point else: description = config_name + # filter vars not prefixed with _ to simplify child values + if child.get('log'): + text = self._parse_dynamic_entry(child['log'], build, dynamic_vars=child_dynamic_vars) + build._log('_run_dynamic', text) + public_child_dynamic_vars = {key: value for key, value in child_dynamic_vars.items() if not key.startswith('_')} child_data = { - 'config_data': {**build.params_id.config_data.dict, "dynamic_vars": dynamic_vars}, + 'config_data': {**build.params_id.config_data.dict, "dynamic_vars": public_child_dynamic_vars}, 'config_id': build.params_id.config_id.id, 'dynamic_active_step_index': 0, 'dynamic_config_position': f'{build.params_id.dynamic_config_position or ""}/{build.dynamic_active_step_index}.{child_index}', @@ -1558,12 +1656,18 @@ def _run_dynamic(self, build): install_modules_pattern = current_step.get('install_modules', '') if install_modules_pattern.split(',', 1)[0] not in ('*', '-*'): install_modules_pattern = '-*,' + install_modules_pattern - config_data['install_module_pattern'] = self._parse_dynamic_entry(install_modules_pattern, build) + config_data['install_module_pattern'] = self._parse_dynamic_entry(install_modules_pattern, build, dynamic_vars) if 'test_tags' in current_step: - config_data['test_tags'] = self._parse_dynamic_entry(current_step.get('test_tags'), build) + config_data['test_tags'] = self._parse_dynamic_entry(current_step.get('test_tags'), build, dynamic_vars) config_data['test_enable'] = bool(current_step.get('test_enable') or current_step.get('test_tags')) + if 'extra_params' in current_step: + config_data['extra_params'] = self._parse_dynamic_entry(current_step.get('extra_params'), build, dynamic_vars) + + if 'cpu_limit' in current_step: + config_data['cpu_limit'] = int(current_step.get('cpu_limit')) + for key in ('screencast', 'demo_mode', 'enable_auto_tags'): if key in current_step: value = current_step[key] @@ -1584,6 +1688,7 @@ def _run_dynamic(self, build): 'addons_path': ",".join(build._get_addons_path()), 'exports': ",".join(exports.keys()), 'exports_paths': ",".join(exports.values()), + **dynamic_vars, } command = [shlex.quote(self._parse_dynamic_entry(part, build, values)) for part in command] pres = [] @@ -1605,22 +1710,23 @@ def _get_dynamic_db_suffix(self, step): db_suffix = re.sub(r'[^a-z0-9_\-]', '_', db_suffix.lower()) return db_suffix - def _parse_dynamic_entry(self, entry, build, additional_dynamic_vars=None): + def _parse_dynamic_entry(self, entry, build, dynamic_vars): """ transforms a module/test-tags entry dynamically """ - dynamic_config = build.dynamic_config - expression_filters = { 'filter_all_modules': filter_all_modules, 'filter_default_modules': filter_default_modules, 'make_module_test_tags': make_module_test_tags, + 'select_existing_modules': select_existing_modules, + 'get_dependencies': get_dependencies, + 'get_dependant': get_dependant, 'prepend': prepend_string, 'append': append_string, 'modified_modules': keep_modified_modules, - 'modified_modules_or_base': keep_modified_modules_or_base, + 'union': union, } - dynamic_vars = {**dynamic_config.get('vars', {}), **build.params_id.config_data.get('dynamic_vars', {}), **(additional_dynamic_vars or {})} + dynamic_vars = dynamic_vars or {} def parse_expression(match): # inspired by jinja but with limited features diff --git a/runbot/models/build_error.py b/runbot/models/build_error.py index b165914d6..470e64774 100644 --- a/runbot/models/build_error.py +++ b/runbot/models/build_error.py @@ -9,16 +9,15 @@ from dateutil import rrule from dateutil.relativedelta import relativedelta from markupsafe import Markup - from werkzeug.urls import url_join from odoo import api, fields, models from odoo.exceptions import AccessError, UserError, ValidationError -from odoo.tools import SQL, lazy, ormcache from odoo.fields import Domain +from odoo.tools import SQL, lazy, ormcache +from ..common import TestTagsParser, transactioncache from ..fields import JsonDictField -from ..common import transactioncache, TestTagsParser _logger = logging.getLogger(__name__) @@ -215,7 +214,6 @@ class BuildError(models.Model): _inherit = ('mail.thread', 'mail.activity.mixin', 'runbot.build.error.seen.mixin') _mail_post_access = 'read' - name = fields.Char("Name") active = fields.Boolean('Open (not fixed)', default=True, tracking=True) description = fields.Text("Description", store=True, compute='_compute_description') @@ -242,9 +240,10 @@ class BuildError(models.Model): breaking_bundle_id = fields.Many2one('runbot.bundle', 'Breaking bundle', tracking=True, help="Bundle that introduced the error", related='breaking_pr_id.bundle_id') breaking_bundle_url = fields.Char('Breaking bundle url', related='breaking_bundle_id.frontend_url') breaking_pr_date = fields.Datetime('Breaking date', related="breaking_pr_id.close_date", help="Date of the merge of the first pr") + duplicate_breaking_pr_count = fields.Integer('Same Breaking PR', compute='_compute_duplicate_breaking_pr_count', help='Other errors with same breaking PR') - test_tags = fields.Char(string='Test tags', help="Comma separated list of test_tags to use to reproduce/remove this error", tracking=True) - canonical_tags = fields.Char('Canonical tag', compute='_compute_canonical_tags', store=True) + test_tags = fields.Text(string='Test tags', help="Comma separated list of test_tags to use to reproduce/remove this error", tracking=True) + canonical_tags = fields.Text('Canonical tag', compute='_compute_canonical_tags', store=True) tags_match_count = fields.Integer('Nb errors matching the test_tags', compute='_compute_tags_match_count') tags_min_version_excluded_id = fields.Many2one('runbot.version', 'Tag min version (excluded)') tags_min_version_id = fields.Many2one('runbot.version', 'Tags Min version', compute="_compute_tags_min_version_id", inverse="_inverse_tags_min_version_id", help="Minimal version where the test tags will be applied.", tracking=True) @@ -289,7 +288,7 @@ def _inverse_tags_min_version_id(self): def _compute_canonical_tags(self): for record in self: canonical_tags = sorted(set(record.error_content_ids.filtered('canonical_tag').mapped('canonical_tag'))) - record.canonical_tags = ','.join(canonical_tags) + record.canonical_tags = '\n'.join(canonical_tags) @api.depends('tags_min_version_id') def _compute_tags_min_version_id(self): @@ -322,6 +321,25 @@ def _compute_fixing_bundle_id(self): for record in self: record.fixing_bundle_id = record.fixing_pr_id.bundle_id if record.fixing_pr_id else False + @api.depends('breaking_pr_id') + def _compute_duplicate_breaking_pr_count(self): + breaking_counts = self.env["runbot.build.error"]._read_group( + domain=[ + ("breaking_pr_id", "in", self.breaking_pr_id.ids), + ("active", "=", True), + ], + groupby=["breaking_pr_id"], + aggregates=["id:count"], + having=[('id:count', '>', 1)], + ) + + count_by_pr = {pr_count[0]: pr_count[1] for pr_count in breaking_counts} + + for record in self: + # remove 1 to not count the current error + record.duplicate_breaking_pr_count = count_by_pr.get(record.breaking_pr_id, 1) - 1 + + @api.depends('error_content_ids.version_ids') def _compute_version_ids(self): for record in self: @@ -431,10 +449,10 @@ def _compute_unique_qualifiers(self): @api.depends('common_qualifiers') def _compute_similar_ids(self): for record in self: - if record.common_qualifiers: + if record.common_qualifiers and (record.id or record.id.origin): query = SQL( r"""SELECT id FROM runbot_build_error WHERE id != %s AND common_qualifiers @> %s""", - record.id, + record.id or record.id.origin, json.dumps(record.common_qualifiers.dict), ) self.env.cr.execute(query) @@ -445,10 +463,10 @@ def _compute_similar_ids(self): @api.depends('common_qualifiers') def _compute_similar_content_ids(self): for record in self: - if record.common_qualifiers: + if record.common_qualifiers and (record.id or record.id.origin): query = SQL( r"""SELECT id FROM runbot_build_error_content WHERE error_id != %s AND qualifiers @> %s""", - record.id, + record.id or record.id.origin, json.dumps(record.common_qualifiers.dict), ) self.env.cr.execute(query) @@ -459,10 +477,10 @@ def _compute_similar_content_ids(self): @api.depends('common_qualifiers') def _compute_analogous_ids(self): for record in self: - if record.common_qualifiers: + if record.common_qualifiers and (record.id or record.id.origin): query = SQL( r"""SELECT id FROM runbot_build_error WHERE id != %s AND unique_qualifiers @> %s""", - record.id, + record.id or record.id.origin, json.dumps(record.unique_qualifiers.dict), ) self.env.cr.execute(query) @@ -473,10 +491,10 @@ def _compute_analogous_ids(self): @api.depends('common_qualifiers') def _compute_analogous_content_ids(self): for record in self: - if record.common_qualifiers: + if record.common_qualifiers and (record.id or record.id.origin): query = SQL( r"""SELECT id FROM runbot_build_error_content WHERE error_id != %s AND qualifiers @> %s""", - record.id, + record.id or record.id.origin, json.dumps(record.unique_qualifiers.dict), ) self.env.cr.execute(query) @@ -489,16 +507,20 @@ def _compute_tags_match_count(self): for record in self: record.tags_match_count = 0 if record.test_tags: - tags_parser = TestTagsParser(record.test_tags) - search_domain = tags_parser.test_tags_to_search_domain(exclude_error_id=record.id) - if search_domain: - record.tags_match_count = self.env['runbot.build.error'].with_context(active_test=True).search_count(search_domain) + try: + tags_parser = TestTagsParser(record.test_tags.replace('\n', ',')) + search_domain = tags_parser.test_tags_to_search_domain(exclude_error_id=record.id or record.id.origin) + if search_domain: + record.tags_match_count = self.env['runbot.build.error'].with_context(active_test=True).search_count(search_domain) + except Exception as e: # noqa: BLE001 + record.tags_match_count = -1 + _logger.warning("Error while computing tags_match_count for error %s with test_tags %s: %s", record.id, record.test_tags, e) def action_view_impacted_by_tag(self): self.ensure_one() if not self.test_tags: return - tags_parser = TestTagsParser(self.test_tags) + tags_parser = TestTagsParser(self.test_tags.replace('\n', ',')) return { 'type': 'ir.actions.act_window', 'views': [(False, 'list'), (False, 'form')], @@ -511,8 +533,15 @@ def action_view_impacted_by_tag(self): @api.constrains('test_tags') def _check_test_tags(self): for build_error in self: - if build_error.test_tags and '-' in build_error.test_tags: - raise ValidationError('Build error test_tags should not be negated') + if build_error.test_tags: + try: + test_tags = build_error.test_tags.replace('\n', ',') + tags_parser = TestTagsParser(test_tags) + tags_parser = TestTagsParser(test_tags, keep_escape=False) + except Exception as e: # noqa: BLE001 + raise ValidationError(f'Invalid test_tags format: {e}') + if tags_parser.exclude or any(params[0] == '-' for p, params in tags_parser.parameters): + raise ValidationError('Build error test_tags should not be negated') @api.onchange('test_tags') def _onchange_test_tags(self): @@ -587,12 +616,12 @@ def _merge(self, others): error.sudo().test_tags = previous_error.test_tags previous_error.sudo().test_tags = False elif self.env.su: - test_tags = error.test_tags.split(',') - previous_error - for tag in previous_error.test_tags.split(','): + test_tags = TestTagsParser(error.test_tags.replace('\n', ',')).filter_specs + previous_error_tags = TestTagsParser(previous_error.test_tags.replace('\n', ',')).filter_specs + for tag in previous_error_tags: if tag not in test_tags: test_tags.append(tag) - error.test_tags = ','.join(test_tags) + error.test_tags = '\n'.join(test_tags) previous_error.test_tags = False for field in fields_to_merge + fields_to_copy: if previous_error[field]: @@ -623,7 +652,16 @@ def filter_tags(e): return True test_tag_list = self.search([('test_tags', '!=', False)]).filtered(filter_tags).mapped('test_tags') - return [test_tag for error_tags in test_tag_list for test_tag in (error_tags).split(',')] + parsed_test_tags = [] + for error_tags in test_tag_list: + try: + # we cannot rely only on '\n' since old test-tags or user defined ones could be comma separated + error_tags = error_tags.replace('\n', ',') + tags_parser = TestTagsParser(error_tags) + parsed_test_tags.extend(tags_parser.filter_specs) + except Exception as e: # noqa: BLE001 + _logger.warning('Error while parsing test_tags for error with id %s: %s', self.id, e) + return parsed_test_tags @api.model def _disabling_tags(self, build_id=False): @@ -701,6 +739,19 @@ def action_view_analogous_qualified_contents(self): 'name': 'Similary Qualified Contents' } + def action_view_duplicate_breaking_pr(self): + self.ensure_one() + return { + 'type': 'ir.actions.act_window', + 'res_model': 'runbot.build.error', + 'domain': [ + ('breaking_pr_id', '=', self.breaking_pr_id.id), + ('active', '=', True), + ], + 'view_mode': 'list,form', + 'name': 'Errors with same breaking PR', + } + @api.depends('manual_team_id', 'auto_team_id') def _compute_team_id(self): for error in self: @@ -854,7 +905,7 @@ class BuildErrorContent(models.Model): breaking_pr_id = fields.Many2one(related='error_id.breaking_pr_id') fixing_pr_alive = fields.Boolean(related='error_id.fixing_pr_alive') fixing_pr_url = fields.Char(related='error_id.fixing_pr_url') - test_tags = fields.Char(related='error_id.test_tags') + test_tags = fields.Text(related='error_id.test_tags') tags_min_version_id = fields.Many2one(related='error_id.tags_min_version_id') tags_max_version_id = fields.Many2one(related='error_id.tags_max_version_id') @@ -992,10 +1043,10 @@ def _compute_error_display_id(self): def _compute_similar_ids(self): """error contents having the exactly the same qualifiers""" for record in self: - if record.qualifiers: + if record.qualifiers and (record.id or record.id.origin): query = SQL( r"""SELECT id FROM runbot_build_error_content WHERE id != %s AND qualifiers @> %s AND qualifiers <@ %s""", - record.id, + record.id or record.id.origin, json.dumps(record.qualifiers.dict), json.dumps(record.qualifiers.dict), ) diff --git a/runbot/models/bundle.py b/runbot/models/bundle.py index 747440072..3e8073e89 100644 --- a/runbot/models/bundle.py +++ b/runbot/models/bundle.py @@ -277,7 +277,7 @@ def _consistency_warning(self): warnings.append(('info', 'PR %s targeting a non base branch: %s' % (branch.dname, branch.target_branch_name))) else: warnings.append(('warning' if branch.alive else 'info', 'PR %s targeting wrong version: %s (expecting %s)' % (branch.dname, branch.target_branch_name, self.base_id.name))) - elif not branch.is_pr and not branch.name.startswith(self.base_id.name) and not self.defined_base_id: + elif not branch.is_pr and not branch.name.startswith(self.base_id.name) and not self.defined_base_id and branch.remote_id.repo_id.enforce_version: warnings.append(('warning', 'Branch %s not starting with version name (%s)' % (branch.dname, self.base_id.name))) return warnings @@ -322,7 +322,12 @@ def action_generate_custom_trigger_restore_action(self): return self._generate_custom_trigger_action(context) def action_disable_all_triggers(self): - triggers_to_disable = ( + self._configure_custom_trigger_start_mode('disable') + + def _configure_custom_trigger_start_mode(self, mode): + self.ensure_one() + + triggers_to_create = ( self.env["runbot.trigger"] .search([ ("id", "not in", self.trigger_custom_ids.trigger_id.ids), @@ -335,13 +340,26 @@ def action_disable_all_triggers(self): ) ) vals = [] - for trigger in triggers_to_disable: - vals.append({ - 'bundle_id': self.id, - 'trigger_id': trigger.id, - 'start_mode': 'disabled', - }) + bundle_repos = self.branch_ids.remote_id.repo_id + for trigger in triggers_to_create: + if trigger.repo_ids & bundle_repos or trigger.dependency_ids & bundle_repos: + vals.append({ + 'bundle_id': self.id, + 'trigger_id': trigger.id, + }) self.env['runbot.bundle.trigger.custom'].create(vals) + for custom_trigger in self.trigger_custom_ids: + trigger_mode = mode + if mode == 'light' and not custom_trigger.trigger_id.light_config_id: + trigger_mode = 'auto' + custom_trigger.start_mode = trigger_mode + + def _force_ci(self): + for bundle in self: + bundle._configure_custom_trigger_start_mode('force') + # we need to create a new batch in case some of the triggers were in minimal mode + batch = bundle._force() or bundle.last_batch + batch._log("Batch was requested for ci") class BundleTag(models.Model): diff --git a/runbot/models/commit.py b/runbot/models/commit.py index 255bb44f3..b0aba479a 100644 --- a/runbot/models/commit.py +++ b/runbot/models/commit.py @@ -3,7 +3,6 @@ import subprocess from ..common import os, RunbotException, make_github_session, transactioncache -import glob import shutil from odoo import models, fields, api @@ -66,22 +65,14 @@ def _rebase_on(self, commit): return self return self._get(self.name, self.repo_id.id, self.read()[0], commit.id) - def _get_available_modules(self): - for manifest_file_name in self.repo_id.manifest_files.split(','): # '__manifest__.py' '__openerp__.py' - for addons_path in (self.repo_id.addons_paths or '').split(','): # '' 'addons' 'odoo/addons' - sep = os.path.join(addons_path, '*') - for manifest_path in glob.glob(self._source_path(sep, manifest_file_name)): - module = os.path.basename(os.path.dirname(manifest_path)) - yield (addons_path, module, manifest_file_name) - def _list_files(self, patterns): #example: git ls-files --with-tree=abcf390f90dbdd39fd61abc53f8516e7278e0931 ':(glob)addons/*/*.py' ':(glob)odoo/addons/*/*.py' # note that glob is needed to avoid the star matching ** self.ensure_one() - return self.repo_id._git(['ls-files', '--with-tree', self.name, *patterns]).split('\n') + self._fetch() + return self.repo_id._git(['ls-files', '--with-tree', self.tree_hash, *patterns]).split('\n') def _list_available_modules(self): - # beta version, may replace _get_available_modules latter addons_paths = (self.repo_id.addons_paths or '').split(',') patterns = [] for manifest_file_name in self.repo_id.manifest_files.split(','): # '__manifest__.py' '__openerp__.py' @@ -98,18 +89,24 @@ def _list_available_modules(self): module, manifest_file_name = elems yield (addons_path, module, manifest_file_name) + @transactioncache # hack to avoid to fetch two time the same commit inside the same transaction + def _fetch(self): + try: + self.repo_id._fetch(self.name) + except RunbotException: + self.repo_id._fetch(self.tree_hash) def _export(self, build): """Export a git repo into a sources""" # TODO add automated tests self.ensure_one() - self.repo_id._fetch(self.tree_hash) + self._fetch() if not self.env['runbot.commit.export'].search([('build_id', '=', build.id), ('commit_id', '=', self.id)]): self.env['runbot.commit.export'].create({'commit_id': self.id, 'build_id': build.id}) export_path = self._source_path() if os.path.isdir(export_path): - _logger.info('git export: exporting to %s (already exists)', export_path) + _logger.debug('git export: exporting to %s (already exists)', export_path) return export_path _logger.info('git export: exporting to %s (new)', export_path) @@ -166,12 +163,43 @@ def _read_source(self, file, mode='r'): @transactioncache def _git_show_file(self, file): + return self._git_show_files([file])[0] + + def _git_show_files(self, files): self.ensure_one() + if not files: + return [] + self.repo_id._fetch(self.name) + + queries = "\n".join([f"{self.name}:{f}" for f in files]) + "\n" + try: - return self.repo_id._git(['show', '%s:%s' % (self.name, file)]) + buffer = self.repo_id._git( + ['cat-file', '--batch'], + input_data=queries, + raw=True, + ) except subprocess.CalledProcessError: - return False + return [False] * len(files) + + results = [] + offset = 0 + buffer_len = len(buffer) + while offset < buffer_len: + newline_idx = buffer.find(b'\n', offset) + if newline_idx == -1: + break + header = buffer[offset:newline_idx].decode('utf-8') + offset = newline_idx + 1 + try: + size_in_bytes = int(header.rsplit(' ', 1)[-1]) + except ValueError: # most likely missing + results.append(False) + continue + results.append(buffer[offset : offset + size_in_bytes].decode('utf-8', errors='replace')) + offset += size_in_bytes + 1 + return results def _source_path(self, *paths): if not self.tree_hash: diff --git a/runbot/models/custom_trigger.py b/runbot/models/custom_trigger.py index 53cc74281..a31d7fa55 100644 --- a/runbot/models/custom_trigger.py +++ b/runbot/models/custom_trigger.py @@ -8,7 +8,7 @@ class BundleTriggerCustomization(models.Model): _description = 'Custom trigger' trigger_id = fields.Many2one('runbot.trigger') - start_mode = fields.Selection([('disabled', 'Disabled'), ('auto', 'Auto'), ('force', 'Force')], required=True, default='auto') + start_mode = fields.Selection([('disabled', 'Disabled'), ('auto', 'Auto'), ('light', 'Light'), ('force', 'Force')], required=True, default='auto') use_base_commits = fields.Boolean("Use base commits", help="Allow to test a trigger without the branch changes", default=False) bundle_id = fields.Many2one('runbot.bundle') config_id = fields.Many2one('runbot.build.config') diff --git a/runbot/models/docker.py b/runbot/models/docker.py index 547a3c6e0..d7115e3d5 100644 --- a/runbot/models/docker.py +++ b/runbot/models/docker.py @@ -145,6 +145,7 @@ class Dockerfile(models.Model): dockerfile = fields.Text(compute='_compute_dockerfile', recursive=True, tracking=True) in_error = fields.Boolean('In error', help='The last build failed.', default=False) to_build = fields.Boolean('To Build', help='Build Dockerfile. Check this when the Dockerfile is ready.', default=True) + nocache = fields.Boolean('No Cache', help='Force a full rebuild on next build, bypassing the Docker layer cache. Automatically reset to False after the build.', copy=False) always_pull = fields.Boolean('Always pull', help='Always Pull on the hosts, not only at the use time', default=False, tracking=True, copy=False) version_ids = fields.One2many('runbot.version', 'dockerfile_id', string='Versions') description = fields.Text('Description') @@ -351,6 +352,7 @@ def _get_cached_content(self, docker_build_path): destination = add_match.group('destination') # Use the destination name as hardlink name to avoid rebuild if file content is the same but not the url hardlink_name = re.sub(r'[^a-zA-Z0-9]', '_', destination) + lines[i] = f'# CACHED {lines[i + 1]}' lines[i + 1] = f'COPY {hardlink_name} {destination}' cache_file_path = cache_dir / filename if not cache_file_path.exists() or time.time() - cache_file_path.lstat().st_mtime > cache_duration: @@ -383,7 +385,7 @@ def _build(self, host=None): content = self._get_cached_content(docker_build_path) with open(self.env['runbot.runbot']._path('docker', tag_dir, 'Dockerfile'), 'w', encoding="utf-8") as Dockerfile: Dockerfile.write(content) - result = docker_build(docker_build_path, self.image_future_tag, self.pull_on_build) + result = docker_build(docker_build_path, self.image_future_tag, self.pull_on_build, self.nocache) duration = result['duration'] msg = result['msg'] success = image_id = result.get('image_id') @@ -427,6 +429,9 @@ def clean_output(output): message = f'Build failure, check results for more info ({result.summary})' self.message_post(body=message) _logger.error(message) + + if self.nocache: + self.nocache = False return image_id diff --git a/runbot/models/host.py b/runbot/models/host.py index 7dea8fba1..6b9055ef5 100644 --- a/runbot/models/host.py +++ b/runbot/models/host.py @@ -341,7 +341,7 @@ def _get_builds(self, domain, order=None): return self.env['runbot.build'].search(self._get_build_domain(domain), order=order) def _process_messages(self): - self.host_message_ids._process() + return self.host_message_ids._process() class MessageQueue(models.Model): @@ -351,14 +351,25 @@ class MessageQueue(models.Model): _log_access = False create_date = fields.Datetime('Create date', default=fields.Datetime.now) - host_id = fields.Many2one('runbot.host', required=True, ondelete='cascade') - build_id = fields.Many2one('runbot.build') + host_id = fields.Many2one('runbot.host', required=True, ondelete='cascade', index=True) + build_id = fields.Many2one('runbot.build', index=True) message = fields.Char('Message') def _process(self): records = self + processed = False # todo consume messages here if records: + processed = True for record in records: - self.env['runbot.runbot']._warning(f'Host {record.host_id.name} got an unexpected message {record.message}') + if record.message == 'kill': + if record.build_id: + build = record.build_id + result = None + if build.local_state != 'running' and build.global_result not in ('warn', 'ko'): + result = 'killed' + build._kill(result=result) + else: + self.env['runbot.runbot']._warning(f'Host {record.host_id.name} got an unexpected message {record.message}') self.unlink() + return processed diff --git a/runbot/models/ir_qweb.py b/runbot/models/ir_qweb.py index e9fa061ce..c8ea0eb37 100644 --- a/runbot/models/ir_qweb.py +++ b/runbot/models/ir_qweb.py @@ -1,5 +1,5 @@ -from ..common import s2human, s2human_long, precise_s2human -from odoo import models +from ..common import s2human, s2human_long, precise_s2human, transactioncache +from odoo import models, tools from odoo.http import request from odoo.addons.website.controllers.main import QueryURL @@ -12,3 +12,10 @@ def _prepare_frontend_environment(self, values): values['s2human_long'] = s2human_long values['precise_s2human'] = precise_s2human return response + + @tools.conditional( + 'xml' in tools.config['dev_mode'], + transactioncache, + ) # replace ormcache by transaction cache to avoid reading the same template multiple times in the same requests. Context is ignored but should be the same for each call in the same request + def _generate_code_cached(self, ref: int): + return super()._generate_code_cached(ref) diff --git a/runbot/models/project.py b/runbot/models/project.py index f60844bfb..7032754c4 100644 --- a/runbot/models/project.py +++ b/runbot/models/project.py @@ -25,6 +25,9 @@ class Project(models.Model): active = fields.Boolean("Active", default=True) process_delay = fields.Integer('Process delay', default=60, required=True, help="Delay between a push and a batch starting its process.") next_freeze_tag_id = fields.Many2one('runbot.bundle.tag', string="Next freeze tag") + use_light_default = fields.Boolean('Use light config by default', help="Use the light config when possible for all triggers") + use_light_draft = fields.Boolean('Use light config for draft PRs', help="Use the light config when possible for bundle having draft pr") + use_light_no_pr = fields.Boolean('Use light config when no PR', help="Use the light config when possible for all bundles not having any pr") @api.constrains('process_delay') def _constraint_process_delay(self): diff --git a/runbot/models/repo.py b/runbot/models/repo.py index ef0c3a10a..32c12acb6 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -6,6 +6,7 @@ import re import subprocess import time +import psutil import requests import markupsafe import shlex @@ -73,6 +74,7 @@ class Trigger(models.Model): ) module_filters = fields.One2many('runbot.module.filter', 'trigger_id', string="Module filters", help='Will be combined with repo module filters when used with this trigger') config_id = fields.Many2one('runbot.build.config', string="Config", required=True) + light_config_id = fields.Many2one('runbot.build.config', string="Light config", help="Alternative config to use when light mode is enabled") config_data = JsonDictField('Config Data') network_enabled = fields.Boolean('Network Enabled') batch_dependent = fields.Boolean('Batch Dependent', help="Force adding batch in build parameters to make it unique and give access to bundle") @@ -263,6 +265,11 @@ def _compute_remote_name(self): for remote in self: remote.remote_name = sanitize(remote.short_name) + def _get_fetch_url(self): + if not self.name.startswith('https://') or not self.token: + return self.name + return self.name.replace('https://', 'https://%s@' % self.token, 1) + def create(self, values_list): remote = super().create(values_list) if not remote.repo_id.main_remote_id: @@ -429,8 +436,10 @@ class Repo(models.Model): get_ref_time = fields.Float('Last refs db update', compute='_compute_get_ref_time') trigger_ids = fields.Many2many('runbot.trigger', relation='runbot_trigger_triggers', readonly=True) single_version = fields.Many2one('runbot.version', "Single version", help="Limit the repo to a single version for non versionned repo") + enforce_version = fields.Boolean('Force version', help="Force all bundle containing branch from this repo to be prefixed with the correct version", default=True) forbidden_regex = fields.Char('Forbidden regex', help="Regex that forid bundle creation if branch name is matching", tracking=True) invalid_branch_message = fields.Char('Forbidden branch message', tracking=True) + allow_slashes = fields.Boolean('Allow slashes in branch names', help="Allow branches with slashes in their name (e.g. odoo/tests/my_branch). If unchecked, only one level of branches is allowed (e.g. odoo/my_branch)", default=True) def _compute_get_ref_time(self): self.env.cr.execute(""" @@ -500,25 +509,56 @@ def _get_git_command(self, cmd, errors='strict'): cmd = ['git', '-C', self.path] + config_args + cmd return cmd - def _git(self, cmd, errors='strict', quiet=False): + def _git(self, cmd, errors='strict', quiet=False, input_data=None, raw=False): cmd = self._get_git_command(cmd, errors) - if not quiet: - _logger.info("git command: %s", shlex.join(cmd)) - return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode(errors=errors) + if input_data is not None and isinstance(input_data, str): + input_data = input_data.encode('utf-8') + + fetch_timeout = int(self.env['ir.config_parameter'].get_param('runbot.runbot_fetch_timeout', default=30)) + for i in range(3): # retry in case of timeout + if not quiet: + _logger.info("git command: %s", shlex.join(cmd)) + + killed_fetch = False + stdin = subprocess.PIPE if input_data is not None else None + process = subprocess.Popen(cmd, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + while True: + try: + output, _ = process.communicate(input=input_data, timeout=fetch_timeout) + break + except subprocess.TimeoutExpired: + try: + parent = psutil.Process(process.pid) + for child in parent.children(recursive=True): + if 'git-upload-pack' in str(child.cmdline()) and child.status() == psutil.STATUS_SLEEPING: + child.kill() + _logger.info("Killed sleeping git subprocess (pid: %s)", child.pid) + killed_fetch = True + fetch_timeout *= 10 # increase the timeout for the next try + except psutil.NoSuchProcess: + pass + + if not killed_fetch: + break + if process.returncode: + raise subprocess.CalledProcessError(process.returncode, cmd, output=output) + + if raw: + return output + return output.decode(errors=errors) def _fetch(self, sha): + self._git_init() if not self._hash_exists(sha): - self._update(force=True) + for remote in self.remote_ids: + try: + self._git(['fetch', remote.remote_name, sha]) + break + except subprocess.CalledProcessError: + pass if not self._hash_exists(sha): - for remote in self.remote_ids: - try: - self._git(['fetch', remote.remote_name, sha]) - _logger.info('Success fetching specific head %s on %s', sha, remote) - break - except subprocess.CalledProcessError: - pass - if not self._hash_exists(sha): - raise RunbotException("Commit %s is unreachable. Did you force push the branch?" % sha) + raise RunbotException("Commit %s is unreachable, most likely because it is not attached to any branch anymore" % sha) def _hash_exists(self, commit_hash): """ Verify that a commit hash exists in the repo """ @@ -555,7 +595,10 @@ def _get_refs(self, max_age=30, ignore=None): self._set_ref_time(get_ref_time) fields = ['refname', 'objectname', 'committerdate:unix', 'authorname', 'authoremail', 'subject', 'committername', 'committeremail', 'tree'] fmt = "%00".join(["%(" + field + ")" for field in fields]) - cmd = ['for-each-ref', '--format', fmt, '--sort=-committerdate', 'refs/*/heads/*'] + refs_desc = 'refs/*/heads/*' + if self.allow_slashes: + refs_desc = 'refs/*/heads/**' + cmd = ['for-each-ref', '--format', fmt, '--sort=-committerdate', refs_desc] if any(remote.fetch_pull for remote in self.remote_ids): cmd.append('refs/*/pull/*') git_refs = self._git(cmd) @@ -582,7 +625,7 @@ def _find_or_create_branches(self, refs): """ # FIXME WIP - names = [r[0].split('/')[-1] for r in refs] + names = [r[0].split('/', 3)[-1] for r in refs] branches = self.env['runbot.branch'].search([('name', 'in', names), ('remote_id', 'in', self.remote_ids.ids)]) ref_branches = {branch._ref(): branch for branch in branches} new_branch_values = [] @@ -591,7 +634,7 @@ def _find_or_create_branches(self, refs): # format example: # refs/ruodoo-dev/heads/12.0-must-fail # refs/ruodoo/pull/1 - _, remote_name, branch_type, name = ref_name.split('/') + _, remote_name, branch_type, name = ref_name.split('/', 3) remote_id = self.remote_ids.filtered(lambda r: r.remote_name == remote_name).id if not remote_id: _logger.warning('Remote %s not found', remote_name) @@ -671,6 +714,9 @@ def _update_batches(self, force=False, ignore=None): def _update_git_config(self): """ Update repo git config file """ + if not self: + return + _logger.info('Updating git config for %s repos', len(self)) for repo in self: if repo.mode == 'disabled': _logger.info(f'skipping disabled repo {repo.name}') @@ -684,6 +730,7 @@ def _update_git_config(self): _logger.info('Config updated for repo %s' % repo.name) else: _logger.info('Repo not cloned, skiping config update for %s' % repo.name) + return max(self.mapped('write_date')) def _git_init(self): """ Clone the remote repo if needed """ diff --git a/runbot/models/runbot.py b/runbot/models/runbot.py index 047e3090b..195efe4a7 100644 --- a/runbot/models/runbot.py +++ b/runbot/models/runbot.py @@ -53,10 +53,10 @@ def _scheduler(self, host): processed += 1 build._process_requested_actions() self._commit() + if host._process_messages(): + self._commit() host._process_logs() self._commit() - host._process_messages() - self._commit() for build in host._get_builds([('local_state', 'in', ['testing', 'running'])]) | self._get_builds_to_init(host): build = build.browse(build.id) # remove preftech ids, manage build one by one result = build._schedule() @@ -104,7 +104,7 @@ def _get_builds_to_init(self, host): def _gc_running(self, host): running_max = host._get_running_max() Build = self.env['runbot.build'] - cannot_be_killed_ids = host._get_builds([('keep_running', '=', True)]).ids + cannot_be_killed_ids = host._get_builds([('gc_running_date', '>', fields.Date.today())]).ids sticky_bundles = self.env['runbot.bundle'].search([('sticky', '=', True), ('project_id.keep_sticky_running', '=', True)]) cannot_be_killed_ids += [ build.id @@ -113,14 +113,14 @@ def _gc_running(self, host): ][:running_max] build_ids = host._get_builds([('local_state', '=', 'running'), ('id', 'not in', cannot_be_killed_ids)], order='job_start desc').ids for build in Build.browse(build_ids)[running_max:]: - build._kill() + build._kill(None) def _gc_testing(self, host): """garbage collect builds that could be killed""" # decide if we need room Build = self.env['runbot.build'] domain_host = host._get_build_domain() - testing_builds = Build.search(domain_host + [('local_state', 'in', ['testing', 'pending']), ('requested_action', '!=', 'deathrow')]) + testing_builds = Build.search(domain_host + [('local_state', 'in', ['testing', 'pending']), ('message_ids', '=', False)]) used_slots = len(testing_builds) available_slots = host.nb_worker - used_slots nb_pending = Build.search_count([('local_state', '=', 'pending'), ('host', '=', False)]) diff --git a/runbot/security/ir.model.access.csv b/runbot/security/ir.model.access.csv index 6c233c654..f0ca84182 100644 --- a/runbot/security/ir.model.access.csv +++ b/runbot/security/ir.model.access.csv @@ -68,6 +68,9 @@ access_runbot_error_regex_manager,runbot_error_regex_manager,runbot.model_runbot access_runbot_host_public,runbot_host_public,runbot.model_runbot_host,runbot.base_runbot_model_access,1,0,0,0 access_runbot_host_manager,runbot_host_manager,runbot.model_runbot_host,runbot.group_runbot_admin,1,1,1,1 +access_runbot_host_message_public,runbot_host_message_public,runbot.model_runbot_host_message,runbot.base_runbot_model_access,1,0,0,0 +access_runbot_host_message_admin,runbot_host_message_admin,runbot.model_runbot_host_message,runbot.group_runbot_admin,1,1,1,1 + access_runbot_repo_hooktime,runbot_repo_hooktime,runbot.model_runbot_repo_hooktime,group_user,1,0,0,0 access_runbot_repo_referencetime,runbot_repo_referencetime,runbot.model_runbot_repo_reftime,group_user,1,0,0,0 access_runbot_build_stat_admin,runbot_build_stat_admin,runbot.model_runbot_build_stat,runbot.group_runbot_admin,1,1,1,1 @@ -106,7 +109,6 @@ access_runbot_bundle_public,access_runbot_bundle_public,runbot.model_runbot_bund access_runbot_bundle_runbot_bundle_manager,access_runbot_bundle_runbot_manager,runbot.model_runbot_bundle,runbot.group_runbot_bundle_manager,1,1,0,0 access_runbot_bundle_runbot_admin,access_runbot_bundle_runbot_admin,runbot.model_runbot_bundle,runbot.group_runbot_admin,1,1,1,1 - access_runbot_batch_public,access_runbot_batch_public,runbot.model_runbot_batch,runbot.base_runbot_model_access,1,0,0,0 access_runbot_batch_runbot_admin,access_runbot_batch_runbot_admin,runbot.model_runbot_batch,runbot.group_runbot_admin,1,1,1,1 diff --git a/runbot/static/src/css/runbot.css b/runbot/static/src/css/runbot.css index 46415f421..39df923c8 100644 --- a/runbot/static/src/css/runbot.css +++ b/runbot/static/src/css/runbot.css @@ -1,5 +1,9 @@ :root { --gray: #6c757d; /* used for batch limitation */ + --btn-default-color: var(--bs-body-color); + --btn-default-bg: var(--bs-body-bg); + --btn-default-border: #ccc; + --active-project-color: #777; } /* @@ -13,7 +17,8 @@ --bs-info-bg-subtle: #d9edf7; --bs-info-rgb: 23, 162, 184; } -:root[data-bs-theme=red404] { + +:root[data-bs-theme=red404] { --bs-success-bg-subtle: #cdffb9; --bs-danger-bg-subtle: #e67ecf; --bs-warning-bg-subtle: #fae9b1; @@ -21,6 +26,11 @@ --bs-info-rgb: 23, 162, 184; } +:root[data-bs-theme=dark] { + --btn-default-border: #333; + --active-project-color: #CCC; +} + [data-bs-theme=legacy] .text-bg-info { color: #fff !important; /* black by default, changes from previous version, color forced to fit with --bs-info-rgb*/ } @@ -59,37 +69,28 @@ --bs-btn-disabled-border-color: #b90e6c; } - -:root { - --alternative:#ccc; - --btn-default-color: var(--bs-body-color); - --btn-default-border:#ccc; - --bs-default-rgb: var(--bs-body-color-rgb); - --active-project-color: #777; - -} - -:root[data-bs-theme=dark] { - --btn-default-border:#333; - --btn-default-color: var(--bs-body-color); - --active-project-color: #CCC; -} - .btn-default { --bs-btn-color: var(--btn-default-color); - --bs-btn-bg: var(--bs-body-bg); + --bs-btn-bg: var(--btn-default-bg); --bs-btn-border-color: var(--btn-default-border); --bs-btn-hover-color: var(--btn-default-color); - --bs-btn-hover-bg: var(--btn-default-border); - --bs-btn-hover-border-color: var(--btn-default-border); + --bs-btn-hover-bg: color-mix(in lab, var(--btn-default-bg), black 15%); + --bs-btn-hover-border-color: color-mix(in lab, var(--btn-default-border), black 10%); --bs-btn-focus-shadow-rgb: 60, 153, 110; --bs-btn-active-color: var(--btn-default-color); - --bs-btn-active-bg: var(--bs-body-bg); - --bs-btn-active-border-color: var(--bs-body-bg); + --bs-btn-active-bg: color-mix(in lab, var(--btn-default-bg), black 20%); + --bs-btn-active-border-color: color-mix(in lab, var(--btn-default-border), black 15%); --bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125); --bs-btn-disabled-color: var(--btn-default-color); - --bs-btn-disabled-bg: var(--bs-body-bg); - --bs-btn-disabled-border-color: var(--btn-default-border);; + --bs-btn-disabled-bg: var(--btn-default-bg); + --bs-btn-disabled-border-color: var(--btn-default-border); +} + +[data-bs-theme=dark] .btn-default { + --bs-btn-hover-bg: color-mix(in lab, var(--btn-default-bg), white 15%); + --bs-btn-hover-border-color: color-mix(in lab, var(--btn-default-border), white 10%); + --bs-btn-active-bg: color-mix(in lab, var(--btn-default-bg), white 20%); + --bs-btn-active-border-color: color-mix(in lab, var(--btn-default-border), white 15%); } .btn-info { @@ -152,6 +153,22 @@ table { font-size: 0.875rem; } +dialog.modal { + --bs-modal-zindex: auto; + margin: 0; + padding: 0; + border: none; + background-color: transparent; + + &::backdrop { + background-color: rgba(0, 0, 0, 0.5); + } + + &[open] { + display: block; + } +} + .fa { line-height: inherit; /* reset fa icon line height to body height*/ } @@ -184,7 +201,7 @@ a.slots_infos:hover { } .separator { - border-top: 2px solid #666; + border-top: 0.2em solid #666; } body, .table { @@ -426,3 +443,31 @@ code { .hide-success tr.bg-success-subtle { display: none; } + +.pre { + display: block; + font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; + white-space: pre; + overflow: auto; + font-size: 0.875em; + margin:0; + padding:0; + margin-top: 0.2em; + border: none; + +} + +.subtle_link { + color: var(--bs-body-color); + text-decoration: underline; +} + +.table-condensed .log-server td, .table-condensed .log-details td { + padding-top: 0; + padding-bottom: 0; + border: none; +} + +.log-details td { + padding-left: 20px; +} diff --git a/runbot/static/src/js/polyfill_command_api.js b/runbot/static/src/js/polyfill_command_api.js new file mode 100644 index 000000000..701498808 --- /dev/null +++ b/runbot/static/src/js/polyfill_command_api.js @@ -0,0 +1,28 @@ +// @odoo-module ignore +(function () { + if ( + typeof HTMLButtonElement !== "undefined" && + "command" in HTMLButtonElement.prototype && + // eslint-disable-next-line no-undef + "source" in ((CommandEvent || {}).prototype || {}) + ) { + return; + } + const SUPPORTED_COMMANDS = { + "show-modal": "showModal", + "close": "close", + }; + document.addEventListener("click", (ev) => { + const commandEl = ev.target.closest("[commandfor]"); + if (!commandEl) { + return; + } + const forTarget = document.getElementById(commandEl.getAttribute("commandfor")); + const command = commandEl.getAttribute("command"); + if (command in SUPPORTED_COMMANDS) { + forTarget[SUPPORTED_COMMANDS[command]](); + } else { + throw new Error(`UnsupportedCommand: ${command} is not a supported command.`); + } + }); +})(); diff --git a/runbot/static/src/js/runbot.js b/runbot/static/src/js/runbot.js index 58a902976..76b2a129b 100644 --- a/runbot/static/src/js/runbot.js +++ b/runbot/static/src/js/runbot.js @@ -38,3 +38,19 @@ function copyToClipboard(text) { } navigator.clipboard.writeText(text); } + +document.addEventListener('DOMContentLoaded', function() { + const collapseElement = document.getElementById('customTriggers'); + if (collapseElement) { + collapseElement.addEventListener('show.bs.collapse', function () { + const url = new URL(window.location); + url.searchParams.set('expand_custom', '1'); + window.history.replaceState({}, '', url); + }); + collapseElement.addEventListener('hide.bs.collapse', function () { + const url = new URL(window.location); + url.searchParams.delete('expand_custom'); + window.history.replaceState({}, '', url); + }); + } +}); diff --git a/runbot/templates/build.xml b/runbot/templates/build.xml index b751ae50b..9f57f309c 100644 --- a/runbot/templates/build.xml +++ b/runbot/templates/build.xml @@ -44,7 +44,7 @@
  • - +
    • @@ -68,7 +68,7 @@
    -
    +
    This build is referenced in bundles
    @@ -114,6 +114,9 @@
    + + Mode: Light Configure
    +     Version: @@ -222,7 +225,7 @@ - + Build @@ -265,25 +268,27 @@
    - + - - - + - + - - + @@ -291,54 +296,22 @@ - - + diff --git a/runbot/templates/build_stats.xml b/runbot/templates/build_stats.xml index 8f36b6850..7721b8490 100644 --- a/runbot/templates/build_stats.xml +++ b/runbot/templates/build_stats.xml @@ -4,7 +4,7 @@ diff --git a/runbot/templates/utils.xml b/runbot/templates/utils.xml index 1687e18e7..1b9554322 100644 --- a/runbot/templates/utils.xml +++ b/runbot/templates/utils.xml @@ -15,7 +15,7 @@ - + @@ -31,7 +31,7 @@ - + @@ -67,11 +67,11 @@
  • - -
  • - + +
  • +
  • @@ -147,68 +147,78 @@ -
    -
    - -
    -
    - - -
    -
    - -
    - - - - -
    -
    - -
    - - - - -
    -
    - -
    - -
    - -
    -
    - - - - -

    -
    - -
    - -
    +
    +
    + + + + +
    + +

    + @@ -273,6 +283,7 @@ + Light @@ -329,7 +340,7 @@ default - + killed @@ -364,12 +375,12 @@ Database selector - + Rebuild - + Kill diff --git a/runbot/tests/common.py b/runbot/tests/common.py index 9f6ba86c8..81cc8e784 100644 --- a/runbot/tests/common.py +++ b/runbot/tests/common.py @@ -12,7 +12,7 @@ class RunbotCase(TransactionCase): - def mock_git_helper(self, repo, cmd): + def mock_git_helper(self, repo, cmd, input_data=None, raw=False): """Helper that returns a mock for repo._git()""" if cmd[:2] == ['show', '-s'] or cmd[:3] == ['show', '--pretty="%H -- %s"', '-s']: return 'commit message for %s' % cmd[-1] @@ -82,7 +82,9 @@ def setUp(self): self.repo_odoo: [ ('odoo/addons', 'base', '__manifest__.py'), ('odoo/addons', 'test_lint', '__manifest__.py'), + ('addons', 'account', '__manifest__.py'), ('addons', 'mail', '__manifest__.py'), + ('addons', 'test_mail', '__manifest__.py'), ('addons', 'web', '__manifest__.py'), ('addons', 'crm', '__manifest__.py'), ('addons', 'project', '__manifest__.py'), @@ -194,8 +196,8 @@ def setUp(self): self.docker_run_calls = [] self.diff = '' - def mock_git(repo, cmd, quiet=False): - return self.mock_git_helper(repo, cmd) + def mock_git(repo, cmd, quiet=False, input_data=None, raw=False): + return self.mock_git_helper(repo, cmd, input_data=input_data, raw=raw) self.start_patcher('git_patcher', 'odoo.addons.runbot.models.repo.Repo._git', new=mock_git) self.start_patcher('hostname_patcher', 'odoo.addons.runbot.common.socket.gethostname', 'host.runbot.com') @@ -232,10 +234,10 @@ def mock_git(repo, cmd, quiet=False): self.start_patcher('_write_file', 'odoo.addons.runbot.models.build.BuildResult._write_file', None) self.start_patcher('_parse_config', 'odoo.addons.runbot.models.build.BuildResult._parse_config', {'--test-enable', '--test-tags', '--with-demo'}) - def get_available_modules(self_commit): + def _list_available_modules(self_commit): return self.addons_per_repo.get(self_commit.repo_id, []) - self.start_patcher('_get_available_modules', 'odoo.addons.runbot.models.commit.Commit._get_available_modules', new=get_available_modules) + self.start_patcher('_list_available_modules', 'odoo.addons.runbot.models.commit.Commit._list_available_modules', new=_list_available_modules) def no_commit(*_args, **_kwargs): _logger.info('Skipping commit') diff --git a/runbot/tests/test_build.py b/runbot/tests/test_build.py index e55aab30a..2cf84f65e 100644 --- a/runbot/tests/test_build.py +++ b/runbot/tests/test_build.py @@ -705,6 +705,32 @@ def test_build_cmd_faketime(self): second_child_cmd = second_child._cmd(py_version=3) self.assertIn('faketime "2024-02-04 04:42 UTC" python3 odoo/server.py', str(second_child_cmd)) + def test_format_message(self): + def get_log(message): + return self.env['ir.logging'].create({ + 'build_id': build.id, + 'type': 'server', + 'message': message, + 'level': 'INFO', + 'name': 'odoo.addons.web.tests.test_web', + 'path': '', + 'func': '', + 'line': '', + }) + build = self.Build.create({ + 'params_id': self.server_params.id, + 'description': 'A nice **description** with a link to odoo.com', + }) + self.assertEqual( + build._format_message(get_log('File "/data/build/odoo/addons/web/tests/test_web.py", line 42, in test_web')), + 'File "/data/build/odoo/addons/web/tests/test_web.py", line 42, in test_web' + ) + + self.assertEqual( + build._format_message(get_log('File "/data/build/odoo/addons/web/tests/test_web.py", in test_web')), + 'File "/data/build/odoo/addons/web/tests/test_web.py", in test_web' + ) + class TestGc(RunbotCaseMinimalSetup): def test_repo_gc_testing(self): @@ -744,7 +770,6 @@ def test_repo_gc_testing(self): bundle_b = self.env['runbot.bundle'].search([('name', '=', branch_b_name)]) bundle_b.last_batch._process() - build_b = bundle_b.last_batch.slot_ids[0].build_id # the two builds are starting tests on two different hosts @@ -754,8 +779,8 @@ def test_repo_gc_testing(self): # no room needed, verify that nobody got killed self.Runbot._gc_testing(host) - self.assertFalse(build_a.requested_action) - self.assertFalse(build_b.requested_action) + self.assertFalse(build_a.to_kill) + self.assertFalse(build_b.to_kill) # a new commit is pushed on branch_a self.push_commit(self.remote_odoo_dev, branch_a_name, 'new subject', sha='d0cad0ca') @@ -775,10 +800,10 @@ def test_repo_gc_testing(self): # no room needed, verify that nobody got killed self.Runbot._gc_testing(host) - self.assertFalse(build_a.requested_action) - self.assertFalse(build_b.requested_action) - self.assertFalse(build_a_last.requested_action) - self.assertFalse(children_b.requested_action) + self.assertFalse(build_a.to_kill) + self.assertFalse(build_b.to_kill) + self.assertFalse(build_a_last.to_kill) + self.assertFalse(children_b.to_kill) # now children_b starts on runbot_xxx children_b.write({'local_state': 'testing', 'host': host.name}) @@ -789,10 +814,10 @@ def test_repo_gc_testing(self): self.Runbot._gc_testing(host) # the killable build should have been marked to be killed - self.assertEqual(build_a.requested_action, 'deathrow') - self.assertFalse(build_b.requested_action) - self.assertFalse(build_a_last.requested_action) - self.assertFalse(children_b.requested_action) + self.assertTrue(build_a.to_kill) + self.assertFalse(build_b.to_kill) + self.assertFalse(build_a_last.to_kill) + self.assertFalse(children_b.to_kill) class TestGithubStatus(RunbotCase): diff --git a/runbot/tests/test_build_config_step.py b/runbot/tests/test_build_config_step.py index 063c318ec..1e66d26fc 100644 --- a/runbot/tests/test_build_config_step.py +++ b/runbot/tests/test_build_config_step.py @@ -439,25 +439,58 @@ def setUp(self): }).id, 'local_result': 'ok', }) + self.module_dependencies = { + "test_mail": ["mail"], + "mail": ["web"], + "account": ["web"], + "crm": ["web"], + "project": ["web"], + "test_l10n": ["l10n_be", "l10n_in"], + "l10n_be": ["account"], + "l10n_in": ["account"], + "web_enterprise": ["web"], + } + + def mock_git_helper(self, repo, cmd, input_data=None, raw=False): + def make_catfile_output(commit, content): + content_bytes = content.encode('utf-8') + header = f"{commit} blob {len(content_bytes)}\n".encode() + result = header + content_bytes + b"\n" + return result + + if cmd == ['cat-file', '--batch']: + if repo == self.repo_odoo and input_data == 'dfdfcfcf0000ffffffffffffffffffffffffffff:odoo/tests/.runbot/parallel_testing.json\n': + return make_catfile_output('dfdfcfcf0000ffffffffffffffffffffffffffff', self.config_file) + if repo == self.repo_odoo and input_data == 'dfdfcfcf0000ffffffffffffffffffffffffffff:odoo/tests/.runbot/l10n_standalone_testing.json\n': + return make_catfile_output('dfdfcfcf0000ffffffffffffffffffffffffffff', self.l10n_standalone_testing_file) + + if "__manifest__.py" in input_data: + modules_info = [ + (line, line.split(':')[-1].split('/')[-2]) + for line in input_data.splitlines() + if line.endswith('__manifest__.py') + ] + result = b"" + for original_query, module in modules_info: + content = '''{'name': '%s', 'depends': %s}''' % (module, self.module_dependencies.get(module, [])) + result += make_catfile_output(original_query.split(':')[0], content) + return result - def mock_git_helper(self, repo, cmd): - if repo == self.repo_odoo and cmd == ['show', 'dfdfcfcf0000ffffffffffffffffffffffffffff:odoo/tests/.runbot/parallel_testing.json']: - return self.config_file - elif repo == self.repo_odoo and cmd == ['show', 'dfdfcfcf0000ffffffffffffffffffffffffffff:odoo/tests/.runbot/l10n_standalone_testing.json']: - return self.l10n_standalone_testing_file - elif 'show' in cmd: + if cmd == ['cat-file', '--batch']: raise subprocess.CalledProcessError(cmd, 128) - return super().mock_git_helper(repo, cmd) + elif 'diff' in cmd: + return 'odoo/addons/crm/some/file.py\nodoo/addons/project/some/file.py' + return super().mock_git_helper(repo, cmd, input_data, raw) def test_module_filters(self): - self.assertEqual(self.build._get_modules_to_test('-> !mail'), ['base', 'crm', 'documents']) - self.assertEqual(self.build._get_modules_to_test('mail -> !web'), ['mail', 'project', 'test_l10n', 'test_lint']) + self.assertEqual(self.build._get_modules_to_test('-> !mail'), ['account', 'base', 'crm', 'documents']) + self.assertEqual(self.build._get_modules_to_test('mail -> !web'), ['mail', 'project', 'test_l10n', 'test_lint', 'test_mail']) self.assertEqual(self.build._get_modules_to_test('web -> web'), ['web']) self.assertEqual(self.build._get_modules_to_test('!web ->'), ['web_enterprise']) - self.assertEqual(self.build._get_modules_to_test('-> !mail, -crm'), ['base', 'documents']) - self.assertEqual(self.build._get_modules_to_test('mail -> !web, !project'), ['mail', 'test_l10n', 'test_lint']) - self.assertEqual(self.build._get_modules_to_test('-*,odoo/*'), ['base', 'crm', 'hw_drivers', 'mail', 'project', 'test_l10n', 'test_lint', 'web']) - self.assertEqual(self.build._get_modules_to_test('-*,odoo/test_*'), ['test_l10n', 'test_lint']) + self.assertEqual(self.build._get_modules_to_test('-> !mail, -crm'), ['account', 'base', 'documents']) + self.assertEqual(self.build._get_modules_to_test('mail -> !web, !project'), ['mail', 'test_l10n', 'test_lint', 'test_mail']) + self.assertEqual(self.build._get_modules_to_test('-*,odoo/*'), ['account', 'base', 'crm', 'hw_drivers', 'mail', 'project', 'test_l10n', 'test_lint', 'test_mail', 'web']) + self.assertEqual(self.build._get_modules_to_test('-*,odoo/test_*'), ['test_l10n', 'test_lint', 'test_mail']) self.assertEqual(self.build._get_modules_to_test('-*,enterprise/*'), ['documents', 'l10n_be', 'l10n_in', 'web_enterprise']) self.assertEqual(self.build._get_modules_to_test('-*,web*'), ['web', 'web_enterprise']) self.assertEqual(self.build._get_modules_to_test('-*,web*,-enterprise/web*'), ['web']) @@ -467,6 +500,35 @@ def test_config_extension(self): self.assertEqual(json.loads(self.config.default_dynamic_config)['vars']['module_filter'], '*,-hw_*') self.assertEqual(self.build.dynamic_config['vars']['module_filter'], '*,-hw_*,-l10n_*') + def test_parse_dynamic_entry(self): + Step = self.env['runbot.build.config.step'] + + def check_parse(entry, expected): + res = Step._parse_dynamic_entry(entry, self.build, {'key': 'value', 'test_method': '.test_method'}) + self.assertEqual(res, expected) + check_parse('{{-test_*|filter_all_modules}}', 'account,base,crm,documents,hw_drivers,l10n_be,l10n_in,mail,project,web,web_enterprise') + check_parse('{{-*,web*|filter_all_modules}}', 'web,web_enterprise') + check_parse('{{-*,web*|filter_all_modules|make_module_test_tags}}', '/web,/web_enterprise') + check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|prepend("some_tag")}}', 'some_tag/web,some_tag/web_enterprise') + check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|prepend(key)}}', 'value/web,value/web_enterprise') + check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|append(".test_method")}}', '/web.test_method,/web_enterprise.test_method') + check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|append(test_method)}}', '/web.test_method,/web_enterprise.test_method') + + self.patch(type(self.build), '_modified_modules', lambda cl, defaults=None: {'crm'}) + + check_parse('{{*|filter_all_modules|modified_modules}}', 'crm') + + def test_modules_dependencies(self): + self.assertEqual(self.build._get_modules_dependencies(['test_mail'], 1), ['mail', 'test_mail']) + self.assertEqual(self.build._get_modules_dependencies(['test_mail']), ['base', 'mail', 'test_mail', 'web']) + self.assertEqual(self.build._get_modules_dependencies(['test_l10n']), ['account', 'base', 'l10n_be', 'l10n_in', 'test_l10n', 'web']) + self.assertEqual(self.build._get_modules_dependencies(['test_mail', 'test_l10n']), ['account', 'base', 'l10n_be', 'l10n_in', 'mail', 'test_l10n', 'test_mail', 'web']) + self.assertEqual(self.build._get_modules_dependencies(['test_mail', 'test_l10n'], 1), ['l10n_be', 'l10n_in', 'mail', 'test_l10n', 'test_mail']) + + self.assertEqual(self.build._get_dependant_modules(['account'], 1), ['account', 'l10n_be', 'l10n_in']) + self.assertEqual(self.build._get_dependant_modules(['account']), ['account', 'l10n_be', 'l10n_in', 'test_l10n']) + self.assertEqual(self.build._get_dependant_modules(['base']), ['account', 'base', 'crm', 'documents', 'hw_drivers', 'l10n_be', 'l10n_in', 'mail', 'project', 'test_l10n', 'test_lint', 'test_mail', 'web', 'web_enterprise']) + def check_server_cmd(self, cmd, install, test_enable, test_tags, db=None): self.assertIn('odoo/server.py', cmd) if install: @@ -537,7 +599,7 @@ def test_dynamic_step_parallel_testing(self): cmd = self.docker_run_calls[0][0] odoo_cmd = cmd.cmd self.check_server_cmd(odoo_cmd, - install=['base', 'crm', 'documents', 'mail', 'project', 'test_l10n', 'test_lint', 'web', 'web_enterprise'], + install=['account', 'base', 'crm', 'documents', 'mail', 'project', 'test_l10n', 'test_lint', 'test_mail', 'web', 'web_enterprise'], test_enable=False, test_tags=None, db=f'{build.dest}-all', @@ -572,7 +634,7 @@ def test_dynamic_step_parallel_testing(self): cmd = self.docker_run_calls[0][0] odoo_cmd = cmd.cmd self.check_server_cmd(odoo_cmd, - install=['base', 'crm', 'documents', 'mail', 'project', 'test_l10n', 'test_lint', 'web', 'web_enterprise'], + install=['account', 'base', 'crm', 'documents', 'mail', 'project', 'test_l10n', 'test_lint', 'test_mail', 'web', 'web_enterprise'], test_enable=True, test_tags='-post_install,-/test_lint', ) @@ -589,8 +651,8 @@ def test_dynamic_step_parallel_testing(self): ) for post_install, expected_tags in [ - (post_install_1, '-at_install,/base,/crm,/documents,/hw_drivers,/l10n_be,/l10n_in'), # we need the blacklisted modules here - (post_install_2, '-at_install,/mail,/project,/test_l10n,/test_lint'), + (post_install_1, '-at_install,/account,/base,/crm,/documents,/hw_drivers,/l10n_be,/l10n_in'), # we need the blacklisted modules here + (post_install_2, '-at_install,/mail,/project,/test_l10n,/test_lint,/test_mail'), (post_install_3, '-at_install,/web'), (post_install_4, '-at_install,/web_enterprise'), ]: @@ -662,11 +724,14 @@ def test_dynamic_step_l10n_standalone(self): ) # 0.2 run standalone l10n script + self.config_step.check_exit_status = True self.docker_run_calls = [] build._schedule()() self.assertEqual(len(self.docker_run_calls), 1, "One docker run should have been called for install_all step") cmd = self.docker_run_calls[0][0] - self.assertEqual(cmd.build(), f'odoo/odoo/tests/test_module_operations.py -d {build.dest}-l10n --data-dir /data/build/datadir/ --addons-path odoo/addons,odoo/core/addons,enterprise --standalone all_l10n') + expected_cmd = f'odoo/odoo/tests/test_module_operations.py -d {build.dest}-l10n --data-dir /data/build/datadir/ --addons-path odoo/addons,odoo/core/addons,enterprise --standalone all_l10n' + expected_cmd += r' ; echo $? > /data/build/logs/running_standalone_exit_status.txt' + self.assertEqual(cmd.build(), expected_cmd) # 0.3. create post install builds build._schedule() @@ -694,7 +759,7 @@ def test_dynamic_step_l10n_standalone(self): (post_install_1, '-external,-external_l10n,post_install_l10n/l10n_hr_payroll_be,post_install_l10n/l10n_hr_payroll_in'), # we need the blacklisted modules here (post_install_2, '-external,-external_l10n,post_install_l10n/l10n_edi_be,post_install_l10n/l10n_edi_in'), (post_install_3, '-external,-external_l10n,post_install_l10n/l10n_reports_be,post_install_l10n/l10n_reports_in'), - (post_install_4, Like('-external,-external_l10n,post_install_l10n/base,post_install_l10n/crm,...')), + (post_install_4, Like('-external,-external_l10n,post_install_l10n/account,post_install_l10n/base,post_install_l10n/crm,...')), ]: with self.subTest(post_install=expected_tags): # 4.1 post install restore @@ -738,6 +803,7 @@ def test_foreach_module(self): self.config.step_ids[0]._run_dynamic(self.build) self.assertEqual(self.build.children_ids.mapped('description'), [ + 'Post install tests for **account**', 'Post install tests for **base**', 'Post install tests for **crm**', 'Post install tests for **documents**', @@ -767,31 +833,154 @@ def test_foreach_modified_module(self): }] }''' - self.patch(type(self.build), '_modified_modules', lambda cl: {'crm'}) + self.patch(type(self.build), '_modified_modules', lambda cl, defaults=None: {'crm'}) self.config.default_dynamic_config = dynamic_config self.config.step_ids[0]._run_dynamic(self.build) self.assertEqual(self.build.children_ids.mapped('description'), - [ - 'Post install tests for **crm**', + [ + 'Post install tests for **crm**', ]) - def test_parse_dynamic_entry(self): - Step = self.env['runbot.build.config.step'] + def test_modified_existing_module(self): + dynamic_config = '''{ + "vars": { + "modified_modules": "{{*|filter_all_modules|modified_modules}}", + "test_modules": "{{modified_modules|prepend('test_')|select_existing_modules}}", + "modules_to_test": "{{modified_modules|union(test_modules)}}" + }, + "name": "Foreach module testing", + "steps": [{ + "name": "Create module builds", + "job_type": "create_build", + "children": [{ + "name": "Test single module", + "description": "Post install tests for **{{modules_to_test}}**", + "steps": [{ + "name": "Start single module test", + "job_type": "odoo", + "install_modules": "{{modules_to_test}}", + "test_tags": "{{modules_to_test|make_module_test_tags}}" + }] + }] + }] + }''' - def check_parse(entry, expected): - res = Step._parse_dynamic_entry(entry, self.build, {'key': 'value', 'test_method': '.test_method'}) - self.assertEqual(res, expected) - check_parse('{{-test_*|filter_all_modules}}', 'base,crm,documents,hw_drivers,l10n_be,l10n_in,mail,project,web,web_enterprise') - check_parse('{{-*,web*|filter_all_modules}}', 'web,web_enterprise') - check_parse('{{-*,web*|filter_all_modules|make_module_test_tags}}', '/web,/web_enterprise') - check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|prepend("some_tag")}}', 'some_tag/web,some_tag/web_enterprise') - check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|prepend(key)}}', 'value/web,value/web_enterprise') - check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|append(".test_method")}}', '/web.test_method,/web_enterprise.test_method') - check_parse('{{-*,web*|filter_all_modules|make_module_test_tags|append(test_method)}}', '/web.test_method,/web_enterprise.test_method') + self.patch(type(self.build), '_modified_modules', lambda cl, defaults=None: {'crm', 'mail'}) + self.config.default_dynamic_config = dynamic_config + self.config.step_ids[0]._run_dynamic(self.build) + self.assertEqual(self.build.children_ids.mapped('description'), + [ + 'Post install tests for **crm,mail,test_mail**', + ]) + child_dynamic_vars = self.build.children_ids.params_id.config_data['dynamic_vars'] + self.assertEqual(child_dynamic_vars, { + 'modified_modules': 'crm,mail', + 'test_modules': 'test_mail', + 'modules_to_test': 'crm,mail,test_mail', + }) - self.patch(type(self.build), '_modified_modules', lambda cl: {'crm'}) + def test_modified_existing_module_parallel(self): + dynamic_config = '''{ + "vars": { + "modified_modules": "{{*|filter_all_modules|modified_modules}}", + "modules_to_test": "{{modified_modules|prepend('test_')|select_existing_modules|union(modified_modules)}}" + }, + "name": "Parallel split modified", + "steps": [{ + "name": "Create module builds", + "job_type": "create_build", + "for_each_vars": [{ + "test_module_filter": "{{modules_to_test}},->!mail" + }, + { + "test_module_filter": "{{modules_to_test}},mail->!website" + }, + { + "test_module_filter": "{{modules_to_test}},website->" + } + ], + "if": "{{child_modules_to_test}}", + "children": [{ + "vars": { + "child_modules_to_test": "{{test_module_filter|select_existing_modules}}" + }, + "name": "Test single module", + "description": "Post install tests for **{{child_modules_to_test}}**", + "steps": [{ + "name": "Start single module test", + "job_type": "odoo", + "install_modules": "{{child_modules_to_test}}", + "test_tags": "{{child_modules_to_test|make_module_test_tags}}" + }] + }] + }] + }''' - check_parse('{{*|filter_all_modules|modified_modules}}', 'crm') + self.patch(type(self.build), '_modified_modules', lambda cl, defaults=None: {'crm', 'mail'}) + self.config.default_dynamic_config = dynamic_config + self.config.step_ids[0]._run_dynamic(self.build) + self.assertEqual(self.build.children_ids.mapped('description'), + [ + 'Post install tests for **crm**', + 'Post install tests for **mail,test_mail**', + ]) + + self.assertEqual(self.build.children_ids[0].params_id.config_data['dynamic_vars']['child_modules_to_test'], 'crm') + self.assertEqual(self.build.children_ids[1].params_id.config_data['dynamic_vars']['child_modules_to_test'], 'mail,test_mail') + + def test_modified_existing_module_parallel_relations(self): + dynamic_config = '''{ + "vars": [ + {"module_filter": "*,-hw_*,-*l10n_*,-theme_*,-account_bacs,-account_reports_cash_basis,-auth_ldap,-base_gengo,-document_ftp,-iot_drivers,-note_pad,-odoo_referral,-odoo_referral_portal,-pad,-pad_project,-pos_blackbox_be,-pos_cache,-pos_six,-social_demo,-website_gengo,-website_instantclick,test_l10n_be_hr_payroll_account,test_l10n_us_hr_payroll_account"}, + {"_modified_modules": "{{module_filter|filter_all_modules|modified_modules}}"}, + {"_modules_dependencies": "{{_modified_modules|get_dependencies(1)}}"}, + {"_dependant_modules": "{{_modified_modules|get_dependant(1)}}"}, + {"_test_modules": "{{_modified_modules|prepend('test_')|select_existing_modules}}"}, + {"_modules_to_test": "{{_modified_modules|union(_test_modules)|union(_dependant_modules)|union(_modules_dependencies)}}"} + ], + "name": "Parallel split modified", + "steps": [{ + "name": "Create module builds", + "job_type": "create_build", + "for_each_vars": [{ + "_test_module_filter": "{{_modules_to_test}},->!mail" + }, + { + "_test_module_filter": "{{_modules_to_test}},mail->!website" + }, + { + "_test_module_filter": "{{_modules_to_test}},website->" + } + ], + "if": "{{child_modules_to_test}}", + "log": "Modified modules: {{_modified_modules}}\\nDepenencies: {{_modules_dependencies}}\\nDependant: {{_dependant_modules}}\\nTest modules: {{_test_modules}}", + "children": [{ + "vars": { + "child_modules_to_test": "{{_test_module_filter|select_existing_modules}}" + }, + "name": "Test single module", + "description": "Post install tests for **{{child_modules_to_test}}**", + "steps": [{ + "name": "Start single module test", + "job_type": "odoo", + "install_modules": "{{child_modules_to_test}}", + "test_tags": "{{child_modules_to_test|make_module_test_tags}}" + }] + }] + }] + }''' + + self.patch(type(self.build), '_modified_modules', lambda cl, defaults=None: {'crm', 'mail'}) + self.config.default_dynamic_config = dynamic_config + self.config.step_ids[0]._run_dynamic(self.build) + self.assertEqual(self.build.children_ids.mapped('description'), + [ + 'Post install tests for **crm**', + 'Post install tests for **mail,test_mail,web**', + ]) + self.assertEqual(self.build.children_ids[0].params_id.config_data['dynamic_vars']['child_modules_to_test'], 'crm') + self.assertEqual(self.build.children_ids[1].params_id.config_data['dynamic_vars']['child_modules_to_test'], 'mail,test_mail,web') + self.assertEqual(list(self.build.children_ids[0].params_id.config_data['dynamic_vars'].keys()), ['module_filter', 'child_modules_to_test']) class TestBuildConfigStep(TestBuildConfigStepCommon): @@ -864,12 +1053,13 @@ def test_coverage(self, mock_checkout): 'name': 'coverage', 'job_type': 'install_odoo', 'coverage': True, + 'coverage_make_report': True }) cmd = config_step._run_install_odoo(self.parent_build)['cmd'] - self.assertEqual(cmd.cmd[:10], ['python3', '-m', 'coverage', 'run', '--branch', '--source', '/data/build', '--omit', '*__manifest__.py,odoo/addons/hw_drivers/*', 'odoo/server.py']) - self.assertIn(['python3', '-m', 'coverage', 'html', '-d', '/data/build/coverage', '--ignore-errors'], cmd.finals) - self.assertIn(['python3', '-m', 'coverage', 'xml', '-o', '/data/build/logs/coverage.xml', '--ignore-errors'], cmd.finals) + self.assertEqual(cmd.cmd[:7], ['python3', '-m', 'coverage', 'run', '--source', '/data/build', 'odoo/server.py']) + self.assertIn(['python3', '-m', 'coverage', 'html', '-d', '/data/build/logs/coverage', '--ignore-errors'], cmd.finals) + self.assertIn(['python3', '-m', 'coverage', 'json', '-o', '/data/build/logs/coverage.json', '--ignore-errors'], cmd.finals) @patch('odoo.addons.runbot.models.build.BuildResult._checkout') @@ -1185,22 +1375,15 @@ def test_network_can_be_enable(self, mock_checkout): 'job_type': 'install_odoo', }) - # by default, network is disabled - def first_docker_run(cmd, log_path, *args, **kwargs): - self.assertFalse(kwargs['network_enabled']) - - self.docker_run_patch = first_docker_run config_step._run_step(self.parent_build)() - - def second_docker_run(cmd, log_path, *args, **kwargs): - self.assertTrue(kwargs['network_enabled']) - - self.docker_run_patch = second_docker_run + self.assertFalse(self.docker_run_calls[0][3]['network_enabled']) parent_build_params = self.parent_build.params_id.copy({'config_data': {'network_enabled': True}}) parent_build = self.parent_build.copy({'params_id': parent_build_params.id}) config_step._run_step(parent_build)() + self.assertTrue(self.docker_run_calls[1][3]['network_enabled']) + @patch('odoo.addons.runbot.models.build.BuildResult._checkout') def test_run_python_networkcan_be_enabled(self, mock_checkout): @@ -1241,6 +1424,21 @@ def test_install_custom_parametric_tags(self, mock_checkout, parse_config): tags = self.get_test_tags(params) self.assertEqual(tags, '"-at_install,/web/foo.py:WebSuite.test_unit_desktop[@bar/test with spaces],-/web/bar.py[@snafu/other test with spaces]"') + @patch('odoo.addons.runbot.models.build.BuildResult._checkout') + def test_memory_limit(self, mock_checkout): + """ test that network can be disabled with config_data """ + config_step = self.ConfigStep.create({ + 'name': 'default', + 'job_type': 'install_odoo', + }) + self.env['ir.config_parameter'].sudo().set_param('runbot.runbot_containers_memory', 10) + config_step._run_step(self.parent_build)() + self.assertEqual(self.docker_run_calls[0][3]['memory'], 10 * 1024 ** 3) + + parent_build_params = self.parent_build.params_id.copy({'config_data': {'memory_limit_factor': 0.9}}) + parent_build = self.parent_build.copy({'params_id': parent_build_params.id}) + config_step._run_step(parent_build)() + self.assertEqual(self.docker_run_calls[1][3]['memory'], 9 * 1024 ** 3) class TestMakeResult(RunbotCase): def setUp(self): @@ -1474,3 +1672,103 @@ def make_warn(build): mock_make_odoo_results.side_effect = make_warn config_step._make_results(build) self.assertEqual(build.local_result, 'warn') + + def test_check_exit_status_ok(self): + self.config_step.write({'check_exit_status': True}) + file_content = """ +Loading stuff +odoo.stuff.modules.loading: Modules loaded. +Some post install stuff +Initiating shutdown + """ + exit_status_content = "0\n" + + def mock_open_files(filename, mode='r', *args, **kwargs): + if filename.endswith('_exit_status.txt'): + return mock_open(read_data=exit_status_content)() + return mock_open(read_data=file_content)() + + with patch('builtins.open', mock_open_files): + with patch('os.path.exists', return_value=True): + self.config_step._make_results(self.build) + + self.assertEqual(self.build.local_result, 'ok') + + def test_check_exit_status_ko(self): + self.config_step.write({'check_exit_status': True}) + file_content = """ +Loading stuff +odoo.stuff.modules.loading: Modules loaded. +Some post install stuff +Initiating shutdown + """ + exit_status_content = "1\n" + + def mock_open_files(filename, mode='r', *args, **kwargs): + if filename.endswith('_exit_status.txt'): + return mock_open(read_data=exit_status_content)() + return mock_open(read_data=file_content)() + + with patch('builtins.open', mock_open_files): + with patch('os.path.exists', return_value=True): + self.config_step._make_results(self.build) + + self.assertEqual(self.build.local_result, 'ko') + self.assertIn(('ERROR', 'Main command exited with status code 1'), self.logs) + + def test_check_exit_status_file_empty(self): + self.config_step.write({'check_exit_status': True}) + file_content = """ +Loading stuff +odoo.stuff.modules.loading: Modules loaded. +Some post install stuff +Initiating shutdown + """ + exit_status_content = "" + + def mock_open_files(filename, mode='r', *args, **kwargs): + if filename.endswith('_exit_status.txt'): + return mock_open(read_data=exit_status_content)() + return mock_open(read_data=file_content)() + + with patch('builtins.open', mock_open_files): + with patch('os.path.exists', return_value=True): + self.config_step._make_results(self.build) + + self.assertEqual(self.build.local_result, 'ko') + self.assertIn(('ERROR', 'Exception or file empty while reading status file "all_exit_status.txt"'), self.logs) + self.assertIn(('ERROR', 'Main command exited with status code -241'), self.logs) + + def test_check_exit_status_file_non_integer(self): + self.config_step.write({'check_exit_status': True}) + file_content = """ +Loading stuff +odoo.stuff.modules.loading: Modules loaded. +Some post install stuff +Initiating shutdown + """ + exit_status_content = "d0d0caca" + + def mock_open_files(filename, mode='r', *args, **kwargs): + if filename.endswith('_exit_status.txt'): + return mock_open(read_data=exit_status_content)() + return mock_open(read_data=file_content)() + + with patch('builtins.open', mock_open_files): + with patch('os.path.exists', return_value=True): + self.config_step._make_results(self.build) + + self.assertEqual(self.build.local_result, 'ko') + self.assertIn(('ERROR', 'Status file "all_exit_status.txt" does not contain an integer'), self.logs) + self.assertIn(('ERROR', 'Main command exited with status code -242'), self.logs) + + def test_check_exit_status_file_not_found(self): + config_step = self.ConfigStep.create({ + 'name': 'test', + 'job_type': 'install_odoo', + 'check_exit_status': True, + }) + self.patchers['file_exist'].return_value = False + config_step._make_results(self.build) + self.assertEqual(self.build.local_result, 'ko') + self.assertIn(('ERROR', 'Exit status file "test_exit_status.txt" not found'), self.logs) diff --git a/runbot/tests/test_build_error.py b/runbot/tests/test_build_error.py index 24e9ab6a2..0217e3182 100644 --- a/runbot/tests/test_build_error.py +++ b/runbot/tests/test_build_error.py @@ -219,6 +219,44 @@ def test_merge_pr_ids(self): self.assertEqual(error_a.fixing_pr_id, self.dev_pr) self.assertEqual(error_a.breaking_pr_id, self.dev_pr) + def test_duplicate_breaking_pr(self): + pr_branch = self.Branch.create({ + 'name': '242', + 'is_pr': True, + 'alive': True, + 'remote_id': self.remote_odoo.id, + 'target_branch_name': self.branch_odoo.name, + 'pull_head_name': f'{self.remote_odoo.owner}:{self.dev_branch.name}', + }) + + error_a = self.BuildError.create({'content': 'error A', 'breaking_pr_id': pr_branch.id}) + error_b = self.BuildError.create({'content': 'error B', 'breaking_pr_id': pr_branch.id}) + error_c = self.BuildError.create({'content': 'error C', 'breaking_pr_id': pr_branch.id}) + error_d = self.BuildError.create({'content': 'error D'}) + + # Test compute count (excludes self) + self.assertEqual(error_a.duplicate_breaking_pr_count, 2) + self.assertEqual(error_b.duplicate_breaking_pr_count, 2) + self.assertEqual(error_c.duplicate_breaking_pr_count, 2) + self.assertEqual(error_d.duplicate_breaking_pr_count, 0) + + # Test action returns all errors including self + action = error_a.action_view_duplicate_breaking_pr() + self.assertEqual(action['type'], 'ir.actions.act_window') + self.assertEqual(action['res_model'], 'runbot.build.error') + errors_in_action = self.BuildError.search(action['domain']) + self.assertIn(error_a, errors_in_action) + self.assertEqual(len(errors_in_action), 3) + + # Test count update on deactivation + error_b.active = False + error_a.invalidate_recordset(['duplicate_breaking_pr_count']) + self.assertEqual(error_a.duplicate_breaking_pr_count, 1) + + # Test count update on breaking PR removal + error_a.breaking_pr_id = False + self.assertEqual(error_a.duplicate_breaking_pr_count, 0) + def test_relink_contents(self): build_a = self.create_test_build({'local_result': 'ko', 'local_state': 'done'}) error_content_a = self.BuildErrorContent.create({'content': 'foo bar'}) @@ -266,7 +304,7 @@ def test_relink_simple(self): build_b = self.create_test_build({'local_result': 'ko', 'local_state': 'done'}) error_content_b = self.BuildErrorContent.create({'content': 'foo bar'}) error_b = error_content_b.error_id - error_b.test_tags = 'footag' + error_b.test_tags = 'footag[@test, comma]\nfootag2[@test, comma],footag3[@test, comma]' self.BuildErrorLink.create({'build_id': build_b.id, 'error_content_id': error_content_b.id}) self.assertEqual(self.BuildErrorContent.search([('fingerprint', '=', error_content_a.fingerprint)]), error_content_a | error_content_b) @@ -275,7 +313,7 @@ def test_relink_simple(self): self.assertFalse(error_b.error_content_ids) self.assertTrue(error_a.active, 'The merged error without test tags should have been deactivated') - self.assertEqual(error_a.test_tags, 'footag', 'Tags should have been transfered from b to a') + self.assertEqual(error_a.test_tags, 'footag[@test, comma]\nfootag2[@test, comma],footag3[@test, comma]', 'Tags should have been transfered from b to a') self.assertFalse(error_b.active, 'The merged error with test tags should remain active') self.assertIn(build_a, error_content_a.build_ids) self.assertIn(build_b, error_content_a.build_ids) @@ -284,9 +322,9 @@ def test_relink_simple(self): tagged_error_content = self.BuildErrorContent.create({'content': 'foo bar'}) tagged_error = tagged_error_content.error_id - tagged_error.test_tags = 'bartag' + tagged_error.test_tags = 'bartag[@test, comma]\nbartag2[@test, comma],bartag3[@test, comma]' (error_content_a | tagged_error_content)._relink() - self.assertEqual(error_a.test_tags, 'footag,bartag') + self.assertEqual(error_a.test_tags, 'footag[@test, comma]\nfootag2[@test, comma]\nfootag3[@test, comma]\nbartag[@test, comma]\nbartag2[@test, comma]\nbartag3[@test, comma]') self.assertTrue(error_a.active) self.assertFalse(tagged_error.active) @@ -869,7 +907,38 @@ def test_error_content_multiple_canonical_tags(self): self.assertEqual(error_content_2.canonical_tag, '/web/tests/test_file.py:TestUi.TestUi') self.assertNotEqual(error_content_1, error_content_2) self.assertEqual(error_content_1.error_id, error_content_2.error_id) - self.assertEqual(error_content_1.error_id.canonical_tags, '/base/tests/test_file.py:TestUi.TestUi,/web/tests/test_file.py:TestUi.TestUi') + self.assertEqual(error_content_1.error_id.canonical_tags, '/base/tests/test_file.py:TestUi.TestUi\n/web/tests/test_file.py:TestUi.TestUi') + error_content_1.error_id.test_tags = error_content_1.error_id.canonical_tags + self.assertEqual(error_content_1.error_id._disabling_tags(), [ + '-/base/tests/test_file.py:TestUi.TestUi', + '-/web/tests/test_file.py:TestUi.TestUi', + ]) + error_content_3 = self.env['runbot.build.error.content'].create({ + 'content': 'Tour foobar_tour failed at step step_14 in mode mode', + 'metadata': {'test': {'canonical_tag': '/web/tests/test_file.py:TestJs.test_unit_desktop[@web/some , comma\\[\\] and brackets]'}}, + }) + self.assertEqual(error_content_3.canonical_tag, '/web/tests/test_file.py:TestJs.test_unit_desktop[@web/some , comma\\[\\] and brackets]') + self.assertNotEqual(error_content_1, error_content_2) + self.assertEqual(error_content_1.error_id, error_content_2.error_id) + self.assertEqual(error_content_1.error_id.canonical_tags, """/base/tests/test_file.py:TestUi.TestUi\n/web/tests/test_file.py:TestJs.test_unit_desktop[@web/some , comma\\[\\] and brackets]\n/web/tests/test_file.py:TestUi.TestUi""") + error_content_1.error_id.test_tags = error_content_1.error_id.canonical_tags + self.assertEqual(error_content_1.error_id._disabling_tags(), [ + '-/base/tests/test_file.py:TestUi.TestUi', + '-/web/tests/test_file.py:TestJs.test_unit_desktop[@web/some , comma\\[\\] and brackets]', + '-/web/tests/test_file.py:TestUi.TestUi', + ], "disabling tags must keep the escaping and correct format") + error_content_1.error_id.test_tags = error_content_1.error_id.test_tags.replace('\n', ',') + self.assertEqual(error_content_1.error_id._disabling_tags(), [ + '-/base/tests/test_file.py:TestUi.TestUi', + '-/web/tests/test_file.py:TestJs.test_unit_desktop[@web/some , comma\\[\\] and brackets]', + '-/web/tests/test_file.py:TestUi.TestUi', + ], "_disabling_tags shoudl also work fine with comma separted tags") + error_content_1.error_id.test_tags = error_content_1.error_id.test_tags.replace('\\', '') + self.assertEqual(error_content_1.error_id._disabling_tags(), [ + '-/base/tests/test_file.py:TestUi.TestUi', + '-/web/tests/test_file.py:TestJs.test_unit_desktop[@web/some , comma[] and brackets]', + '-/web/tests/test_file.py:TestUi.TestUi', + ]) class TestCodeOwner(RunbotCase): diff --git a/runbot/tests/test_dockerfile.py b/runbot/tests/test_dockerfile.py index cb7539bc2..b04ea6fb2 100644 --- a/runbot/tests/test_dockerfile.py +++ b/runbot/tests/test_dockerfile.py @@ -44,7 +44,7 @@ def test_docker_default(self): && rm -rf /var/lib/apt/lists/* RUN set -x ; \ apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends publicsuffix python3 flake8 python3-dbfread python3-dev python3-gevent python3-pip python3-setuptools python3-wheel python3-markdown python3-mock python3-phonenumbers python3-websocket python3-google-auth libpq-dev pylint python3-jwt python3-asn1crypto python3-html2text python3-suds python3-xmlsec python3-markdown2 python3-aiosmtpd \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends publicsuffix python3 flake8 python3-dbfread python3-dev python3-gevent python3-pip python3-setuptools python3-wheel python3-markdown python3-mock python3-phonenumbers python3-websocket python3-google-auth libpq-dev pylint python3-jwt python3-asn1crypto python3-html2text python3-suds python3-xmlsec python3-markdown2 python3-aiosmtpd python3-paramiko \ && rm -rf /var/lib/apt/lists/* RUN curl -sSL https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox_0.12.6.1-2.jammy_amd64.deb -o /tmp/wkhtml.deb \ && apt-get update \ @@ -204,7 +204,7 @@ def test_dockerfile_get_cached_content(self): expected_content = """# CacheAddTest FROM ubuntu:noble -# CACHE 60 +# CACHED ADD https://nowhere.example.org/nothing.txt /data/nothing.txt COPY _data_nothing_txt /data/nothing.txt @@ -270,7 +270,7 @@ def test_dockerfile_build_with_cached_content(self): expected_content = """# Cache Test FROM ubuntu:noble -# CACHE 60 +# CACHED ADD https://nowhere.example.org/nothing.txt /data/nothing.txt COPY _data_nothing_txt /data/nothing.txt diff --git a/runbot/tests/test_repo.py b/runbot/tests/test_repo.py index 81c940ab0..d4c40fcae 100644 --- a/runbot/tests/test_repo.py +++ b/runbot/tests/test_repo.py @@ -372,7 +372,7 @@ def setUp(self): self.fetch_count = 0 self.force_failure = False - def mock_git_helper(self, repo, cmd): + def mock_git_helper(self, repo, cmd, input_data=None, raw=False): self.assertIn('fetch', cmd) self.fetch_count += 1 if self.fetch_count < 3 or self.force_failure: @@ -383,26 +383,28 @@ def mock_git_helper(self, repo, cmd): class TestIdentityFile(RunbotCase): - def check_output_helper(self): - """Helper that returns a mock for repo._git()""" - def mock_check_output(cmd, *args, **kwargs): - expected_option = r'-c core.sshCommand=ssh -i \/.+\/\.ssh\/fake_identity' - git_cmd = ' '.join(cmd) - self.assertTrue(re.search(expected_option, git_cmd), '%s did not match %s' % (git_cmd, expected_option)) - return Mock() + def popen_helper(self): + """Helper that returns a mock for repo._git()""" + def mock_popen(cmd, *args, **kwargs): + expected_option = r'-c core.sshCommand=ssh -i \/.+\/\.ssh\/fake_identity' + git_cmd = ' '.join(cmd) + self.assertTrue(re.search(expected_option, git_cmd), '%s did not match %s' % (git_cmd, expected_option)) + popen_mock = Mock() + attrs = {"communicate.return_value": (b"", b"")} + popen_mock.configure_mock(**attrs) + return popen_mock - return mock_check_output + return mock_popen - def test_identity_file(self): - """test that the identity file is used in git command""" + def test_identity_file(self): + """test that the identity file is used in git command""" - self.patcher_objects['git_patcher'].stop() - self.start_patcher('check_output_patcher', 'odoo.addons.runbot.models.repo.subprocess.check_output', new=self.check_output_helper()) + self.patcher_objects['git_patcher'].stop() + self.start_patcher('popen_patcher', 'odoo.addons.runbot.models.repo.subprocess.Popen', new=self.popen_helper()) + self.repo_odoo.identity_file = 'fake_identity' - self.repo_odoo.identity_file = 'fake_identity' - - with mute_logger("odoo.addons.runbot.models.repo"): - self.repo_odoo._update_fetch_cmd() + with mute_logger("odoo.addons.runbot.models.repo"): + self.repo_odoo._update_fetch_cmd() class TestRepoScheduler(RunbotCase): @@ -457,7 +459,7 @@ def setUp(self): super().setUp() self.test_refs = [] - def mock_git_helper(self, repo, cmd): + def mock_git_helper(self, repo, cmd, input_data=None, raw=False): self.assertIn('for-each-ref', cmd) self.assertIn('refs/*/pull/*', cmd) return '\n'.join(['\x00'.join(ref_data) for ref_data in self.test_refs]) diff --git a/runbot/views/build_error_views.xml b/runbot/views/build_error_views.xml index cee4d2d8e..e521b5eb4 100644 --- a/runbot/views/build_error_views.xml +++ b/runbot/views/build_error_views.xml @@ -10,6 +10,9 @@ +
    diff --git a/runbot/views/build_views.xml b/runbot/views/build_views.xml index 3a6064cfa..71a1b49e5 100644 --- a/runbot/views/build_views.xml +++ b/runbot/views/build_views.xml @@ -12,6 +12,8 @@ + + @@ -63,7 +65,7 @@ - + @@ -80,7 +82,7 @@ - + diff --git a/runbot/views/bundle_views.xml b/runbot/views/bundle_views.xml index 1a0a107cd..ff76a3fb4 100644 --- a/runbot/views/bundle_views.xml +++ b/runbot/views/bundle_views.xml @@ -18,6 +18,9 @@ + + + diff --git a/runbot/views/config_views.xml b/runbot/views/config_views.xml index 11637d564..11d52ac7c 100644 --- a/runbot/views/config_views.xml +++ b/runbot/views/config_views.xml @@ -76,6 +76,10 @@ + + + + diff --git a/runbot/views/dockerfile_views.xml b/runbot/views/dockerfile_views.xml index dab232b01..27d45ac5e 100644 --- a/runbot/views/dockerfile_views.xml +++ b/runbot/views/dockerfile_views.xml @@ -23,6 +23,7 @@ + @@ -107,6 +108,8 @@ + + diff --git a/runbot/views/repo_views.xml b/runbot/views/repo_views.xml index 6cb2f00d9..41b620f6e 100644 --- a/runbot/views/repo_views.xml +++ b/runbot/views/repo_views.xml @@ -17,6 +17,7 @@ + @@ -199,6 +200,7 @@ + diff --git a/runbot_builder/builder.py b/runbot_builder/builder.py index 90637ee64..5ca51d0c8 100755 --- a/runbot_builder/builder.py +++ b/runbot_builder/builder.py @@ -13,6 +13,7 @@ class BuilderClient(RunbotClient): def on_start(self): + self.last_update = datetime(1970, 1, 1) self.last_docker_updates = None if self.host.is_builder: builds_path = self.env['runbot.runbot']._path('build') @@ -30,6 +31,8 @@ def loop_turn(self): self.host._docker_update_images() self.env.cr.commit() if self.host.is_builder: + self.last_update = self.env['runbot.repo'].search([('write_date', '>', self.last_update)])._update_git_config() + self.env.cr.commit() if self.count == 1: # cleanup at second iteration self.env['runbot.runbot']._source_cleanup() self.env.cr.commit() @@ -39,8 +42,6 @@ def loop_turn(self): self.env.cr.commit() self.host._set_psql_conn_count() self.env.cr.commit() - self.env['runbot.repo']._update_git_config() - self.env.cr.commit() self.git_gc() self.env.cr.commit() return self.env['runbot.runbot']._scheduler_loop_turn(self.host) diff --git a/runbot_builder/leader.py b/runbot_builder/leader.py index c39cb8ea5..106d83eb1 100755 --- a/runbot_builder/leader.py +++ b/runbot_builder/leader.py @@ -1,23 +1,27 @@ #!/usr/bin/python3 from tools import RunbotClient, run import logging -import time + +from datetime import datetime + _logger = logging.getLogger(__name__) -class LeaderClient(RunbotClient): # Conductor, Director, Main, Maestro, Lead +class LeaderClient(RunbotClient): def __init__(self, env): self.pull_info_failures = {} + self.last_update = datetime(1970, 1, 1) super().__init__(env) def loop_turn(self): if not self.host.is_leader: _logger.warning('Leader client is not a leader host, skipping loop_turn') return 10 + + self.last_update = self.env['runbot.repo'].search([('write_date', '>', self.last_update)])._update_git_config() + self.env.cr.commit() if self.count == 0: - self.env['runbot.repo']._update_git_config() - self.env.cr.commit() self.git_gc() self.env.cr.commit()
    • DateDate (UTC) LevelType Message
    - -
    - + - + + + + + + + + - - - - Build # - - - - - - - - - - - - - : - + + + + Build # + + + - - - - - - - - - - - - - - - - - - - - - -
    -     + + +     		- + @@ -348,17 +321,16 @@ -
    +
    - This error is already . - - - - - () + + This error is already . + + + + () +