From 7f9eaef3f481ed9750261576abc69e8350b72872 Mon Sep 17 00:00:00 2001 From: Xavier-Do Date: Wed, 27 May 2026 10:13:39 +0200 Subject: [PATCH] [IMP] runbot: move some directory from static Most runbot working dirs are historically stored in static. Even if it is usefull for builds to be able to give access to log and test files, it is not needed for the repos, sources, nginx config, ... This pr moves some of those directories from runbot static to .local/share/runbot for a cleaner deployment --- README.md | 2 +- runbot/models/build.py | 23 +++++------------------ runbot/models/build_config.py | 11 +++++------ runbot/models/commit.py | 2 +- runbot/models/host.py | 16 ++++++++++++---- runbot/models/repo.py | 6 +++--- runbot/models/runbot.py | 25 ++++++++++++++++++++----- runbot/tests/common.py | 1 - runbot/tests/test_build.py | 26 +++++++++++++------------- 9 files changed, 60 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index fd6c9afe0..6184f528d 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,7 @@ Create a repo for your custom addons repo - **Remotes**: `git@github.com:odoo/runbot.git` - The remote *PR* option can be checked if needed to fetch pull request too. Will work only if a github token is given for this repo. -A config file with your remotes should be created for each repo. You can check the content in `/runbot/static/repo/(runbot|odoo)/config`. The repo will be fetched, this operation may take some time too. After that, you should start seeing empty batches in both projects on the frontend (`/` or `/runbot`) +A config file with your remotes should be created for each repo. You can check the content in `~/.local/share/runbot/repo/(runbot|odoo)/config`. The repo will be fetched, this operation may take some time too. After that, you should start seeing empty batches in both projects on the frontend (`/` or `/runbot`) #### Triggers and config At this point, runbot will discover new branches, new commits, create bundle, but no build will be created. diff --git a/runbot/models/build.py b/runbot/models/build.py index dea856f12..a5d5092c9 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -1381,27 +1381,23 @@ def _get_server_info(self, commit=None): _logger.error('None of %s found in commit, actual commit content:\n %s' % (commit.repo_id.server_files, os.listdir(commit._source_path()))) raise RunbotException('No server found in %s' % commit.dname) - def _make_pip_command(self, py_version=None): - if not py_version: - py_version = self._get_py_version() + def _make_pip_command(self): pres = [] if not self.params_id.skip_requirements and not self.params_id.config_data.get('skip_requirements'): for commit_id in self.env.context.get('defined_commit_ids') or self.params_id.commit_ids.sorted(lambda c: (c.repo_id.sequence, c.repo_id.id)): if os.path.isfile(commit_id._source_path('requirements.txt')): repo_dir = self._docker_source_folder(commit_id) requirement_path = os.sep.join([repo_dir, 'requirements.txt']) - pres.append([f'python{py_version}', '-m', 'pip', 'install', '--progress-bar', 'off', '-r', f'{requirement_path}']) + pres.append([f'python3', '-m', 'pip', 'install', '--progress-bar', 'off', '-r', f'{requirement_path}']) return pres - def _cmd(self, python_params=None, py_version=None, local_only=True, sub_command=None, enable_log_db=True): + def _cmd(self, python_params=None, local_only=True, sub_command=None, enable_log_db=True): """Return a list describing the command to start the build """ self.ensure_one() build = self python_params = python_params or [] - py_version = py_version if py_version is not None else build._get_py_version() - - pres = self._make_pip_command(py_version) + pres = self._make_pip_command() faketime = [] if faketime_params := self.params_id.config_data.get('faketime'): @@ -1415,7 +1411,7 @@ def _cmd(self, python_params=None, py_version=None, local_only=True, sub_command server_dir = self._docker_source_folder(server_commit) # commandline - cmd = faketime + ['python%s' % py_version] + python_params + [os.sep.join([server_dir, server_file])] + cmd = faketime + ['python3'] + python_params + [os.sep.join([server_dir, server_file])] if sub_command: cmd += [sub_command] @@ -1477,15 +1473,6 @@ def _cmd_check(self, cmd): 'build_id': self.id }) - def _get_py_version(self): - """return the python name to use from build batch""" - (server_commit, server_file) = self._get_server_info() - server_path = server_commit._source_path(server_file) - with file_open(server_path, 'r') as f: - if f.readline().strip().endswith('python3'): - return '3' - return '' - def _parse_logs(self): """ Parse build logs to classify errors """ # only parse logs from builds in error and not already scanned diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py index 85ff7af8d..e4f70a83d 100644 --- a/runbot/models/build_config.py +++ b/runbot/models/build_config.py @@ -757,7 +757,6 @@ def _run_install_odoo(self, build, config_data=None): modules_to_install = build._get_modules_to_test(install_module_pattern) mods = ",".join(modules_to_install) python_params = [] - py_version = build._get_py_version() if self.coverage or config_data.get('coverage'): build.coverage = True python_params = ['-m', 'coverage', 'run', '--source', '/data/build'] @@ -768,7 +767,7 @@ def _run_install_odoo(self, build, config_data=None): python_params += self._coverage_params(build, config_data) elif self.flamegraph: python_params = ['-m', 'flamegraph', '-o', self._perfs_data_path(build)] - cmd = build._cmd(python_params, py_version, sub_command=self.sub_command, enable_log_db=self.enable_log_db) + cmd = build._cmd(python_params, sub_command=self.sub_command, enable_log_db=self.enable_log_db) # create db if needed db_suffix = config_data.get('db_name') or (build.params_id.dump_db.db_suffix if not self.create_db else False) or self._get_db_name(build) db_suffix = re.sub(r'[^a-z0-9\-_]', '_', db_suffix.lower()) @@ -834,7 +833,7 @@ def _run_install_odoo(self, build, config_data=None): if extra_params: cmd.extend(shlex.split(extra_params)) - cmd.finals.extend(self._post_install_commands(build, config_data, py_version)) # coverage post, extra-checks, ... + cmd.finals.extend(self._post_install_commands(build, config_data)) # coverage post, extra-checks, ... if config_data.get('export_database', True): self._add_zip_generation(build, cmd, db_name) @@ -1231,11 +1230,11 @@ def _log_end(self, build): message = 'Flamegraph report: [data @icon-download](%s), [svg @icon-eye](%s)' build._log('end_job', message, dat_url, svg_url, log_type='markdown') - def _post_install_commands(self, build, config_data, py_version): + def _post_install_commands(self, build, config_data): cmds = [] if config_data.get('coverage_make_report', (self.coverage and self.coverage_make_report)): - cmds.append(['python%s' % py_version, "-m", "coverage", "html", "-d", "/data/build/logs/coverage", "--ignore-errors"]) - cmds.append(['python%s' % py_version, "-m", "coverage", "json", "-o", "/data/build/logs/coverage.json", "--ignore-errors"]) + cmds.append(['python3', "-m", "coverage", "html", "-d", "/data/build/logs/coverage", "--ignore-errors"]) + cmds.append(['python3', "-m", "coverage", "json", "-o", "/data/build/logs/coverage.json", "--ignore-errors"]) if config_data.get('coverage', self.coverage): cmds.append(['mv', "/data/build/.coverage", f"/data/build/logs/coverage.{build.id}.{int(time.time())}"]) return cmds diff --git a/runbot/models/commit.py b/runbot/models/commit.py index b0aba479a..b2c50d1b9 100644 --- a/runbot/models/commit.py +++ b/runbot/models/commit.py @@ -156,7 +156,7 @@ def _export(self, build): def _read_source(self, file, mode='r'): file_path = self._source_path(file) try: - with file_open(file_path, mode) as f: + with file_open(file_path, mode, env=self.env) as f: return f.read() except: return False diff --git a/runbot/models/host.py b/runbot/models/host.py index 6d657d90f..d15681d7a 100644 --- a/runbot/models/host.py +++ b/runbot/models/host.py @@ -127,11 +127,19 @@ def _bootstrap_db_template(self): def _bootstrap(self): """ Create needed directories in static """ - dirs = ['build', 'nginx', 'repo', 'sources', 'src', 'docker'] - static_path = self.env['runbot.runbot']._root() - static_dirs = {d: self.env['runbot.runbot']._path(d) for d in dirs} - for dir, path in static_dirs.items(): + for local_dir in ['nginx', 'repo', 'sources', 'docker']: + path = self.env['runbot.runbot']._local_path(local_dir) + if not os.path.exists(path): # migration of existing statics dirs, todo remove in 21.0 + static_path = self.env['runbot.runbot']._path(local_dir) + if os.path.exists(static_path): + _logger.info('Moving %s to %s', static_path, path) + os.rename(static_path, path) os.makedirs(path, exist_ok=True) + + for static_dir in ['build', 'src']: + path = self.env['runbot.runbot']._path(static_dir) + os.makedirs(path, exist_ok=True) + self._bootstrap_db_template() self._bootstrap_local_logs_db() diff --git a/runbot/models/repo.py b/runbot/models/repo.py index 51f6a9c8c..9ec5ed58d 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -495,10 +495,10 @@ def _compute_path(self): repo.path = repo._path() def _path(self, *path_parts): - return self.env['runbot.runbot']._path('repo', sanitize(self.name), *path_parts) + return self.env['runbot.runbot']._local_path('repo', sanitize(self.name), *path_parts) def _source_path(self, *path_parts): - return self.env['runbot.runbot']._path('sources', sanitize(self.name), *path_parts) + return self.env['runbot.runbot']._local_path('sources', sanitize(self.name), *path_parts) def _get_git_command(self, cmd, errors='strict'): """Execute a git command 'cmd'""" @@ -718,7 +718,7 @@ def _update_git_config(self): git_config_path = repo._path('config') template_params = {'repo': repo} git_config = self.env['ir.ui.view']._render_template("runbot.git_config", template_params) - with file_open(git_config_path, 'w') as config_file: + with file_open(git_config_path, 'w', env=self.env) as config_file: config_file.write(str(git_config)) _logger.info('Config updated for repo %s' % repo.name) else: diff --git a/runbot/models/runbot.py b/runbot/models/runbot.py index 195efe4a7..03c861d41 100644 --- a/runbot/models/runbot.py +++ b/runbot/models/runbot.py @@ -15,7 +15,7 @@ from odoo import fields, models from odoo.exceptions import UserError from odoo.fields import Domain -from odoo.tools import config, file_open, SQL +from odoo.tools import config, SQL from ..common import dest_reg, os, sanitize from ..container import docker_ps, docker_stop @@ -36,6 +36,13 @@ def _root(self): """Return root directory of repository""" return os.path.abspath(os.sep.join([os.path.dirname(__file__), '../static'])) + def _local_root(self): + """Return local root directory""" + local_root = os.path.expanduser('~/.local/share/runbot') + if not local_root in self.env.transaction._Transaction__file_open_tmp_paths: + self.env.transaction._Transaction__file_open_tmp_paths.append(local_root) + return local_root + def _path(self, *path_parts): """Return the repo build path""" root = self.env['runbot.runbot']._root() @@ -44,6 +51,14 @@ def _path(self, *path_parts): raise UserError('Invalid path') return file_path + def _local_path(self, *path_parts): + """Return the local repo build path""" + root = self.env['runbot.runbot']._local_root() + file_path = os.path.normpath(os.sep.join([root] + [sanitize(path) for path_part in path_parts for path in path_part.split(os.sep) if path])) + if not file_path.startswith(root): + raise UserError('Invalid path') + return file_path + def _scheduler(self, host): self._gc_testing(host) self._commit() @@ -158,7 +173,7 @@ def _reload_nginx(self): settings['port'] = config.get('http_port') settings['runbot_static'] = self.env['runbot.runbot']._root() + os.sep settings['base_url'] = self.get_base_url() - nginx_dir = self.env['runbot.runbot']._path('nginx') + nginx_dir = self.env['runbot.runbot']._local_path('nginx') settings['nginx_dir'] = nginx_dir settings['re_escape'] = re.escape host_name = self.env['runbot.host']._get_current_name() @@ -169,17 +184,17 @@ def _reload_nginx(self): nginx_config = env['ir.ui.view']._render_template("runbot.nginx_config", settings) os.makedirs(nginx_dir, exist_ok=True) content = None - nginx_conf_path = self.env['runbot.runbot']._path('nginx', 'nginx.conf') + nginx_conf_path = self.env['runbot.runbot']._local_path('nginx', 'nginx.conf') content = '' if os.path.isfile(nginx_conf_path): - with file_open(nginx_conf_path, 'r') as f: + with open(nginx_conf_path, 'r') as f: content = f.read() if content != nginx_config: _logger.info('reload nginx') with open(nginx_conf_path, 'w') as f: f.write(str(nginx_config)) try: - pid = int(file_open(self.env['runbot.runbot']._path('nginx', 'nginx.pid')).read().strip(' \n')) + pid = int(open(self.env['runbot.runbot']._local_path('nginx', 'nginx.pid')).read().strip(' \n')) os.kill(pid, signal.SIGHUP) except Exception: _logger.info('start nginx') diff --git a/runbot/tests/common.py b/runbot/tests/common.py index 81cc8e784..a42cc0d73 100644 --- a/runbot/tests/common.py +++ b/runbot/tests/common.py @@ -230,7 +230,6 @@ def mock_git(repo, cmd, quiet=False, input_data=None, raw=False): self.start_patcher('_local_pg_createdb', 'odoo.addons.runbot.models.build.BuildResult._local_pg_createdb', True) self.start_patcher('getmtime', 'odoo.addons.runbot.common.os.path.getmtime', datetime.datetime.now().timestamp()) self.start_patcher('file_exist', 'odoo.tools.misc.os.path.exists', True) - self.start_patcher('_get_py_version', 'odoo.addons.runbot.models.build.BuildResult._get_py_version', 3) self.start_patcher('_write_file', 'odoo.addons.runbot.models.build.BuildResult._write_file', None) self.start_patcher('_parse_config', 'odoo.addons.runbot.models.build.BuildResult._parse_config', {'--test-enable', '--test-tags', '--with-demo'}) diff --git a/runbot/tests/test_build.py b/runbot/tests/test_build.py index 2cf84f65e..0598bebb0 100644 --- a/runbot/tests/test_build.py +++ b/runbot/tests/test_build.py @@ -384,7 +384,7 @@ def test_build_cmd_log_db(self): build = self.Build.create({ 'params_id': self.server_params.id, }) - cmd = build._cmd(py_version=3) + cmd = build._cmd() self.assertIn('log_db = runbot_logs', cmd.get_config()) @@ -400,7 +400,7 @@ def test_build_cmd_custom_pre_post(self): build = self.Build.create({ 'params_id': self.server_params.id, }) - cmd = build._cmd(py_version=3) + cmd = build._cmd() self.assertIn(['python3', '-m', 'pip', 'install', '--progress-bar', 'off', '-r', 'odoo/requirements.txt'], cmd.pres) self.assertIn(custom_pre, cmd.pres) self.assertIn(custom_post, cmd.posts) @@ -411,7 +411,7 @@ def test_build_cmd_server_path_no_dep(self): build = self.Build.create({ 'params_id': self.server_params.id, }) - cmd = build._cmd(py_version=3) + cmd = build._cmd() self.assertEqual('python3', cmd[0]) self.assertEqual('odoo/server.py', cmd[1]) self.assertIn('--addons-path', cmd) @@ -424,13 +424,13 @@ def test_build_cmd_server_path_with_dep(self): def is_file(file): self.assertIn(file, [ - self.env['runbot.runbot']._path('sources/enterprise/0d0d0caca0000fffffffffffffffffffffffffff/requirements.txt'), - self.env['runbot.runbot']._path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/requirements.txt'), - self.env['runbot.runbot']._path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/server.py'), - self.env['runbot.runbot']._path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/odoo/tools/config.py'), - self.env['runbot.runbot']._path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/odoo/sql_db.py'), + self.env['runbot.runbot']._local_path('sources/enterprise/0d0d0caca0000fffffffffffffffffffffffffff/requirements.txt'), + self.env['runbot.runbot']._local_path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/requirements.txt'), + self.env['runbot.runbot']._local_path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/server.py'), + self.env['runbot.runbot']._local_path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/odoo/tools/config.py'), + self.env['runbot.runbot']._local_path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/odoo/sql_db.py'), ]) - return file != self.env['runbot.runbot']._path('static/sources/enterprise/0d0d0caca0000fffffffffffffffffffffffffff/requirements.txt') + return file != self.env['runbot.runbot']._local_path('sources/enterprise/0d0d0caca0000fffffffffffffffffffffffffff/requirements.txt') def is_dir(file): paths = [ @@ -448,7 +448,7 @@ def is_dir(file): 'params_id': self.addons_params.id, }) - cmd = build._cmd(py_version=3) + cmd = build._cmd() self.assertIn('--addons-path', cmd) addons_path_pos = cmd.index('--addons-path') + 1 self.assertEqual(cmd[addons_path_pos], 'odoo/addons,odoo/core/addons,enterprise') @@ -690,19 +690,19 @@ def test_build_cmd_faketime(self): build = self.Build.create({ 'params_id': self.server_params.id, }) - cmd = build._cmd(py_version=3) + cmd = build._cmd() self.assertIn('faketime "2024-02-04 02:42 UTC" python3 odoo/server.py', str(cmd)) # let's ensure that a time offset is added to a child build build.build_start = datetime.datetime(2025, 1, 1, 12, 00) child_build = build._add_child({}) child_build.create_date = datetime.datetime(2025, 1, 1, 13, 00) - child_cmd = child_build._cmd(py_version=3) + child_cmd = child_build._cmd() self.assertIn('faketime "2024-02-04 03:42 UTC" python3 odoo/server.py', str(child_cmd)) build.build_end = datetime.datetime(2025, 1, 1, 14, 00) second_child = build._add_child({}) - second_child_cmd = second_child._cmd(py_version=3) + second_child_cmd = second_child._cmd() self.assertIn('faketime "2024-02-04 04:42 UTC" python3 odoo/server.py', str(second_child_cmd)) def test_format_message(self):