diff --git a/README.md b/README.md index fd6c9afe0..6184f528d 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,7 @@ Create a repo for your custom addons repo - **Remotes**: `git@github.com:odoo/runbot.git` - The remote *PR* option can be checked if needed to fetch pull request too. Will work only if a github token is given for this repo. -A config file with your remotes should be created for each repo. You can check the content in `/runbot/static/repo/(runbot|odoo)/config`. The repo will be fetched, this operation may take some time too. After that, you should start seeing empty batches in both projects on the frontend (`/` or `/runbot`) +A config file with your remotes should be created for each repo. You can check the content in `~/.local/share/runbot/repo/(runbot|odoo)/config`. The repo will be fetched, this operation may take some time too. After that, you should start seeing empty batches in both projects on the frontend (`/` or `/runbot`) #### Triggers and config At this point, runbot will discover new branches, new commits, create bundle, but no build will be created. diff --git a/runbot/models/build.py b/runbot/models/build.py index dea856f12..a5d5092c9 100644 --- a/runbot/models/build.py +++ b/runbot/models/build.py @@ -1381,27 +1381,23 @@ def _get_server_info(self, commit=None): _logger.error('None of %s found in commit, actual commit content:\n %s' % (commit.repo_id.server_files, os.listdir(commit._source_path()))) raise RunbotException('No server found in %s' % commit.dname) - def _make_pip_command(self, py_version=None): - if not py_version: - py_version = self._get_py_version() + def _make_pip_command(self): pres = [] if not self.params_id.skip_requirements and not self.params_id.config_data.get('skip_requirements'): for commit_id in self.env.context.get('defined_commit_ids') or self.params_id.commit_ids.sorted(lambda c: (c.repo_id.sequence, c.repo_id.id)): if os.path.isfile(commit_id._source_path('requirements.txt')): repo_dir = self._docker_source_folder(commit_id) requirement_path = os.sep.join([repo_dir, 'requirements.txt']) - pres.append([f'python{py_version}', '-m', 'pip', 'install', '--progress-bar', 'off', '-r', f'{requirement_path}']) + pres.append([f'python3', '-m', 'pip', 'install', '--progress-bar', 'off', '-r', f'{requirement_path}']) return pres - def _cmd(self, python_params=None, py_version=None, local_only=True, sub_command=None, enable_log_db=True): + def _cmd(self, python_params=None, local_only=True, sub_command=None, enable_log_db=True): """Return a list describing the command to start the build """ self.ensure_one() build = self python_params = python_params or [] - py_version = py_version if py_version is not None else build._get_py_version() - - pres = self._make_pip_command(py_version) + pres = self._make_pip_command() faketime = [] if faketime_params := self.params_id.config_data.get('faketime'): @@ -1415,7 +1411,7 @@ def _cmd(self, python_params=None, py_version=None, local_only=True, sub_command server_dir = self._docker_source_folder(server_commit) # commandline - cmd = faketime + ['python%s' % py_version] + python_params + [os.sep.join([server_dir, server_file])] + cmd = faketime + ['python3'] + python_params + [os.sep.join([server_dir, server_file])] if sub_command: cmd += [sub_command] @@ -1477,15 +1473,6 @@ def _cmd_check(self, cmd): 'build_id': self.id }) - def _get_py_version(self): - """return the python name to use from build batch""" - (server_commit, server_file) = self._get_server_info() - server_path = server_commit._source_path(server_file) - with file_open(server_path, 'r') as f: - if f.readline().strip().endswith('python3'): - return '3' - return '' - def _parse_logs(self): """ Parse build logs to classify errors """ # only parse logs from builds in error and not already scanned diff --git a/runbot/models/build_config.py b/runbot/models/build_config.py index 85ff7af8d..e4f70a83d 100644 --- a/runbot/models/build_config.py +++ b/runbot/models/build_config.py @@ -757,7 +757,6 @@ def _run_install_odoo(self, build, config_data=None): modules_to_install = build._get_modules_to_test(install_module_pattern) mods = ",".join(modules_to_install) python_params = [] - py_version = build._get_py_version() if self.coverage or config_data.get('coverage'): build.coverage = True python_params = ['-m', 'coverage', 'run', '--source', '/data/build'] @@ -768,7 +767,7 @@ def _run_install_odoo(self, build, config_data=None): python_params += self._coverage_params(build, config_data) elif self.flamegraph: python_params = ['-m', 'flamegraph', '-o', self._perfs_data_path(build)] - cmd = build._cmd(python_params, py_version, sub_command=self.sub_command, enable_log_db=self.enable_log_db) + cmd = build._cmd(python_params, sub_command=self.sub_command, enable_log_db=self.enable_log_db) # create db if needed db_suffix = config_data.get('db_name') or (build.params_id.dump_db.db_suffix if not self.create_db else False) or self._get_db_name(build) db_suffix = re.sub(r'[^a-z0-9\-_]', '_', db_suffix.lower()) @@ -834,7 +833,7 @@ def _run_install_odoo(self, build, config_data=None): if extra_params: cmd.extend(shlex.split(extra_params)) - cmd.finals.extend(self._post_install_commands(build, config_data, py_version)) # coverage post, extra-checks, ... + cmd.finals.extend(self._post_install_commands(build, config_data)) # coverage post, extra-checks, ... if config_data.get('export_database', True): self._add_zip_generation(build, cmd, db_name) @@ -1231,11 +1230,11 @@ def _log_end(self, build): message = 'Flamegraph report: [data @icon-download](%s), [svg @icon-eye](%s)' build._log('end_job', message, dat_url, svg_url, log_type='markdown') - def _post_install_commands(self, build, config_data, py_version): + def _post_install_commands(self, build, config_data): cmds = [] if config_data.get('coverage_make_report', (self.coverage and self.coverage_make_report)): - cmds.append(['python%s' % py_version, "-m", "coverage", "html", "-d", "/data/build/logs/coverage", "--ignore-errors"]) - cmds.append(['python%s' % py_version, "-m", "coverage", "json", "-o", "/data/build/logs/coverage.json", "--ignore-errors"]) + cmds.append(['python3', "-m", "coverage", "html", "-d", "/data/build/logs/coverage", "--ignore-errors"]) + cmds.append(['python3', "-m", "coverage", "json", "-o", "/data/build/logs/coverage.json", "--ignore-errors"]) if config_data.get('coverage', self.coverage): cmds.append(['mv', "/data/build/.coverage", f"/data/build/logs/coverage.{build.id}.{int(time.time())}"]) return cmds diff --git a/runbot/models/commit.py b/runbot/models/commit.py index b0aba479a..b2c50d1b9 100644 --- a/runbot/models/commit.py +++ b/runbot/models/commit.py @@ -156,7 +156,7 @@ def _export(self, build): def _read_source(self, file, mode='r'): file_path = self._source_path(file) try: - with file_open(file_path, mode) as f: + with file_open(file_path, mode, env=self.env) as f: return f.read() except: return False diff --git a/runbot/models/host.py b/runbot/models/host.py index 6d657d90f..d15681d7a 100644 --- a/runbot/models/host.py +++ b/runbot/models/host.py @@ -127,11 +127,19 @@ def _bootstrap_db_template(self): def _bootstrap(self): """ Create needed directories in static """ - dirs = ['build', 'nginx', 'repo', 'sources', 'src', 'docker'] - static_path = self.env['runbot.runbot']._root() - static_dirs = {d: self.env['runbot.runbot']._path(d) for d in dirs} - for dir, path in static_dirs.items(): + for local_dir in ['nginx', 'repo', 'sources', 'docker']: + path = self.env['runbot.runbot']._local_path(local_dir) + if not os.path.exists(path): # migration of existing statics dirs, todo remove in 21.0 + static_path = self.env['runbot.runbot']._path(local_dir) + if os.path.exists(static_path): + _logger.info('Moving %s to %s', static_path, path) + os.rename(static_path, path) os.makedirs(path, exist_ok=True) + + for static_dir in ['build', 'src']: + path = self.env['runbot.runbot']._path(static_dir) + os.makedirs(path, exist_ok=True) + self._bootstrap_db_template() self._bootstrap_local_logs_db() diff --git a/runbot/models/repo.py b/runbot/models/repo.py index 51f6a9c8c..9ec5ed58d 100644 --- a/runbot/models/repo.py +++ b/runbot/models/repo.py @@ -495,10 +495,10 @@ def _compute_path(self): repo.path = repo._path() def _path(self, *path_parts): - return self.env['runbot.runbot']._path('repo', sanitize(self.name), *path_parts) + return self.env['runbot.runbot']._local_path('repo', sanitize(self.name), *path_parts) def _source_path(self, *path_parts): - return self.env['runbot.runbot']._path('sources', sanitize(self.name), *path_parts) + return self.env['runbot.runbot']._local_path('sources', sanitize(self.name), *path_parts) def _get_git_command(self, cmd, errors='strict'): """Execute a git command 'cmd'""" @@ -718,7 +718,7 @@ def _update_git_config(self): git_config_path = repo._path('config') template_params = {'repo': repo} git_config = self.env['ir.ui.view']._render_template("runbot.git_config", template_params) - with file_open(git_config_path, 'w') as config_file: + with file_open(git_config_path, 'w', env=self.env) as config_file: config_file.write(str(git_config)) _logger.info('Config updated for repo %s' % repo.name) else: diff --git a/runbot/models/runbot.py b/runbot/models/runbot.py index 195efe4a7..03c861d41 100644 --- a/runbot/models/runbot.py +++ b/runbot/models/runbot.py @@ -15,7 +15,7 @@ from odoo import fields, models from odoo.exceptions import UserError from odoo.fields import Domain -from odoo.tools import config, file_open, SQL +from odoo.tools import config, SQL from ..common import dest_reg, os, sanitize from ..container import docker_ps, docker_stop @@ -36,6 +36,13 @@ def _root(self): """Return root directory of repository""" return os.path.abspath(os.sep.join([os.path.dirname(__file__), '../static'])) + def _local_root(self): + """Return local root directory""" + local_root = os.path.expanduser('~/.local/share/runbot') + if not local_root in self.env.transaction._Transaction__file_open_tmp_paths: + self.env.transaction._Transaction__file_open_tmp_paths.append(local_root) + return local_root + def _path(self, *path_parts): """Return the repo build path""" root = self.env['runbot.runbot']._root() @@ -44,6 +51,14 @@ def _path(self, *path_parts): raise UserError('Invalid path') return file_path + def _local_path(self, *path_parts): + """Return the local repo build path""" + root = self.env['runbot.runbot']._local_root() + file_path = os.path.normpath(os.sep.join([root] + [sanitize(path) for path_part in path_parts for path in path_part.split(os.sep) if path])) + if not file_path.startswith(root): + raise UserError('Invalid path') + return file_path + def _scheduler(self, host): self._gc_testing(host) self._commit() @@ -158,7 +173,7 @@ def _reload_nginx(self): settings['port'] = config.get('http_port') settings['runbot_static'] = self.env['runbot.runbot']._root() + os.sep settings['base_url'] = self.get_base_url() - nginx_dir = self.env['runbot.runbot']._path('nginx') + nginx_dir = self.env['runbot.runbot']._local_path('nginx') settings['nginx_dir'] = nginx_dir settings['re_escape'] = re.escape host_name = self.env['runbot.host']._get_current_name() @@ -169,17 +184,17 @@ def _reload_nginx(self): nginx_config = env['ir.ui.view']._render_template("runbot.nginx_config", settings) os.makedirs(nginx_dir, exist_ok=True) content = None - nginx_conf_path = self.env['runbot.runbot']._path('nginx', 'nginx.conf') + nginx_conf_path = self.env['runbot.runbot']._local_path('nginx', 'nginx.conf') content = '' if os.path.isfile(nginx_conf_path): - with file_open(nginx_conf_path, 'r') as f: + with open(nginx_conf_path, 'r') as f: content = f.read() if content != nginx_config: _logger.info('reload nginx') with open(nginx_conf_path, 'w') as f: f.write(str(nginx_config)) try: - pid = int(file_open(self.env['runbot.runbot']._path('nginx', 'nginx.pid')).read().strip(' \n')) + pid = int(open(self.env['runbot.runbot']._local_path('nginx', 'nginx.pid')).read().strip(' \n')) os.kill(pid, signal.SIGHUP) except Exception: _logger.info('start nginx') diff --git a/runbot/tests/common.py b/runbot/tests/common.py index 81cc8e784..a42cc0d73 100644 --- a/runbot/tests/common.py +++ b/runbot/tests/common.py @@ -230,7 +230,6 @@ def mock_git(repo, cmd, quiet=False, input_data=None, raw=False): self.start_patcher('_local_pg_createdb', 'odoo.addons.runbot.models.build.BuildResult._local_pg_createdb', True) self.start_patcher('getmtime', 'odoo.addons.runbot.common.os.path.getmtime', datetime.datetime.now().timestamp()) self.start_patcher('file_exist', 'odoo.tools.misc.os.path.exists', True) - self.start_patcher('_get_py_version', 'odoo.addons.runbot.models.build.BuildResult._get_py_version', 3) self.start_patcher('_write_file', 'odoo.addons.runbot.models.build.BuildResult._write_file', None) self.start_patcher('_parse_config', 'odoo.addons.runbot.models.build.BuildResult._parse_config', {'--test-enable', '--test-tags', '--with-demo'}) diff --git a/runbot/tests/test_build.py b/runbot/tests/test_build.py index 2cf84f65e..0598bebb0 100644 --- a/runbot/tests/test_build.py +++ b/runbot/tests/test_build.py @@ -384,7 +384,7 @@ def test_build_cmd_log_db(self): build = self.Build.create({ 'params_id': self.server_params.id, }) - cmd = build._cmd(py_version=3) + cmd = build._cmd() self.assertIn('log_db = runbot_logs', cmd.get_config()) @@ -400,7 +400,7 @@ def test_build_cmd_custom_pre_post(self): build = self.Build.create({ 'params_id': self.server_params.id, }) - cmd = build._cmd(py_version=3) + cmd = build._cmd() self.assertIn(['python3', '-m', 'pip', 'install', '--progress-bar', 'off', '-r', 'odoo/requirements.txt'], cmd.pres) self.assertIn(custom_pre, cmd.pres) self.assertIn(custom_post, cmd.posts) @@ -411,7 +411,7 @@ def test_build_cmd_server_path_no_dep(self): build = self.Build.create({ 'params_id': self.server_params.id, }) - cmd = build._cmd(py_version=3) + cmd = build._cmd() self.assertEqual('python3', cmd[0]) self.assertEqual('odoo/server.py', cmd[1]) self.assertIn('--addons-path', cmd) @@ -424,13 +424,13 @@ def test_build_cmd_server_path_with_dep(self): def is_file(file): self.assertIn(file, [ - self.env['runbot.runbot']._path('sources/enterprise/0d0d0caca0000fffffffffffffffffffffffffff/requirements.txt'), - self.env['runbot.runbot']._path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/requirements.txt'), - self.env['runbot.runbot']._path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/server.py'), - self.env['runbot.runbot']._path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/odoo/tools/config.py'), - self.env['runbot.runbot']._path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/odoo/sql_db.py'), + self.env['runbot.runbot']._local_path('sources/enterprise/0d0d0caca0000fffffffffffffffffffffffffff/requirements.txt'), + self.env['runbot.runbot']._local_path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/requirements.txt'), + self.env['runbot.runbot']._local_path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/server.py'), + self.env['runbot.runbot']._local_path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/odoo/tools/config.py'), + self.env['runbot.runbot']._local_path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/odoo/sql_db.py'), ]) - return file != self.env['runbot.runbot']._path('static/sources/enterprise/0d0d0caca0000fffffffffffffffffffffffffff/requirements.txt') + return file != self.env['runbot.runbot']._local_path('sources/enterprise/0d0d0caca0000fffffffffffffffffffffffffff/requirements.txt') def is_dir(file): paths = [ @@ -448,7 +448,7 @@ def is_dir(file): 'params_id': self.addons_params.id, }) - cmd = build._cmd(py_version=3) + cmd = build._cmd() self.assertIn('--addons-path', cmd) addons_path_pos = cmd.index('--addons-path') + 1 self.assertEqual(cmd[addons_path_pos], 'odoo/addons,odoo/core/addons,enterprise') @@ -690,19 +690,19 @@ def test_build_cmd_faketime(self): build = self.Build.create({ 'params_id': self.server_params.id, }) - cmd = build._cmd(py_version=3) + cmd = build._cmd() self.assertIn('faketime "2024-02-04 02:42 UTC" python3 odoo/server.py', str(cmd)) # let's ensure that a time offset is added to a child build build.build_start = datetime.datetime(2025, 1, 1, 12, 00) child_build = build._add_child({}) child_build.create_date = datetime.datetime(2025, 1, 1, 13, 00) - child_cmd = child_build._cmd(py_version=3) + child_cmd = child_build._cmd() self.assertIn('faketime "2024-02-04 03:42 UTC" python3 odoo/server.py', str(child_cmd)) build.build_end = datetime.datetime(2025, 1, 1, 14, 00) second_child = build._add_child({}) - second_child_cmd = second_child._cmd(py_version=3) + second_child_cmd = second_child._cmd() self.assertIn('faketime "2024-02-04 04:42 UTC" python3 odoo/server.py', str(second_child_cmd)) def test_format_message(self):