[IMP] runbot: move some directory from static

README.md

@@ -238,7 +238,7 @@ Create a repo for your custom addons repo
 - **Remotes**: `git@github.com:odoo/runbot.git` 
 - The remote *PR* option can be checked if needed to fetch pull request too. Will work only if a github token is given for this repo.
 
-A config file with your remotes should be created for each repo. You can check the content in `/runbot/static/repo/(runbot|odoo)/config`. The repo will be fetched, this operation may take some time too. After that, you should start seeing empty batches in both projects on the frontend (`/` or `/runbot`)
+A config file with your remotes should be created for each repo. You can check the content in `~/.local/share/runbot/repo/(runbot|odoo)/config`. The repo will be fetched, this operation may take some time too. After that, you should start seeing empty batches in both projects on the frontend (`/` or `/runbot`)
 
 #### Triggers and config
 At this point, runbot will discover new branches, new commits, create bundle, but no build will be created.

runbot/models/build.py

@@ -1381,27 +1381,23 @@ def _get_server_info(self, commit=None):
         _logger.error('None of %s found in commit, actual commit content:\n %s' % (commit.repo_id.server_files, os.listdir(commit._source_path())))
         raise RunbotException('No server found in %s' % commit.dname)
 
-    def _make_pip_command(self, py_version=None):
-        if not py_version:
-            py_version = self._get_py_version()
+    def _make_pip_command(self):
         pres = []
         if not self.params_id.skip_requirements and not self.params_id.config_data.get('skip_requirements'):
             for commit_id in self.env.context.get('defined_commit_ids') or self.params_id.commit_ids.sorted(lambda c: (c.repo_id.sequence, c.repo_id.id)):
                 if os.path.isfile(commit_id._source_path('requirements.txt')):
                     repo_dir = self._docker_source_folder(commit_id)
                     requirement_path = os.sep.join([repo_dir, 'requirements.txt'])
-                    pres.append([f'python{py_version}', '-m', 'pip', 'install', '--progress-bar', 'off', '-r', f'{requirement_path}'])
+                    pres.append([f'python3', '-m', 'pip', 'install', '--progress-bar', 'off', '-r', f'{requirement_path}'])
         return pres
 
-    def _cmd(self, python_params=None, py_version=None, local_only=True, sub_command=None, enable_log_db=True):
+    def _cmd(self, python_params=None, local_only=True, sub_command=None, enable_log_db=True):
         """Return a list describing the command to start the build
         """
         self.ensure_one()
         build = self
         python_params = python_params or []
-        py_version = py_version if py_version is not None else build._get_py_version()
-
-        pres = self._make_pip_command(py_version)
+        pres = self._make_pip_command()
 
         faketime = []
         if faketime_params := self.params_id.config_data.get('faketime'):
@@ -1415,7 +1411,7 @@ def _cmd(self, python_params=None, py_version=None, local_only=True, sub_command
         server_dir = self._docker_source_folder(server_commit)
 
         # commandline
-        cmd = faketime + ['python%s' % py_version] + python_params + [os.sep.join([server_dir, server_file])]
+        cmd = faketime + ['python3'] + python_params + [os.sep.join([server_dir, server_file])]
         if sub_command:
             cmd += [sub_command]
 
@@ -1477,15 +1473,6 @@ def _cmd_check(self, cmd):
                 'build_id': self.id
             })
 
-    def _get_py_version(self):
-        """return the python name to use from build batch"""
-        (server_commit, server_file) = self._get_server_info()
-        server_path = server_commit._source_path(server_file)
-        with file_open(server_path, 'r') as f:
-            if f.readline().strip().endswith('python3'):
-                return '3'
-        return ''
-
     def _parse_logs(self):
         """ Parse build logs to classify errors """
         # only parse logs from builds in error and not already scanned

runbot/models/build_config.py

@@ -757,7 +757,6 @@ def _run_install_odoo(self, build, config_data=None):
         modules_to_install = build._get_modules_to_test(install_module_pattern)
         mods = ",".join(modules_to_install)
         python_params = []
-        py_version = build._get_py_version()
         if self.coverage or config_data.get('coverage'):
             build.coverage = True
             python_params = ['-m', 'coverage', 'run', '--source', '/data/build']
@@ -768,7 +767,7 @@ def _run_install_odoo(self, build, config_data=None):
             python_params += self._coverage_params(build, config_data)
         elif self.flamegraph:
             python_params = ['-m', 'flamegraph', '-o', self._perfs_data_path(build)]
-        cmd = build._cmd(python_params, py_version, sub_command=self.sub_command, enable_log_db=self.enable_log_db)
+        cmd = build._cmd(python_params, sub_command=self.sub_command, enable_log_db=self.enable_log_db)
         # create db if needed
         db_suffix = config_data.get('db_name') or (build.params_id.dump_db.db_suffix if not self.create_db else False) or self._get_db_name(build)
         db_suffix = re.sub(r'[^a-z0-9\-_]', '_', db_suffix.lower())
@@ -834,7 +833,7 @@ def _run_install_odoo(self, build, config_data=None):
         if extra_params:
             cmd.extend(shlex.split(extra_params))
 
-        cmd.finals.extend(self._post_install_commands(build, config_data, py_version))  # coverage post, extra-checks, ...
+        cmd.finals.extend(self._post_install_commands(build, config_data))  # coverage post, extra-checks, ...
 
         if config_data.get('export_database', True):
             self._add_zip_generation(build, cmd, db_name)
@@ -1231,11 +1230,11 @@ def _log_end(self, build):
             message = 'Flamegraph report: [data @icon-download](%s), [svg @icon-eye](%s)'
             build._log('end_job', message, dat_url, svg_url, log_type='markdown')
 
-    def _post_install_commands(self, build, config_data, py_version):
+    def _post_install_commands(self, build, config_data):
         cmds = []
         if config_data.get('coverage_make_report', (self.coverage and self.coverage_make_report)):
-            cmds.append(['python%s' % py_version, "-m", "coverage", "html", "-d", "/data/build/logs/coverage", "--ignore-errors"])
-            cmds.append(['python%s' % py_version, "-m", "coverage", "json", "-o", "/data/build/logs/coverage.json", "--ignore-errors"])
+            cmds.append(['python3', "-m", "coverage", "html", "-d", "/data/build/logs/coverage", "--ignore-errors"])
+            cmds.append(['python3', "-m", "coverage", "json", "-o", "/data/build/logs/coverage.json", "--ignore-errors"])
         if config_data.get('coverage', self.coverage):
             cmds.append(['mv', "/data/build/.coverage", f"/data/build/logs/coverage.{build.id}.{int(time.time())}"])
         return cmds

runbot/models/commit.py

@@ -156,7 +156,7 @@ def _export(self, build):
     def _read_source(self, file, mode='r'):
         file_path = self._source_path(file)
         try:
-            with file_open(file_path, mode) as f:
+            with file_open(file_path, mode, env=self.env) as f:
                 return f.read()
         except:
             return False

runbot/models/host.py

@@ -127,11 +127,19 @@ def _bootstrap_db_template(self):
 
     def _bootstrap(self):
         """ Create needed directories in static """
-        dirs = ['build', 'nginx', 'repo', 'sources', 'src', 'docker']
-        static_path = self.env['runbot.runbot']._root()
-        static_dirs = {d: self.env['runbot.runbot']._path(d) for d in dirs}
-        for dir, path in static_dirs.items():
+        for local_dir in ['nginx', 'repo', 'sources', 'docker']:
+            path = self.env['runbot.runbot']._local_path(local_dir)
+            if not os.path.exists(path):  # migration of existing statics dirs, todo remove in 21.0
+                static_path = self.env['runbot.runbot']._path(local_dir)
+                if os.path.exists(static_path):
+                    _logger.info('Moving %s to %s', static_path, path)
+                    os.rename(static_path, path)
             os.makedirs(path, exist_ok=True)
+
+        for static_dir in ['build', 'src']:
+            path = self.env['runbot.runbot']._path(static_dir)
+            os.makedirs(path, exist_ok=True)
+
         self._bootstrap_db_template()
         self._bootstrap_local_logs_db()
 

runbot/models/repo.py

@@ -495,10 +495,10 @@ def _compute_path(self):
             repo.path = repo._path()
 
     def _path(self, *path_parts):
-        return self.env['runbot.runbot']._path('repo', sanitize(self.name), *path_parts)
+        return self.env['runbot.runbot']._local_path('repo', sanitize(self.name), *path_parts)
 
     def _source_path(self, *path_parts):
-        return self.env['runbot.runbot']._path('sources', sanitize(self.name), *path_parts)
+        return self.env['runbot.runbot']._local_path('sources', sanitize(self.name), *path_parts)
 
     def _get_git_command(self, cmd, errors='strict'):
         """Execute a git command 'cmd'"""
@@ -718,7 +718,7 @@ def _update_git_config(self):
                 git_config_path = repo._path('config')
                 template_params = {'repo': repo}
                 git_config = self.env['ir.ui.view']._render_template("runbot.git_config", template_params)
-                with file_open(git_config_path, 'w') as config_file:
+                with file_open(git_config_path, 'w', env=self.env) as config_file:
                     config_file.write(str(git_config))
                 _logger.info('Config updated for repo %s' % repo.name)
             else:

runbot/models/runbot.py

@@ -15,7 +15,7 @@
 from odoo import fields, models
 from odoo.exceptions import UserError
 from odoo.fields import Domain
-from odoo.tools import config, file_open, SQL
+from odoo.tools import config, SQL
 
 from ..common import dest_reg, os, sanitize
 from ..container import docker_ps, docker_stop
@@ -36,6 +36,13 @@ def _root(self):
         """Return root directory of repository"""
         return os.path.abspath(os.sep.join([os.path.dirname(__file__), '../static']))
 
+    def _local_root(self):
+        """Return local root directory"""
+        local_root = os.path.expanduser('~/.local/share/runbot')
+        if not local_root in self.env.transaction._Transaction__file_open_tmp_paths:
+            self.env.transaction._Transaction__file_open_tmp_paths.append(local_root)
+        return local_root
+
     def _path(self, *path_parts):
         """Return the repo build path"""
         root = self.env['runbot.runbot']._root()
@@ -44,6 +51,14 @@ def _path(self, *path_parts):
             raise UserError('Invalid path')
         return file_path
 
+    def _local_path(self, *path_parts):
+        """Return the local repo build path"""
+        root = self.env['runbot.runbot']._local_root()
+        file_path = os.path.normpath(os.sep.join([root] + [sanitize(path) for path_part in path_parts for path in path_part.split(os.sep) if path]))
+        if not file_path.startswith(root):
+            raise UserError('Invalid path')
+        return file_path
+
     def _scheduler(self, host):
         self._gc_testing(host)
         self._commit()
@@ -158,7 +173,7 @@ def _reload_nginx(self):
         settings['port'] = config.get('http_port')
         settings['runbot_static'] = self.env['runbot.runbot']._root() + os.sep
         settings['base_url'] = self.get_base_url()
-        nginx_dir = self.env['runbot.runbot']._path('nginx')
+        nginx_dir = self.env['runbot.runbot']._local_path('nginx')
         settings['nginx_dir'] = nginx_dir
         settings['re_escape'] = re.escape
         host_name = self.env['runbot.host']._get_current_name()
@@ -169,17 +184,17 @@ def _reload_nginx(self):
         nginx_config = env['ir.ui.view']._render_template("runbot.nginx_config", settings)
         os.makedirs(nginx_dir, exist_ok=True)
         content = None
-        nginx_conf_path = self.env['runbot.runbot']._path('nginx', 'nginx.conf')
+        nginx_conf_path = self.env['runbot.runbot']._local_path('nginx', 'nginx.conf')
         content = ''
         if os.path.isfile(nginx_conf_path):
-            with file_open(nginx_conf_path, 'r') as f:
+            with open(nginx_conf_path, 'r') as f:
                 content = f.read()
         if content != nginx_config:
             _logger.info('reload nginx')
             with open(nginx_conf_path, 'w') as f:
                 f.write(str(nginx_config))
             try:
-                pid = int(file_open(self.env['runbot.runbot']._path('nginx', 'nginx.pid')).read().strip(' \n'))
+                pid = int(open(self.env['runbot.runbot']._local_path('nginx', 'nginx.pid')).read().strip(' \n'))
                 os.kill(pid, signal.SIGHUP)
             except Exception:
                 _logger.info('start nginx')

runbot/tests/common.py

@@ -230,7 +230,6 @@ def mock_git(repo, cmd, quiet=False, input_data=None, raw=False):
         self.start_patcher('_local_pg_createdb', 'odoo.addons.runbot.models.build.BuildResult._local_pg_createdb', True)
         self.start_patcher('getmtime', 'odoo.addons.runbot.common.os.path.getmtime', datetime.datetime.now().timestamp())
         self.start_patcher('file_exist', 'odoo.tools.misc.os.path.exists', True)
-        self.start_patcher('_get_py_version', 'odoo.addons.runbot.models.build.BuildResult._get_py_version', 3)
         self.start_patcher('_write_file', 'odoo.addons.runbot.models.build.BuildResult._write_file', None)
         self.start_patcher('_parse_config', 'odoo.addons.runbot.models.build.BuildResult._parse_config', {'--test-enable', '--test-tags', '--with-demo'})
 

runbot/tests/test_build.py

@@ -384,7 +384,7 @@ def test_build_cmd_log_db(self):
         build = self.Build.create({
             'params_id': self.server_params.id,
         })
-        cmd = build._cmd(py_version=3)
+        cmd = build._cmd()
         self.assertIn('log_db = runbot_logs', cmd.get_config())
 
 
@@ -400,7 +400,7 @@ def test_build_cmd_custom_pre_post(self):
         build = self.Build.create({
             'params_id': self.server_params.id,
         })
-        cmd = build._cmd(py_version=3)
+        cmd = build._cmd()
         self.assertIn(['python3', '-m', 'pip', 'install', '--progress-bar', 'off', '-r', 'odoo/requirements.txt'], cmd.pres)
         self.assertIn(custom_pre, cmd.pres)
         self.assertIn(custom_post, cmd.posts)
@@ -411,7 +411,7 @@ def test_build_cmd_server_path_no_dep(self):
         build = self.Build.create({
             'params_id': self.server_params.id,
         })
-        cmd = build._cmd(py_version=3)
+        cmd = build._cmd()
         self.assertEqual('python3', cmd[0])
         self.assertEqual('odoo/server.py', cmd[1])
         self.assertIn('--addons-path', cmd)
@@ -424,13 +424,13 @@ def test_build_cmd_server_path_with_dep(self):
 
         def is_file(file):
             self.assertIn(file, [
-                self.env['runbot.runbot']._path('sources/enterprise/0d0d0caca0000fffffffffffffffffffffffffff/requirements.txt'),
-                self.env['runbot.runbot']._path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/requirements.txt'),
-                self.env['runbot.runbot']._path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/server.py'),
-                self.env['runbot.runbot']._path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/odoo/tools/config.py'),
-                self.env['runbot.runbot']._path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/odoo/sql_db.py'),
+                self.env['runbot.runbot']._local_path('sources/enterprise/0d0d0caca0000fffffffffffffffffffffffffff/requirements.txt'),
+                self.env['runbot.runbot']._local_path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/requirements.txt'),
+                self.env['runbot.runbot']._local_path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/server.py'),
+                self.env['runbot.runbot']._local_path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/odoo/tools/config.py'),
+                self.env['runbot.runbot']._local_path('sources/odoo/0dfdfcfcf0000fffffffffffffffffffffffffff/odoo/sql_db.py'),
             ])
-            return file != self.env['runbot.runbot']._path('static/sources/enterprise/0d0d0caca0000fffffffffffffffffffffffffff/requirements.txt')
+            return file != self.env['runbot.runbot']._local_path('sources/enterprise/0d0d0caca0000fffffffffffffffffffffffffff/requirements.txt')
 
         def is_dir(file):
             paths = [
@@ -448,7 +448,7 @@ def is_dir(file):
             'params_id': self.addons_params.id,
         })
 
-        cmd = build._cmd(py_version=3)
+        cmd = build._cmd()
         self.assertIn('--addons-path', cmd)
         addons_path_pos = cmd.index('--addons-path') + 1
         self.assertEqual(cmd[addons_path_pos], 'odoo/addons,odoo/core/addons,enterprise')
@@ -690,19 +690,19 @@ def test_build_cmd_faketime(self):
         build = self.Build.create({
             'params_id': self.server_params.id,
         })
-        cmd = build._cmd(py_version=3)
+        cmd = build._cmd()
         self.assertIn('faketime "2024-02-04 02:42 UTC" python3 odoo/server.py', str(cmd))
 
         # let's ensure that a time offset is added to a child build
         build.build_start = datetime.datetime(2025, 1, 1, 12, 00)
         child_build = build._add_child({})
         child_build.create_date = datetime.datetime(2025, 1, 1, 13, 00)
-        child_cmd = child_build._cmd(py_version=3)
+        child_cmd = child_build._cmd()
         self.assertIn('faketime "2024-02-04 03:42 UTC" python3 odoo/server.py', str(child_cmd))
 
         build.build_end = datetime.datetime(2025, 1, 1, 14, 00)
         second_child = build._add_child({})
-        second_child_cmd = second_child._cmd(py_version=3)
+        second_child_cmd = second_child._cmd()
         self.assertIn('faketime "2024-02-04 04:42 UTC" python3 odoo/server.py', str(second_child_cmd))
 
     def test_format_message(self):