[IMP] runbot: limit the size of files to open

runbot/common.py

@@ -19,6 +19,8 @@
 from odoo.fields import Domain
 from odoo.tools.misc import DEFAULT_SERVER_DATETIME_FORMAT, file_open, html_escape, OrderedSet
 
+DEFAULT_MAX_FILE_SIZE = 500 * 1024 * 1024
+
 _logger = logging.getLogger(__name__)
 
 dest_reg = re.compile(r'^\d{5,}-.+$')

runbot/models/build.py

@@ -38,6 +38,7 @@
     sanitize,
     tail,
     transactioncache,
+    DEFAULT_MAX_FILE_SIZE,
 )
 from ..container import Command, docker_pull, docker_run, docker_state, docker_stop
 from ..fields import JsonDictField
@@ -1499,6 +1500,10 @@ def _is_file(self, file, mode='r'):
 
     def _read_file(self, file, mode='r'):
         file_path = self._path(file)
+        max_log_file_size = int(self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_max_log_size', DEFAULT_MAX_FILE_SIZE))
+        if os.path.getsize(file_path) > max_log_file_size:
+            self._log('readfile', f"File size exceeds {max_log_file_size} limit", level="ERROR")
+            return False
         try:
             with file_open(file_path, mode) as f:
                 return f.read()

runbot/models/build_config.py

@@ -26,6 +26,7 @@
     rfind,
     s2human,
     time2str,
+    DEFAULT_MAX_FILE_SIZE,
 )
 from ..container import Command, docker_get_gateway_ip
 
@@ -1339,6 +1340,10 @@ def _check_log(self, build):
         if not os.path.isfile(log_path):
             build._log('_make_tests_results', "Log file not found at the end of test job", level="ERROR")
             return 'ko'
+        max_log_file_size = int(self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_max_log_size', DEFAULT_MAX_FILE_SIZE))
+        if os.path.getsize(log_path) > max_log_file_size:
+            build._log('_make_tests_results', f"Log file exceeds {max_log_file_size} limit", level="ERROR")
+            return 'ko'
         return 'ok'
 
     def _check_module_loaded(self, build):

runbot/models/build_stat_regex.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import logging
 
-from ..common import os
+from ..common import os, DEFAULT_MAX_FILE_SIZE
 import re
 
 from odoo import models, fields, api
@@ -53,6 +53,10 @@ def _find_in_file(self, file_path):
         """
         if not os.path.exists(file_path):
             return {}
+        max_log_file_size = int(self.env['ir.config_parameter'].sudo().get_param('runbot.runbot_max_log_size', DEFAULT_MAX_FILE_SIZE))
+        if os.path.getsize(file_path) > max_log_file_size:
+            _logger.warning("Log file '%s' exceeds %s limit", file_path, max_log_file_size)
+            return {}
         stats_matches = {}
         with file_open(file_path, "r") as log_file:
             data = log_file.read()

runbot/tests/test_build.py

@@ -1,13 +1,13 @@
 # -*- coding: utf-8 -*-
 import datetime
 
-from unittest.mock import patch
+from unittest.mock import patch, MagicMock, mock_open
 
 from odoo import fields
 from odoo.tests import tagged
 from odoo.exceptions import UserError, ValidationError
 from .common import RunbotCase, RunbotCaseMinimalSetup
-from unittest.mock import MagicMock
+from ..common import DEFAULT_MAX_FILE_SIZE
 
 
 def rev_parse(repo, branch_name):

runbot/tests/test_build_config_step.py

@@ -1440,6 +1440,7 @@ def _log(build, func, message, level='INFO', log_type='runbot', path='runbot'):
             self.logs.append((level, message))
 
         self.start_patcher('log_patcher', 'odoo.addons.runbot.models.build.BuildResult._log', new=_log)
+        self.start_patcher('get_size', 'odoo.addons.runbot.models.build_config.os.path.getsize', return_value=100)
 
         self.build = self.Build.create({
             'params_id': self.base_params.id,
@@ -1661,3 +1662,12 @@ def make_warn(build):
         mock_make_odoo_results.side_effect = make_warn
         config_step._make_results(build)
         self.assertEqual(build.local_result, 'warn')
+
+    def test_make_result_large_file(self):
+        custom_limit = 1024
+        self.env['ir.config_parameter'].sudo().set_param('runbot.runbot_max_log_size', custom_limit)
+        self.patchers['get_size'].return_value = custom_limit + 1
+        self.config_step._make_results(self.build)
+        self.assertEqual(str(self.build.job_end), '1970-01-01 02:00:00')
+        self.assertIn(('ERROR', 'Log file exceeds %s limit' % custom_limit), self.logs)
+        self.assertEqual(self.build.local_result, 'ko')

runbot/tests/test_build_stat.py

@@ -9,6 +9,7 @@
 class TestBuildStatRegex(RunbotCase):
     def setUp(self):
         super(TestBuildStatRegex, self).setUp()
+        self.start_patcher('get_size', 'odoo.addons.runbot.models.build_config.os.path.getsize', return_value=100)
         self.StatRegex = self.env["runbot.build.stat.regex"]
         self.ConfigStep = self.env["runbot.build.config.step"]
         self.BuildStat = self.env["runbot.build.stat"]

runbot/tests/test_upgrade.py

@@ -19,6 +19,7 @@ def setUp(self):
 
     def upgrade_flow_setup(self):
         self.start_patcher('find_patcher', 'odoo.addons.runbot.common.find', 0)
+        self.start_patcher('get_size', 'odoo.addons.runbot.models.build_config.os.path.getsize', return_value=100)
         self.additionnal_setup()
 
         self.master_bundle = self.branch_odoo.bundle_id