Skip to content

[IMP] runbot: add a timeout on git fetch commands #1415

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
d-fence wants to merge 1 commit into
base: 19.0
Choose a base branch
from
Draft

[IMP] runbot: add a timeout on git fetch commands #1415

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 12 additions & 5 deletions runbot/models/repo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -508,11 +508,11 @@ def _get_git_command(self, cmd, errors='strict'):
cmd = ['git', '-C', self.path] + config_args + cmd
return cmd

def _git(self, cmd, errors='strict', quiet=False, input_data=None, raw=False):
def _git(self, cmd, errors='strict', quiet=False, input_data=None, raw=False, timeout=None):
cmd = self._get_git_command(cmd, errors)
if not quiet:
_logger.info("git command: %s", shlex.join(cmd))
kwargs = {'stderr': subprocess.STDOUT}
kwargs = {'stderr': subprocess.STDOUT, 'timeout': timeout}
if input_data is not None:
if isinstance(input_data, str):
input_data = input_data.encode('utf-8')
Expand All @@ -526,13 +526,16 @@ def _fetch(self, sha):
if not self._hash_exists(sha):
self._update(force=True)
if not self._hash_exists(sha):
fetch_timeout = int(self.env['ir.config_parameter'].get_param('runbot.runbot_fetch_timeout', default=120))
for remote in self.remote_ids:
try:
self._git(['fetch', remote.remote_name, sha])
self._git(['fetch', remote.remote_name, sha], timeout=fetch_timeout)
_logger.info('Success fetching specific head %s on %s', sha, remote)
break
except subprocess.CalledProcessError:
pass
except subprocess.TimeoutExpired:
_logger.warning('Fetch of %s from %s timed out after %s sec', sha, remote.remote_name, fetch_timeout)
if not self._hash_exists(sha):
raise RunbotException("Commit %s is unreachable. Did you force push the branch?" % sha)

Expand Down Expand Up @@ -748,21 +751,25 @@ def _update_git(self, force=False, poll_delay=5*60):
def _update_fetch_cmd(self):
# Extracted from update_git to be easily overriden in external module
self.ensure_one()
fetch_timeout = int(self.env['ir.config_parameter'].get_param('runbot.runbot_fetch_timeout', default=120))
try_count = 0
success = False
delay = 0
while not success and try_count < 5:
time.sleep(delay)
try:
self._git(['fetch', '-p', '--all', '-j2']) # j2 to use two job and fetch multiple origin at the same time
self._git(['fetch', '-p', '--all', '-j2'], timeout=fetch_timeout) # j2 to use two job and fetch multiple origin at the same time
success = True
except subprocess.CalledProcessError as e:
except (subprocess.CalledProcessError, subprocess.TimeoutException) as e:
try_count += 1
delay = delay * 1.5 if delay else 0.5
if try_count > 4:
message = 'Failed to fetch repo %s: %s' % (self.name, e.output.decode())
host = self.env['runbot.host']._get_current()
host.message_post(body=message)
if isinstance(e, subprocess.TimeoutException):
_logger.warning('Fetching all remotes from repo %s timed out after %s sec', self.name, fetch_timeout)
fetch_timeout += 30
return success

def _update(self, force=False, poll_delay=5*60):
Expand Down