diff --git a/.gitignore b/.gitignore index 7604ea9..4817bdc 100644 --- a/.gitignore +++ b/.gitignore @@ -101,3 +101,6 @@ ENV/ # mypy .mypy_cache/ + +# PyCharm +.idea/ diff --git a/doodad/launch_tools.py b/doodad/launch_tools.py index 292f630..048262f 100644 --- a/doodad/launch_tools.py +++ b/doodad/launch_tools.py @@ -10,10 +10,11 @@ def launch_shell( mode=LOCAL, dry=False, mount_points=None, + verbose=False ): if mount_points is None: mount_points = [] - mode.launch_command(command, dry=dry) + mode.launch_command(command, dry=dry, mount_points=mount_points, verbose=verbose) def launch_python( diff --git a/doodad/mode.py b/doodad/mode.py index 5ac157b..34620c8 100644 --- a/doodad/mode.py +++ b/doodad/mode.py @@ -68,11 +68,12 @@ def launch_command(self, cmd, mount_points=None, dry=False, verbose=False): class DockerMode(LaunchMode): - def __init__(self, image='ubuntu:16.04', gpu=False): + def __init__(self, image='ubuntu:16.04', gpu=False, gpu_id=-1): super(DockerMode, self).__init__() self.docker_image = image self.docker_name = uuid.uuid4() self.gpu = gpu + self.gpu_id = gpu_id # default of -1 results in all local gpus made available def get_docker_cmd(self, main_cmd, extra_args='', use_tty=True, verbose=True, pythonpath=None, pre_cmd=None, post_cmd=None, checkpoint=False, no_root=False): @@ -83,6 +84,8 @@ def get_docker_cmd(self, main_cmd, extra_args='', use_tty=True, verbose=True, py if verbose: if self.gpu: cmd_list.append('echo \"Running in docker (gpu)\"') + if self.gpu_id > -1: + cmd_list.append('echo \"Restricting to gpu_id=%d\"' % self.gpu_id) else: cmd_list.append('echo \"Running in docker\"') if pythonpath: @@ -108,12 +111,15 @@ def get_docker_cmd(self, main_cmd, extra_args='', use_tty=True, verbose=True, py use_tty = False extra_args += ' -d ' # detach is optional + docker_prefix = 'docker run' + if self.gpu: + docker_prefix = 'nvidia-' + docker_prefix + if self.gpu_id > -1: + docker_prefix = '%s -e NVIDIA_VISIBLE_DEVICES=%d' % (docker_prefix, self.gpu_id) if use_tty: - docker_prefix = 'docker run %s -ti %s /bin/bash -c ' % (extra_args, self.docker_image) + docker_prefix = '%s %s -ti %s /bin/bash -c ' % (docker_prefix, extra_args, self.docker_image) else: - docker_prefix = 'docker run %s %s /bin/bash -c ' % (extra_args, self.docker_image) - if self.gpu: - docker_prefix = 'nvidia-'+docker_prefix + docker_prefix = '%s %s %s /bin/bash -c ' % (docker_prefix, extra_args, self.docker_image) main_cmd = cmd_list.to_string() full_cmd = docker_prefix + ("\'%s\'" % main_cmd) return full_cmd @@ -124,7 +130,7 @@ def __init__(self, checkpoints=None, **kwargs): super(LocalDocker, self).__init__(**kwargs) self.checkpoints = checkpoints - def launch_command(self, cmd, mount_points=None, dry=False, verbose=False): + def launch_command(self, cmd, mount_points=None, dry=False, verbose=False, extra_args=''): mnt_args = '' py_path = [] for mount in mount_points: @@ -138,7 +144,7 @@ def launch_command(self, cmd, mount_points=None, dry=False, verbose=False): else: raise NotImplementedError(type(mount)) - full_cmd = self.get_docker_cmd(cmd, extra_args=mnt_args, pythonpath=py_path, + full_cmd = self.get_docker_cmd(cmd, extra_args=extra_args + mnt_args, pythonpath=py_path, checkpoint=self.checkpoints) if verbose: print(full_cmd)