INFO:ssds:syncing submissions/48f9717b-23c6-4045-a33f-10a3df8fc924--UW_HPRC_HiFi/HG01361/PacBio_HiFi_Swap_Fixed/m54329U_200306_185930.subreads.bam from _GSStaging to _S3Staging
DOOM b38574f40a2f5a6c8b32948fcfb5850d-9973 b38574f40a2f5a6c8b32948fcfb5850d-9973
DOOM 5ada6ce0fa3a36a1800ae2489fb28f95-446 5ada6ce0fa3a36a1800ae2489fb28f95-446
Traceback (most recent call last):
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/urllib3/connectionpool.py", line 426, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/urllib3/connectionpool.py", line 421, in _make_request
httplib_response = conn.getresponse()
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/http/client.py", line 1347, in getresponse
response.begin()
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/http/client.py", line 307, in begin
version, status, reason = self._read_status()
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/http/client.py", line 268, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/socket.py", line 669, in readinto
return self._sock.recv_into(b)
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/ssl.py", line 1241, in recv_into
return self.read(nbytes, buffer)
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/ssl.py", line 1099, in read
return self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/requests/adapters.py", line 439, in send
resp = conn.urlopen(
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/urllib3/connectionpool.py", line 726, in urlopen
retries = retries.increment(
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/urllib3/util/retry.py", line 403, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/urllib3/packages/six.py", line 735, in reraise
raise value
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/urllib3/connectionpool.py", line 670, in urlopen
httplib_response = self._make_request(
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/urllib3/connectionpool.py", line 428, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/urllib3/connectionpool.py", line 335, in _raise_timeout
raise ReadTimeoutError(
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='storage.googleapis.com', port=443): Read timed out. (read timeout=60)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/cli_builder/__init__.py", line 93, in __call__
command(args)
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/ssds/cli/staging.py", line 82, in sync_command
for _ in ssds.sync(args.submission_id, src, dst):
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/ssds/__init__.py", line 287, in sync
for part in parts:
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/ssds/blobstore/gs.py", line 97, in __iter__
for chunk_number, data in gscio.for_each_chunk_async(self._blob,
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/gs_chunked_io/reader.py", line 137, in for_each_chunk_async
chunks.put(fetch_chunk, chunk_number)
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/gs_chunked_io/async_collections.py", line 66, in put
self._wait(running - self.concurrency)
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/gs_chunked_io/async_collections.py", line 96, in _wait
f.result()
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/concurrent/futures/_base.py", line 432, in result
return self.__get_result()
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/concurrent/futures/_base.py", line 388, in __get_result
raise self._exception
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/gs_chunked_io/reader.py", line 129, in fetch_chunk
data = reader._fetch_chunk(chunk_number)
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/gs_chunked_io/reader.py", line 55, in _fetch_chunk
data = self.blob.download_as_string(start=start_chunk, end=end_chunk)
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/google/cloud/storage/blob.py", line 1324, in download_as_string
return self.download_as_bytes(
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/google/cloud/storage/blob.py", line 1235, in download_as_bytes
self.download_to_file(
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/google/cloud/storage/blob.py", line 1037, in download_to_file
self._do_download(
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/google/cloud/storage/blob.py", line 919, in _do_download
download.consume_next_chunk(transport, timeout=timeout)
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/google/resumable_media/requests/download.py", line 365, in consume_next_chunk
result = _request_helpers.http_request(
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/google/resumable_media/requests/_request_helpers.py", line 136, in http_request
return _helpers.wait_and_retry(func, RequestsMixin._get_status_code, retry_strategy)
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/google/resumable_media/_helpers.py", line 165, in wait_and_retry
response = func()
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/google/auth/transport/requests.py", line 464, in request
response = super(AuthorizedSession, self).request(
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/requests/sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/requests/sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "/home/linuxbrew/.linuxbrew/opt/python@3.8/lib/python3.8/site-packages/requests/adapters.py", line 529, in send
raise ReadTimeout(e, request=request)
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='storage.googleapis.com', port=443): Read timed out. (read timeout=60)
This will help mitigate against long tail timeouts from S3 and GS
Informational stack trace: