Skip to content

Set default threading state for Client downloads #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
calum-chamberlain wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Set default threading state for Client downloads #3

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions geonet_obspy_utils/clients/aws/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ class Client(object):
(only for GeoNet) data.
"""

def __init__(self, client_name="GEONET"):
def __init__(self, client_name="GEONET", max_threads=1):
"""
Initialize the client with the configuration loaded from a YAML file.

Expand All @@ -44,6 +44,9 @@ def __init__(self, client_name="GEONET"):
:type client_name: str
:param client_name: Name of the client to initialize. Must match a key
in the YAML config file.
:type max_threads: int
:param max_threads: Number of threads to use for downloads. Can be
overloaded later in individual argument calls.

"""

Expand All @@ -64,6 +67,7 @@ def __init__(self, client_name="GEONET"):
self.waveform_dir = self.config["waveform_dir"]
self.year_day_format = self.config["year_day_format"]
self.mseed_file_format = self.config["mseed_file_format"]
self.max_threads = max_threads

@property
def _s3(self):
Expand Down Expand Up @@ -116,7 +120,7 @@ def _list_available_files(self, prefix):
return file_list

def get_waveforms(self, network, station, location, channel, starttime,
endtime, filename=None, max_threads=1):
endtime, filename=None, max_threads=None):
"""
Fetch MiniSEED waveform data from AWS S3 bucket.

Expand Down Expand Up @@ -165,7 +169,10 @@ def get_waveforms(self, network, station, location, channel, starttime,
instead of being parsed to an ObsPy object. Thus it will contain
the raw data from the webservices.
"""

# If max_threads is not set at calltime, then default to the
# objects default threading state.
max_threads = max_threads or self.max_threads

if (not isinstance(starttime, UTCDateTime) or
not isinstance(endtime, UTCDateTime)):
raise ValueError("starttime and endtime must be" +
Expand Down Expand Up @@ -267,7 +274,7 @@ def get_events(self, starttime=None, endtime=None, minlatitude=None,
maxlatitude=None, minlongitude=None, maxlongitude=None,
mindepth=None, maxdepth=None, minmagnitude=None,
maxmagnitude=None, eventid=None, filename=None,
max_threads=1, **kwargs):
max_threads=None, **kwargs):
"""
Get event information from GeoNet AWS S3 bucket.

Expand Down Expand Up @@ -323,6 +330,10 @@ def get_events(self, starttime=None, endtime=None, minlatitude=None,
instead of being parsed to an ObsPy object. Thus it will contain
the raw data from the webservices.
"""
# If max_threads is not set at calltime, then default to the
# objects default threading state.
max_threads = max_threads or self.max_threads

if eventid:
try:
bin_obj = self._s3_event.Object(eventid+".xml") \
Expand Down