Skip to content

Refactor and add get_waveforms_bulk #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
calum-chamberlain wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Refactor and add get_waveforms_bulk #5

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9293fc9
Refactor and add get_waveforms_bulk
calum-chamberlain Apr 15, 2026
06faf0f
cache paths and don't duplicate
calum-chamberlain May 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
179 changes: 113 additions & 66 deletions geonet_obspy_utils/clients/aws/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from io import StringIO, BytesIO
from parse import parse
from itertools import product
from functools import lru_cache


class Client(object):
Expand Down Expand Up @@ -96,6 +97,7 @@ def _s3_event(self):

return event_bob.Bucket(self.event_bucket_name)

@lru_cache(maxsize=365)
def _list_available_files(self, prefix):
"""
List files under the given prefix using boto3 and anonymous access.
Expand All @@ -115,63 +117,33 @@ def _list_available_files(self, prefix):
print(f"Error accessing S3 bucket '{self.bucket_name}': {e}")
return file_list

def get_waveforms(self, network, station, location, channel, starttime,
endtime, filename=None, max_threads=1):
def get_waveforms_bulk(self, bulk, max_threads=1, *args, **kwargs):
"""
Fetch MiniSEED waveform data from AWS S3 bucket.

>>> client = Client("GEONET")
>>> t1 = UTCDateTime("2024-03-20T16:59:00")
>>> t2 = UTCDateTime("2024-03-20T18:05:00")
>>> st = client.get_waveforms("NZ", "DCZ", "10", "LHZ", t1, t2)
>>> print(st)
1 Trace(s) in Stream:
NZ.DCZ.10.LHZ | 2025-09-20T16:59:00.000000Z - .. | 1.0 Hz, 3961 samples

The service can deal with UNIX style wildcards.

>>> st = client.get_waveforms("NZ,IU", "DCZ, ?", "*", "HH*", t1, t2)
>>> print(st)
6 Trace(s) in Stream:
NZ.DCZ.10.HHN | 2025-09-20T16:59:00.0Z - .. | 100.0 Hz, 396001 samples
NZ.DCZ.10.HHE | 2025-09-20T16:59:00.0Z - .. | 100.0 Hz, 396001 samples
IU.SNZO.00.HH1 | 2025-09-20T16:58:59.0Z - .. | 100.0 Hz, 396001 samples
IU.SNZO.00.HHZ | 2025-09-20T16:58:59.0Z - .. | 100.0 Hz, 396001 samples
IU.SNZO.00.HH2 | 2025-09-20T16:58:59.0Z - .. | 100.0 Hz, 396001 samples
NZ.DCZ.10.HHZ | 2025-09-20T16:59:00.0Z - .. | 100.0 Hz, 396001 samples

:type network: str
:param network: Select one or more network codes. Can be SEED network
codes or data center defined codes. Multiple codes are
comma-separated (e.g. ``"NZ,IU"``). Wildcards are allowed.
:type station: str
:param station: Select one or more SEED station codes. Multiple codes
are comma-separated (e.g. ``"LBZ,SNZO"``). Wildcards are allowed.
:type location: str
:param location: Select one or more SEED location identifiers. Multiple
identifiers are comma-separated (e.g. ``"00,01"``). Wildcards are
allowed.
:type channel: str
:param channel: Select one or more SEED channel codes. Multiple codes
are comma-separated (e.g. ``"BHZ,HHZ"``).
:type starttime: :class:`~obspy.core.utcdatetime.UTCDateTime`
:param starttime: Limit results to time series samples on or after the
specified start time
:type endtime: :class:`~obspy.core.utcdatetime.UTCDateTime`
:param endtime: Limit results to time series samples on or before the
specified end time
:type filename: str or file
:param filename: If given, the downloaded data will be saved there
instead of being parsed to an ObsPy object. Thus it will contain
the raw data from the webservices.
Write some docs.
"""
paths_to_download = []
for _bulk in bulk:
n, s, l, c, stime, etime = _bulk
paths_to_download.extend(self._resolve_files(
network=n, station=s, location=l, channel=c,
starttime=stime, endtime=etime))
st = self._get_waveforms(paths_to_download, max_threads=max_threads)
# Trim based on bulk
st_out = Stream()
for _bulk in bulk:
n, s, l, c, stime, etime = _bulk
st_out += st.select(
network=n, station=s, location=l, channel=c).slice(
starttime=stime, endtime=etime)
st_out = st_out.merge(method=-1)
return st_out


def _resolve_files(self, network, station, location, channel, starttime,
endtime):
"""
Resolve file paths for args - supports wildcards.
"""

if (not isinstance(starttime, UTCDateTime) or
not isinstance(endtime, UTCDateTime)):
raise ValueError("starttime and endtime must be" +
" obspy.UTCDateTime objects")

st = Stream()
current_time = starttime

# Create all combinations of network, station, location, and channel
Expand Down Expand Up @@ -233,6 +205,89 @@ def get_waveforms(self, network, station, location, channel, starttime,
raise IndexError("No matching waveform file(s) found in the" +
"bucket. Please check your input parameters. ")

return matching_files


def get_waveforms(self, network, station, location, channel, starttime,
endtime, filename=None, max_threads=1, *args, **kwargs):
"""
Fetch MiniSEED waveform data from AWS S3 bucket.

>>> client = Client("GEONET")
>>> t1 = UTCDateTime("2024-03-20T16:59:00")
>>> t2 = UTCDateTime("2024-03-20T18:05:00")
>>> st = client.get_waveforms("NZ", "DCZ", "10", "LHZ", t1, t2)
>>> print(st)
1 Trace(s) in Stream:
NZ.DCZ.10.LHZ | 2025-09-20T16:59:00.000000Z - .. | 1.0 Hz, 3961 samples

The service can deal with UNIX style wildcards.

>>> st = client.get_waveforms("NZ,IU", "DCZ, ?", "*", "HH*", t1, t2)
>>> print(st)
6 Trace(s) in Stream:
NZ.DCZ.10.HHN | 2025-09-20T16:59:00.0Z - .. | 100.0 Hz, 396001 samples
NZ.DCZ.10.HHE | 2025-09-20T16:59:00.0Z - .. | 100.0 Hz, 396001 samples
IU.SNZO.00.HH1 | 2025-09-20T16:58:59.0Z - .. | 100.0 Hz, 396001 samples
IU.SNZO.00.HHZ | 2025-09-20T16:58:59.0Z - .. | 100.0 Hz, 396001 samples
IU.SNZO.00.HH2 | 2025-09-20T16:58:59.0Z - .. | 100.0 Hz, 396001 samples
NZ.DCZ.10.HHZ | 2025-09-20T16:59:00.0Z - .. | 100.0 Hz, 396001 samples

:type network: str
:param network: Select one or more network codes. Can be SEED network
codes or data center defined codes. Multiple codes are
comma-separated (e.g. ``"NZ,IU"``). Wildcards are allowed.
:type station: str
:param station: Select one or more SEED station codes. Multiple codes
are comma-separated (e.g. ``"LBZ,SNZO"``). Wildcards are allowed.
:type location: str
:param location: Select one or more SEED location identifiers. Multiple
identifiers are comma-separated (e.g. ``"00,01"``). Wildcards are
allowed.
:type channel: str
:param channel: Select one or more SEED channel codes. Multiple codes
are comma-separated (e.g. ``"BHZ,HHZ"``).
:type starttime: :class:`~obspy.core.utcdatetime.UTCDateTime`
:param starttime: Limit results to time series samples on or after the
specified start time
:type endtime: :class:`~obspy.core.utcdatetime.UTCDateTime`
:param endtime: Limit results to time series samples on or before the
specified end time
:type filename: str or file
:param filename: If given, the downloaded data will be saved there
instead of being parsed to an ObsPy object. Thus it will contain
the raw data from the webservices.
"""

if (not isinstance(starttime, UTCDateTime) or
not isinstance(endtime, UTCDateTime)):
raise ValueError("starttime and endtime must be" +
" obspy.UTCDateTime objects")

matching_files = self._resolve_files(
network=network, station=station, location=location,
channel=channel, starttime=starttime, endtime=endtime)

st = self._get_waveforms(
paths_to_download=matching_files, max_threads=max_threads)
st.trim(starttime, endtime)

if len(st) > 0:
if filename:
st.write(filename)
return None
else:
return st

else:
raise IndexError("No waveforms found!")


def _get_waveforms(self, paths_to_download, max_threads=1):
"""
Internal threaded downloader.
"""
st = Stream()
# We use mseedlib here because ObsPy does not apply timing correctly
# when the sampling rate is not uniform with the dataloggers.
def download_file(file):
Expand All @@ -245,23 +300,15 @@ def download_file(file):
return _fix_mseed_timing(mstl.traceids())

with ThreadPoolExecutor(max_workers=max_threads) as executor:
# Work on set to remove possible duplicate files.
futures = [executor.submit(download_file, f)
for f in matching_files]
for f in set(paths_to_download)]
for future in as_completed(futures):
day_st = future.result()
st += day_st

st.trim(starttime, endtime)
return st

if len(st) > 0:
if filename:
st.write(filename)
return None
else:
return st

else:
raise IndexError("No waveforms found!")

def get_events(self, starttime=None, endtime=None, minlatitude=None,
maxlatitude=None, minlongitude=None, maxlongitude=None,
Expand Down