diff --git a/geonet_obspy_utils/clients/aws/client.py b/geonet_obspy_utils/clients/aws/client.py index 431a93f..a4212f8 100644 --- a/geonet_obspy_utils/clients/aws/client.py +++ b/geonet_obspy_utils/clients/aws/client.py @@ -22,6 +22,7 @@ from io import StringIO, BytesIO from parse import parse from itertools import product +from functools import lru_cache class Client(object): @@ -96,6 +97,7 @@ def _s3_event(self): return event_bob.Bucket(self.event_bucket_name) + @lru_cache(maxsize=365) def _list_available_files(self, prefix): """ List files under the given prefix using boto3 and anonymous access. @@ -115,63 +117,33 @@ def _list_available_files(self, prefix): print(f"Error accessing S3 bucket '{self.bucket_name}': {e}") return file_list - def get_waveforms(self, network, station, location, channel, starttime, - endtime, filename=None, max_threads=1): + def get_waveforms_bulk(self, bulk, max_threads=1, *args, **kwargs): """ - Fetch MiniSEED waveform data from AWS S3 bucket. - - >>> client = Client("GEONET") - >>> t1 = UTCDateTime("2024-03-20T16:59:00") - >>> t2 = UTCDateTime("2024-03-20T18:05:00") - >>> st = client.get_waveforms("NZ", "DCZ", "10", "LHZ", t1, t2) - >>> print(st) - 1 Trace(s) in Stream: - NZ.DCZ.10.LHZ | 2025-09-20T16:59:00.000000Z - .. | 1.0 Hz, 3961 samples - - The service can deal with UNIX style wildcards. - - >>> st = client.get_waveforms("NZ,IU", "DCZ, ?", "*", "HH*", t1, t2) - >>> print(st) - 6 Trace(s) in Stream: - NZ.DCZ.10.HHN | 2025-09-20T16:59:00.0Z - .. | 100.0 Hz, 396001 samples - NZ.DCZ.10.HHE | 2025-09-20T16:59:00.0Z - .. | 100.0 Hz, 396001 samples - IU.SNZO.00.HH1 | 2025-09-20T16:58:59.0Z - .. | 100.0 Hz, 396001 samples - IU.SNZO.00.HHZ | 2025-09-20T16:58:59.0Z - .. | 100.0 Hz, 396001 samples - IU.SNZO.00.HH2 | 2025-09-20T16:58:59.0Z - .. | 100.0 Hz, 396001 samples - NZ.DCZ.10.HHZ | 2025-09-20T16:59:00.0Z - .. | 100.0 Hz, 396001 samples - - :type network: str - :param network: Select one or more network codes. Can be SEED network - codes or data center defined codes. Multiple codes are - comma-separated (e.g. ``"NZ,IU"``). Wildcards are allowed. - :type station: str - :param station: Select one or more SEED station codes. Multiple codes - are comma-separated (e.g. ``"LBZ,SNZO"``). Wildcards are allowed. - :type location: str - :param location: Select one or more SEED location identifiers. Multiple - identifiers are comma-separated (e.g. ``"00,01"``). Wildcards are - allowed. - :type channel: str - :param channel: Select one or more SEED channel codes. Multiple codes - are comma-separated (e.g. ``"BHZ,HHZ"``). - :type starttime: :class:`~obspy.core.utcdatetime.UTCDateTime` - :param starttime: Limit results to time series samples on or after the - specified start time - :type endtime: :class:`~obspy.core.utcdatetime.UTCDateTime` - :param endtime: Limit results to time series samples on or before the - specified end time - :type filename: str or file - :param filename: If given, the downloaded data will be saved there - instead of being parsed to an ObsPy object. Thus it will contain - the raw data from the webservices. + Write some docs. + """ + paths_to_download = [] + for _bulk in bulk: + n, s, l, c, stime, etime = _bulk + paths_to_download.extend(self._resolve_files( + network=n, station=s, location=l, channel=c, + starttime=stime, endtime=etime)) + st = self._get_waveforms(paths_to_download, max_threads=max_threads) + # Trim based on bulk + st_out = Stream() + for _bulk in bulk: + n, s, l, c, stime, etime = _bulk + st_out += st.select( + network=n, station=s, location=l, channel=c).slice( + starttime=stime, endtime=etime) + st_out = st_out.merge(method=-1) + return st_out + + + def _resolve_files(self, network, station, location, channel, starttime, + endtime): + """ + Resolve file paths for args - supports wildcards. """ - - if (not isinstance(starttime, UTCDateTime) or - not isinstance(endtime, UTCDateTime)): - raise ValueError("starttime and endtime must be" + - " obspy.UTCDateTime objects") - - st = Stream() current_time = starttime # Create all combinations of network, station, location, and channel @@ -233,6 +205,89 @@ def get_waveforms(self, network, station, location, channel, starttime, raise IndexError("No matching waveform file(s) found in the" + "bucket. Please check your input parameters. ") + return matching_files + + + def get_waveforms(self, network, station, location, channel, starttime, + endtime, filename=None, max_threads=1, *args, **kwargs): + """ + Fetch MiniSEED waveform data from AWS S3 bucket. + + >>> client = Client("GEONET") + >>> t1 = UTCDateTime("2024-03-20T16:59:00") + >>> t2 = UTCDateTime("2024-03-20T18:05:00") + >>> st = client.get_waveforms("NZ", "DCZ", "10", "LHZ", t1, t2) + >>> print(st) + 1 Trace(s) in Stream: + NZ.DCZ.10.LHZ | 2025-09-20T16:59:00.000000Z - .. | 1.0 Hz, 3961 samples + + The service can deal with UNIX style wildcards. + + >>> st = client.get_waveforms("NZ,IU", "DCZ, ?", "*", "HH*", t1, t2) + >>> print(st) + 6 Trace(s) in Stream: + NZ.DCZ.10.HHN | 2025-09-20T16:59:00.0Z - .. | 100.0 Hz, 396001 samples + NZ.DCZ.10.HHE | 2025-09-20T16:59:00.0Z - .. | 100.0 Hz, 396001 samples + IU.SNZO.00.HH1 | 2025-09-20T16:58:59.0Z - .. | 100.0 Hz, 396001 samples + IU.SNZO.00.HHZ | 2025-09-20T16:58:59.0Z - .. | 100.0 Hz, 396001 samples + IU.SNZO.00.HH2 | 2025-09-20T16:58:59.0Z - .. | 100.0 Hz, 396001 samples + NZ.DCZ.10.HHZ | 2025-09-20T16:59:00.0Z - .. | 100.0 Hz, 396001 samples + + :type network: str + :param network: Select one or more network codes. Can be SEED network + codes or data center defined codes. Multiple codes are + comma-separated (e.g. ``"NZ,IU"``). Wildcards are allowed. + :type station: str + :param station: Select one or more SEED station codes. Multiple codes + are comma-separated (e.g. ``"LBZ,SNZO"``). Wildcards are allowed. + :type location: str + :param location: Select one or more SEED location identifiers. Multiple + identifiers are comma-separated (e.g. ``"00,01"``). Wildcards are + allowed. + :type channel: str + :param channel: Select one or more SEED channel codes. Multiple codes + are comma-separated (e.g. ``"BHZ,HHZ"``). + :type starttime: :class:`~obspy.core.utcdatetime.UTCDateTime` + :param starttime: Limit results to time series samples on or after the + specified start time + :type endtime: :class:`~obspy.core.utcdatetime.UTCDateTime` + :param endtime: Limit results to time series samples on or before the + specified end time + :type filename: str or file + :param filename: If given, the downloaded data will be saved there + instead of being parsed to an ObsPy object. Thus it will contain + the raw data from the webservices. + """ + + if (not isinstance(starttime, UTCDateTime) or + not isinstance(endtime, UTCDateTime)): + raise ValueError("starttime and endtime must be" + + " obspy.UTCDateTime objects") + + matching_files = self._resolve_files( + network=network, station=station, location=location, + channel=channel, starttime=starttime, endtime=endtime) + + st = self._get_waveforms( + paths_to_download=matching_files, max_threads=max_threads) + st.trim(starttime, endtime) + + if len(st) > 0: + if filename: + st.write(filename) + return None + else: + return st + + else: + raise IndexError("No waveforms found!") + + + def _get_waveforms(self, paths_to_download, max_threads=1): + """ + Internal threaded downloader. + """ + st = Stream() # We use mseedlib here because ObsPy does not apply timing correctly # when the sampling rate is not uniform with the dataloggers. def download_file(file): @@ -245,23 +300,15 @@ def download_file(file): return _fix_mseed_timing(mstl.traceids()) with ThreadPoolExecutor(max_workers=max_threads) as executor: + # Work on set to remove possible duplicate files. futures = [executor.submit(download_file, f) - for f in matching_files] + for f in set(paths_to_download)] for future in as_completed(futures): day_st = future.result() st += day_st - st.trim(starttime, endtime) + return st - if len(st) > 0: - if filename: - st.write(filename) - return None - else: - return st - - else: - raise IndexError("No waveforms found!") def get_events(self, starttime=None, endtime=None, minlatitude=None, maxlatitude=None, minlongitude=None, maxlongitude=None,