From 32963230dfcab52344f67dc6ba085c0dd71aed57 Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 9 Jan 2026 14:43:23 +1300 Subject: [PATCH 01/72] add type checking --- .github/workflows/types.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/types.yml diff --git a/.github/workflows/types.yml b/.github/workflows/types.yml new file mode 100644 index 00000000..bc12dc22 --- /dev/null +++ b/.github/workflows/types.yml @@ -0,0 +1,23 @@ +name: Type Check +on: [pull_request] +jobs: + typecheck: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + + - name: Install project with types + run: uv sync --all-extras --dev + + - name: Run type checking with ty + run: uv run ty check --exclude setup.py \ No newline at end of file From 569cac64a907826abd7d7f381d1eca228ae49cbe Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 9 Jan 2026 15:04:02 +1300 Subject: [PATCH 02/72] change numpy version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3f53afa2..9f3e6c87 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ typer>0.12.3 pandas -numpy<2.0.0 +numpy>=2.0 obspy mseedlib scipy From 00d391c31cdd872e2be7deb6e24cb72b9f61c56e Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 9 Jan 2026 15:16:54 +1300 Subject: [PATCH 03/72] install ty --- .github/workflows/types.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/types.yml b/.github/workflows/types.yml index bc12dc22..5c0ab230 100644 --- a/.github/workflows/types.yml +++ b/.github/workflows/types.yml @@ -16,8 +16,8 @@ jobs: with: enable-cache: true - - name: Install project with types - run: uv sync --all-extras --dev + - name: Ensure ty is installed into the uv environment + run: uv run python -m pip install ty - name: Run type checking with ty run: uv run ty check --exclude setup.py \ No newline at end of file From 78037d6f248cfe4f6fff376c682c21b81d2c5849 Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 9 Jan 2026 15:28:04 +1300 Subject: [PATCH 04/72] adjust type check --- .github/workflows/types.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/types.yml b/.github/workflows/types.yml index 5c0ab230..dafa9395 100644 --- a/.github/workflows/types.yml +++ b/.github/workflows/types.yml @@ -10,14 +10,11 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 - - - name: Install uv - uses: astral-sh/setup-uv@v5 with: - enable-cache: true + python-version: '3.11' - - name: Ensure ty is installed into the uv environment - run: uv run python -m pip install ty + - name: Install ty on runner + run: pip install ty - - name: Run type checking with ty - run: uv run ty check --exclude setup.py \ No newline at end of file + - name: Run type checking + run: ty check --exclude setup.py \ No newline at end of file From ffe5299808793361e306cbb95746673f8d2e83fe Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 9 Jan 2026 15:33:12 +1300 Subject: [PATCH 05/72] install dependencies --- .github/workflows/types.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/types.yml b/.github/workflows/types.yml index dafa9395..b84913cb 100644 --- a/.github/workflows/types.yml +++ b/.github/workflows/types.yml @@ -16,5 +16,8 @@ jobs: - name: Install ty on runner run: pip install ty + - name: Install dependencies + run: pip install -r requirements.txt + - name: Run type checking - run: ty check --exclude setup.py \ No newline at end of file + run: ty check --exclude setup.py --exclude nzgmdb/CCLD/ccldpy.py \ No newline at end of file From 0fad64e64eac1d4ddb90c0344d282c7724217371 Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 9 Jan 2026 15:36:56 +1300 Subject: [PATCH 06/72] ensure same env --- .github/workflows/types.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/types.yml b/.github/workflows/types.yml index b84913cb..fe5f5770 100644 --- a/.github/workflows/types.yml +++ b/.github/workflows/types.yml @@ -3,7 +3,6 @@ on: [pull_request] jobs: typecheck: runs-on: ubuntu-latest - steps: - name: Checkout code uses: actions/checkout@v4 @@ -13,11 +12,13 @@ jobs: with: python-version: '3.11' - - name: Install ty on runner - run: pip install ty + - name: Install project dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt - - name: Install dependencies - run: pip install -r requirements.txt + - name: Install ty into same interpreter + run: python -m pip install ty - name: Run type checking - run: ty check --exclude setup.py --exclude nzgmdb/CCLD/ccldpy.py \ No newline at end of file + run: python -m ty check --exclude setup.py --exclude nzgmdb/CCLD/ccldpy.py From 4358ff3518a4aa167351a92b1583c3e34c62cfc4 Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 12 Jan 2026 11:23:06 +1300 Subject: [PATCH 07/72] revert to uv --- .github/workflows/types.yml | 19 +++++++++---------- pyproject.toml | 4 ++++ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/types.yml b/.github/workflows/types.yml index fe5f5770..0b5e69d6 100644 --- a/.github/workflows/types.yml +++ b/.github/workflows/types.yml @@ -3,22 +3,21 @@ on: [pull_request] jobs: typecheck: runs-on: ubuntu-latest + steps: - name: Checkout code uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Install project dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true - - name: Install ty into same interpreter - run: python -m pip install ty + - name: Install project with types + run: uv sync --all-extras --dev - - name: Run type checking - run: python -m ty check --exclude setup.py --exclude nzgmdb/CCLD/ccldpy.py + - name: Run type checking with ty + run: uv run ty check --exclude setup.py --exclude nzgmdb/CCLD/ccldpy.py diff --git a/pyproject.toml b/pyproject.toml index 5b7fbd9d..60330ec6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,10 @@ readme = "README.md" requires-python = "==3.12.*" dynamic = ["version", "dependencies"] +[project.optional-dependencies] +test = ["pytest"] +dev = ["ty"] + [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]} From 56b7cc6c50928376df44bcf110d7b3105b98e04b Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 14 Jan 2026 13:23:15 +1300 Subject: [PATCH 08/72] sites vs30 z addition --- nzgmdb/data_retrieval/sites.py | 259 +++++++++++++++++++++++- nzgmdb/management/data_registry.py | 2 + tests/test_sites.py | 304 +++++++++++++++++++++++++++++ 3 files changed, 559 insertions(+), 6 deletions(-) create mode 100644 tests/test_sites.py diff --git a/nzgmdb/data_retrieval/sites.py b/nzgmdb/data_retrieval/sites.py index ee7616ac..14303ddb 100644 --- a/nzgmdb/data_retrieval/sites.py +++ b/nzgmdb/data_retrieval/sites.py @@ -6,14 +6,179 @@ from pathlib import Path import fiona +import numpy as np import pandas as pd +import rasterio from obspy.clients.fdsn import Client as FDSN_Client +from pyproj import Transformer +from scipy.spatial import cKDTree from nzgmdb.data_retrieval import tect_domain from nzgmdb.management import config as cfg from nzgmdb.management.data_registry import NZGMDB_DATA from qcore import point_in_polygon -from velocity_modelling import registry +from velocity_modelling import registry, threshold + + +def fill_gaps_with_nearest( + coords: np.ndarray, + values: np.ndarray, + invalid_mask: np.ndarray = None, + k: int = 8, +): + """ + Fill NaN or invalid values using nearest-neighbour averaging. + + Parameters + ---------- + coords : (N, 2) array_like + Coordinates of the points (e.g., [x, y] or [lon, lat]). + values : (N,) array_like + Values at the points, with NaN for invalid/missing values. + invalid_mask : (N,) array_like, optional + Boolean mask indicating invalid points. If None, NaNs in `values` are used. + k : int, default=8 + Number of nearest neighbors to consider for averaging. + + Returns + ------- + filled_values : (N,) ndarray + Values with NaNs filled using nearest-neighbour averaging. + """ + + coords = np.asarray(coords) + values = np.asarray(values).astype(float) + + # ---- Enforce correct shapes ---- + if values.ndim == 2 and values.shape[1] == 1: + values = values.ravel() + + if coords.ndim != 2 or coords.shape[1] != 2: + raise ValueError("coords must be of shape (N, 2)") + + if invalid_mask is None: + invalid_mask = np.isnan(values) + else: + invalid_mask = np.asarray(invalid_mask) + if invalid_mask.ndim == 2 and invalid_mask.shape[1] == 1: + invalid_mask = invalid_mask.ravel() + + valid_mask = ~invalid_mask + + if not valid_mask.any(): + return np.full_like(values, np.nan) + + # ---- Build KDTree ---- + tree = cKDTree(coords[valid_mask]) + valid_values = values[valid_mask] + + # ---- Fill invalid points ---- + for idx in np.where(invalid_mask)[0]: + coord = coords[idx] + kk = min(k, len(valid_values)) + _, nn = tree.query(coord, k=kk) + values[idx] = np.nanmean(valid_values[nn]) + + return values + + +def sample_points_from_geotiff( + file_path: Path, + latlon_points: np.ndarray, + band: int = 1, +): + """ + Sample a GeoTIFF raster at given latitude/longitude points. + + Parameters + ---------- + file_path : Path + Path to the GeoTIFF file. + latlon_points : (N, 2) array_like + Input points as [lat, lon] in EPSG:4326. + band : int, default=1 + Raster band to sample (1-based index). + + Returns + ------- + samples : (N, 1) ndarray + Sampled raster values. NaN where points fall outside the raster + or where raster contains nodata. + """ + + # ---- Normalize inputs ---- + file_path = Path(file_path) + latlon_points = np.asarray(latlon_points, dtype=float) + + lat = latlon_points[:, 0] + lon = latlon_points[:, 1] + + # Prepare output (NaN by default) + samples = np.full(lat.shape, np.nan, dtype=float) + + # ---- Open raster ---- + with rasterio.open(file_path) as ds: + + if ds.crs is None: + raise ValueError("Raster CRS is undefined.") + + # CRS of input coordinates (WGS84 lat/lon) + input_crs = rasterio.crs.CRS.from_epsg(4326) + + # ---- Transform coordinates if needed ---- + if ds.crs == input_crs: + # Raster already in lat/lon + x = lon + y = lat + else: + # Transform lat/lon → raster CRS + transformer = Transformer.from_crs( + input_crs, + ds.crs, + always_xy=True, + ) + x, y = transformer.transform(lon, lat) + + # ---- Determine which points lie inside raster bounds ---- + bounds = ds.bounds + inside = ( + (x >= bounds.left) + & (x <= bounds.right) + & (y >= bounds.bottom) + & (y <= bounds.top) + ) + + if not np.any(inside): + return samples.reshape(-1, 1) + + # ---- Sample raster at valid points ---- + coords = list(zip(x[inside], y[inside])) + + raw_values = np.array( + [v[0] for v in ds.sample(coords, indexes=band)], + dtype=float, + ) + + # ---- Handle nodata ---- + if ds.nodata is not None: + raw_values[raw_values == ds.nodata] = np.nan + + # ---- Apply scale and offset if defined ---- + scale = 1.0 + offset = 0.0 + + if ds.scales is not None: + scale = ds.scales[band - 1] + + if ds.offsets is not None: + offset = ds.offsets[band - 1] + + values = raw_values * scale + offset + + # ---- Insert values back into output ---- + samples[inside] = values + + return samples.reshape(-1, 1) def create_site_table_response() -> pd.DataFrame: @@ -51,7 +216,26 @@ def create_site_table_response() -> pd.DataFrame: station_info, columns=["net", "sta", "lat", "lon", "elev", "creation_date", "end_date"], ) - sta_df = sta_df.drop_duplicates(["net", "sta"]).reset_index(drop=True) + sta_df = sta_df.drop_duplicates(["net", "sta"]) + + bbox = config.get_value("bbox") # [min_lon, min_lat, max_lon, max_lat] + min_lon, min_lat, max_lon, max_lat = bbox + + # Ensure lat/lon are present and within latitude bounds + mask_lat = ( + sta_df["lat"].notna() + & sta_df["lon"].notna() + & (sta_df["lat"] >= min_lat) + & (sta_df["lat"] <= max_lat) + ) + + # Handle antimeridian crossing: if min_lon > max_lon use OR + if min_lon <= max_lon: + mask_lon = (sta_df["lon"] >= min_lon) & (sta_df["lon"] <= max_lon) + else: + mask_lon = (sta_df["lon"] >= min_lon) | (sta_df["lon"] <= max_lon) + + sta_df = sta_df.loc[mask_lat & mask_lon] # Get the Geonet metadata summary information geo_meta_summary_df = pd.read_csv( @@ -62,8 +246,6 @@ def create_site_table_response() -> pd.DataFrame: geo_meta_summary_df = geo_meta_summary_df.rename( columns={ "Name": "sta", - "Lat": "lat", - "Long": "lon", "NZS1170SiteClass": "site_class", "Vs30_median": "Vs30", "Sigmaln_Vs30": "Vs30_std", @@ -82,12 +264,14 @@ def create_site_table_response() -> pd.DataFrame: ) merged_df = geo_meta_summary_df.merge( - sta_df[["net", "elev", "sta", "creation_date", "end_date"]], + sta_df[["net", "sta", "lat", "lon", "elev", "creation_date", "end_date"]], on="sta", how="outer", ) - # Fill Elevation NaN values from sta_df + # Fill Lat, Lon, Elevation NaN values from sta_df merged_df["elev"] = merged_df["Elevation"].combine_first(merged_df["elev"]) + merged_df["lat"] = merged_df["Lat"].combine_first(merged_df["lat"]) + merged_df["lon"] = merged_df["Long"].combine_first(merged_df["lon"]) # Specify the required files for fiona NZGMDB_DATA.fetch("nt_domains_kiran.shp") NZGMDB_DATA.fetch("nt_domains_kiran.dbf") @@ -100,6 +284,69 @@ def create_site_table_response() -> pd.DataFrame: # Rename the domain column tect_merged_df = tect_merged_df.rename(columns={"domain_no": "site_domain_no"}) + # Only compute thresholds for stations where Z1.0 is missing + mask_missing_z1 = tect_merged_df["Z1.0"].isna() + if mask_missing_z1.any(): + # Prepare stations DataFrame for only missing rows, indexed by station code + stations = tect_merged_df.loc[mask_missing_z1, ["sta", "lon", "lat"]].set_index( + "sta" + )[["lon", "lat"]] + try: + nzcvm_version = config.get_value("nzcvm_version") + thresholds = threshold.compute_station_thresholds( + stations, model_version=nzcvm_version + ) + # Merge computed thresholds back (computed columns will be suffixed) + tect_merged_df = tect_merged_df.merge( + thresholds[["Z1.0(km)", "Z2.5(km)", "sigma"]], + left_on="sta", + right_index=True, + how="left", + ) + + # Add in the computed values where missing + tect_merged_df["Z1.0"] = tect_merged_df["Z1.0"].combine_first( + tect_merged_df.get("Z1.0(km)") * 1000.0 + ) + tect_merged_df["Z2.5"] = tect_merged_df["Z2.5"].combine_first( + tect_merged_df.get("Z2.5(km)") * 1000.0 + ) + tect_merged_df["Z1.0_std"] = tect_merged_df["Z1.0_std"].combine_first( + tect_merged_df.get("sigma") + ) + tect_merged_df["Z2.5_std"] = tect_merged_df["Z2.5_std"].combine_first( + tect_merged_df.get("sigma") + ) + + # Set extra ref / quality fields + tect_merged_df.loc[ + mask_missing_z1, ["Z1.0_ref", "Z2.5_ref", "Q_Z1.0", "Q_Z2.5"] + ] = ["NZCVM (2026)", "NZCVM (2026)", "Q3", "Q3"] + + # Get the file path to the combined MVN GeoTIFF + NZGMDB_DATA.fetch("combined_mvn_wgs84.tif") + file_path = Path(NZGMDB_DATA.abspath) / "combined_mvn_wgs84.tif" + + # Compute Vs30 for missing values + points = tect_merged_df.loc[mask_missing_z1, ["lat", "lon"]].to_numpy() + vs30_values = sample_points_from_geotiff(file_path, points).ravel() + + # Fill missing gaps in Vs30 using nearest-neighbour averaging + coords = np.column_stack([points[:, 1], points[:, 0]]) + vs30_values_filled = fill_gaps_with_nearest(coords, vs30_values) + vs30_values_filled_rounded = np.round(vs30_values_filled) + + # Update Vs30 and related fields + tect_merged_df.loc[mask_missing_z1, "Vs30"] = vs30_values_filled_rounded + + # Ensure reference and quality fields are set for Vs30 where filled + vs30_mask = mask_missing_z1 & ~tect_merged_df["Vs30"].isna() + tect_merged_df.loc[vs30_mask, "Vs30_Ref"] = "Foster et al. (2019)" + tect_merged_df.loc[vs30_mask, "Q_Vs30"] = "Q3" + + except (FileNotFoundError, ValueError, RuntimeError) as e: + print(f"Warning: Could not compute thresholds for missing Z1.0 values: {e}") + # Select specific columns site_df = tect_merged_df[ [ diff --git a/nzgmdb/management/data_registry.py b/nzgmdb/management/data_registry.py index 56703a47..6abc76ad 100644 --- a/nzgmdb/management/data_registry.py +++ b/nzgmdb/management/data_registry.py @@ -5,6 +5,7 @@ REGISTRY = { "hik_kerm_fault_300km_wgs84_poslon.txt": "sha256:1a199978b6c9c608f8473539b639a8825c1091167da3d14b07c7268528320e03", "Geonet_Metadata_Summary_v1.4.csv": "sha256:7884422c3fcae0810c02948ba1a3bd39ba5793ba28189e90d730541be1c207c0", + "combined_mvn_wgs84.tif": "sha256:b0aed1d441a3c441d784a5dd9016314ee74df164dcafee6e233211a43cbfba0f", "puy_slab2_dep_02.26.18.xyz": "sha256:9ebe4feab4ee3b80e3fe403f2d873f94e4d7f06d937d721cc9e154ecee83e3c0", "reyners_relocations.csv": "sha256:7795c60dae67af14eb590b0d919fa850e2f2fe2fb3beb077cbac14d27eb8faf5", "focal_mech_tectonic_domain_v1.csv": "sha256:1f1e0c4b7f9ca1b87fb2ca4883e587f330fe82b5bbf9ebb6eb8f4d12aa2e1936", @@ -40,6 +41,7 @@ URLS = { "focal_mech_tectonic_domain_v1.csv": "https://www.dropbox.com/scl/fi/zseg304cbjmti7gg5tdyv/focal_mech_tectonic_domain_v1.csv?rlkey=kfb9ttvnv9yi9zftixw6kmz4v&st=4j9pgpgj&dl=1", "Geonet_Metadata_Summary_v1.4.csv": "https://www.dropbox.com/scl/fi/iev7qmoqqzvc5quhf8mk8/Geonet-Metadata-Summary_v1.4.csv?rlkey=7twwwck5iy5zao7lwao6xodvm&st=6m3elzuu&dl=1", + "combined_mvn_wgs84.tif": "https://www.dropbox.com/scl/fi/lzqijoivcg4wzybj06rsh/combined_mvn_wgs84.tif?rlkey=3g4milzk41c2lcsdggvy2ieql&st=re0izlz4&dl=1", "GeoNet_CMT_solutions_20201129_PreferredNodalPlane_v1.csv": "https://www.dropbox.com/scl/fi/fq28jx8jlbozj0d1x5tnq/GeoNet_CMT_solutions_20201129_PreferredNodalPlane_v1.csv?rlkey=30xj6n7ara0vz4t8kg4pz8w5s&st=63x7nr3j&dl=1", "hik_kerm_fault_300km_wgs84_poslon.txt": "https://www.dropbox.com/scl/fi/ig3ajufpv4xg2qjfxxuup/hik_kerm_fault_300km_wgs84_poslon.txt?rlkey=9jajfkq2elrzwzzgh6px17k8e&st=6ham2oox&dl=1", "Mw_rrup.txt": "https://www.dropbox.com/scl/fi/e3o9v9ze9e4955xxtrl14/Mw_rrup.txt?rlkey=c663zntx7gaeyxt04i97r62nu&st=6ri3c620&dl=1", diff --git a/tests/test_sites.py b/tests/test_sites.py new file mode 100644 index 00000000..e8b46cbb --- /dev/null +++ b/tests/test_sites.py @@ -0,0 +1,304 @@ +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +from rasterio.io import MemoryFile +from rasterio.transform import from_origin + +from nzgmdb.data_retrieval import sites + + +def _make_test_geotiff( + width: int = 10, + height: int = 10, + nodata: float = -9999.0, +) -> MemoryFile: + """ + Create an in-memory single-band GeoTIFF for testing. + + Parameters + ---------- + width : int, default=10 + Raster width in pixels. + height : int, default=10 + Raster height in pixels. + nodata : float, default=-9999.0 + NoData value written to the dataset (and to one pixel in the raster). + + Returns + ------- + memfile : rasterio.io.MemoryFile + In-memory GeoTIFF containing a simple gradient field with a NoData pixel. + """ + data = np.arange(width * height, dtype=np.float32).reshape(height, width) + transform = from_origin(0.0, 10.0, 1.0, 1.0) + data[0, 0] = nodata + + memfile = MemoryFile() + with memfile.open( + driver="GTiff", + height=height, + width=width, + count=1, + dtype=data.dtype, + crs="EPSG:4326", + transform=transform, + nodata=nodata, + ) as ds: + ds.write(data, 1) + + return memfile + + +def test_sample_points_from_geotiff_inside_outside_and_nodata() -> None: + """ + Test that GeoTIFF sampling returns finite values for in-bounds points, and + returns NaN for out-of-bounds or NoData pixels. + + Returns + ------- + None + """ + memfile = _make_test_geotiff() + with memfile.open() as ds: + points = np.array( + [ + [9.5, 0.5], + [5.5, 5.5], + [20.0, 5.0], + [-5.0, 5.0], + [5.0, 20.0], + ], + dtype=float, + ) + out = sites.sample_points_from_geotiff(ds.name, points).ravel() + + assert out.shape == (len(points),) + assert np.isnan(out[0]) + assert np.isfinite(out[1]) + assert np.isnan(out[2]) + assert np.isnan(out[3]) + assert np.isnan(out[4]) + + +def test_fill_gaps_with_nearest_fills_nans_and_preserves_finite() -> None: + """ + Test that `fill_gaps_with_nearest` fills NaN entries while preserving + existing finite values. + + Returns + ------- + None + """ + coords = np.array( + [[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [1.0, 1.0]], + dtype=float, + ) + values = np.array([10.0, 20.0, np.nan, 40.0], dtype=float) + + filled = sites.fill_gaps_with_nearest(coords, values, k=3) + + assert filled.shape == values.shape + assert np.isfinite(filled[2]) + assert filled[0] == 10.0 + assert filled[1] == 20.0 + assert filled[3] == 40.0 + + +@pytest.mark.parametrize( + "points", + [ + np.array( + [ + [-36.8485, 174.7633], + [-41.2865, 174.7762], + [-43.5321, 172.6362], + [-45.0312, 168.6626], + ], + dtype=float, + ), + np.array( + [[-34.0, 172.5], [-47.5, 166.0], [-41.0, 179.9], [-41.0, 166.5]], + dtype=float, + ), + np.array( + [[-30.0, 174.0], [-50.0, 170.0], [-41.0, 160.0], [-41.0, -175.0]], + dtype=float, + ), + np.array( + [ + [48.8566, 2.3522], + [51.5074, -0.1278], + [52.5200, 13.4050], + [41.9028, 12.4964], + ], + dtype=float, + ), + np.vstack( + [ + np.array([[-36.8485, 174.7633], [-41.2865, 174.7762]], dtype=float), + np.array([[48.8566, 2.3522], [51.5074, -0.1278]], dtype=float), + np.array([[-43.5321, 172.6362], [-45.0312, 168.6626]], dtype=float), + ] + ), + ], +) +def test_site_updates_vs30_and_z1_fields_only_when_available( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + points: np.ndarray, +) -> None: + """ + Integration-style test of site table population logic using monkeypatches. + + Verifies expected output columns exist and that Vs30 and Z1\.0/Z2\.5 values + (and their reference/quality fields) are only set when data are available. + + Parameters + ---------- + monkeypatch : pytest.MonkeyPatch + Pytest fixture used to patch external dependencies. + tmp_path : pathlib.Path + Pytest fixture providing a temporary directory for test artifacts. + points : numpy.ndarray, shape (N, 2) + Input points as \[lat, lon\] used to construct a minimal metadata table. + + Returns + ------- + None + """ + + class _Cfg: + def get_value(self, key): + if key == "channel_codes": + return "HHZ" + if key == "bbox": + return [0.0, -90.0, 360.0, 90.0] + if key == "nzcvm_version": + return "test" + return None + + monkeypatch.setattr(sites.cfg, "Config", _Cfg) + + class _DummyInv(list): + pass + + class _DummyClient: + def __init__(self, *_args, **_kwargs): + pass + + def get_stations(self, *args, **kwargs): + return _DummyInv() + + monkeypatch.setattr(sites, "FDSN_Client", _DummyClient) + + monkeypatch.setattr(sites.fiona, "open", lambda *_a, **_k: []) + + # Do NOT monkeypatch NZGMDB_DATA.abspath (read-only property). + # Instead, patch fetch() to return a temp file path for the tif and anything else requested. + combined_tif = tmp_path / "combined_mvn_wgs84.tif" + combined_tif.touch() + + def _fetch(name, *args, **kwargs): + p = tmp_path / name + p.parent.mkdir(parents=True, exist_ok=True) + p.touch(exist_ok=True) + return str(p) + + monkeypatch.setattr(sites.NZGMDB_DATA, "fetch", _fetch) + + geo = pd.DataFrame( + { + "Name": [f"S{i}" for i in range(len(points))], + "Lat": points[:, 0], + "Long": points[:, 1], + "Elevation": np.zeros(len(points)), + "Vs30_median": [np.nan] * len(points), + "Sigmaln_Vs30": [np.nan] * len(points), + "T_median": [np.nan] * len(points), + "sigmaln_T": [np.nan] * len(points), + "Q_T": [None] * len(points), + "D_T": [None] * len(points), + "T_Ref": [None] * len(points), + "Z1.0_median": [np.nan] * len(points), + "sigmaln_Z1.0": [np.nan] * len(points), + "Z1.0_Ref": [None] * len(points), + "Z2.5_median": [np.nan] * len(points), + "sigmaln_Z2.5": [np.nan] * len(points), + "Z2.5_Ref": [None] * len(points), + "NZS1170SiteClass": [None] * len(points), + } + ) + monkeypatch.setattr(pd, "read_csv", lambda *_a, **_k: geo) + + def _find_domain_from_shapes(df, _shapes): + out = df.copy() + out["domain_no"] = 1 + return out + + monkeypatch.setattr( + sites.tect_domain, "find_domain_from_shapes", _find_domain_from_shapes + ) + + def _compute_station_thresholds(stations, model_version=None): + n = len(stations) + return pd.DataFrame( + { + "Z1.0(km)": np.full(n, 0.5), + "Z2.5(km)": np.full(n, 2.0), + "sigma": np.full(n, 0.25), + }, + index=stations.index, + ) + + monkeypatch.setattr( + sites.threshold, "compute_station_thresholds", _compute_station_thresholds + ) + + def _sample_points_from_geotiff(_file_path, latlon_points, band=1): + n = len(latlon_points) + vals = np.linspace(100.0, 500.0, n).astype(float) + if n >= 2: + vals[0] = np.nan + return vals.reshape(-1, 1) + + monkeypatch.setattr( + sites, "sample_points_from_geotiff", _sample_points_from_geotiff + ) + + def _fill_gaps_with_nearest(coords, values, invalid_mask=None, k=8): + values = np.asarray(values, dtype=float).copy() + values[np.isnan(values)] = 250.0 + return values + + monkeypatch.setattr(sites, "fill_gaps_with_nearest", _fill_gaps_with_nearest) + + site_df = sites.create_site_table_response() + + for col in [ + "sta", + "lat", + "lon", + "Vs30", + "Q_Vs30", + "Vs30_Ref", + "Z1.0", + "Z2.5", + "Z1.0_ref", + "Z2.5_ref", + "Q_Z1.0", + "Q_Z2.5", + ]: + assert col in site_df.columns + + assert site_df["Z1.0"].notna().any() + assert site_df["Z2.5"].notna().any() + assert (site_df["Z1.0_ref"].dropna() == "NZCVM (2026)").all() + assert (site_df["Z2.5_ref"].dropna() == "NZCVM (2026)").all() + + vs30_non_nan = site_df["Vs30"].notna() + assert (site_df.loc[vs30_non_nan, "Vs30_Ref"] == "Foster et al. (2019)").all() + assert (site_df.loc[vs30_non_nan, "Q_Vs30"] == "Q3").all() + assert site_df.loc[~vs30_non_nan, "Vs30_Ref"].isna().all() + assert site_df.loc[~vs30_non_nan, "Q_Vs30"].isna().all() From 02fd89d18dd2667591ba4c05feb8076f38cc67df Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 16 Jan 2026 11:43:55 +1300 Subject: [PATCH 09/72] changelog update --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 8beb442a..e22d40f0 100644 --- a/changelog.md +++ b/changelog.md @@ -5,7 +5,6 @@ * New Quality Filter to remove Broadband data during certain time periods due to sensitivity issues * New Quality Filter to compare against an empirical GMPE (Atkinson 2022) to remove significant outliers * Add ability to generate a report to compare NZGMDB versions -* Automatic report creation * Add GMC skipped reasons * Update to use the new NZCVM 2.09 basins * TPVZ calculation fix @@ -22,6 +21,7 @@ * Change processing to use remove response instead of remove sensitivity * Updated CMT solutions for domain regions * Increased date range to end of 2025 +* Vs30 and Z1.0 / 2.5 for missing station metadata ## Version 4.3 - July 25 **2000-01-01 to 2024-12-31** * Sensitivity Fix (previously always taking first value not for actual datetime expected) From 5dd07e56bdaf69631e5cdd27d017cad9c5e005b6 Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 19 Jan 2026 15:54:03 +1300 Subject: [PATCH 10/72] formatting --- nzgmdb/data_retrieval/sites.py | 4 ++-- nzgmdb/data_retrieval/waveform_extraction.py | 2 +- nzgmdb/phase_arrival/run_phasenet.py | 1 - requirements.txt | 1 + tests/test_sites.py | 21 ++++++++++++-------- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/nzgmdb/data_retrieval/sites.py b/nzgmdb/data_retrieval/sites.py index 14303ddb..da6878a3 100644 --- a/nzgmdb/data_retrieval/sites.py +++ b/nzgmdb/data_retrieval/sites.py @@ -42,7 +42,7 @@ def fill_gaps_with_nearest( Returns ------- - filled_values : (N,) ndarray + ndarray Values with NaNs filled using nearest-neighbour averaging. """ @@ -101,7 +101,7 @@ def sample_points_from_geotiff( Returns ------- - samples : (N, 1) ndarray + ndarray Sampled raster values. NaN where points fall outside the raster or where raster contains nodata. """ diff --git a/nzgmdb/data_retrieval/waveform_extraction.py b/nzgmdb/data_retrieval/waveform_extraction.py index 789fe60b..3f829609 100644 --- a/nzgmdb/data_retrieval/waveform_extraction.py +++ b/nzgmdb/data_retrieval/waveform_extraction.py @@ -24,8 +24,8 @@ from obspy.io.mseed import InternalMSEEDError, ObsPyMSEEDFilesizeTooSmallError from pandas.errors import EmptyDataError -from nzgmdb.data_processing import filtering, multi_event from nzgmdb.data_processing import filtering +from nzgmdb.data_processing import multi_event from nzgmdb.data_retrieval import inventory_xml from nzgmdb.management import config as cfg from nzgmdb.management import custom_errors, file_structure diff --git a/nzgmdb/phase_arrival/run_phasenet.py b/nzgmdb/phase_arrival/run_phasenet.py index 2dc03422..453790b2 100644 --- a/nzgmdb/phase_arrival/run_phasenet.py +++ b/nzgmdb/phase_arrival/run_phasenet.py @@ -4,7 +4,6 @@ import argparse from pathlib import Path -from typing import Union import h5py import mseedlib diff --git a/requirements.txt b/requirements.txt index e987d2c4..86e8edba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ gmprocess h5py pooch matplotlib +rasterio oq_wrapper>=2025.12.3, qcore-utils>=2025.12.1, source_modelling>=2025.12.1, diff --git a/tests/test_sites.py b/tests/test_sites.py index e8b46cbb..e449bc58 100644 --- a/tests/test_sites.py +++ b/tests/test_sites.py @@ -152,7 +152,7 @@ def test_site_updates_vs30_and_z1_fields_only_when_available( """ Integration-style test of site table population logic using monkeypatches. - Verifies expected output columns exist and that Vs30 and Z1\.0/Z2\.5 values + Verifies expected output columns exist and that Vs30 and Z1.0 / Z2.5 values (and their reference/quality fields) are only set when data are available. Parameters @@ -162,7 +162,7 @@ def test_site_updates_vs30_and_z1_fields_only_when_available( tmp_path : pathlib.Path Pytest fixture providing a temporary directory for test artifacts. points : numpy.ndarray, shape (N, 2) - Input points as \[lat, lon\] used to construct a minimal metadata table. + Input points as [lat, lon] used to construct a minimal metadata table. Returns ------- @@ -170,7 +170,7 @@ def test_site_updates_vs30_and_z1_fields_only_when_available( """ class _Cfg: - def get_value(self, key): + def get_value(self, key: str): if key == "channel_codes": return "HHZ" if key == "bbox": @@ -200,7 +200,7 @@ def get_stations(self, *args, **kwargs): combined_tif = tmp_path / "combined_mvn_wgs84.tif" combined_tif.touch() - def _fetch(name, *args, **kwargs): + def _fetch(name: str, *args, **kwargs): p = tmp_path / name p.parent.mkdir(parents=True, exist_ok=True) p.touch(exist_ok=True) @@ -232,7 +232,7 @@ def _fetch(name, *args, **kwargs): ) monkeypatch.setattr(pd, "read_csv", lambda *_a, **_k: geo) - def _find_domain_from_shapes(df, _shapes): + def _find_domain_from_shapes(df: pd.DataFrame, _shapes: list) -> pd.DataFrame: out = df.copy() out["domain_no"] = 1 return out @@ -241,7 +241,7 @@ def _find_domain_from_shapes(df, _shapes): sites.tect_domain, "find_domain_from_shapes", _find_domain_from_shapes ) - def _compute_station_thresholds(stations, model_version=None): + def _compute_station_thresholds(stations: pd.DataFrame) -> pd.DataFrame: n = len(stations) return pd.DataFrame( { @@ -256,7 +256,10 @@ def _compute_station_thresholds(stations, model_version=None): sites.threshold, "compute_station_thresholds", _compute_station_thresholds ) - def _sample_points_from_geotiff(_file_path, latlon_points, band=1): + def _sample_points_from_geotiff( + geotiff_path: str, + latlon_points: np.ndarray, + ) -> np.ndarray: n = len(latlon_points) vals = np.linspace(100.0, 500.0, n).astype(float) if n >= 2: @@ -267,7 +270,9 @@ def _sample_points_from_geotiff(_file_path, latlon_points, band=1): sites, "sample_points_from_geotiff", _sample_points_from_geotiff ) - def _fill_gaps_with_nearest(coords, values, invalid_mask=None, k=8): + def _fill_gaps_with_nearest( + values: np.ndarray, + ) -> np.ndarray: values = np.asarray(values, dtype=float).copy() values[np.isnan(values)] = 250.0 return values From 3e6ff8e964e5dde86593873c0ce6aa846a63f6fd Mon Sep 17 00:00:00 2001 From: joelridden Date: Tue, 20 Jan 2026 10:20:47 +1300 Subject: [PATCH 11/72] fix tests --- nzgmdb/data_retrieval/waveform_extraction.py | 3 +-- nzgmdb/mseed_management/reading.py | 1 - tests/test_sites.py | 10 ++++++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/nzgmdb/data_retrieval/waveform_extraction.py b/nzgmdb/data_retrieval/waveform_extraction.py index 3f829609..1db881cd 100644 --- a/nzgmdb/data_retrieval/waveform_extraction.py +++ b/nzgmdb/data_retrieval/waveform_extraction.py @@ -24,8 +24,7 @@ from obspy.io.mseed import InternalMSEEDError, ObsPyMSEEDFilesizeTooSmallError from pandas.errors import EmptyDataError -from nzgmdb.data_processing import filtering -from nzgmdb.data_processing import multi_event +from nzgmdb.data_processing import filtering, multi_event from nzgmdb.data_retrieval import inventory_xml from nzgmdb.management import config as cfg from nzgmdb.management import custom_errors, file_structure diff --git a/nzgmdb/mseed_management/reading.py b/nzgmdb/mseed_management/reading.py index 786b1d34..91c16992 100644 --- a/nzgmdb/mseed_management/reading.py +++ b/nzgmdb/mseed_management/reading.py @@ -6,7 +6,6 @@ import mseedlib import numpy as np -from obspy import Inventory from obspy.core import Stream, Trace, UTCDateTime from obspy.core.inventory import Inventory diff --git a/tests/test_sites.py b/tests/test_sites.py index e449bc58..cd03d5aa 100644 --- a/tests/test_sites.py +++ b/tests/test_sites.py @@ -241,7 +241,9 @@ def _find_domain_from_shapes(df: pd.DataFrame, _shapes: list) -> pd.DataFrame: sites.tect_domain, "find_domain_from_shapes", _find_domain_from_shapes ) - def _compute_station_thresholds(stations: pd.DataFrame) -> pd.DataFrame: + def _compute_station_thresholds( + stations: pd.DataFrame, model_version: str = None + ) -> pd.DataFrame: n = len(stations) return pd.DataFrame( { @@ -257,8 +259,9 @@ def _compute_station_thresholds(stations: pd.DataFrame) -> pd.DataFrame: ) def _sample_points_from_geotiff( - geotiff_path: str, + file_path: str, latlon_points: np.ndarray, + band: int = 1, ) -> np.ndarray: n = len(latlon_points) vals = np.linspace(100.0, 500.0, n).astype(float) @@ -271,7 +274,10 @@ def _sample_points_from_geotiff( ) def _fill_gaps_with_nearest( + coords: np.ndarray, values: np.ndarray, + invalid_mask: np.ndarray = None, + k: int = 8, ) -> np.ndarray: values = np.asarray(values, dtype=float).copy() values[np.isnan(values)] = 250.0 From 70f4bb75eab20e7ede295f96e976fa851cd6078a Mon Sep 17 00:00:00 2001 From: joelridden Date: Tue, 20 Jan 2026 15:45:52 +1300 Subject: [PATCH 12/72] add tmp array code --- nzgmdb/config/config.yaml | 6 + nzgmdb/data_retrieval/inventory_xml.py | 10 + nzgmdb/scripts/run_nzgmdb.py | 21 ++ nzgmdb/temp_arrays/backup_to_dropbox.py | 225 ++++++++++++++++ nzgmdb/temp_arrays/check_data.py | 147 ++++++++++ nzgmdb/temp_arrays/get_stations.py | 166 ++++++++++++ nzgmdb/temp_arrays/mass_download_data.py | 329 +++++++++++++++++++++++ 7 files changed, 904 insertions(+) create mode 100755 nzgmdb/temp_arrays/backup_to_dropbox.py create mode 100644 nzgmdb/temp_arrays/check_data.py create mode 100644 nzgmdb/temp_arrays/get_stations.py create mode 100644 nzgmdb/temp_arrays/mass_download_data.py diff --git a/nzgmdb/config/config.yaml b/nzgmdb/config/config.yaml index b66b1e90..9795bff5 100644 --- a/nzgmdb/config/config.yaml +++ b/nzgmdb/config/config.yaml @@ -27,6 +27,12 @@ priority_phase_list: channel_codes: "HN?,BN?,HH?,BH?" percentage_gap_allowed: 0.1 is_large_overlap: 0.5 +# Provider / Network Filters +main_providers_networks: + - GEONET: + - NZ +tmp_array_providers_networks: + - IRIS # Mseed Variables vs30: 500 pre_event_time_difference: 15 diff --git a/nzgmdb/data_retrieval/inventory_xml.py b/nzgmdb/data_retrieval/inventory_xml.py index 824786cd..5da0c1c5 100644 --- a/nzgmdb/data_retrieval/inventory_xml.py +++ b/nzgmdb/data_retrieval/inventory_xml.py @@ -11,6 +11,16 @@ from nzgmdb.management import file_structure +def fetch_inventory( + add_tmp_arrays: bool = False, + level: str = "response", + channel_codes: str | None = None, + starttime: str = "2000-01-01", + endtime: str = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d"), +): + pass + + def fetch_and_save_inventory( main_dir: Path, stations: list[str], diff --git a/nzgmdb/scripts/run_nzgmdb.py b/nzgmdb/scripts/run_nzgmdb.py index 8a5d2c46..132f7878 100644 --- a/nzgmdb/scripts/run_nzgmdb.py +++ b/nzgmdb/scripts/run_nzgmdb.py @@ -837,6 +837,17 @@ def run_full_nzgmdb( dir_okay=False, ), ] = None, + add_tmp_arrays: Annotated[ + bool, + typer.Option(), + ] = False, + tmp_array_data_dir: Annotated[ + Path, + typer.Option( + exists=True, + file_okay=False, + ), + ] = None, machine: Annotated[ cfg.MachineName, typer.Option( @@ -907,12 +918,22 @@ def run_full_nzgmdb( If True, the function will create a quality database (default is False). bypass_records_ffp : Path, optional The full file path to the bypass records file, if applicable. + add_tmp_arrays : bool, optional + If True, temporary arrays will be added to the database run (default is False). + tmp_array_data_dir : Path, optional + The directory containing temporary array data, required if add_tmp_arrays is True. machine : cfg.MachineName, optional The machine name to use for process configuration (default is None). """ main_dir.mkdir(parents=True, exist_ok=True) config = cfg.Config() + # Check that if add_tmp_arrays is True, tmp_array_data_dir is provided + if add_tmp_arrays and tmp_array_data_dir is None: + raise ValueError( + "tmp_array_data_dir must be provided if add_tmp_arrays is True." + ) + # Generate the site basin flatfile flatfile_dir = file_structure.get_flatfile_dir(main_dir) flatfile_dir.mkdir(parents=True, exist_ok=True) diff --git a/nzgmdb/temp_arrays/backup_to_dropbox.py b/nzgmdb/temp_arrays/backup_to_dropbox.py new file mode 100755 index 00000000..63b487eb --- /dev/null +++ b/nzgmdb/temp_arrays/backup_to_dropbox.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 + +import csv +import subprocess +import sys +from pathlib import Path +from typing import Dict + +import typer + +app = typer.Typer(pretty_exceptions_enable=False) + +DROPBOX_PATH = "dropbox:/QuakeCoRE/Public/NZGMDB/tmp_array" +MANIFEST_HEADER = [ + "type", + "net", + "name", + "local_path", + "zip_name", + "status", + "bytes", +] + + +# ---------------------------- +# RCLONE / ZIP HELPERS +# ---------------------------- + +def zip_directory(src_dir: Path, out_dir: Path) -> Path: + """ + Zip an entire directory using system zip (fast + reliable). + """ + out_dir.mkdir(parents=True, exist_ok=True) + zip_path = out_dir / f"{src_dir.name}.zip" + + if zip_path.exists(): + return zip_path + + subprocess.check_call( + ["zip", "-r", "-1", str(zip_path), src_dir.name], + cwd=src_dir.parent, + ) + + return zip_path + + +def upload_and_verify(local_file: Path, dropbox_dir: str) -> bool: + """ + Upload using rclone and verify by file size. + """ + subprocess.check_call( + ["rclone", "copy", str(local_file), dropbox_dir], + ) + + local_size = local_file.stat().st_size + + out = subprocess.check_output( + [ + "rclone", + "lsf", + "--format=s", + f"{dropbox_dir}/{local_file.name}", + ] + ).decode().strip() + + return bool(out) and int(out) == local_size + + +# ---------------------------- +# MANIFEST LOGIC +# ---------------------------- + +def load_manifest(path: Path) -> Dict[str, dict]: + """ + Load manifest into dict keyed by zip_name. + """ + rows = {} + + if not path.exists(): + return rows + + with path.open() as f: + reader = csv.DictReader(f) + for row in reader: + rows[row["zip_name"]] = row + + return rows + + +def append_manifest_row(path: Path, row: dict): + new_file = not path.exists() + + with path.open("a", newline="") as f: + writer = csv.DictWriter(f, fieldnames=MANIFEST_HEADER) + if new_file: + writer.writeheader() + writer.writerow(row) + + +def update_manifest_status(path: Path, zip_name: str, status: str, size: int): + rows = load_manifest(path) + rows[zip_name]["status"] = status + rows[zip_name]["bytes"] = size + + with path.open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=MANIFEST_HEADER) + writer.writeheader() + writer.writerows(rows.values()) + + +# ---------------------------- +# DISCOVERY +# ---------------------------- + +def discover_stationxml(stationxml_dir: Path): + yield { + "type": "stationxml", + "net": "", + "name": stationxml_dir.name, + "local_path": str(stationxml_dir), + "zip_name": f"{stationxml_dir.name}.zip", + "status": "PENDING", + "bytes": 0, + } + + +def discover_waveforms(waveforms_root: Path): + for net_dir in sorted(waveforms_root.iterdir()): + if not net_dir.is_dir(): + continue + + for leaf in sorted(net_dir.iterdir()): + if not leaf.is_dir(): + continue + + yield { + "type": "waveforms", + "net": net_dir.name, + "name": leaf.name, + "local_path": str(leaf), + "zip_name": f"{leaf.name}.zip", + "status": "PENDING", + "bytes": 0, + } + + +# ---------------------------- +# MAIN PIPELINE +# ---------------------------- + +def process_entry( + entry: dict, + tmp_zip_dir: Path, + manifest_path: Path, +): + src = Path(entry["local_path"]) + zip_path = zip_directory(src, tmp_zip_dir) + + if entry["type"] == "stationxml": + dropbox_target = f"{DROPBOX_PATH}/stationxml" + else: + dropbox_target = f"{DROPBOX_PATH}/waveforms/{entry['net']}" + + try: + ok = upload_and_verify(zip_path, dropbox_target) + except Exception as e: + print(f"ERROR uploading {zip_path.name}: {e}") + update_manifest_status(manifest_path, entry["zip_name"], "FAILED", 0) + return + + if ok: + size = zip_path.stat().st_size + update_manifest_status(manifest_path, entry["zip_name"], "DONE", size) + zip_path.unlink() + print(f"DONE {zip_path.name}") + else: + update_manifest_status(manifest_path, entry["zip_name"], "FAILED", 0) + print(f"FAILED {zip_path.name}") + + +# ---------------------------- +# CLI +# ---------------------------- + +@app.command() +def run( + data_root: Path = typer.Argument(..., help="Root directory containing waveforms/ and stationxml/"), +): + """ + Resume-safe Dropbox backup with manifest tracking. + """ + manifest = data_root / "dropbox_manifest.csv" + manifest_rows = load_manifest(manifest) + + # Discover + register new entries + stationxml = data_root / "stationxml" + waveforms = data_root / "waveforms" + + tmp_zip_dir = data_root / "tmp_zips" + tmp_zip_dir.mkdir(parents=True, exist_ok=True) + + for entry in discover_stationxml(stationxml): + if entry["zip_name"] not in manifest_rows: + append_manifest_row(manifest, entry) + + for entry in discover_waveforms(waveforms): + if entry["zip_name"] not in manifest_rows: + append_manifest_row(manifest, entry) + + # Reload after discovery + manifest_rows = load_manifest(manifest) + + pending = [ + row for row in manifest_rows.values() + if row["status"] != "DONE" + ] + + print(f"Pending uploads: {len(pending)}") + + for entry in pending: + process_entry(entry, tmp_zip_dir, manifest) + + +if __name__ == "__main__": + app() diff --git a/nzgmdb/temp_arrays/check_data.py b/nzgmdb/temp_arrays/check_data.py new file mode 100644 index 00000000..29170772 --- /dev/null +++ b/nzgmdb/temp_arrays/check_data.py @@ -0,0 +1,147 @@ +import pandas as pd +import os +from obspy import UTCDateTime + + +OUTPUT_DIR = "/scratch/jobs/jri83/runs/tmp_array/mass_data_row_mp" +MSEED_DIR = "waveforms" +STATIONXML_DIR = "stationxml" + +# month length in seconds (30 days) +MONTH_SECONDS = 30 * 24 * 3600 + + +def _format_end_for_filename(end_dt): + """Format an obspy UTCDateTime or parseable date string to the filename timestamp form.""" + if not isinstance(end_dt, UTCDateTime): + end_dt = UTCDateTime(end_dt) + return end_dt.strftime("%Y%m%dT%H%M%SZ") + + +def is_row_done(row): + """ + Check the mseed output directory for this row to see if a file exists + whose final `__` timestamp equals the row end_date. + Returns True if done, False otherwise. + """ + net = str(row["net"]).strip() + sta = str(row["sta"]).strip() + loc_field = str(row["loc"]) + # loc_field = "" if loc_field == "NA" else loc_field + chan_prefix = str(row["chan"]).strip() + + record_sub = f"{net}_{sta}_{chan_prefix}_{loc_field}" + mseed_path = os.path.join(OUTPUT_DIR, MSEED_DIR, net, record_sub) + + if not os.path.isdir(mseed_path): + return False + + target_end = _format_end_for_filename(row["end_date"]) + + try: + for fname in os.listdir(mseed_path): + if not fname.endswith(".mseed"): + continue + stem = os.path.splitext(fname)[0] + + # Check the CHAN field that it ends in Z + chan_check = stem.split(".")[3] + chan_check = chan_check.split("__")[0] # remove any suffix after __ + if not chan_check.endswith("Z"): + continue + + # filename parts expected like: NET.STA..CHAN__START__END + # take last segment after the final '__' + if "__" in stem: + last = stem.rsplit("__", 1)[-1] + if last == target_end: + return True + except Exception: + # any filesystem error -> treat as not done so row will be retried + return False + + return False + + +def is_row_started(row): + """ + Check whether *any* mseed file exists for this row. + Returns True if at least one .mseed file is present. + """ + net = str(row["net"]).strip() + sta = str(row["sta"]).strip() + loc_field = str(row["loc"]) + chan_prefix = str(row["chan"]).strip() + + record_sub = f"{net}_{sta}_{chan_prefix}_{loc_field}" + mseed_path = os.path.join(OUTPUT_DIR, MSEED_DIR, net, record_sub) + + if not os.path.isdir(mseed_path): + return False + + try: + for fname in os.listdir(mseed_path): + if fname.endswith(".mseed"): + return True + except Exception: + return False + + return False + + +def evaluate_download_completeness(csv_file, output_csv): + """ + Evaluate download state with two levels: + - completed: full end_date reached + - started: at least one file exists + """ + df = pd.read_csv(csv_file, dtype={"loc": str}, keep_default_na=False) + df = df[df["provider"] == "IRIS"].reset_index(drop=True) + + print(f"Evaluating {len(df)} rows for completeness...") + + # Two independent checks + df["completed"] = df.apply(is_row_done, axis=1) + df["started"] = df.apply(is_row_started, axis=1) + + total = len(df) + + completed = int(df["completed"].sum()) + started = int(df["started"].sum()) + + partial = int(((df["started"]) & (~df["completed"])).sum()) + none = int((~df["started"]).sum()) + + print("\n===== DOWNLOAD SUMMARY =====") + print(f"Total rows : {total}") + print(f"Fully completed : {completed}") + print(f"Started (any data) : {started}") + print(f"Partial (started only): {partial}") + print(f"No data at all : {none}") + + # Rows with no data whatsoever + if none > 0: + print("\n===== ROWS WITH NO DATA =====") + cols = ["provider", "net", "sta", "loc", "chan", "start_date", "end_date"] + print(df.loc[~df["started"], cols].to_string(index=False)) + + # Rows with partial data (useful for retries) + if partial > 0: + print("\n===== PARTIALLY DOWNLOADED ROWS =====") + cols = ["provider", "net", "sta", "loc", "chan", "start_date", "end_date"] + print(df.loc[df["started"] & ~df["completed"], cols].to_string(index=False)) + + df.to_csv(output_csv, index=False) + print(f"\nWrote evaluation CSV to:\n {output_csv}") + + return df + + +CSV_FILE = "/scratch/jobs/jri83/runs/tmp_array/all_nz_sta_providers_desired_channels_mustang.csv" +OUTPUT_CSV = "/scratch/jobs/jri83/runs/tmp_array/download_completeness_evaluation.csv" + +if __name__ == "__main__": + evaluate_download_completeness( + csv_file=CSV_FILE, + output_csv=OUTPUT_CSV, + ) diff --git a/nzgmdb/temp_arrays/get_stations.py b/nzgmdb/temp_arrays/get_stations.py new file mode 100644 index 00000000..99ad22ec --- /dev/null +++ b/nzgmdb/temp_arrays/get_stations.py @@ -0,0 +1,166 @@ +from obspy.clients.fdsn import Client as FDSN_Client +from obspy import UTCDateTime + +URL_MAPPINGS = { + "AUSPASS": "http://auspass.edu.au", + "BGR": "http://eida.bgr.de", + "EIDA": "http://eida-federator.ethz.ch", + "ETH": "http://eida.ethz.ch", + "EMSC": "http://www.seismicportal.eu", + "GEONET": "http://service.geonet.org.nz", + "GEOFON": "http://geofon.gfz-potsdam.de", + "GFZ": "http://geofon.gfz-potsdam.de", + "ICGC": "http://ws.icgc.cat", + "IESDMC": "http://batsws.earth.sinica.edu.tw", + "INGV": "http://webservices.ingv.it", + "IPGP": "http://ws.ipgp.fr", + "IRIS": "http://service.iris.edu", + "IRISPH5": "http://service.iris.edu", + "ISC": "http://www.isc.ac.uk", + "KNMI": "http://rdsa.knmi.nl", + "KOERI": "http://eida.koeri.boun.edu.tr", + "LMU": "https://erde.geophysik.uni-muenchen.de", + "NCEDC": "https://service.ncedc.org", + "NIEP": "http://eida-sc3.infp.ro", + "NOA": "http://eida.gein.noa.gr", + "ODC": "http://www.orfeus-eu.org", + "ORFEUS": "http://www.orfeus-eu.org", + "RESIF": "http://ws.resif.fr", + "RESIFPH5": "http://ph5ws.resif.fr", + "RASPISHAKE": "https://data.raspberryshake.org", + "SCEDC": "http://service.scedc.caltech.edu", + "TEXNET": "http://rtserve.beg.utexas.edu", + "UIB-NORSAR": "http://eida.geo.uib.no", + "USGS": "http://earthquake.usgs.gov", + "USP": "http://sismo.iag.usp.br", +} + +# Define rough NZ bounding box (adjust as needed) +min_lat, max_lat = -49, -32.0 +min_lon, max_lon = 165.0, -176.9 + +# Time window for station metadata +starttime = UTCDateTime("2000-01-01") +endtime = UTCDateTime() # now + +import pandas as pd + +all_station_info = [] + +import nzgeom.coastlines +from shapely.geometry import Point + +# Load NZ coastline polygons once (efficient) +_NZ_COAST = nzgeom.coastlines.get_NZ_coastlines().to_crs("EPSG:4326") + + +def is_point_inside_nz(lat, lon): + """ + Returns True if the given latitude/longitude lies inside + the NZ mainland or island coastline polygons. + """ + # shapely uses (lon, lat) + p = Point(lon, lat) + # test containment against all polygons + return _NZ_COAST.geometry.apply(lambda g: g.contains(p)).any() + + +for provider, _ in URL_MAPPINGS.items(): + try: + client = FDSN_Client(base_url=provider) + networks = client.get_stations( + starttime=starttime, + endtime=endtime, + minlatitude=min_lat, + maxlatitude=max_lat, + minlongitude=min_lon, + maxlongitude=max_lon, + level="network", + ) + network_codes = [net.code for net in networks] + + print("Processing provider:", provider, "with networks:", len(network_codes)) + + for net_code in network_codes: + try: + inv = client.get_stations( + network=net_code, + level="channel", + minlatitude=min_lat, + maxlatitude=max_lat, + minlongitude=min_lon, + maxlongitude=max_lon, + ) + + for network in inv: + print( + " Network:", + network.code, + "with stations:", + len(network.stations), + ) + + for station in network: + lat = station.latitude + lon = station.longitude + if not is_point_inside_nz(lat, lon): + continue + current_channels = set() + for channel in station: + chan_id = (channel.location_code, channel.code[:2]) + if chan_id in current_channels: + continue + current_channels.add(chan_id) + all_station_info.append( + [ + provider, # provider as first column + network.code, + station.code, + lat, + lon, + station.elevation, + channel.code[:2], + channel.location_code, + channel.start_date, + channel.end_date, + ] + ) + except Exception: + # continue to next network code on failure + continue + except Exception: + # continue to next provider on failure + continue + +# build dataframe (provider is first column) +station_df = pd.DataFrame( + all_station_info, + columns=[ + "provider", + "net", + "sta", + "lat", + "lon", + "elev", + "chan", + "loc", + "start_date", + "end_date", + ], +) +station_df = station_df.drop_duplicates( + ["provider", "net", "sta", "chan", "loc"] +).reset_index(drop=True) + +# write outputs +station_df.to_csv( + "/media/joel/data/nzgmdb/tmp_arrays/nz_mainland_stations_all_provider_networks_channels.csv", + index=False, +) + +desired_channels = ["HH", "BH", "HN", "BN"] +filtered_df = station_df[station_df["chan"].isin(desired_channels)] +filtered_df.to_csv( + "/media/joel/data/nzgmdb/tmp_arrays/nz_mainland_stations_all_provider_networks_desired_channels.csv", + index=False, +) diff --git a/nzgmdb/temp_arrays/mass_download_data.py b/nzgmdb/temp_arrays/mass_download_data.py new file mode 100644 index 00000000..7fef4492 --- /dev/null +++ b/nzgmdb/temp_arrays/mass_download_data.py @@ -0,0 +1,329 @@ +import os +import pandas as pd +from obspy.clients.fdsn.mass_downloader import ( + MassDownloader, + Restrictions, + GlobalDomain, +) +from obspy import UTCDateTime +import multiprocessing + + +# ---------------- USER SETTINGS ---------------- # +# CSV_FILE = '/media/joel/data/nzgmdb/tmp_arrays/all_nz_sta_providers_desired_channels_mustang.csv' +CSV_FILE = "/media/joel/data/nzgmdb/tmp_arrays/HR1_inventory.csv" +# CSV_FILE = '/scratch/jobs/jri83/runs/tmp_array/download_completeness_evaluation_3.csv' + +OUTPUT_DIR = "/media/joel/data/nzgmdb/tmp_arrays/hr1" +# OUTPUT_DIR = '/scratch/jobs/jri83/runs/tmp_array/mass_data_row_mp' +MSEED_DIR = "waveforms" +STATIONXML_DIR = "stationxml" + +RESULTS_CSV = os.path.join(OUTPUT_DIR, "download_results.csv") + +# month length in seconds (15 days) +MONTH_SECONDS = 15 * 24 * 3600 +# Minimum chunk size (seconds) when backing off after 413 / manifest-too-large. +# 3600s = 1 hour. +MIN_CHUNK_SECONDS: int = 3600 + +# ------------------------------------------------ # + + +def _format_end_for_filename(end_dt): + """Format an obspy UTCDateTime or parseable date string to the filename timestamp form.""" + if not isinstance(end_dt, UTCDateTime): + end_dt = UTCDateTime(end_dt) + return end_dt.strftime("%Y%m%dT%H%M%SZ") + + +def save_results(results, results_csv=RESULTS_CSV): + """ + Create one CSV from a list of result dicts. + - Finds all unique keys across results. + - Uses a preferred column order for common fields. + - Normalizes missing keys to empty string and converts non-scalar values to strings. + """ + import os + import json + import pandas as pd + + if not results: + # ensure output dir exists and write an empty file + out_dir = os.path.dirname(results_csv) + if out_dir: + os.makedirs(out_dir, exist_ok=True) + pd.DataFrame().to_csv(results_csv, index=False) + return + + # collect all keys + all_keys = set() + for r in results: + if isinstance(r, dict): + all_keys.update(r.keys()) + else: + # non-dict entries will be recorded under 'value' + all_keys.add("value") + + # preferred column order for readability + preferred = [ + "idx", + "provider", + "net", + "sta", + "status", + "error", + "attempts", + "chunklength", + "timestamp", + "mseed_path", + "xml_path", + ] + cols = [k for k in preferred if k in all_keys] + sorted( + k for k in all_keys if k not in preferred + ) + + # normalize rows + norm_rows = [] + for r in results: + if not isinstance(r, dict): + row = {"value": str(r)} + else: + row = {} + for k in cols: + v = r.get(k, "") + # convert lists/dicts/other non-primitives to JSON or string + if isinstance(v, (dict, list)): + try: + v = json.dumps(v, ensure_ascii=False) + except Exception: + v = str(v) + elif v is None: + v = "" + else: + # keep numbers/strings as-is; cover other types + if not isinstance(v, (str, int, float, bool)): + v = str(v) + row[k] = v + norm_rows.append(row) + + # ensure output directory exists + out_dir = os.path.dirname(results_csv) + if out_dir: + os.makedirs(out_dir, exist_ok=True) + + df = pd.DataFrame(norm_rows, columns=cols) + df.to_csv(results_csv, index=False, encoding="utf-8") + + +def is_row_done(row): + """ + Check the mseed output directory for this row to see if a file exists + whose final `__` timestamp equals the row end_date. + Returns True if done, False otherwise. + """ + net = str(row["net"]).strip() + sta = str(row["sta"]).strip() + loc_field = str(row["loc"]) + # loc_field = "" if loc_field == "NA" else loc_field + chan_prefix = str(row["chan"]).strip() + + record_sub = f"{net}_{sta}_{chan_prefix}_{loc_field}" + mseed_path = os.path.join(OUTPUT_DIR, MSEED_DIR, net, record_sub) + + if not os.path.isdir(mseed_path): + return False + + target_end = _format_end_for_filename(row["end_date"]) + + try: + for fname in os.listdir(mseed_path): + if not fname.endswith(".mseed"): + continue + stem = os.path.splitext(fname)[0] + + # Check the CHAN field that it ends in Z + chan_check = stem.split(".")[3] + chan_check = chan_check.split("__")[0] # remove any suffix after __ + if not chan_check.endswith("Z"): + continue + + # filename parts expected like: NET.STA..CHAN__START__END + # take last segment after the final '__' + if "__" in stem: + last = stem.rsplit("__", 1)[-1] + if last == target_end: + return True + except Exception: + # any filesystem error -> treat as not done so row will be retried + return False + + return False + + +def create_output_dirs(net, sta, chan_prefix, loc): + """ + Create directories for the network if needed. + """ + record_sub = f"{net}_{sta}_{chan_prefix}_{loc}" + + mseed_path = os.path.join(OUTPUT_DIR, MSEED_DIR, net, record_sub) + xml_path = os.path.join(OUTPUT_DIR, STATIONXML_DIR, net, record_sub) + + os.makedirs(mseed_path, exist_ok=True) + os.makedirs(xml_path, exist_ok=True) + + return mseed_path, xml_path + + +def worker(task): + """ + Worker that performs a single row download. + task: (idx, provider, row_dict) + """ + idx, provider, row = task + try: + net = row["net"] + sta = row["sta"] + loc_field = str(row["loc"]) + loc = "*" if loc_field == "NA" else loc_field.strip() + chan_prefix = str(row["chan"]).strip() + channel = f"{chan_prefix}?" # add ? automatically + + start = UTCDateTime(row["start_date"]) + end = UTCDateTime(row["end_date"]) + + # Chunk length: month (30 days) but not longer than the full requested window + total_window = end - start + chunk_base = int(min(total_window, MONTH_SECONDS)) + + max_attempts = 4 + attempt = 1 + + print( + f"[{idx}] Provider={provider} Downloading {net}.{sta} {channel} {start} -> {end} chunk={chunk_base}s" + ) + + # create output dirs using the raw loc field for naming (keeps 'NA' if present) + mseed_path, xml_path = create_output_dirs(net, sta, chan_prefix, loc_field) + + while attempt <= max_attempts: + chunklength = int(min(total_window, chunk_base)) + if chunklength < 1: + chunklength = 1 + + domain = GlobalDomain() + restrictions = Restrictions( + starttime=start, + endtime=end, + chunklength_in_sec=chunklength, + network=net, + station=sta, + location=loc, + channel=channel, + reject_channels_with_gaps=False, + minimum_length=0.0, + minimum_interstation_distance_in_m=0.0, + ) + + try: + mdl = MassDownloader(providers=[provider]) + mdl.download( + domain, + restrictions, + mseed_storage=mseed_path, + stationxml_storage=xml_path, + ) + print(f"[{idx}] Done") + return { + "idx": idx, + "status": "ok", + "provider": provider, + "net": net, + "sta": sta, + } + except Exception as e: + err_text = repr(e) + " " + str(e) + # Detect 413 / manifest-too-large responses from server text + is_manifest_too_large = ( + "Estimated manifest size" in err_text + or "Request Entity Too Large" in err_text + or "413" in err_text + ) + + if is_manifest_too_large: + # halve the base chunk and retry, unless already at minimum + if chunk_base <= MIN_CHUNK_SECONDS: + print( + f"[{idx}] Server denied request and chunk is already at minimum ({chunk_base}s). Giving up." + ) + raise + old = chunk_base + chunk_base = max(MIN_CHUNK_SECONDS, chunk_base // 2) + print( + f"[{idx}] Server denied request (413). Reducing chunk base {old}s -> {chunk_base}s and retrying." + ) + attempt += 1 + continue + raise e + + except Exception as e: + print( + f"[{idx}] ERROR provider={provider} net={row.get('net')} sta={row.get('sta')}: {e}" + ) + return {"idx": idx, "status": "error", "error": str(e), "provider": provider} + + +def main(): + # Use explicit start method to avoid forking issues in some environments + try: + multiprocessing.set_start_method("spawn") + except RuntimeError: + # start method already set + pass + + df = pd.read_csv(CSV_FILE, dtype={"loc": str}, keep_default_na=False) + + # Filter down for ones that are False in completed column + # df = df[df["completed"] == False] + + # Filter down to ones thar are True in started column + # df = df[df["started"] == False] + + # Filter net to Y3 net (kept from original script) + # df = df[df["provider"] == "IRIS"] + + required_cols = {"net", "sta", "loc", "chan", "start_date", "end_date", "provider"} + + if not required_cols.issubset(df.columns): + raise ValueError(f"CSV must contain: {required_cols}") + + # Build tasks for all rows (one task per CSV row) + tasks = [] + skipped = 0 + for idx, row in df.reset_index(drop=True).iterrows(): + if is_row_done(row): + skipped += 1 + continue + tasks.append((int(idx), row["provider"], row.to_dict())) + + processes = 1 + print( + f"Starting multiprocessing pool with {processes} processes for {len(tasks)} tasks (skipped {skipped} already done)" + ) + + with multiprocessing.Pool(processes=processes) as pool: + results = pool.map(worker, tasks) + + save_results(results, results_csv=RESULTS_CSV) + + # Simple summary + oks = sum(1 for r in results if r.get("status") == "ok") + errs = sum(1 for r in results if r.get("status") == "error") + print( + f"Completed: {oks} succeeded, {errs} failed (skipped {skipped} previously done)" + ) + + +if __name__ == "__main__": + main() From 03f382af558c1bb229af7a5266276c6476a9205b Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 21 Jan 2026 09:47:12 +1300 Subject: [PATCH 13/72] fix upload script --- nzgmdb/temp_arrays/backup_to_dropbox.py | 49 ++++++++++++--------- nzgmdb/temp_arrays/mass_download_data.py | 55 ++++++++++++++---------- 2 files changed, 62 insertions(+), 42 deletions(-) diff --git a/nzgmdb/temp_arrays/backup_to_dropbox.py b/nzgmdb/temp_arrays/backup_to_dropbox.py index 63b487eb..52ecff11 100755 --- a/nzgmdb/temp_arrays/backup_to_dropbox.py +++ b/nzgmdb/temp_arrays/backup_to_dropbox.py @@ -10,7 +10,6 @@ app = typer.Typer(pretty_exceptions_enable=False) -DROPBOX_PATH = "dropbox:/QuakeCoRE/Public/NZGMDB/tmp_array" MANIFEST_HEADER = [ "type", "net", @@ -26,6 +25,7 @@ # RCLONE / ZIP HELPERS # ---------------------------- + def zip_directory(src_dir: Path, out_dir: Path) -> Path: """ Zip an entire directory using system zip (fast + reliable). @@ -54,14 +54,18 @@ def upload_and_verify(local_file: Path, dropbox_dir: str) -> bool: local_size = local_file.stat().st_size - out = subprocess.check_output( - [ - "rclone", - "lsf", - "--format=s", - f"{dropbox_dir}/{local_file.name}", - ] - ).decode().strip() + out = ( + subprocess.check_output( + [ + "rclone", + "lsf", + "--format=s", + f"{dropbox_dir}/{local_file.name}", + ] + ) + .decode() + .strip() + ) return bool(out) and int(out) == local_size @@ -70,6 +74,7 @@ def upload_and_verify(local_file: Path, dropbox_dir: str) -> bool: # MANIFEST LOGIC # ---------------------------- + def load_manifest(path: Path) -> Dict[str, dict]: """ Load manifest into dict keyed by zip_name. @@ -112,6 +117,7 @@ def update_manifest_status(path: Path, zip_name: str, status: str, size: int): # DISCOVERY # ---------------------------- + def discover_stationxml(stationxml_dir: Path): yield { "type": "stationxml", @@ -148,18 +154,17 @@ def discover_waveforms(waveforms_root: Path): # MAIN PIPELINE # ---------------------------- + def process_entry( - entry: dict, - tmp_zip_dir: Path, - manifest_path: Path, + entry: dict, tmp_zip_dir: Path, manifest_path: Path, dropbox_path: str ): src = Path(entry["local_path"]) zip_path = zip_directory(src, tmp_zip_dir) if entry["type"] == "stationxml": - dropbox_target = f"{DROPBOX_PATH}/stationxml" + dropbox_target = f"{dropbox_path}/stationxml" else: - dropbox_target = f"{DROPBOX_PATH}/waveforms/{entry['net']}" + dropbox_target = f"{dropbox_path}/waveforms/{entry['net']}" try: ok = upload_and_verify(zip_path, dropbox_target) @@ -182,9 +187,16 @@ def process_entry( # CLI # ---------------------------- + @app.command() def run( - data_root: Path = typer.Argument(..., help="Root directory containing waveforms/ and stationxml/"), + data_root: Path = typer.Argument( + ..., help="Root directory containing waveforms/ and stationxml/" + ), + dropbox_path: str = typer.Argument( + ..., + help="Rclone Dropbox path to upload to.", + ), ): """ Resume-safe Dropbox backup with manifest tracking. @@ -210,15 +222,12 @@ def run( # Reload after discovery manifest_rows = load_manifest(manifest) - pending = [ - row for row in manifest_rows.values() - if row["status"] != "DONE" - ] + pending = [row for row in manifest_rows.values() if row["status"] != "DONE"] print(f"Pending uploads: {len(pending)}") for entry in pending: - process_entry(entry, tmp_zip_dir, manifest) + process_entry(entry, tmp_zip_dir, manifest, dropbox_path) if __name__ == "__main__": diff --git a/nzgmdb/temp_arrays/mass_download_data.py b/nzgmdb/temp_arrays/mass_download_data.py index 7fef4492..574764f9 100644 --- a/nzgmdb/temp_arrays/mass_download_data.py +++ b/nzgmdb/temp_arrays/mass_download_data.py @@ -176,10 +176,36 @@ def create_output_dirs(net, sta, chan_prefix, loc): return mseed_path, xml_path -def worker(task): +def download_task( + task: tuple[int, str, dict[str, object]] +) -> None | dict[str, int | str | object] | dict[str, int | str]: """ - Worker that performs a single row download. - task: (idx, provider, row_dict) + Download waveform and StationXML data for a single CSV row task. + + Parameters + ---------- + task + A 3-tuple of `(idx, provider, row_dict)` where: + - `idx` is the zero-based row index in the input CSV. + - `provider` is the FDSN provider name. + - `row_dict` contains the CSV row fields such as `net`, `sta`, `loc`, + `chan`, `start_date`, and `end_date`. + + Returns + ------- + dict[str, object] + A result dictionary containing at least: + - `idx`: int + - `status`: `ok` or `error` + - `provider`: str + And, on success, `net` and `sta`. On error, includes `error`. + + Raises + ------ + Exception + Re-raises exceptions encountered during download attempts (for example + when the server repeatedly rejects the request as too large after + backoff to the minimum chunk size). """ idx, provider, row = task try: @@ -275,24 +301,9 @@ def worker(task): def main(): - # Use explicit start method to avoid forking issues in some environments - try: - multiprocessing.set_start_method("spawn") - except RuntimeError: - # start method already set - pass df = pd.read_csv(CSV_FILE, dtype={"loc": str}, keep_default_na=False) - # Filter down for ones that are False in completed column - # df = df[df["completed"] == False] - - # Filter down to ones thar are True in started column - # df = df[df["started"] == False] - - # Filter net to Y3 net (kept from original script) - # df = df[df["provider"] == "IRIS"] - required_cols = {"net", "sta", "loc", "chan", "start_date", "end_date", "provider"} if not required_cols.issubset(df.columns): @@ -307,13 +318,13 @@ def main(): continue tasks.append((int(idx), row["provider"], row.to_dict())) - processes = 1 print( - f"Starting multiprocessing pool with {processes} processes for {len(tasks)} tasks (skipped {skipped} already done)" + f"Starting sequential run for {len(tasks)} tasks (skipped {skipped} already done)" ) - with multiprocessing.Pool(processes=processes) as pool: - results = pool.map(worker, tasks) + results = [] + for task in tasks: + results.append(download_task(task)) save_results(results, results_csv=RESULTS_CSV) From d7d787aa407e118fabec539323d31e7cf3f3b6d2 Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 21 Jan 2026 14:55:46 +1300 Subject: [PATCH 14/72] comply scripts --- .../data_processing/waveform_manipulation.py | 2 +- nzgmdb/temp_arrays/backup_to_dropbox.py | 232 +++++++--- nzgmdb/temp_arrays/check_data.py | 296 +++++++++--- nzgmdb/temp_arrays/get_stations.py | 429 ++++++++++++----- nzgmdb/temp_arrays/mass_download_data.py | 435 ++++++++++++------ 5 files changed, 1004 insertions(+), 390 deletions(-) diff --git a/nzgmdb/data_processing/waveform_manipulation.py b/nzgmdb/data_processing/waveform_manipulation.py index 8da4420b..e35276c6 100644 --- a/nzgmdb/data_processing/waveform_manipulation.py +++ b/nzgmdb/data_processing/waveform_manipulation.py @@ -18,7 +18,7 @@ def initial_preprocessing( apply_taper: bool = True, apply_zero_padding: bool = True, inventory: Inventory = None, -): +) -> Stream: """ Basic pre-processing of the waveform data This performs the following: diff --git a/nzgmdb/temp_arrays/backup_to_dropbox.py b/nzgmdb/temp_arrays/backup_to_dropbox.py index 52ecff11..c6864e87 100755 --- a/nzgmdb/temp_arrays/backup_to_dropbox.py +++ b/nzgmdb/temp_arrays/backup_to_dropbox.py @@ -1,10 +1,8 @@ -#!/usr/bin/env python3 - import csv import subprocess -import sys +from collections.abc import Iterator from pathlib import Path -from typing import Dict +from typing import TypedDict import typer @@ -21,14 +19,35 @@ ] -# ---------------------------- -# RCLONE / ZIP HELPERS -# ---------------------------- +class ManifestRow(TypedDict): + """A single row in the Dropbox backup manifest CSV.""" + + type: str + net: str + name: str + local_path: str + zip_name: str + status: str + bytes: int def zip_directory(src_dir: Path, out_dir: Path) -> Path: - """ - Zip an entire directory using system zip (fast + reliable). + """Create a zip file for a directory. + + This uses the system `zip` executable (fast + reliable) and skips work if the + expected zip already exists. + + Parameters + ---------- + src_dir : Path + Source directory to zip. + out_dir : Path + Output directory where the zip file will be written. + + Returns + ------- + Path + Path to the created (or existing) zip file. """ out_dir.mkdir(parents=True, exist_ok=True) zip_path = out_dir / f"{src_dir.name}.zip" @@ -45,8 +64,19 @@ def zip_directory(src_dir: Path, out_dir: Path) -> Path: def upload_and_verify(local_file: Path, dropbox_dir: str) -> bool: - """ - Upload using rclone and verify by file size. + """Upload a file to Dropbox using rclone and verify by file size. + + Parameters + ---------- + local_file : Path + File to upload. + dropbox_dir : str + Rclone remote path (directory) to upload into. + + Returns + ------- + bool + True if the remote file exists and its size matches the local file. """ subprocess.check_call( ["rclone", "copy", str(local_file), dropbox_dir], @@ -70,29 +100,62 @@ def upload_and_verify(local_file: Path, dropbox_dir: str) -> bool: return bool(out) and int(out) == local_size -# ---------------------------- -# MANIFEST LOGIC -# ---------------------------- +def load_manifest(path: Path) -> dict[str, ManifestRow]: + """Load an upload manifest keyed by ``zip_name``. + Parameters + ---------- + path : Path + Path to the manifest CSV. -def load_manifest(path: Path) -> Dict[str, dict]: - """ - Load manifest into dict keyed by zip_name. + Returns + ------- + dict[str, ManifestRow] + Manifest rows keyed by ``zip_name``. Returns an empty dict if the file + does not exist. """ - rows = {} + rows: dict[str, ManifestRow] = {} if not path.exists(): return rows - with path.open() as f: - reader = csv.DictReader(f) + with path.open(newline="") as f: + reader: csv.DictReader[str] = csv.DictReader(f) for row in reader: - rows[row["zip_name"]] = row + # DictReader gives us strings; coerce the known int column. + bytes_value = int(row.get("bytes") or 0) + zip_name = row.get("zip_name") + if not zip_name: + # Keep behavior conservative: skip malformed rows. + continue + + rows[zip_name] = ManifestRow( + type=row.get("type", ""), + net=row.get("net", ""), + name=row.get("name", ""), + local_path=row.get("local_path", ""), + zip_name=zip_name, + status=row.get("status", ""), + bytes=bytes_value, + ) return rows -def append_manifest_row(path: Path, row: dict): +def append_manifest_row(path: Path, row: ManifestRow) -> None: + """Append a single row to the manifest CSV, creating it if needed. + + Parameters + ---------- + path : Path + Path to the manifest CSV. + row : ManifestRow + The manifest row to append. + + Returns + ------- + None + """ new_file = not path.exists() with path.open("a", newline="") as f: @@ -102,7 +165,20 @@ def append_manifest_row(path: Path, row: dict): writer.writerow(row) -def update_manifest_status(path: Path, zip_name: str, status: str, size: int): +def update_manifest_status(path: Path, zip_name: str, status: str, size: int) -> None: + """Update an existing manifest row with a new status and size. + + Parameters + ---------- + path : Path + Path to the manifest CSV. + zip_name : str + Zip filename key for the entry to update. + status : str + New status string (e.g. "DONE", "FAILED"). + size : int + Size of the zip file in bytes. + """ rows = load_manifest(path) rows[zip_name]["status"] = status rows[zip_name]["bytes"] = size @@ -113,24 +189,47 @@ def update_manifest_status(path: Path, zip_name: str, status: str, size: int): writer.writerows(rows.values()) -# ---------------------------- -# DISCOVERY -# ---------------------------- +def discover_stationxml(stationxml_dir: Path) -> Iterator[ManifestRow]: + """Discover the StationXML directory entry. + Parameters + ---------- + stationxml_dir : Path + Path to the directory containing StationXML files. + + Returns + ------- + Iterator[ManifestRow] + An iterator yielding a single manifest row representing the StationXML + directory. + """ + yield ManifestRow( + type="stationxml", + net="", + name=stationxml_dir.name, + local_path=str(stationxml_dir), + zip_name=f"{stationxml_dir.name}.zip", + status="PENDING", + bytes=0, + ) -def discover_stationxml(stationxml_dir: Path): - yield { - "type": "stationxml", - "net": "", - "name": stationxml_dir.name, - "local_path": str(stationxml_dir), - "zip_name": f"{stationxml_dir.name}.zip", - "status": "PENDING", - "bytes": 0, - } +def discover_waveforms(waveforms_root: Path) -> Iterator[ManifestRow]: + """Discover waveform leaf directories to back up. -def discover_waveforms(waveforms_root: Path): + The expected directory structure is ``waveforms///`` where each + ``leaf`` directory is zipped independently. + + Parameters + ---------- + waveforms_root : Path + Root directory containing per-network waveform directories. + + Returns + ------- + Iterator[ManifestRow] + Iterator of manifest rows for each leaf directory. + """ for net_dir in sorted(waveforms_root.iterdir()): if not net_dir.is_dir(): continue @@ -139,25 +238,33 @@ def discover_waveforms(waveforms_root: Path): if not leaf.is_dir(): continue - yield { - "type": "waveforms", - "net": net_dir.name, - "name": leaf.name, - "local_path": str(leaf), - "zip_name": f"{leaf.name}.zip", - "status": "PENDING", - "bytes": 0, - } - - -# ---------------------------- -# MAIN PIPELINE -# ---------------------------- + yield ManifestRow( + type="waveforms", + net=net_dir.name, + name=leaf.name, + local_path=str(leaf), + zip_name=f"{leaf.name}.zip", + status="PENDING", + bytes=0, + ) def process_entry( - entry: dict, tmp_zip_dir: Path, manifest_path: Path, dropbox_path: str -): + entry: ManifestRow, tmp_zip_dir: Path, manifest_path: Path, dropbox_path: str +) -> None: + """Zip, upload, verify, and update manifest for a single entry. + + Parameters + ---------- + entry : ManifestRow + Manifest entry describing what to back up. + tmp_zip_dir : Path + Directory where zips are created temporarily. + manifest_path : Path + Path to the manifest CSV. + dropbox_path : str + Base rclone Dropbox remote path. + """ src = Path(entry["local_path"]) zip_path = zip_directory(src, tmp_zip_dir) @@ -168,7 +275,7 @@ def process_entry( try: ok = upload_and_verify(zip_path, dropbox_target) - except Exception as e: + except (OSError, subprocess.CalledProcessError) as e: print(f"ERROR uploading {zip_path.name}: {e}") update_manifest_status(manifest_path, entry["zip_name"], "FAILED", 0) return @@ -183,11 +290,6 @@ def process_entry( print(f"FAILED {zip_path.name}") -# ---------------------------- -# CLI -# ---------------------------- - - @app.command() def run( data_root: Path = typer.Argument( @@ -197,9 +299,15 @@ def run( ..., help="Rclone Dropbox path to upload to.", ), -): - """ - Resume-safe Dropbox backup with manifest tracking. +) -> None: + """Resume-safe Dropbox backup with manifest tracking. + + Parameters + ---------- + data_root : Path + Root directory containing ``waveforms/`` and ``stationxml/``. + dropbox_path : str + Base rclone Dropbox remote path to upload to. """ manifest = data_root / "dropbox_manifest.csv" manifest_rows = load_manifest(manifest) diff --git a/nzgmdb/temp_arrays/check_data.py b/nzgmdb/temp_arrays/check_data.py index 29170772..da58933e 100644 --- a/nzgmdb/temp_arrays/check_data.py +++ b/nzgmdb/temp_arrays/check_data.py @@ -1,111 +1,220 @@ +from pathlib import Path + import pandas as pd -import os +import typer from obspy import UTCDateTime +app = typer.Typer(pretty_exceptions_enable=False) -OUTPUT_DIR = "/scratch/jobs/jri83/runs/tmp_array/mass_data_row_mp" -MSEED_DIR = "waveforms" -STATIONXML_DIR = "stationxml" - -# month length in seconds (30 days) +# Month length in seconds (30 days) MONTH_SECONDS = 30 * 24 * 3600 -def _format_end_for_filename(end_dt): - """Format an obspy UTCDateTime or parseable date string to the filename timestamp form.""" +def _format_end_for_filename(end_dt: UTCDateTime | str) -> str: + """Format an end datetime into the MiniSEED filename timestamp form. + + Parameters + ---------- + end_dt : UTCDateTime | str + An ObsPy ``UTCDateTime`` or any string parseable by ``UTCDateTime``. + + Returns + ------- + str + Timestamp string formatted like ``YYYYMMDDTHHMMSSZ``. + + Raises + ------ + ValueError + If ``end_dt`` cannot be parsed by ``UTCDateTime``. + """ if not isinstance(end_dt, UTCDateTime): end_dt = UTCDateTime(end_dt) return end_dt.strftime("%Y%m%dT%H%M%SZ") -def is_row_done(row): - """ - Check the mseed output directory for this row to see if a file exists - whose final `__` timestamp equals the row end_date. - Returns True if done, False otherwise. +def _row_mseed_dir(output_dir: Path, mseed_dirname: str, row: pd.Series) -> Path: + """Build the expected output MiniSEED directory for a CSV row. + + Parameters + ---------- + output_dir : Path + Root output directory containing the MiniSEED subdirectory. + mseed_dirname : str + Name of the MiniSEED subdirectory (typically ``"waveforms"``). + row : pandas.Series + A row from the input CSV containing at least ``net``, ``sta``, ``loc``, + and ``chan``. + + Returns + ------- + Path + Path like ``///___``. """ net = str(row["net"]).strip() sta = str(row["sta"]).strip() loc_field = str(row["loc"]) - # loc_field = "" if loc_field == "NA" else loc_field chan_prefix = str(row["chan"]).strip() record_sub = f"{net}_{sta}_{chan_prefix}_{loc_field}" - mseed_path = os.path.join(OUTPUT_DIR, MSEED_DIR, net, record_sub) + return output_dir / mseed_dirname / net / record_sub + + +def is_row_done(row: pd.Series, *, output_dir: Path, mseed_dirname: str) -> bool: + """Check whether a CSV row has fully completed waveform download. + + A row is considered **done** if the expected MiniSEED output directory + contains at least one ``.mseed`` file whose final ``__END`` timestamp equals + the row's ``end_date``. + + Notes + ----- + This check is filesystem-driven and intentionally conservative: - if not os.path.isdir(mseed_path): + - Only files whose channel code ends with ``"Z"`` are considered. + - Any filesystem error or unexpected filename format causes this function + to return ``False`` (so the row can be retried). + + Parameters + ---------- + row : pandas.Series + A row of the input CSV. + output_dir : Path + Root output directory. + mseed_dirname : str + Name of the MiniSEED subdirectory (typically ``"waveforms"``). + + Returns + ------- + bool + ``True`` if the row appears fully completed, otherwise ``False``. + """ + mseed_path = _row_mseed_dir(output_dir, mseed_dirname, row) + + if not mseed_path.is_dir(): return False target_end = _format_end_for_filename(row["end_date"]) try: - for fname in os.listdir(mseed_path): - if not fname.endswith(".mseed"): + for path in mseed_path.iterdir(): + if path.suffix != ".mseed": + continue + + stem = path.stem + + # Filename parts expected like: NET.STA..CHAN__START__END + parts = stem.split(".") + if len(parts) <= 3: continue - stem = os.path.splitext(fname)[0] - # Check the CHAN field that it ends in Z - chan_check = stem.split(".")[3] - chan_check = chan_check.split("__")[0] # remove any suffix after __ + chan_check = parts[3].split("__", 1)[0] # remove any suffix after __ if not chan_check.endswith("Z"): continue - # filename parts expected like: NET.STA..CHAN__START__END - # take last segment after the final '__' - if "__" in stem: - last = stem.rsplit("__", 1)[-1] - if last == target_end: - return True - except Exception: - # any filesystem error -> treat as not done so row will be retried + if "__" not in stem: + continue + + last = stem.rsplit("__", 1)[-1] + if last == target_end: + return True + except OSError: + # Any filesystem error -> treat as not done so row will be retried. return False return False -def is_row_started(row): - """ - Check whether *any* mseed file exists for this row. - Returns True if at least one .mseed file is present. - """ - net = str(row["net"]).strip() - sta = str(row["sta"]).strip() - loc_field = str(row["loc"]) - chan_prefix = str(row["chan"]).strip() +def is_row_started(row: pd.Series, *, output_dir: Path, mseed_dirname: str) -> bool: + """Check whether any waveform data exists for a CSV row. - record_sub = f"{net}_{sta}_{chan_prefix}_{loc_field}" - mseed_path = os.path.join(OUTPUT_DIR, MSEED_DIR, net, record_sub) + Parameters + ---------- + row : pandas.Series + A row of the input CSV. + output_dir : Path + Root output directory. + mseed_dirname : str + Name of the MiniSEED subdirectory (typically ``"waveforms"``). - if not os.path.isdir(mseed_path): - return False + Returns + ------- + bool + ``True`` if at least one ``.mseed`` file exists for the row. + """ + mseed_path = _row_mseed_dir(output_dir, mseed_dirname, row) - try: - for fname in os.listdir(mseed_path): - if fname.endswith(".mseed"): - return True - except Exception: + if not mseed_path.is_dir(): return False - return False - - -def evaluate_download_completeness(csv_file, output_csv): - """ - Evaluate download state with two levels: - - completed: full end_date reached - - started: at least one file exists + return any(p.suffix == ".mseed" for p in mseed_path.iterdir()) + + +def evaluate_download_completeness( + csv_file: Path, + output_csv: Path, + output_dir: Path, + provider: str = "IRIS", +) -> pd.DataFrame: + """Evaluate and summarize download completeness for waveform requests. + + The evaluation is performed at two levels: + + - **completed**: at least one MiniSEED file exists with a final ``__END`` + timestamp equal to the row ``end_date``. + - **started**: at least one MiniSEED file exists for the row. + + Parameters + ---------- + csv_file : Path + Input CSV describing waveform requests. + output_csv : Path + Output CSV to write the evaluation results to. + output_dir : Path + Root output directory containing the MiniSEED data. + provider : str, optional + Provider name to filter by, by default "IRIS". + + Returns + ------- + pandas.DataFrame + The evaluated dataframe including the ``completed`` and ``started`` + boolean columns. + + Raises + ------ + ValueError + If required columns are missing from the input CSV. """ df = pd.read_csv(csv_file, dtype={"loc": str}, keep_default_na=False) - df = df[df["provider"] == "IRIS"].reset_index(drop=True) + + required_cols = { + "provider", + "net", + "sta", + "loc", + "chan", + "start_date", + "end_date", + } + missing = sorted(required_cols - set(df.columns)) + if missing: + raise ValueError(f"Input CSV missing required columns: {missing}") + + df = df[df["provider"] == provider].reset_index(drop=True) print(f"Evaluating {len(df)} rows for completeness...") - # Two independent checks - df["completed"] = df.apply(is_row_done, axis=1) - df["started"] = df.apply(is_row_started, axis=1) + mseed_dirname = "waveforms" - total = len(df) + df["completed"] = df.apply( + is_row_done, axis=1, output_dir=output_dir, mseed_dirname=mseed_dirname + ) + df["started"] = df.apply( + is_row_started, axis=1, output_dir=output_dir, mseed_dirname=mseed_dirname + ) + total = len(df) completed = int(df["completed"].sum()) started = int(df["started"].sum()) @@ -113,35 +222,72 @@ def evaluate_download_completeness(csv_file, output_csv): none = int((~df["started"]).sum()) print("\n===== DOWNLOAD SUMMARY =====") - print(f"Total rows : {total}") - print(f"Fully completed : {completed}") - print(f"Started (any data) : {started}") + print(f"Total rows : {total}") + print(f"Fully completed : {completed}") + print(f"Started (any data) : {started}") print(f"Partial (started only): {partial}") - print(f"No data at all : {none}") + print(f"No data at all : {none}") + + cols = ["provider", "net", "sta", "loc", "chan", "start_date", "end_date"] - # Rows with no data whatsoever if none > 0: print("\n===== ROWS WITH NO DATA =====") - cols = ["provider", "net", "sta", "loc", "chan", "start_date", "end_date"] print(df.loc[~df["started"], cols].to_string(index=False)) - # Rows with partial data (useful for retries) if partial > 0: print("\n===== PARTIALLY DOWNLOADED ROWS =====") - cols = ["provider", "net", "sta", "loc", "chan", "start_date", "end_date"] print(df.loc[df["started"] & ~df["completed"], cols].to_string(index=False)) + output_csv.parent.mkdir(parents=True, exist_ok=True) df.to_csv(output_csv, index=False) print(f"\nWrote evaluation CSV to:\n {output_csv}") return df -CSV_FILE = "/scratch/jobs/jri83/runs/tmp_array/all_nz_sta_providers_desired_channels_mustang.csv" -OUTPUT_CSV = "/scratch/jobs/jri83/runs/tmp_array/download_completeness_evaluation.csv" - -if __name__ == "__main__": +@app.command() +def run( + csv_file: Path = typer.Argument( + ..., exists=True, dir_okay=False, help="Input waveform request CSV." + ), + output_csv: Path = typer.Argument( + ..., dir_okay=False, help="Output CSV to write evaluation results to." + ), + output_dir: Path = typer.Argument( + ..., + exists=False, + file_okay=False, + help="Root output directory containing the MiniSEED data.", + ), + provider: str = typer.Option( + "IRIS", + help="Provider name to filter by.", + ), +) -> None: + """Evaluate MiniSEED download completeness for a request table. + + Parameters + ---------- + csv_file : Path + Input CSV describing waveform requests. + output_csv : Path + Output CSV to write the evaluation results. + output_dir : Path + Root output directory containing the MiniSEED data. + provider : str + Provider name to filter by. + + Returns + ------- + None + """ evaluate_download_completeness( - csv_file=CSV_FILE, - output_csv=OUTPUT_CSV, + csv_file=csv_file, + output_csv=output_csv, + output_dir=output_dir, + provider=provider, ) + + +if __name__ == "__main__": + app() diff --git a/nzgmdb/temp_arrays/get_stations.py b/nzgmdb/temp_arrays/get_stations.py index 99ad22ec..d44c093d 100644 --- a/nzgmdb/temp_arrays/get_stations.py +++ b/nzgmdb/temp_arrays/get_stations.py @@ -1,5 +1,18 @@ -from obspy.clients.fdsn import Client as FDSN_Client +"""Fetch station/channel metadata from FDSN providers and write to CSV.""" + +from collections.abc import Iterable +from pathlib import Path +from typing import Annotated + +import geopandas as gpd +import nzgeom.coastlines +import pandas as pd +import typer from obspy import UTCDateTime +from obspy.clients.fdsn import Client as FDSN_Client +from shapely.geometry import Point + +app = typer.Typer(pretty_exceptions_enable=False) URL_MAPPINGS = { "AUSPASS": "http://auspass.edu.au", @@ -35,132 +48,334 @@ "USP": "http://sismo.iag.usp.br", } -# Define rough NZ bounding box (adjust as needed) -min_lat, max_lat = -49, -32.0 -min_lon, max_lon = 165.0, -176.9 -# Time window for station metadata -starttime = UTCDateTime("2000-01-01") -endtime = UTCDateTime() # now +def is_point_inside_nz(lat: float, lon: float, *, nz_coast: gpd.GeoDataFrame) -> bool: + """Return True if the given point lies within NZ coastline polygons. -import pandas as pd + Parameters + ---------- + lat : float + Latitude in degrees. + lon : float + Longitude in degrees. + nz_coast : geopandas.GeoDataFrame + Coastline polygons in EPSG:4326. -all_station_info = [] + Returns + ------- + bool + True if the point is contained in any polygon. + """ -import nzgeom.coastlines -from shapely.geometry import Point + p = Point(lon, lat) + + # Iterate geometries directly so we don't rely on pandas/geopandas methods here. + return any(getattr(g, "contains")(p) for g in nz_coast.geometry) -# Load NZ coastline polygons once (efficient) -_NZ_COAST = nzgeom.coastlines.get_NZ_coastlines().to_crs("EPSG:4326") +def _parse_time(value: str) -> UTCDateTime: + """Parse a time value into ``UTCDateTime``. -def is_point_inside_nz(lat, lon): + Parameters + ---------- + value : str + Input time string parseable by ObsPy. If "now" (case-insensitive), the + current time is used. + + Returns + ------- + UTCDateTime + Parsed time. + + Raises + ------ + ValueError + If the value cannot be parsed. """ - Returns True if the given latitude/longitude lies inside - the NZ mainland or island coastline polygons. + if value.strip().lower() == "now": + return UTCDateTime() + return UTCDateTime(value) + + +def _iter_providers(providers: list[str] | None) -> list[str]: + """Build the provider list for querying. + + Parameters + ---------- + providers : list[str] | None + Optional list of provider names. If None or empty, uses all known + providers. + + Returns + ------- + list[str] + Provider names. """ - # shapely uses (lon, lat) - p = Point(lon, lat) - # test containment against all polygons - return _NZ_COAST.geometry.apply(lambda g: g.contains(p)).any() + if not providers: + return sorted(URL_MAPPINGS) + return providers -for provider, _ in URL_MAPPINGS.items(): - try: +def collect_station_rows( + providers: list[str], + starttime: UTCDateTime, + endtime: UTCDateTime, + minlatitude: float, + maxlatitude: float, + minlongitude: float, + maxlongitude: float, + coastline_filter: bool, +) -> list[list[object]]: + """Collect station/channel rows from configured providers. + + Parameters + ---------- + providers : list[str] + Provider names. + starttime : UTCDateTime + Metadata start time. + endtime : UTCDateTime + Metadata end time. + minlatitude, maxlatitude : float + Latitude bounding box. + minlongitude, maxlongitude : float + Longitude bounding box. + coastline_filter : bool + If True, filter stations to those within NZ coastline polygons. + + Returns + ------- + list[list[object]] + Rows matching the output CSV schema. + """ + nz_coast = ( + nzgeom.coastlines.get_NZ_coastlines().to_crs("EPSG:4326") + if coastline_filter + else None + ) + + all_station_info: list[list[object]] = [] + + for provider in providers: client = FDSN_Client(base_url=provider) networks = client.get_stations( starttime=starttime, endtime=endtime, - minlatitude=min_lat, - maxlatitude=max_lat, - minlongitude=min_lon, - maxlongitude=max_lon, + minlatitude=minlatitude, + maxlatitude=maxlatitude, + minlongitude=minlongitude, + maxlongitude=maxlongitude, level="network", ) - network_codes = [net.code for net in networks] + network_codes = [net.code for net in networks] print("Processing provider:", provider, "with networks:", len(network_codes)) for net_code in network_codes: - try: - inv = client.get_stations( - network=net_code, - level="channel", - minlatitude=min_lat, - maxlatitude=max_lat, - minlongitude=min_lon, - maxlongitude=max_lon, - ) - - for network in inv: - print( - " Network:", - network.code, - "with stations:", - len(network.stations), - ) - - for station in network: - lat = station.latitude - lon = station.longitude - if not is_point_inside_nz(lat, lon): + inv = client.get_stations( + network=net_code, + starttime=starttime, + endtime=endtime, + level="channel", + minlatitude=minlatitude, + maxlatitude=maxlatitude, + minlongitude=minlongitude, + maxlongitude=maxlongitude, + ) + + for network in inv: + for station in network: + lat = float(station.latitude) + lon = float(station.longitude) + + if nz_coast is not None and not is_point_inside_nz( + lat, lon, nz_coast=nz_coast + ): + continue + + current_channels: set[tuple[str, str]] = set() + for channel in station: + chan_id = (channel.location_code, channel.code[:2]) + if chan_id in current_channels: continue - current_channels = set() - for channel in station: - chan_id = (channel.location_code, channel.code[:2]) - if chan_id in current_channels: - continue - current_channels.add(chan_id) - all_station_info.append( - [ - provider, # provider as first column - network.code, - station.code, - lat, - lon, - station.elevation, - channel.code[:2], - channel.location_code, - channel.start_date, - channel.end_date, - ] - ) - except Exception: - # continue to next network code on failure - continue - except Exception: - # continue to next provider on failure - continue - -# build dataframe (provider is first column) -station_df = pd.DataFrame( - all_station_info, - columns=[ - "provider", - "net", - "sta", - "lat", - "lon", - "elev", - "chan", - "loc", - "start_date", - "end_date", + + current_channels.add(chan_id) + all_station_info.append( + [ + provider, + network.code, + station.code, + lat, + lon, + getattr(station, "elevation", None), + channel.code[:2], + channel.location_code, + getattr(channel, "start_date", None), + getattr(channel, "end_date", None), + ] + ) + + return all_station_info + + +def build_station_dataframe(rows: Iterable[list[object]]) -> pd.DataFrame: + """Build the station/channel dataframe. + + Parameters + ---------- + rows : Iterable[list[object]] + Station/channel rows. + + Returns + ------- + pandas.DataFrame + DataFrame containing station/channel metadata. + """ + station_df = pd.DataFrame( + list(rows), + columns=[ + "provider", + "net", + "sta", + "lat", + "lon", + "elev", + "chan", + "loc", + "start_date", + "end_date", + ], + ) + + if station_df.empty: + return station_df + + return station_df.drop_duplicates( + ["provider", "net", "sta", "chan", "loc"] + ).reset_index(drop=True) + + +def write_station_csv(df: pd.DataFrame, out_csv: Path) -> None: + """Write station/channel metadata to CSV. + + Parameters + ---------- + df : pandas.DataFrame + DataFrame to write. + out_csv : Path + Output file path. + + Returns + ------- + None + This function returns ``None``. + """ + out_csv.parent.mkdir(parents=True, exist_ok=True) + df.to_csv(out_csv, index=False) + + +@app.command() +def get_stations( + out_all_csv: Annotated[ + Path, + typer.Argument( + dir_okay=False, + help="Output CSV for all channels.", + ), ], -) -station_df = station_df.drop_duplicates( - ["provider", "net", "sta", "chan", "loc"] -).reset_index(drop=True) - -# write outputs -station_df.to_csv( - "/media/joel/data/nzgmdb/tmp_arrays/nz_mainland_stations_all_provider_networks_channels.csv", - index=False, -) - -desired_channels = ["HH", "BH", "HN", "BN"] -filtered_df = station_df[station_df["chan"].isin(desired_channels)] -filtered_df.to_csv( - "/media/joel/data/nzgmdb/tmp_arrays/nz_mainland_stations_all_provider_networks_desired_channels.csv", - index=False, -) + out_filtered_csv: Annotated[ + Path | None, + typer.Option( + dir_okay=False, + help=( + "Optional output CSV for desired channels only. If omitted, the filtered file is not written." + ), + ), + ] = None, + provider: Annotated[ + list[str], + typer.Option( + "--provider", + help="Provider(s) to query. Repeatable. If omitted, queries all known providers.", + ), + ] = None, + starttime: Annotated[ + str, + typer.Option(help="Start time for station metadata (e.g. 2000-01-01)."), + ] = "2000-01-01", + endtime: Annotated[ + str, + typer.Option(help="End time for station metadata (or 'now')."), + ] = "now", + min_latitude: Annotated[float, typer.Option(help="Minimum latitude.")] = -49.0, + max_latitude: Annotated[float, typer.Option(help="Maximum latitude.")] = -32.0, + min_longitude: Annotated[float, typer.Option(help="Minimum longitude.")] = 165.0, + max_longitude: Annotated[float, typer.Option(help="Maximum longitude.")] = -176.9, + coastline_filter: Annotated[ + bool, + typer.Option( + "--coastline-filter/--no-coastline-filter", + help="Filter to stations inside NZ coastline polygons.", + ), + ] = True, + desired_channels: Annotated[ + list[str], + typer.Option( + "--desired-channel", + help="Desired 2-char channel prefixes for the filtered output. Repeatable.", + ), + ] = None, +) -> None: + """Query station/channel metadata and write CSV outputs. + + Parameters + ---------- + out_all_csv : Path + Output CSV for all station/channel rows. + out_filtered_csv : Path | None + Optional output CSV for filtered station/channel rows. + provider : list[str], optional + Provider(s) to query. Repeatable. + starttime : str + Start time for station metadata. + endtime : str + End time for station metadata. + min_latitude : float + Minimum latitude. + max_latitude : float + Maximum latitude. + min_longitude : float + Minimum longitude. + max_longitude : float + Maximum longitude. + coastline_filter : bool + If True, filter to stations in NZ coastline polygons. + desired_channels : list[str], optional + Desired 2-character channel prefixes for ``out_filtered_csv``. + """ + providers = _iter_providers(provider) + + rows = collect_station_rows( + providers=providers, + starttime=_parse_time(starttime), + endtime=_parse_time(endtime), + minlatitude=min_latitude, + maxlatitude=max_latitude, + minlongitude=min_longitude, + maxlongitude=max_longitude, + coastline_filter=coastline_filter, + ) + + station_df = build_station_dataframe(rows) + write_station_csv(station_df, out_all_csv) + + if out_filtered_csv is None: + return + + desired = desired_channels if desired_channels else ["HH", "BH", "HN", "BN"] + filtered_df = station_df[station_df["chan"].isin(desired)].reset_index(drop=True) + write_station_csv(filtered_df, out_filtered_csv) + + +if __name__ == "__main__": + app() diff --git a/nzgmdb/temp_arrays/mass_download_data.py b/nzgmdb/temp_arrays/mass_download_data.py index 574764f9..7168736b 100644 --- a/nzgmdb/temp_arrays/mass_download_data.py +++ b/nzgmdb/temp_arrays/mass_download_data.py @@ -1,71 +1,75 @@ -import os +import json +from pathlib import Path +from typing import Annotated + import pandas as pd +import typer +from obspy import UTCDateTime from obspy.clients.fdsn.mass_downloader import ( + GlobalDomain, MassDownloader, Restrictions, - GlobalDomain, ) -from obspy import UTCDateTime -import multiprocessing - - -# ---------------- USER SETTINGS ---------------- # -# CSV_FILE = '/media/joel/data/nzgmdb/tmp_arrays/all_nz_sta_providers_desired_channels_mustang.csv' -CSV_FILE = "/media/joel/data/nzgmdb/tmp_arrays/HR1_inventory.csv" -# CSV_FILE = '/scratch/jobs/jri83/runs/tmp_array/download_completeness_evaluation_3.csv' - -OUTPUT_DIR = "/media/joel/data/nzgmdb/tmp_arrays/hr1" -# OUTPUT_DIR = '/scratch/jobs/jri83/runs/tmp_array/mass_data_row_mp' -MSEED_DIR = "waveforms" -STATIONXML_DIR = "stationxml" -RESULTS_CSV = os.path.join(OUTPUT_DIR, "download_results.csv") +app = typer.Typer(pretty_exceptions_enable=False) -# month length in seconds (15 days) +# Month length in seconds (15 days) MONTH_SECONDS = 15 * 24 * 3600 # Minimum chunk size (seconds) when backing off after 413 / manifest-too-large. # 3600s = 1 hour. MIN_CHUNK_SECONDS: int = 3600 -# ------------------------------------------------ # +def _format_end_for_filename(end_dt: UTCDateTime | str) -> str: + """Format an end datetime into the MiniSEED filename timestamp form. -def _format_end_for_filename(end_dt): - """Format an obspy UTCDateTime or parseable date string to the filename timestamp form.""" + Parameters + ---------- + end_dt : UTCDateTime | str + An ObsPy ``UTCDateTime`` or any string parseable by ``UTCDateTime``. + + Returns + ------- + str + Timestamp string formatted like ``YYYYMMDDTHHMMSSZ``. + """ if not isinstance(end_dt, UTCDateTime): end_dt = UTCDateTime(end_dt) return end_dt.strftime("%Y%m%dT%H%M%SZ") -def save_results(results, results_csv=RESULTS_CSV): - """ - Create one CSV from a list of result dicts. - - Finds all unique keys across results. - - Uses a preferred column order for common fields. - - Normalizes missing keys to empty string and converts non-scalar values to strings. - """ - import os - import json - import pandas as pd +def save_results( + results: list[dict[str, object] | object], + results_csv: Path, +) -> None: + """Write a single results CSV from per-row download results. + This function is intentionally defensive: + + - It computes the union of keys across result dictionaries. + - It normalizes missing keys to empty strings. + - It serializes nested objects (dict/list) as JSON for readability. + + Parameters + ---------- + results : list[dict[str, object] | object] + Download results. Most entries are dicts, but non-dicts will be written + under a ``value`` column. + results_csv : Path + Output CSV file. + """ if not results: - # ensure output dir exists and write an empty file - out_dir = os.path.dirname(results_csv) - if out_dir: - os.makedirs(out_dir, exist_ok=True) + results_csv.parent.mkdir(parents=True, exist_ok=True) pd.DataFrame().to_csv(results_csv, index=False) return - # collect all keys - all_keys = set() + all_keys: set[str] = set() for r in results: if isinstance(r, dict): all_keys.update(r.keys()) else: - # non-dict entries will be recorded under 'value' all_keys.add("value") - # preferred column order for readability preferred = [ "idx", "provider", @@ -83,145 +87,203 @@ def save_results(results, results_csv=RESULTS_CSV): k for k in all_keys if k not in preferred ) - # normalize rows - norm_rows = [] + norm_rows: list[dict[str, object]] = [] for r in results: if not isinstance(r, dict): - row = {"value": str(r)} - else: - row = {} - for k in cols: - v = r.get(k, "") - # convert lists/dicts/other non-primitives to JSON or string - if isinstance(v, (dict, list)): - try: - v = json.dumps(v, ensure_ascii=False) - except Exception: - v = str(v) - elif v is None: - v = "" - else: - # keep numbers/strings as-is; cover other types - if not isinstance(v, (str, int, float, bool)): - v = str(v) - row[k] = v + row: dict[str, object] = {"value": str(r)} + norm_rows.append(row) + continue + + row = {} + for k in cols: + v = r.get(k, "") + if isinstance(v, (dict, list)): + v = json.dumps(v, ensure_ascii=False) + elif v is None: + v = "" + elif not isinstance(v, (str, int, float, bool)): + v = str(v) + + row[k] = v norm_rows.append(row) - # ensure output directory exists - out_dir = os.path.dirname(results_csv) - if out_dir: - os.makedirs(out_dir, exist_ok=True) + results_csv.parent.mkdir(parents=True, exist_ok=True) df = pd.DataFrame(norm_rows, columns=cols) df.to_csv(results_csv, index=False, encoding="utf-8") -def is_row_done(row): - """ - Check the mseed output directory for this row to see if a file exists - whose final `__` timestamp equals the row end_date. - Returns True if done, False otherwise. +def is_row_done( + row: pd.Series, + output_dir: Path, + mseed_dirname: str, +) -> bool: + """Check whether a CSV row has fully completed waveform download. + + A row is considered **done** if the expected MiniSEED output directory + contains at least one ``.mseed`` file whose final ``__END`` timestamp equals + the row's ``end_date``. + + Notes + ----- + This check is filesystem-driven and intentionally conservative: + + - Only files whose channel code ends with ``"Z"`` are considered. + - Any filesystem error or unexpected filename format causes this function + to return ``False`` (so the row can be retried). + + Parameters + ---------- + row : pandas.Series + A row of the input CSV. + output_dir : Path + Root output directory. + mseed_dirname : str + Name of the MiniSEED subdirectory (typically ``"waveforms"``). + + Returns + ------- + bool + ``True`` if the row appears fully completed, otherwise ``False``. """ net = str(row["net"]).strip() sta = str(row["sta"]).strip() - loc_field = str(row["loc"]) - # loc_field = "" if loc_field == "NA" else loc_field + loc_field = str(row["loc"]).strip() chan_prefix = str(row["chan"]).strip() record_sub = f"{net}_{sta}_{chan_prefix}_{loc_field}" - mseed_path = os.path.join(OUTPUT_DIR, MSEED_DIR, net, record_sub) + mseed_path = output_dir / mseed_dirname / net / record_sub - if not os.path.isdir(mseed_path): + if not mseed_path.is_dir(): return False target_end = _format_end_for_filename(row["end_date"]) try: - for fname in os.listdir(mseed_path): - if not fname.endswith(".mseed"): + for path in mseed_path.iterdir(): + if path.suffix != ".mseed": + continue + + stem = path.stem + + # Filename parts expected like: NET.STA..CHAN__START__END + parts = stem.split(".") + if len(parts) <= 3: continue - stem = os.path.splitext(fname)[0] - # Check the CHAN field that it ends in Z - chan_check = stem.split(".")[3] - chan_check = chan_check.split("__")[0] # remove any suffix after __ + chan_check = parts[3].split("__", 1)[0] if not chan_check.endswith("Z"): continue - # filename parts expected like: NET.STA..CHAN__START__END - # take last segment after the final '__' - if "__" in stem: - last = stem.rsplit("__", 1)[-1] - if last == target_end: - return True - except Exception: - # any filesystem error -> treat as not done so row will be retried + if "__" not in stem: + continue + + last = stem.rsplit("__", 1)[-1] + if last == target_end: + return True + except OSError: return False return False -def create_output_dirs(net, sta, chan_prefix, loc): - """ - Create directories for the network if needed. +def create_output_dirs( + net: str, + sta: str, + chan_prefix: str, + loc: str, + output_dir: Path, + mseed_dirname: str, + stationxml_dirname: str, +) -> tuple[Path, Path]: + """Create output directories for a network/station/channel/location. + + Parameters + ---------- + net : str + Network code. + sta : str + Station code. + chan_prefix : str + 2-character channel prefix (e.g. ``"HH"``). + loc : str + Location code used for naming the output directory. + output_dir : Path + Root output directory. + mseed_dirname : str + MiniSEED subdirectory name. + stationxml_dirname : str + StationXML subdirectory name. + + Returns + ------- + tuple[Path, Path] + Tuple of ``(mseed_path, xml_path)``. """ record_sub = f"{net}_{sta}_{chan_prefix}_{loc}" - mseed_path = os.path.join(OUTPUT_DIR, MSEED_DIR, net, record_sub) - xml_path = os.path.join(OUTPUT_DIR, STATIONXML_DIR, net, record_sub) + mseed_path = output_dir / mseed_dirname / net / record_sub + xml_path = output_dir / stationxml_dirname / net / record_sub - os.makedirs(mseed_path, exist_ok=True) - os.makedirs(xml_path, exist_ok=True) + mseed_path.mkdir(parents=True, exist_ok=True) + xml_path.mkdir(parents=True, exist_ok=True) return mseed_path, xml_path def download_task( - task: tuple[int, str, dict[str, object]] -) -> None | dict[str, int | str | object] | dict[str, int | str]: - """ - Download waveform and StationXML data for a single CSV row task. + task: tuple[int, str, dict[str, object]], + *, + output_dir: Path, + mseed_dirname: str, + stationxml_dirname: str, + month_seconds: int, + min_chunk_seconds: int, +) -> dict[str, int | str] | None: + """Download waveform and StationXML data for a single CSV row. Parameters ---------- - task - A 3-tuple of `(idx, provider, row_dict)` where: - - `idx` is the zero-based row index in the input CSV. - - `provider` is the FDSN provider name. - - `row_dict` contains the CSV row fields such as `net`, `sta`, `loc`, - `chan`, `start_date`, and `end_date`. + task : tuple[int, str, dict[str, object]] + A 3-tuple of ``(idx, provider, row_dict)``. + output_dir : Path + Root output directory. + mseed_dirname : str + MiniSEED subdirectory name. + stationxml_dirname : str + StationXML subdirectory name. + month_seconds : int + Default chunk length (seconds) for large windows. + min_chunk_seconds : int + Minimum chunk length (seconds) when backing off after 413 errors. Returns ------- - dict[str, object] - A result dictionary containing at least: - - `idx`: int - - `status`: `ok` or `error` - - `provider`: str - And, on success, `net` and `sta`. On error, includes `error`. + dict[str, int | str] | None + A result dictionary with keys including ``idx``, ``status``, ``provider``, + ``net``, ``sta``, and optionally ``error``. Raises ------ Exception - Re-raises exceptions encountered during download attempts (for example - when the server repeatedly rejects the request as too large after - backoff to the minimum chunk size). + Re-raises exceptions encountered during download attempts when the + request cannot be reduced any further. """ idx, provider, row = task + try: - net = row["net"] - sta = row["sta"] - loc_field = str(row["loc"]) - loc = "*" if loc_field == "NA" else loc_field.strip() + net = str(row["net"]).strip() + sta = str(row["sta"]).strip() + loc_field = str(row["loc"]).strip() + loc = "*" if loc_field == "NA" else loc_field chan_prefix = str(row["chan"]).strip() - channel = f"{chan_prefix}?" # add ? automatically + channel = f"{chan_prefix}?" start = UTCDateTime(row["start_date"]) end = UTCDateTime(row["end_date"]) - # Chunk length: month (30 days) but not longer than the full requested window total_window = end - start - chunk_base = int(min(total_window, MONTH_SECONDS)) + chunk_base = int(min(total_window, month_seconds)) max_attempts = 4 attempt = 1 @@ -230,8 +292,15 @@ def download_task( f"[{idx}] Provider={provider} Downloading {net}.{sta} {channel} {start} -> {end} chunk={chunk_base}s" ) - # create output dirs using the raw loc field for naming (keeps 'NA' if present) - mseed_path, xml_path = create_output_dirs(net, sta, chan_prefix, loc_field) + mseed_path, xml_path = create_output_dirs( + net, + sta, + chan_prefix, + loc_field, + output_dir=output_dir, + mseed_dirname=mseed_dirname, + stationxml_dirname=stationxml_dirname, + ) while attempt <= max_attempts: chunklength = int(min(total_window, chunk_base)) @@ -257,9 +326,10 @@ def download_task( mdl.download( domain, restrictions, - mseed_storage=mseed_path, - stationxml_storage=xml_path, + mseed_storage=str(mseed_path), + stationxml_storage=str(xml_path), ) + print(f"[{idx}] Done") return { "idx": idx, @@ -268,9 +338,8 @@ def download_task( "net": net, "sta": sta, } - except Exception as e: - err_text = repr(e) + " " + str(e) - # Detect 413 / manifest-too-large responses from server text + except Exception as exc: # noqa: BLE001 + err_text = f"{exc!r} {exc}" is_manifest_too_large = ( "Estimated manifest size" in err_text or "Request Entity Too Large" in err_text @@ -278,57 +347,133 @@ def download_task( ) if is_manifest_too_large: - # halve the base chunk and retry, unless already at minimum - if chunk_base <= MIN_CHUNK_SECONDS: + if chunk_base <= min_chunk_seconds: print( f"[{idx}] Server denied request and chunk is already at minimum ({chunk_base}s). Giving up." ) raise + old = chunk_base - chunk_base = max(MIN_CHUNK_SECONDS, chunk_base // 2) + chunk_base = max(min_chunk_seconds, chunk_base // 2) print( f"[{idx}] Server denied request (413). Reducing chunk base {old}s -> {chunk_base}s and retrying." ) attempt += 1 continue - raise e - except Exception as e: + raise + + except Exception as exc: # noqa: BLE001 print( - f"[{idx}] ERROR provider={provider} net={row.get('net')} sta={row.get('sta')}: {e}" + f"[{idx}] ERROR provider={provider} net={row.get('net')} sta={row.get('sta')}: {exc}" ) - return {"idx": idx, "status": "error", "error": str(e), "provider": provider} + return { + "idx": idx, + "status": "error", + "error": str(exc), + "provider": provider, + } + + +@app.command() +def run( + csv_file: Annotated[ + Path, + typer.Argument( + ..., exists=True, dir_okay=False, help="Input station/channel CSV." + ), + ], + output_dir: Annotated[ + Path, + typer.Argument( + ..., + file_okay=False, + help="Root output directory for waveforms and StationXML.", + ), + ], + results_csv: Annotated[ + Path | None, + typer.Option( + dir_okay=False, + help="Optional output CSV for per-row download results (default: /download_results.csv).", + ), + ] = None, + mseed_dirname: Annotated[ + str, + typer.Option(help="MiniSEED subdirectory name."), + ] = "waveforms", + stationxml_dirname: Annotated[ + str, + typer.Option(help="StationXML subdirectory name."), + ] = "stationxml", +) -> None: + """Download waveforms + StationXML for each row in a station/channel request table. + Parameters + ---------- + csv_file : Path + Input CSV describing requests. Must contain columns: + ``net``, ``sta``, ``loc``, ``chan``, ``start_date``, ``end_date``, ``provider``. + output_dir : Path + Root output directory. + results_csv : Path | None, optional + Output CSV for results. If omitted, defaults to + ``/download_results.csv``. + mseed_dirname : str, optional + MiniSEED subdirectory name. + stationxml_dirname : str, optional + StationXML subdirectory name. -def main(): + Returns + ------- + None + This function returns ``None``. - df = pd.read_csv(CSV_FILE, dtype={"loc": str}, keep_default_na=False) + Raises + ------ + ValueError + If required columns are missing from the input CSV. + """ + df = pd.read_csv(csv_file, dtype={"loc": str}, keep_default_na=False) required_cols = {"net", "sta", "loc", "chan", "start_date", "end_date", "provider"} + missing = sorted(required_cols - set(df.columns)) + if missing: + raise ValueError(f"CSV must contain: {missing}") + + output_dir.mkdir(parents=True, exist_ok=True) - if not required_cols.issubset(df.columns): - raise ValueError(f"CSV must contain: {required_cols}") + if results_csv is None: + results_csv = output_dir / "download_results.csv" - # Build tasks for all rows (one task per CSV row) - tasks = [] + tasks: list[tuple[int, str, dict[str, object]]] = [] skipped = 0 for idx, row in df.reset_index(drop=True).iterrows(): - if is_row_done(row): + if is_row_done(row, output_dir=output_dir, mseed_dirname=mseed_dirname): skipped += 1 continue - tasks.append((int(idx), row["provider"], row.to_dict())) + + tasks.append((int(idx), str(row["provider"]), row.to_dict())) print( f"Starting sequential run for {len(tasks)} tasks (skipped {skipped} already done)" ) - results = [] + results: list[dict[str, object]] = [] for task in tasks: - results.append(download_task(task)) + results.append( + download_task( + task, + output_dir=output_dir, + mseed_dirname=mseed_dirname, + stationxml_dirname=stationxml_dirname, + month_seconds=MONTH_SECONDS, + min_chunk_seconds=MIN_CHUNK_SECONDS, + ) + ) - save_results(results, results_csv=RESULTS_CSV) + save_results(results, results_csv=results_csv) - # Simple summary oks = sum(1 for r in results if r.get("status") == "ok") errs = sum(1 for r in results if r.get("status") == "error") print( @@ -337,4 +482,4 @@ def main(): if __name__ == "__main__": - main() + app() From ffaba4591c20ee3b7073341326315c70fc8e738e Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 21 Jan 2026 15:37:18 +1300 Subject: [PATCH 15/72] unused var --- nzgmdb/data_processing/process_observed.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/nzgmdb/data_processing/process_observed.py b/nzgmdb/data_processing/process_observed.py index f1bdfa8c..f5f3b122 100644 --- a/nzgmdb/data_processing/process_observed.py +++ b/nzgmdb/data_processing/process_observed.py @@ -25,7 +25,6 @@ def process_single_mseed( fmax_df: pd.DataFrame | None = None, bypass_df: pd.DataFrame | None = None, xml_dir: Path | None = None, - inventory: Inventory | None = None, ): """ Process a single mseed file and save the processed data to a txt file @@ -46,8 +45,6 @@ def process_single_mseed( The bypass records containing custom fmin, fmax values xml_dir : Path, optional The directory containing the station xml files for inventory information - inventory : Inventory, optional - The inventory information for the mseed file Returns ------- @@ -247,11 +244,6 @@ def process_mseeds_to_txt( ) bypass_df = None if bypass_records_ffp is None else pd.read_csv(bypass_records_ffp) - config = cfg.Config() - channel_codes = config.get_value("channel_codes") - client = FDSN_Client("GEONET") - inventory = client.get_stations(channel=channel_codes, level="response") - # Use multiprocessing to process the mseed files with multiprocessing.Pool(processes=n_procs) as pool: skipped_records = pool.map( @@ -261,7 +253,6 @@ def process_mseeds_to_txt( fmax_df=fmax_df, bypass_df=bypass_df, xml_dir=xml_dir, - inventory=inventory, ), mseed_files, ) From f1465d1f4e85218d57270b2a29ab7b48ff96d6eb Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 23 Jan 2026 15:07:11 +1300 Subject: [PATCH 16/72] site adjustments --- .../data_processing/waveform_manipulation.py | 10 +- nzgmdb/data_retrieval/geonet.py | 1 + nzgmdb/data_retrieval/inventory_xml.py | 15 +++ nzgmdb/data_retrieval/sites.py | 113 +++++++++++++----- nzgmdb/scripts/run_nzgmdb.py | 12 +- 5 files changed, 114 insertions(+), 37 deletions(-) diff --git a/nzgmdb/data_processing/waveform_manipulation.py b/nzgmdb/data_processing/waveform_manipulation.py index e35276c6..2e3e4792 100644 --- a/nzgmdb/data_processing/waveform_manipulation.py +++ b/nzgmdb/data_processing/waveform_manipulation.py @@ -18,6 +18,8 @@ def initial_preprocessing( apply_taper: bool = True, apply_zero_padding: bool = True, inventory: Inventory = None, + provider: str = "GEONET", + network: str = "NZ", ) -> Stream: """ Basic pre-processing of the waveform data @@ -39,6 +41,10 @@ def initial_preprocessing( Whether to apply zero padding, by default True inventory : Inventory, optional The inventory object to use for sensitivity removal, by default None (Will try to extract from FDSN if not provided) + provider : str, optional + The FDSN provider to use if inventory is not provided, by default "GEONET" + network : str, optional + The network code to use if inventory is not provided, by default "NZ" Returns ------- @@ -82,9 +88,9 @@ def initial_preprocessing( inv = inventory if inv is None: try: - client_NZ = FDSN_Client("GEONET") + client_NZ = FDSN_Client(provider) inv = client_NZ.get_stations( - level="response", network="NZ", station=station, location=location + level="response", network=network, station=station, location=location ) except FDSNNoDataException: raise custom_errors.InventoryNotFoundError( diff --git a/nzgmdb/data_retrieval/geonet.py b/nzgmdb/data_retrieval/geonet.py index a3f3f52f..5714b514 100644 --- a/nzgmdb/data_retrieval/geonet.py +++ b/nzgmdb/data_retrieval/geonet.py @@ -359,6 +359,7 @@ def fetch_sta_extraction( # Create the station_extraction_table station_extraction_table = pd.DataFrame( { + "provider": ["GEONET"], "net": [network.code], "sta": [station.code], "evid": [event_id], diff --git a/nzgmdb/data_retrieval/inventory_xml.py b/nzgmdb/data_retrieval/inventory_xml.py index 5da0c1c5..494488e4 100644 --- a/nzgmdb/data_retrieval/inventory_xml.py +++ b/nzgmdb/data_retrieval/inventory_xml.py @@ -21,6 +21,21 @@ def fetch_inventory( pass +""" +Way of reading all site information from all providers / listed networks +geonet - all +iris - limited +bounding box for all sites to be on land in nz + +station level +response level + + +a specific site channel location extraction - need the provider / net in the function call (maybe) + +""" + + def fetch_and_save_inventory( main_dir: Path, stations: list[str], diff --git a/nzgmdb/data_retrieval/sites.py b/nzgmdb/data_retrieval/sites.py index da6878a3..2751a8d0 100644 --- a/nzgmdb/data_retrieval/sites.py +++ b/nzgmdb/data_retrieval/sites.py @@ -181,61 +181,92 @@ def sample_points_from_geotiff( return samples.reshape(-1, 1) -def create_site_table_response() -> pd.DataFrame: +def create_site_table_response(add_tmp_arrays: bool = False) -> pd.DataFrame: """ Create the site table for the NZGMDB. This function fetches the station information from the FDSN clients, and the Geonet metadata summary information. It then merges the two dataframes and determines the tectonic domain for each station. The final dataframe is saved as a csv file in the flatfile directory. + Parameters + ---------- + add_tmp_arrays : bool, optional + Whether to add temporary arrays to the station information, by default False + Returns ------- pd.DataFrame The site table dataframe with all Z, vs30, domain and location values for each site used in the NZGMDB + pd.DataFrame + The station table dataframe with all channel and location values for each site """ # Fetch the client station information - client_NZ = FDSN_Client("GEONET") config = cfg.Config() channel_codes = config.get_value("channel_codes") - inventory = client_NZ.get_stations(channel=channel_codes, level="station") - station_info = [] - for network in inventory: - for station in network: - station_info.append( - [ - network.code, - station.code, - station.latitude, - station.longitude, - station.elevation, - station.creation_date, - station.end_date, - ] - ) - sta_df = pd.DataFrame( - station_info, - columns=["net", "sta", "lat", "lon", "elev", "creation_date", "end_date"], - ) - sta_df = sta_df.drop_duplicates(["net", "sta"]) + for provider in ["GEONET"]: + client_NZ = FDSN_Client(provider) + inventory = client_NZ.get_stations(channel=channel_codes, level="response") + station_info = [ + [ + provider, + network.code, + station.code, + station.latitude, + station.longitude, + station.elevation, + station.creation_date, + station.end_date, + channel.code[:2], + channel.location_code, + channel.depth, + channel.start_date, + channel.end_date, + ] + for network in inventory + for station in network + for channel in station.channels + ] + all_info_df = pd.DataFrame( + station_info, + columns=[ + "provider", + "net", + "sta", + "lat", + "lon", + "elev", + "creation_date", + "end_date", + "chan", + "loc", + "loc_elev", + "start_time", + "end_time", + ], + ) + + all_info_df = all_info_df.drop_duplicates( + ["provider", "net", "sta", "chan", "loc", "loc_elev"] + ).reset_index(drop=True) bbox = config.get_value("bbox") # [min_lon, min_lat, max_lon, max_lat] min_lon, min_lat, max_lon, max_lat = bbox # Ensure lat/lon are present and within latitude bounds mask_lat = ( - sta_df["lat"].notna() - & sta_df["lon"].notna() - & (sta_df["lat"] >= min_lat) - & (sta_df["lat"] <= max_lat) + all_info_df["lat"].notna() + & all_info_df["lon"].notna() + & (all_info_df["lat"] >= min_lat) + & (all_info_df["lat"] <= max_lat) ) # Handle antimeridian crossing: if min_lon > max_lon use OR if min_lon <= max_lon: - mask_lon = (sta_df["lon"] >= min_lon) & (sta_df["lon"] <= max_lon) + mask_lon = (all_info_df["lon"] >= min_lon) & (all_info_df["lon"] <= max_lon) else: - mask_lon = (sta_df["lon"] >= min_lon) | (sta_df["lon"] <= max_lon) + mask_lon = (all_info_df["lon"] >= min_lon) | (all_info_df["lon"] <= max_lon) - sta_df = sta_df.loc[mask_lat & mask_lon] + all_info_df = all_info_df.loc[mask_lat & mask_lon] # Get the Geonet metadata summary information geo_meta_summary_df = pd.read_csv( @@ -264,11 +295,11 @@ def create_site_table_response() -> pd.DataFrame: ) merged_df = geo_meta_summary_df.merge( - sta_df[["net", "sta", "lat", "lon", "elev", "creation_date", "end_date"]], + all_info_df[["net", "sta", "lat", "lon", "elev", "creation_date", "end_date"]], on="sta", how="outer", ) - # Fill Lat, Lon, Elevation NaN values from sta_df + # Fill Lat, Lon, Elevation NaN values from all_info_df merged_df["elev"] = merged_df["Elevation"].combine_first(merged_df["elev"]) merged_df["lat"] = merged_df["Lat"].combine_first(merged_df["lat"]) merged_df["lon"] = merged_df["Long"].combine_first(merged_df["lon"]) @@ -347,9 +378,25 @@ def create_site_table_response() -> pd.DataFrame: except (FileNotFoundError, ValueError, RuntimeError) as e: print(f"Warning: Could not compute thresholds for missing Z1.0 values: {e}") - # Select specific columns + # Split into station and site dfs + station_df = all_info_df.loc[ + [ + "provider", + "net", + "sta", + "lat", + "lon", + "elev", + "chan", + "loc", + "loc_elev", + "start_time", + "end_time", + ] + ] site_df = tect_merged_df[ [ + "provider", "net", "sta", "lat", @@ -381,7 +428,7 @@ def create_site_table_response() -> pd.DataFrame: site_df = site_df.astype({"Z2.5": float}) site_df.loc[:, "Z2.5"] /= 1000.0 - return site_df + return site_df, station_df def add_site_basins(site_df: pd.DataFrame, nzcvm_data_ffp: Path) -> pd.DataFrame: diff --git a/nzgmdb/scripts/run_nzgmdb.py b/nzgmdb/scripts/run_nzgmdb.py index 132f7878..1ab34d00 100644 --- a/nzgmdb/scripts/run_nzgmdb.py +++ b/nzgmdb/scripts/run_nzgmdb.py @@ -542,6 +542,12 @@ def generate_site_table_basin( file_okay=False, ), ], + add_tmp_arrays: Annotated[ + bool, + typer.Option( + is_flag=True, + ), + ] = False, ): """ Generate the site table basin flatfile. @@ -554,13 +560,15 @@ def generate_site_table_basin( The main directory of the NZGMDB results (Highest level directory). nzcvm_data_ffp : Path The full file path to the nzcvm_data repository that stores the basin information. + add_tmp_arrays : bool, optional + If True, temporary arrays will be added to the site table (default is False). """ main_dir.mkdir(parents=True, exist_ok=True) # Generate the site basin flatfile flatfile_dir = file_structure.get_flatfile_dir(main_dir) flatfile_dir.mkdir(parents=True, exist_ok=True) - site_df = sites.create_site_table_response() + site_df = sites.create_site_table_response(add_tmp_arrays) site_df = sites.add_site_basins(site_df, nzcvm_data_ffp) site_df.to_csv( @@ -942,7 +950,7 @@ def run_full_nzgmdb( and (flatfile_dir / file_structure.PreFlatfileNames.SITE_TABLE).exists() ): print("Generating site table basin flatfile") - generate_site_table_basin(main_dir, nzcvm_data_ffp) + generate_site_table_basin(main_dir, nzcvm_data_ffp, add_tmp_arrays) # Fetch the Geonet data if not ( From 9368b79f3b6cacbc4d7d49f94c7e2e764878bc25 Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 23 Jan 2026 15:53:34 +1300 Subject: [PATCH 17/72] fix tests --- nzgmdb/data_processing/process_observed.py | 3 -- tests/test_sites.py | 42 +++++++++++++++++++++- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/nzgmdb/data_processing/process_observed.py b/nzgmdb/data_processing/process_observed.py index f5f3b122..e0fc7765 100644 --- a/nzgmdb/data_processing/process_observed.py +++ b/nzgmdb/data_processing/process_observed.py @@ -9,12 +9,9 @@ import numpy as np import pandas as pd from obspy import read_inventory -from obspy.clients.fdsn import Client as FDSN_Client -from obspy.core.inventory import Inventory import qcore.timeseries as ts from nzgmdb.data_processing import waveform_manipulation -from nzgmdb.management import config as cfg from nzgmdb.management import custom_errors, file_structure from nzgmdb.mseed_management import reading diff --git a/tests/test_sites.py b/tests/test_sites.py index cd03d5aa..b175eb04 100644 --- a/tests/test_sites.py +++ b/tests/test_sites.py @@ -1,4 +1,7 @@ +from collections.abc import Iterator from pathlib import Path +from types import TracebackType +from typing import Any, Self import numpy as np import pandas as pd @@ -9,6 +12,39 @@ from nzgmdb.data_retrieval import sites +class _DummyFionaCollection: + """ + Minimal context manager that mimics a Fiona Collection. + + Parameters + ---------- + shapes : list + Iterable of shape-like records to yield when iterated. + + Returns + ------- + _DummyFionaCollection + Context manager instance that can be iterated over. + """ + + def __init__(self, shapes: list[Any]) -> None: # noqa: D107 + self._shapes = shapes + + def __enter__(self) -> Self: # noqa: D105 + return self + + def __exit__( # noqa: D105 + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> bool: + return False + + def __iter__(self) -> Iterator[Any]: # noqa: D105 + return iter(self._shapes) + + def _make_test_geotiff( width: int = 10, height: int = 10, @@ -193,7 +229,11 @@ def get_stations(self, *args, **kwargs): monkeypatch.setattr(sites, "FDSN_Client", _DummyClient) - monkeypatch.setattr(sites.fiona, "open", lambda *_a, **_k: []) + monkeypatch.setattr( + sites.fiona, + "open", + lambda *_a, **_k: _DummyFionaCollection([]), + ) # Do NOT monkeypatch NZGMDB_DATA.abspath (read-only property). # Instead, patch fetch() to return a temp file path for the tif and anything else requested. From fea97401b6d870aa9464aceea40c5959ce62df92 Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 26 Jan 2026 11:45:27 +1300 Subject: [PATCH 18/72] net providers --- nzgmdb/config/config.yaml | 35 +++++++++++++++++++++++++++++++--- nzgmdb/data_retrieval/sites.py | 30 +++++++++-------------------- 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/nzgmdb/config/config.yaml b/nzgmdb/config/config.yaml index 9795bff5..f8ee0b9d 100644 --- a/nzgmdb/config/config.yaml +++ b/nzgmdb/config/config.yaml @@ -29,10 +29,39 @@ percentage_gap_allowed: 0.1 is_large_overlap: 0.5 # Provider / Network Filters main_providers_networks: - - GEONET: - - NZ + GEONET: + - IU + - NZ tmp_array_providers_networks: - - IRIS + IRIS: + - 1U + - 2B + - 2L + - 2P + - 3C + - 4A + - 6F + - 6K + - 7D + - 7S + - 9F + - 9G + - IU + - NZ + - QC + - X2 + - XB + - XH + - XQ + - Y3 + - YA + - YO + - YR + - Z1 + - Z8 + - ZP + - ZT + - ZX # Mseed Variables vs30: 500 pre_event_time_difference: 15 diff --git a/nzgmdb/data_retrieval/sites.py b/nzgmdb/data_retrieval/sites.py index 2751a8d0..ac5c06dd 100644 --- a/nzgmdb/data_retrieval/sites.py +++ b/nzgmdb/data_retrieval/sites.py @@ -202,10 +202,17 @@ def create_site_table_response(add_tmp_arrays: bool = False) -> pd.DataFrame: """ # Fetch the client station information config = cfg.Config() + bbox = config.get_value("bbox") # [min_lon, min_lat, max_lon, max_lat] + min_lon, min_lat, max_lon, max_lat = bbox + max_lon = 180 # Due to issues with FDSN of passing barrier (no land past this point for sites that are of interest) channel_codes = config.get_value("channel_codes") - for provider in ["GEONET"]: + provider_networks = config.get_value("main_providers_networks") + if add_tmp_arrays: + provider_networks.update(config.get_value("tmp_array_providers_networks")) + for provider, networks in provider_networks.items(): client_NZ = FDSN_Client(provider) - inventory = client_NZ.get_stations(channel=channel_codes, level="response") + networks = ",".join(networks) + inventory = client_NZ.get_stations(network=networks, channel=channel_codes, level="response", maxlatitude=max_lat, minlatitude=min_lat, maxlongitude=max_lon, minlongitude=min_lon) station_info = [ [ provider, @@ -249,25 +256,6 @@ def create_site_table_response(add_tmp_arrays: bool = False) -> pd.DataFrame: ["provider", "net", "sta", "chan", "loc", "loc_elev"] ).reset_index(drop=True) - bbox = config.get_value("bbox") # [min_lon, min_lat, max_lon, max_lat] - min_lon, min_lat, max_lon, max_lat = bbox - - # Ensure lat/lon are present and within latitude bounds - mask_lat = ( - all_info_df["lat"].notna() - & all_info_df["lon"].notna() - & (all_info_df["lat"] >= min_lat) - & (all_info_df["lat"] <= max_lat) - ) - - # Handle antimeridian crossing: if min_lon > max_lon use OR - if min_lon <= max_lon: - mask_lon = (all_info_df["lon"] >= min_lon) & (all_info_df["lon"] <= max_lon) - else: - mask_lon = (all_info_df["lon"] >= min_lon) | (all_info_df["lon"] <= max_lon) - - all_info_df = all_info_df.loc[mask_lat & mask_lon] - # Get the Geonet metadata summary information geo_meta_summary_df = pd.read_csv( NZGMDB_DATA.fetch("Geonet_Metadata_Summary_v1.4.csv") From 8441f3ef5cadfd227524df1ae8da7b211851b959 Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 26 Jan 2026 16:15:16 +1300 Subject: [PATCH 19/72] inv fetch --- nzgmdb/data_retrieval/geonet.py | 13 +- nzgmdb/data_retrieval/inventory_xml.py | 207 ++++++++++++++++++++++--- nzgmdb/data_retrieval/sites.py | 59 +------ nzgmdb/management/file_structure.py | 2 + nzgmdb/scripts/run_nzgmdb.py | 15 +- 5 files changed, 215 insertions(+), 81 deletions(-) diff --git a/nzgmdb/data_retrieval/geonet.py b/nzgmdb/data_retrieval/geonet.py index 5714b514..2562f308 100644 --- a/nzgmdb/data_retrieval/geonet.py +++ b/nzgmdb/data_retrieval/geonet.py @@ -21,6 +21,7 @@ from pandas.errors import EmptyDataError from scipy.interpolate import interp1d +from nzgmdb.data_retrieval import inventory_xml from nzgmdb.management import config as cfg from nzgmdb.management import custom_errors, file_structure from nzgmdb.management.data_registry import NZGMDB_DATA @@ -731,6 +732,7 @@ def parse_geonet_information( only_record_ids_ffp: Path = None, real_time: bool = False, mp_sites: bool = False, + add_tmp_arrays: bool = False, ): """ Read the geonet information and manage the fetching of more data to create the mseed files @@ -757,6 +759,8 @@ def parse_geonet_information( If the function is being used in real time use a different client, default is False mp_sites : bool (optional) Whether to multiprocess over sites (when not using mp over events) + add_tmp_arrays : bool (optional) + Whether to add temporary array stations to the inventory, default is False """ if only_record_ids_ffp: # Read the only record ids file @@ -781,13 +785,16 @@ def parse_geonet_information( only_record_ids = None config = cfg.Config() - channel_codes = config.get_value("channel_codes") if real_time: + inventory = inventory_xml.get_provider_inventory( + real_time=True, level="station" + ) client_NZ = FDSN_Client(base_url=config.get_value("real_time_url")) else: - # Get Station Information from geonet clients + inventory = inventory_xml.get_full_inventory( + add_tmp_arrays=add_tmp_arrays, level="station" + ) client_NZ = FDSN_Client("GEONET") - inventory = client_NZ.get_stations(channel=channel_codes, level="station") # Get the rrup data mw_rrup_data = np.loadtxt(NZGMDB_DATA.fetch("Mw_rrup.txt")) diff --git a/nzgmdb/data_retrieval/inventory_xml.py b/nzgmdb/data_retrieval/inventory_xml.py index 494488e4..f5b613e4 100644 --- a/nzgmdb/data_retrieval/inventory_xml.py +++ b/nzgmdb/data_retrieval/inventory_xml.py @@ -5,35 +5,202 @@ import datetime from pathlib import Path +import pandas as pd from obspy.clients.fdsn import Client as FDSN_Client from obspy.clients.fdsn.header import FDSNNoDataException +from nzgmdb.management import config as cfg from nzgmdb.management import file_structure -def fetch_inventory( +def get_provider_inventory( + provider: str = None, + networks: list[str] = None, + channel_codes: str | None = None, + stations: str = "*", + level: str = "response", + starttime: str = "2000-01-01", + endtime: str = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d"), + real_time: bool = False, +): + """ + Fetch inventory from a specified FDSN provider within configured bounding box. + + Parameters + ---------- + provider : str, optional + FDSN provider base URL, required if `real_time` is False. + networks : list[str], optional + List of network codes to fetch. If None, fetches all networks. + channel_codes : str or None, optional + Channel codes filter. If None, uses config default. + stations : str, optional + Station selector passed to FDSN, by default "*". + level : str, optional + StationXML detail level to request, by default "response". + starttime : str, optional + Start date (YYYY-MM-DD), by default "2000-01-01". + endtime : str, optional + End date (YYYY-MM-DD), by default today. + real_time : bool, optional + Whether to use real-time data source from config, by default False. + """ + config = cfg.Config() + channel_codes = ( + config.get_value("channel_codes") if channel_codes is None else channel_codes + ) + bbox = config.get_value("bbox") # [min_lon, min_lat, max_lon, max_lat] + min_lon, min_lat, max_lon, max_lat = bbox + max_lon = 180 # Due to issues with FDSN of passing barrier (no land past this point for sites that are of interest) + if real_time: + client = FDSN_Client(base_url=config.get_value("real_time_url")) + # Adjust the start time to be more recent for real-time data to improve speed + starttime = datetime.datetime.strftime( + datetime.datetime.now() - datetime.timedelta(days=14), "%Y-%m-%d" + ) + else: + if provider is None: + raise ValueError("Provider must be specified if not using real-time data.") + client = FDSN_Client(provider) + networks = "*" if networks is None else ",".join(networks) + return client.get_stations( + network=networks, + station=stations, + channel=channel_codes, + level=level, + maxlatitude=max_lat, + minlatitude=min_lat, + maxlongitude=max_lon, + minlongitude=min_lon, + starttime=starttime, + endtime=endtime, + ) + + +def get_full_inventory( add_tmp_arrays: bool = False, level: str = "response", channel_codes: str | None = None, + stations: str = "*", starttime: str = "2000-01-01", endtime: str = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d"), + return_df: bool = False, ): - pass - - -""" -Way of reading all site information from all providers / listed networks -geonet - all -iris - limited -bounding box for all sites to be on land in nz - -station level -response level - - -a specific site channel location extraction - need the provider / net in the function call (maybe) + """ + Fetch inventories from all configured providers and optionally return station/channel metadata. -""" + Parameters + ---------- + add_tmp_arrays : bool, optional + Whether to include temporary array providers, by default False. + level : str, optional + StationXML detail level to request, by default "response". + channel_codes : str or None, optional + Channel codes filter. If None, uses config default. + stations : str, optional + Station selector passed to FDSN, by default "*". + starttime : str, optional + Start date (YYYY-MM-DD), by default "2000-01-01". + endtime : str, optional + End date (YYYY-MM-DD), by default today. + return_df : bool, optional + If True, return a DataFrame of station/channel info; otherwise return merged Inventory. + + Returns + ------- + pandas.DataFrame or obspy.core.inventory.inventory.Inventory + DataFrame when `return_df=True`, else the merged ObsPy Inventory. + """ + config = cfg.Config() + provider_networks = config.get_value("main_providers_networks") + if add_tmp_arrays: + provider_networks.update(config.get_value("tmp_array_providers_networks")) + return_inv = None + info_dfs = [] + for provider, networks in provider_networks.items(): + inventory = get_provider_inventory( + provider=provider, + networks=networks, + stations=stations, + channel_codes=channel_codes, + level=level, + starttime=starttime, + endtime=endtime, + ) + if return_inv is None: + return_inv = inventory + else: + return_inv += inventory + + if return_df: + station_info = [ + [ + provider, + network.code, + station.code, + station.latitude, + station.longitude, + station.elevation, + station.creation_date, + station.end_date, + channel.code[:2], + channel.location_code, + channel.depth, + channel.start_date, + channel.end_date, + ] + for network in inventory + for station in network + for channel in station.channels + ] + + info_dfs.append( + pd.DataFrame( + station_info, + columns=[ + "provider", + "net", + "sta", + "lat", + "lon", + "elev", + "creation_date", + "end_date", + "chan", + "loc", + "loc_elev", + "start_time", + "end_time", + ], + ) + ) + + if return_df: + if not info_dfs: + return pd.DataFrame( + columns=[ + "provider", + "net", + "sta", + "lat", + "lon", + "elev", + "creation_date", + "end_date", + "chan", + "loc", + "loc_elev", + "start_time", + "end_time", + ] + ) + + all_info_df = pd.concat(info_dfs, ignore_index=True) + return all_info_df.drop_duplicates( + ["provider", "net", "sta", "chan", "loc", "loc_elev"] + ).reset_index(drop=True) + + return all_info_df if return_df else return_inv def fetch_and_save_inventory( @@ -56,20 +223,16 @@ def fetch_and_save_inventory( endtime : str, optional The end time for the inventory data, by default the current date. """ - client = FDSN_Client("GEONET") - xml_dir = file_structure.get_stationxml_dir(main_dir) xml_dir.mkdir(parents=True, exist_ok=True) all_stations = ",".join(stations) try: - inv = client.get_stations( - network="NZ", - station=all_stations, + inv = get_full_inventory( + stations=all_stations, starttime=starttime, endtime=endtime, - level="response", ) for sta in stations: sel = inv.select(station=sta) diff --git a/nzgmdb/data_retrieval/sites.py b/nzgmdb/data_retrieval/sites.py index ac5c06dd..f1b5c77e 100644 --- a/nzgmdb/data_retrieval/sites.py +++ b/nzgmdb/data_retrieval/sites.py @@ -9,11 +9,10 @@ import numpy as np import pandas as pd import rasterio -from obspy.clients.fdsn import Client as FDSN_Client from pyproj import Transformer from scipy.spatial import cKDTree -from nzgmdb.data_retrieval import tect_domain +from nzgmdb.data_retrieval import tect_domain, inventory_xml from nzgmdb.management import config as cfg from nzgmdb.management.data_registry import NZGMDB_DATA from qcore import point_in_polygon @@ -202,59 +201,9 @@ def create_site_table_response(add_tmp_arrays: bool = False) -> pd.DataFrame: """ # Fetch the client station information config = cfg.Config() - bbox = config.get_value("bbox") # [min_lon, min_lat, max_lon, max_lat] - min_lon, min_lat, max_lon, max_lat = bbox - max_lon = 180 # Due to issues with FDSN of passing barrier (no land past this point for sites that are of interest) - channel_codes = config.get_value("channel_codes") - provider_networks = config.get_value("main_providers_networks") - if add_tmp_arrays: - provider_networks.update(config.get_value("tmp_array_providers_networks")) - for provider, networks in provider_networks.items(): - client_NZ = FDSN_Client(provider) - networks = ",".join(networks) - inventory = client_NZ.get_stations(network=networks, channel=channel_codes, level="response", maxlatitude=max_lat, minlatitude=min_lat, maxlongitude=max_lon, minlongitude=min_lon) - station_info = [ - [ - provider, - network.code, - station.code, - station.latitude, - station.longitude, - station.elevation, - station.creation_date, - station.end_date, - channel.code[:2], - channel.location_code, - channel.depth, - channel.start_date, - channel.end_date, - ] - for network in inventory - for station in network - for channel in station.channels - ] - all_info_df = pd.DataFrame( - station_info, - columns=[ - "provider", - "net", - "sta", - "lat", - "lon", - "elev", - "creation_date", - "end_date", - "chan", - "loc", - "loc_elev", - "start_time", - "end_time", - ], - ) - - all_info_df = all_info_df.drop_duplicates( - ["provider", "net", "sta", "chan", "loc", "loc_elev"] - ).reset_index(drop=True) + all_info_df = inventory_xml.get_full_inventory( + add_tmp_arrays=add_tmp_arrays, return_df=True + ) # Get the Geonet metadata summary information geo_meta_summary_df = pd.read_csv( diff --git a/nzgmdb/management/file_structure.py b/nzgmdb/management/file_structure.py index 01d686c4..f4d28b6f 100644 --- a/nzgmdb/management/file_structure.py +++ b/nzgmdb/management/file_structure.py @@ -23,6 +23,7 @@ class PreFlatfileNames(StrEnum): EARTHQUAKE_SOURCE_GEOMETRY = "earthquake_source_geometry_all.csv" PHASE_ARRIVAL_TABLE = "phase_arrival_table_all.csv" SITE_TABLE = "site_table_all.csv" + STATION_TABLE = "station_table_all.csv" PROPAGATION_TABLE = "propagation_path_table_all.csv" GROUND_MOTION_IM_CATALOGUE = "ground_motion_im_catalogue.csv" PROB_SERIES = "prob_series.h5" @@ -39,6 +40,7 @@ class FlatfileNames(StrEnum): STATION_EXTRACTION_TABLE = "station_extraction_table.csv" MULTI_EVENT_TABLE = "multi_event_table.csv" SITE_TABLE = "site_table.csv" + STATION_TABLE = "station_table.csv" PHASE_ARRIVAL_TABLE = "phase_arrival_table.csv" PROPAGATION_TABLE = "propagation_path_table.csv" GMC_PREDICTIONS = "gmc_predictions.csv" diff --git a/nzgmdb/scripts/run_nzgmdb.py b/nzgmdb/scripts/run_nzgmdb.py index 1ab34d00..aa0b6ed9 100644 --- a/nzgmdb/scripts/run_nzgmdb.py +++ b/nzgmdb/scripts/run_nzgmdb.py @@ -39,6 +39,12 @@ def fetch_geonet_data( ] = None, real_time: Annotated[bool, typer.Option()] = False, mp_sites: Annotated[bool, typer.Option()] = False, + add_tmp_arrays: Annotated[ + bool, + typer.Option( + is_flag=True, + ), + ] = False, ): """ Fetch earthquake data from Geonet and generate the earthquake source and station magnitude tables. @@ -65,6 +71,8 @@ def fetch_geonet_data( If True, the function will run in real-time mode by using a different client (default is False). mp_sites : bool, optional If True, the function will use multiprocessing over sites instead of events (default is False). + add_tmp_arrays : bool, optional + If True, temporary arrays will be added to the database run (default is False). """ geonet.parse_geonet_information( main_dir, @@ -77,6 +85,7 @@ def fetch_geonet_data( only_record_ids_ffp, real_time, mp_sites, + add_tmp_arrays, ) @@ -568,12 +577,15 @@ def generate_site_table_basin( flatfile_dir = file_structure.get_flatfile_dir(main_dir) flatfile_dir.mkdir(parents=True, exist_ok=True) - site_df = sites.create_site_table_response(add_tmp_arrays) + site_df, station_df = sites.create_site_table_response(add_tmp_arrays) site_df = sites.add_site_basins(site_df, nzcvm_data_ffp) site_df.to_csv( flatfile_dir / file_structure.PreFlatfileNames.SITE_TABLE, index=False ) + station_df.to_csv( + flatfile_dir / file_structure.PreFlatfileNames.STATION_TABLE, index=False + ) @cli.from_docstring(app) @@ -976,6 +988,7 @@ def run_full_nzgmdb( only_sites, only_record_ids_ffp, real_time, + add_tmp_arrays, ) # Extract Waveforms From e068850d75114243afc0ecb6d07eeb604472ee3f Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 4 Feb 2026 11:11:44 +1300 Subject: [PATCH 20/72] use site table --- nzgmdb/calculation/distances.py | 40 +++++---------- nzgmdb/calculation/snr.py | 2 +- nzgmdb/config/config.yaml | 60 +++++++++++----------- nzgmdb/data_processing/merge_flatfiles.py | 51 ++++-------------- nzgmdb/data_processing/process_observed.py | 2 +- nzgmdb/data_retrieval/inventory_xml.py | 4 +- 6 files changed, 56 insertions(+), 103 deletions(-) diff --git a/nzgmdb/calculation/distances.py b/nzgmdb/calculation/distances.py index 762b740f..74d107ad 100644 --- a/nzgmdb/calculation/distances.py +++ b/nzgmdb/calculation/distances.py @@ -614,7 +614,7 @@ def get_nodal_plane_info( def compute_distances_for_event( event_row: pd.Series, im_df: pd.DataFrame, - station_df: pd.DataFrame, + site_df: pd.DataFrame, cmt_df: pd.DataFrame, domain_focal_df: pd.DataFrame, taupo_polygon: Polygon, @@ -633,8 +633,8 @@ def compute_distances_for_event( The event row from the earthquake source table im_df : pd.DataFrame The full IM data from the catalogue - station_df : pd.DataFrame - The full station data + site_df : pd.DataFrame + The full site data cmt_df : pd.DataFrame The Centroid Moment Tensor data domain_focal_df : pd.DataFrame @@ -671,8 +671,8 @@ def compute_distances_for_event( if im_event_df.empty: return None, None, None - # Get the station data - event_sta_df = station_df[station_df["sta"].isin(im_event_df["sta"])].reset_index() + # Get the site data + event_sta_df = site_df[site_df["sta"].isin(im_event_df["sta"])].reset_index() stations = event_sta_df[["lon", "lat", "depth"]].to_numpy() # Get the nodal plane information @@ -1270,38 +1270,24 @@ def calc_distances(main_dir: Path, n_procs: int = 1): usecols=["evid", "sta"], ) - # Get the station information - client_NZ = FDSN_Client("GEONET") - channel_codes = config.get_value("channel_codes") - inventory = client_NZ.get_stations(channel=channel_codes, level="station") - station_info = [] - for network in inventory: - for station in network: - station_info.append( - [ - network.code, - station.code, - station.latitude, - station.longitude, - station.elevation, - ] - ) - station_df = pd.DataFrame( - station_info, columns=["net", "sta", "lat", "lon", "elev"] + # Get the site information + site_df = pd.read_csv( + flatfile_dir / file_structure.PreFlatfileNames.SITE_TABLE, + dtype={"sta": str}, ) - station_df = station_df.drop_duplicates().reset_index(drop=True) + site_df = site_df.loc[:, ["sta", "lat", "lon", "elev"]] # Select unique stations from IM data and merge im_station_df = im_df[["sta"]].drop_duplicates() - station_df = pd.merge(im_station_df, station_df, on="sta", how="left") - station_df["depth"] = station_df["elev"] / -1000 + site_df = pd.merge(im_station_df, site_df, on="sta", how="left") + site_df["depth"] = site_df["elev"] / -1000 with mp.Pool(n_procs) as p: result_dfs = p.map( functools.partial( compute_distances_for_event, im_df=im_df, - station_df=station_df, + site_df=site_df, cmt_df=cmt_df, domain_focal_df=domain_focal_df, taupo_polygon=taupo_polygon, diff --git a/nzgmdb/calculation/snr.py b/nzgmdb/calculation/snr.py index faea1ee2..9441745c 100644 --- a/nzgmdb/calculation/snr.py +++ b/nzgmdb/calculation/snr.py @@ -66,7 +66,7 @@ def compute_snr_for_single_mseed( inventory = None if xml_dir: # Load the inventory information - inventory_file = xml_dir / f"NZ.{station}.xml" + inventory_file = xml_dir / f"{station}.xml" if inventory_file.is_file(): inventory = read_inventory(inventory_file) diff --git a/nzgmdb/config/config.yaml b/nzgmdb/config/config.yaml index f8ee0b9d..7f54e993 100644 --- a/nzgmdb/config/config.yaml +++ b/nzgmdb/config/config.yaml @@ -30,38 +30,38 @@ is_large_overlap: 0.5 # Provider / Network Filters main_providers_networks: GEONET: - - IU - - NZ + - "IU" + - "NZ" tmp_array_providers_networks: IRIS: - - 1U - - 2B - - 2L - - 2P - - 3C - - 4A - - 6F - - 6K - - 7D - - 7S - - 9F - - 9G - - IU - - NZ - - QC - - X2 - - XB - - XH - - XQ - - Y3 - - YA - - YO - - YR - - Z1 - - Z8 - - ZP - - ZT - - ZX + - "1U" + - "2B" + - "2L" + - "2P" + - "3C" + - "4A" + - "6F" + - "6K" + - "7D" + - "7S" + - "9F" + - "9G" + - "IU" + - "NZ" + - "QC" + - "X2" + - "XB" + - "XH" + - "XQ" + - "Y3" + - "YA" + - "YO" + - "YR" + - "Z1" + - "Z8" + - "ZP" + - "ZT" + - "ZX" # Mseed Variables vs30: 500 pre_event_time_difference: 15 diff --git a/nzgmdb/data_processing/merge_flatfiles.py b/nzgmdb/data_processing/merge_flatfiles.py index d6609b3b..e5bb4068 100644 --- a/nzgmdb/data_processing/merge_flatfiles.py +++ b/nzgmdb/data_processing/merge_flatfiles.py @@ -154,13 +154,16 @@ def merge_im_data( def add_ground_level( + station_df: pd.DataFrame, gm_im_df_flat: pd.DataFrame, ): """ - Add in the ground level location elevation information to the gm_im_df_flat dataframe + Add in the is ground level location elevation information to the gm_im_df_flat dataframe Parameters ---------- + station_df : pd.DataFrame + The station dataframe containing the station information such as loc_elev gm_im_df_flat : pd.DataFrame The ground motion IM dataframe to add the ground level information to @@ -169,46 +172,6 @@ def add_ground_level( pd.DataFrame The ground motion IM dataframe with the ground level information added """ - # Find the station location information with the inventory lat, lon and elev - config = cfg.Config() - channel_codes = config.get_value("channel_codes") - client_NZ = FDSN_Client("GEONET") - inventory = client_NZ.get_stations(channel=channel_codes, level="response") - station_info = [ - [ - station.code, - station.latitude, - station.longitude, - station.elevation, - channel.code[:2], - channel.location_code, - channel.depth, - channel.start_date, - channel.end_date, - ] - for network in inventory - for station in network - for channel in station.channels - ] - station_df = pd.DataFrame( - station_info, - columns=[ - "sta", - "sta_lat", - "sta_lon", - "sta_elev", - "chan", - "loc", - "loc_elev", - "start_time", - "end_time", - ], - ) - # Remove duplicates - station_df = station_df.drop_duplicates( - ["sta", "chan", "loc", "loc_elev"] - ).reset_index(drop=True) - # Get the recorders information for location codes config = cfg.Config() locations_url = config.get_value("locations_url") @@ -442,6 +405,9 @@ def merge_flatfiles(main_dir: Path, bypass_records_ffp: Path = None): site_basin_df = pd.read_csv( flatfile_dir / file_structure.PreFlatfileNames.SITE_TABLE ) + station_df = pd.read_csv( + flatfile_dir / file_structure.PreFlatfileNames.STATION_TABLE + ) station_extraction_df = pd.read_csv( flatfile_dir / file_structure.PreFlatfileNames.STATION_EXTRACTION_TABLE_GEONET, dtype={"evid": str}, @@ -471,6 +437,7 @@ def merge_flatfiles(main_dir: Path, bypass_records_ffp: Path = None): # Ensure that the site_basin_df only has the unique sites found in the im_df unique_sites = im_df["sta"].unique() site_basin_df = site_basin_df[site_basin_df["sta"].isin(unique_sites)] + station_df = station_df[station_df["sta"].isin(unique_sites)] # Ensure the station magnitude table only has values of events and station pairs available in the im_df unique_pairs_df = im_df[["evid", "sta"]].drop_duplicates() @@ -578,7 +545,7 @@ def merge_flatfiles(main_dir: Path, bypass_records_ffp: Path = None): ) # Add in the ground level location elevation information - gm_im_df_flat = add_ground_level(gm_im_df_flat) + gm_im_df_flat = add_ground_level(station_df, gm_im_df_flat) # Add in multi_event information gm_im_df_flat = gm_im_df_flat.merge( diff --git a/nzgmdb/data_processing/process_observed.py b/nzgmdb/data_processing/process_observed.py index f5f3b122..31caaf90 100644 --- a/nzgmdb/data_processing/process_observed.py +++ b/nzgmdb/data_processing/process_observed.py @@ -75,7 +75,7 @@ def process_single_mseed( inventory = None if xml_dir: # Load the inventory information - inventory_file = xml_dir / f"NZ.{station}.xml" + inventory_file = xml_dir / f"{station}.xml" if inventory_file.is_file(): inventory = read_inventory(inventory_file) diff --git a/nzgmdb/data_retrieval/inventory_xml.py b/nzgmdb/data_retrieval/inventory_xml.py index f5b613e4..6e461f28 100644 --- a/nzgmdb/data_retrieval/inventory_xml.py +++ b/nzgmdb/data_retrieval/inventory_xml.py @@ -200,7 +200,7 @@ def get_full_inventory( ["provider", "net", "sta", "chan", "loc", "loc_elev"] ).reset_index(drop=True) - return all_info_df if return_df else return_inv + return return_inv def fetch_and_save_inventory( @@ -239,7 +239,7 @@ def fetch_and_save_inventory( if not sel.networks: print(f"Warning: No inventory data found for station {sta}. Skipping.") continue - fname = xml_dir / f"NZ.{sta}.xml" + fname = xml_dir / f"{sta}.xml" sel.write(fname, format="STATIONXML") except FDSNNoDataException: print("No inventory data found for the specified stations and time range.") From f148282dca4762cbbbe4d7cd62883a769e28b9be Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 4 Feb 2026 11:33:51 +1300 Subject: [PATCH 21/72] pr comments --- nzgmdb/data_retrieval/sites.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nzgmdb/data_retrieval/sites.py b/nzgmdb/data_retrieval/sites.py index 37f968ab..e525a091 100644 --- a/nzgmdb/data_retrieval/sites.py +++ b/nzgmdb/data_retrieval/sites.py @@ -23,9 +23,9 @@ def fill_gaps_with_nearest( coords: np.ndarray, values: np.ndarray, - invalid_mask: np.ndarray = None, + invalid_mask: np.ndarray | None = None, k: int = 8, -): +) -> np.ndarray: """ Fill NaN or invalid values using nearest-neighbour averaging. @@ -86,7 +86,7 @@ def sample_points_from_geotiff( file_path: Path, latlon_points: np.ndarray, band: int = 1, -): +) -> np.ndarray: """ Sample a GeoTIFF raster at given latitude/longitude points. @@ -345,8 +345,10 @@ def create_site_table_response() -> pd.DataFrame: tect_merged_df.loc[vs30_mask, "Vs30_Ref"] = "Foster et al. (2019)" tect_merged_df.loc[vs30_mask, "Q_Vs30"] = "Q3" - except (FileNotFoundError, ValueError, RuntimeError) as e: - print(f"Warning: Could not compute thresholds for missing Z1.0 values: {e}") + except (FileNotFoundError, ValueError, RuntimeError): + raise UserWarning( + "Could not compute thresholds for missing Z1.0 values, check correct setup for NZCVM" + ) # Select specific columns site_df = tect_merged_df[ From 09d2e770d7238351e2914ffeab1bdf1ec5665ca9 Mon Sep 17 00:00:00 2001 From: joelridden Date: Tue, 10 Feb 2026 11:22:44 +1300 Subject: [PATCH 22/72] run working --- nzgmdb/calculation/distances.py | 4 +- nzgmdb/data_processing/merge_flatfiles.py | 120 +++++++++------------- nzgmdb/data_retrieval/sites.py | 31 ++++-- nzgmdb/management/shell_commands.py | 4 +- 4 files changed, 76 insertions(+), 83 deletions(-) diff --git a/nzgmdb/calculation/distances.py b/nzgmdb/calculation/distances.py index 1167f09b..24d343ce 100644 --- a/nzgmdb/calculation/distances.py +++ b/nzgmdb/calculation/distances.py @@ -496,6 +496,7 @@ def get_nodal_plane_info( hik_strike_rbf, hik_dip_rbf, hik_footprint = hik_objs puy_strike_rbf, puy_dip_rbf, puy_footprint = puy_objs domain_no_backup = event_row["domain_no_backup"] + nodal_plane_info["f_type"] = "domain" if event_row["tect_class"] == "Crustal": # First assume strike-slip to estimate length @@ -870,6 +871,7 @@ def compute_distances_for_event( [ { "evid": event_id, + "provider": station.provider, "net": station.net, "sta": station.sta, "r_epi": r_epis[station_index], @@ -1265,7 +1267,7 @@ def calc_distances(main_dir: Path, n_procs: int = 1): flatfile_dir / file_structure.PreFlatfileNames.SITE_TABLE, dtype={"sta": str}, ) - site_df = site_df.loc[:, ["sta", "lat", "lon", "elev"]] + site_df = site_df.loc[:, ["sta", "provider", "net", "lat", "lon", "elev"]] # Select unique stations from IM data and merge im_station_df = im_df[["sta"]].drop_duplicates() diff --git a/nzgmdb/data_processing/merge_flatfiles.py b/nzgmdb/data_processing/merge_flatfiles.py index e5bb4068..2446c16c 100644 --- a/nzgmdb/data_processing/merge_flatfiles.py +++ b/nzgmdb/data_processing/merge_flatfiles.py @@ -203,50 +203,56 @@ def add_ground_level( station_df["end_time"] = station_df["end_time"].fillna(pd.Timestamp.max) # Ensure datetime dtypes - def _to_py_datetime(val: object) -> object: - """ - Convert UTCDateTime to python datetime.datetime if needed. - - Parameters - ---------- - val : object - The value to convert. - - Returns - ------- - object - The converted value. - """ - if isinstance(val, UTCDateTime): - return val.datetime - return val - - station_df["start_time"] = station_df["start_time"].apply(_to_py_datetime) - station_df["end_time"] = station_df["end_time"].apply(_to_py_datetime) - - def ensure_utc(series: pd.Series) -> pd.Series: - """ - Ensure a pandas Series of datetimes is timezone-aware in UTC. - - Parameters - ---------- - series : pd.Series - The pandas Series to ensure is timezone-aware in UTC. - - Returns - ------- - pd.Series - The timezone-aware pandas Series in UTC. - """ - # coerce to datetime first, then ensure UTC tz (convert if already tz-aware, localize if naive) - s = pd.to_datetime(series, errors="coerce") - if pd.api.types.is_datetime64tz_dtype(s.dtype): - return s.dt.tz_convert("UTC") - return s.dt.tz_localize("UTC") + # def _to_py_datetime(val: object) -> object: + # """ + # Convert UTCDateTime to python datetime.datetime if needed. + # + # Parameters + # ---------- + # val : object + # The value to convert. + # + # Returns + # ------- + # object + # The converted value. + # """ + # if isinstance(val, UTCDateTime): + # return val.datetime + # return val + # + # station_df["start_time"] = station_df["start_time"].apply(_to_py_datetime) + # station_df["end_time"] = station_df["end_time"].apply(_to_py_datetime) + + # def ensure_utc(series: pd.Series) -> pd.Series: + # """ + # Ensure a pandas Series of datetimes is timezone-aware in UTC. + # + # Parameters + # ---------- + # series : pd.Series + # The pandas Series to ensure is timezone-aware in UTC. + # + # Returns + # ------- + # pd.Series + # The timezone-aware pandas Series in UTC. + # """ + # # coerce to datetime first, then ensure UTC tz (convert if already tz-aware, localize if naive) + # s = pd.to_datetime(series, errors="coerce") + # if pd.api.types.is_datetime64tz_dtype(s.dtype): + # return s.dt.tz_convert("UTC") + # return s.dt.tz_localize("UTC") # Normalize both frames to UTC before sorting / merge_asof - station_df["start_time"] = ensure_utc(station_df["start_time"]) - station_df["end_time"] = ensure_utc(station_df["end_time"]) + # station_df["start_time"] = ensure_utc(station_df["start_time"]) + # station_df["end_time"] = ensure_utc(station_df["end_time"]) + station_df["start_time"] = pd.to_datetime( + station_df["start_time"], errors="coerce", utc=True + ) + station_df["end_time"] = pd.to_datetime( + station_df["end_time"], errors="coerce", utc=True + ) station_df["start_time"] = pd.to_datetime(station_df["start_time"]) station_df["end_time"] = pd.to_datetime(station_df["end_time"]) @@ -317,33 +323,6 @@ def custom_idxmin(group: pd.DataFrame): ["is_ground_level", "loc_elev"], ] = [True, 0.0] - # remove duplicates of sta in the station_df - station_df = station_df.drop_duplicates(subset=["sta"]) - - # Merge missing station lat / lon / elev information into the gm_im_df_flat - gm_im_df_flat = gm_im_df_flat.merge( - station_df[["sta", "sta_lat", "sta_lon", "sta_elev"]], - on="sta", - how="left", - suffixes=("", "_new"), - ) - - # Find where sta_lat is nan and replace with the inventory's lat, lon and elev - gm_im_df_flat["sta_lat"] = gm_im_df_flat["sta_lat"].fillna( - gm_im_df_flat["sta_lat_new"] - ) - gm_im_df_flat["sta_lon"] = gm_im_df_flat["sta_lon"].fillna( - gm_im_df_flat["sta_lon_new"] - ) - gm_im_df_flat["sta_elev"] = gm_im_df_flat["sta_elev"].fillna( - gm_im_df_flat["sta_elev_new"] - ) - - # Drop the new columns - gm_im_df_flat = gm_im_df_flat.drop( - columns=["sta_lat_new", "sta_lon_new", "sta_elev_new"] - ) - return gm_im_df_flat @@ -813,6 +792,9 @@ def merge_flatfiles(main_dir: Path, bypass_records_ffp: Path = None): site_basin_df.to_csv( flatfile_dir / file_structure.FlatfileNames.SITE_TABLE, index=False ) + station_df.to_csv( + flatfile_dir / file_structure.FlatfileNames.STATION_TABLE, index=False + ) prop_df.to_csv( flatfile_dir / file_structure.FlatfileNames.PROPAGATION_TABLE, index=False ) diff --git a/nzgmdb/data_retrieval/sites.py b/nzgmdb/data_retrieval/sites.py index 61b23af1..1b5479a2 100644 --- a/nzgmdb/data_retrieval/sites.py +++ b/nzgmdb/data_retrieval/sites.py @@ -180,7 +180,9 @@ def sample_points_from_geotiff( return samples.reshape(-1, 1) -def create_site_table_response(add_tmp_arrays: bool = False) -> pd.DataFrame: +def create_site_table_response( + add_tmp_arrays: bool = False, +) -> tuple[pd.DataFrame, pd.DataFrame]: """ Create the site table for the NZGMDB. This function fetches the station information from the FDSN clients, and the Geonet metadata summary information. It then merges the two dataframes and determines the tectonic domain for each @@ -231,15 +233,19 @@ def create_site_table_response(add_tmp_arrays: bool = False) -> pd.DataFrame: } ) - merged_df = geo_meta_summary_df.merge( - all_info_df[["net", "sta", "lat", "lon", "elev", "creation_date", "end_date"]], + # separate into site and sta here to avoid merging issues exploding + site_df = all_info_df[ + ["provider", "net", "sta", "lat", "lon", "elev", "creation_date", "end_date"] + ] + # Remove duplicate stations (keep first occurrence) + site_df = site_df.drop_duplicates(subset=["provider", "net", "sta"]) + + merged_df = site_df.merge( + geo_meta_summary_df, on="sta", - how="outer", + how="left", ) - # Fill Lat, Lon, Elevation NaN values from all_info_df - merged_df["elev"] = merged_df["Elevation"].combine_first(merged_df["elev"]) - merged_df["lat"] = merged_df["Lat"].combine_first(merged_df["lat"]) - merged_df["lon"] = merged_df["Long"].combine_first(merged_df["lon"]) + # Specify the required files for fiona NZGMDB_DATA.fetch("nt_domains_kiran.shp") NZGMDB_DATA.fetch("nt_domains_kiran.dbf") @@ -320,6 +326,7 @@ def create_site_table_response(add_tmp_arrays: bool = False) -> pd.DataFrame: # Split into station and site dfs station_df = all_info_df.loc[ + :, [ "provider", "net", @@ -332,9 +339,11 @@ def create_site_table_response(add_tmp_arrays: bool = False) -> pd.DataFrame: "loc_elev", "start_time", "end_time", - ] + ], ] - site_df = tect_merged_df[ + + site_df = tect_merged_df.loc[ + :, [ "provider", "net", @@ -363,7 +372,7 @@ def create_site_table_response(add_tmp_arrays: bool = False) -> pd.DataFrame: "Q_Z2.5", "Z2.5_ref", "site_domain_no", - ] + ], ] site_df = site_df.astype({"Z2.5": float}) site_df.loc[:, "Z2.5"] /= 1000.0 diff --git a/nzgmdb/management/shell_commands.py b/nzgmdb/management/shell_commands.py index 679c6497..e7d752f8 100644 --- a/nzgmdb/management/shell_commands.py +++ b/nzgmdb/management/shell_commands.py @@ -31,11 +31,11 @@ def run_command( """ with open(log_file_path, "w") as log_file: # Create the command to source conda.sh, activate the environment, and execute the full command - command = f"source {env_sh} && {env_activate_command} && {command}" + bash_command = f"source {env_sh} && {env_activate_command} && {command}" env = os.environ.copy() try: subprocess.check_call( - command, + bash_command, stdout=log_file, stderr=log_file, shell=True, From 6570f83443585a5fafa47229aa3ae1952c5a7b3a Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 16 Feb 2026 13:28:07 +1300 Subject: [PATCH 23/72] remove clipnet filter --- nzgmdb/data_processing/filtering.py | 84 +++++++++++++++-------------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/nzgmdb/data_processing/filtering.py b/nzgmdb/data_processing/filtering.py index 79929b8b..a7a721ca 100644 --- a/nzgmdb/data_processing/filtering.py +++ b/nzgmdb/data_processing/filtering.py @@ -3,10 +3,11 @@ """ import numpy as np -from gmprocess.waveform_processing.clipping.clipping_ann import clipNet -from gmprocess.waveform_processing.clipping.histogram import Histogram -from gmprocess.waveform_processing.clipping.max_amp import MaxAmp -from gmprocess.waveform_processing.clipping.ping import Ping + +# from gmprocess.waveform_processing.clipping.clipping_ann import clipNet +# from gmprocess.waveform_processing.clipping.histogram import Histogram +# from gmprocess.waveform_processing.clipping.max_amp import MaxAmp +# from gmprocess.waveform_processing.clipping.ping import Ping from obspy import Stream from nzgmdb.management import config as cfg @@ -31,43 +32,44 @@ def get_clip_probability(event_mag: float, dist: float, mseed: Stream) -> float: The clip probability from ClipNet """ # Get the config values - config = cfg.Config() - mag_clip_low = config.get_value("mag_clip_low") - mag_clip_high = config.get_value("mag_clip_high") - dist_clip_low = config.get_value("dist_clip_low") - dist_clip_high = config.get_value("dist_clip_high") - - # Ensure numeric inputs - try: - event_mag = float(event_mag) - except (TypeError, ValueError) as exc: - raise TypeError(f"event_mag must be a number, got {event_mag!r}") from exc - - try: - dist = float(dist) - except (TypeError, ValueError) as exc: - raise TypeError(f"dist must be a number, got {dist!r}") from exc - - # Clip the event_mag and dist values - event_mag = np.clip(event_mag, mag_clip_low, mag_clip_high) - dist = np.clip(dist, dist_clip_low, dist_clip_high) - - # Get different methods for clipping - max_amp_method = MaxAmp(mseed) - hist_method = Histogram(mseed) - ping_method = Ping(mseed) - - # Define the inputs for the clipNet - inputs = [ - event_mag, - dist, - max_amp_method.is_clipped, - hist_method.is_clipped, - ping_method.is_clipped, - ] - # Get the clip probability - clip_nnet = clipNet() - return clip_nnet.evaluate(inputs)[0][0] + # config = cfg.Config() + # mag_clip_low = config.get_value("mag_clip_low") + # mag_clip_high = config.get_value("mag_clip_high") + # dist_clip_low = config.get_value("dist_clip_low") + # dist_clip_high = config.get_value("dist_clip_high") + # + # # Ensure numeric inputs + # try: + # event_mag = float(event_mag) + # except (TypeError, ValueError) as exc: + # raise TypeError(f"event_mag must be a number, got {event_mag!r}") from exc + # + # try: + # dist = float(dist) + # except (TypeError, ValueError) as exc: + # raise TypeError(f"dist must be a number, got {dist!r}") from exc + # + # # Clip the event_mag and dist values + # event_mag = np.clip(event_mag, mag_clip_low, mag_clip_high) + # dist = np.clip(dist, dist_clip_low, dist_clip_high) + # + # # Get different methods for clipping + # max_amp_method = MaxAmp(mseed) + # hist_method = Histogram(mseed) + # ping_method = Ping(mseed) + # + # # Define the inputs for the clipNet + # inputs = [ + # event_mag, + # dist, + # max_amp_method.is_clipped, + # hist_method.is_clipped, + # ping_method.is_clipped, + # ] + # # Get the clip probability + # clip_nnet = clipNet() + # return clip_nnet.evaluate(inputs)[0][0] + return 0.0 def get_jerk(mseed: Stream) -> bool: From 30efa79abf9917e3662dfa74ad4b52558c45b276 Mon Sep 17 00:00:00 2001 From: joelridden Date: Tue, 17 Feb 2026 10:36:30 +1300 Subject: [PATCH 24/72] rch update --- nzgmdb/config/machine_config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nzgmdb/config/machine_config.yaml b/nzgmdb/config/machine_config.yaml index d33466c8..f770d393 100644 --- a/nzgmdb/config/machine_config.yaml +++ b/nzgmdb/config/machine_config.yaml @@ -38,10 +38,10 @@ rch: geonet: 128 extraction: 32 tec_domain: 128 - phase_table: 128 - snr: 64 + phase_table: 16 + snr: 32 fmax: 128 - gmc: 64 + gmc: 16 process: 128 im: 64 distances: 128 From 8923ab56cf32e8f786061659c2742b0274f5d712 Mon Sep 17 00:00:00 2001 From: joelridden Date: Tue, 17 Feb 2026 10:45:26 +1300 Subject: [PATCH 25/72] rch less cores --- nzgmdb/config/machine_config.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nzgmdb/config/machine_config.yaml b/nzgmdb/config/machine_config.yaml index f770d393..22394b8b 100644 --- a/nzgmdb/config/machine_config.yaml +++ b/nzgmdb/config/machine_config.yaml @@ -35,14 +35,14 @@ hypocentre: distances: 44 upload: 44 rch: - geonet: 128 + geonet: 32 extraction: 32 - tec_domain: 128 - phase_table: 16 - snr: 32 - fmax: 128 - gmc: 16 - process: 128 - im: 64 - distances: 128 - upload: 128 \ No newline at end of file + tec_domain: 32 + phase_table: 8 + snr: 8 + fmax: 32 + gmc: 8 + process: 32 + im: 16 + distances: 32 + upload: 32 \ No newline at end of file From af131c38cc897682bdb7b193396b7d5a3719fed6 Mon Sep 17 00:00:00 2001 From: joelridden Date: Tue, 17 Feb 2026 14:24:10 +1300 Subject: [PATCH 26/72] fix snr --- nzgmdb/mseed_management/reading.py | 2 +- nzgmdb/scripts/run_nzgmdb.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/nzgmdb/mseed_management/reading.py b/nzgmdb/mseed_management/reading.py index 91c16992..7d36341a 100644 --- a/nzgmdb/mseed_management/reading.py +++ b/nzgmdb/mseed_management/reading.py @@ -130,7 +130,7 @@ def create_waveform_from_mseed( data = np.stack([tr.data for tr in mseed], axis=1) data = data.astype(np.float32) # Reshape the waveform to have the correct shape for the IM calculation - reshaped_waveform = data[np.newaxis, :, :] + reshaped_waveform = data.T[:, None, :] except ValueError: raise custom_errors.InvalidTraceLengthError( f"Error reading data from {mseed_file}" diff --git a/nzgmdb/scripts/run_nzgmdb.py b/nzgmdb/scripts/run_nzgmdb.py index aa0b6ed9..636cb19d 100644 --- a/nzgmdb/scripts/run_nzgmdb.py +++ b/nzgmdb/scripts/run_nzgmdb.py @@ -502,9 +502,17 @@ def run_im_calculation( ), ] = False, intensity_measures: Annotated[ - list[IM], + str | None, typer.Option( - callback=lambda x: [IM(i) for i in x[0].split(",")], + callback=lambda x: ( + None + if x is None or (isinstance(x, (list, tuple)) and not x) + else [ + IM(i.strip()) + for i in (x[0] if isinstance(x, (list, tuple)) else x).split(",") + if i.strip() + ] + ), ), ] = None, ): From 716472af31a26a0b4d27baf3067bdf12f46809fd Mon Sep 17 00:00:00 2001 From: joelridden Date: Tue, 17 Feb 2026 14:24:44 +1300 Subject: [PATCH 27/72] double cores --- nzgmdb/config/machine_config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/nzgmdb/config/machine_config.yaml b/nzgmdb/config/machine_config.yaml index 22394b8b..3c57fd3b 100644 --- a/nzgmdb/config/machine_config.yaml +++ b/nzgmdb/config/machine_config.yaml @@ -35,14 +35,14 @@ hypocentre: distances: 44 upload: 44 rch: - geonet: 32 - extraction: 32 - tec_domain: 32 - phase_table: 8 - snr: 8 - fmax: 32 - gmc: 8 - process: 32 - im: 16 - distances: 32 - upload: 32 \ No newline at end of file + geonet: 64 + extraction: 64 + tec_domain: 64 + phase_table: 16 + snr: 16 + fmax: 64 + gmc: 16 + process: 64 + im: 32 + distances: 64 + upload: 64 \ No newline at end of file From b0235a25d305659ef46f8577ba5337e0329e94e8 Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 18 Feb 2026 11:44:48 +1300 Subject: [PATCH 28/72] gmc update --- nzgmdb/scripts/run_gmc.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nzgmdb/scripts/run_gmc.py b/nzgmdb/scripts/run_gmc.py index 8b7b39f3..c8c96356 100644 --- a/nzgmdb/scripts/run_gmc.py +++ b/nzgmdb/scripts/run_gmc.py @@ -73,6 +73,7 @@ def process_batch( gmc_predict_activate: str, phase_arrival_table_ffp: Path, prob_series_ffp: Path, + xml_dir: Path, ): """ Process a single subfolder: extract features and run predictions. @@ -99,6 +100,8 @@ def process_batch( The full file path to the phase arrival table prob_series_ffp : Path The full file path to the prob_series hdf5 file. + xml_dir : Path + The directory containing the stationxml files. Raises ------ @@ -128,7 +131,7 @@ def process_batch( ) else: # Activate gmc environment and extract features for the subfolder - features_command = f"python {gmc_scripts_path}/extract_features.py {gmc_dir} {waveform_dir} mseed --ko_matrices_dir {ko_matrices_dir} --record_list_ffp {batch_txt} --phase_arrival_table {phase_arrival_table_ffp} --prob_series {prob_series_ffp}" + features_command = f"python {gmc_scripts_path}/extract_features.py {gmc_dir} {waveform_dir} mseed --ko_matrices_dir {ko_matrices_dir} --record_list_ffp {batch_txt} --phase_arrival_table {phase_arrival_table_ffp} --prob_series {prob_series_ffp} --xml_dir {xml_dir}" shell_commands.run_command( features_command, conda_sh, gmc_activate, log_file_path_features ) @@ -287,6 +290,7 @@ def run_gmc_processing( gmc_predict_activate=gmc_predict_activate, phase_arrival_table_ffp=phase_arrival_table_ffp, prob_series_ffp=prob_series_ffp, + xml_dir=file_structure.get_stationxml_dir(main_dir), ) # Use multiprocessing with starmap and the partial function From a280e23018f580be763a36dee16e96b37316584d Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 18 Feb 2026 13:24:52 +1300 Subject: [PATCH 29/72] gmc reduce procs --- nzgmdb/config/machine_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nzgmdb/config/machine_config.yaml b/nzgmdb/config/machine_config.yaml index 3c57fd3b..99be786d 100644 --- a/nzgmdb/config/machine_config.yaml +++ b/nzgmdb/config/machine_config.yaml @@ -41,7 +41,7 @@ rch: phase_table: 16 snr: 16 fmax: 64 - gmc: 16 + gmc: 4 process: 64 im: 32 distances: 64 From 3342a82cf743b87f4dd2928d1f52bf8928ced63c Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 18 Feb 2026 14:53:32 +1300 Subject: [PATCH 30/72] increase n_procs --- nzgmdb/config/machine_config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/nzgmdb/config/machine_config.yaml b/nzgmdb/config/machine_config.yaml index 99be786d..a0f8433a 100644 --- a/nzgmdb/config/machine_config.yaml +++ b/nzgmdb/config/machine_config.yaml @@ -35,14 +35,14 @@ hypocentre: distances: 44 upload: 44 rch: - geonet: 64 - extraction: 64 - tec_domain: 64 - phase_table: 16 - snr: 16 - fmax: 64 - gmc: 4 - process: 64 - im: 32 - distances: 64 - upload: 64 \ No newline at end of file + geonet: 128 + extraction: 128 + tec_domain: 128 + phase_table: 32 + snr: 32 + fmax: 128 + gmc: 32 + process: 128 + im: 64 + distances: 128 + upload: 128 \ No newline at end of file From e09dd658a6f916fbad134db344a5bb792d76e543 Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 18 Feb 2026 15:11:10 +1300 Subject: [PATCH 31/72] reduce n_procs --- nzgmdb/config/machine_config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/nzgmdb/config/machine_config.yaml b/nzgmdb/config/machine_config.yaml index a0f8433a..3c57fd3b 100644 --- a/nzgmdb/config/machine_config.yaml +++ b/nzgmdb/config/machine_config.yaml @@ -35,14 +35,14 @@ hypocentre: distances: 44 upload: 44 rch: - geonet: 128 - extraction: 128 - tec_domain: 128 - phase_table: 32 - snr: 32 - fmax: 128 - gmc: 32 - process: 128 - im: 64 - distances: 128 - upload: 128 \ No newline at end of file + geonet: 64 + extraction: 64 + tec_domain: 64 + phase_table: 16 + snr: 16 + fmax: 64 + gmc: 16 + process: 64 + im: 32 + distances: 64 + upload: 64 \ No newline at end of file From 5a801ad1b8de7822d331d2365683c36238a72387 Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 20 Feb 2026 14:26:03 +1300 Subject: [PATCH 32/72] none check --- nzgmdb/data_retrieval/geonet.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/nzgmdb/data_retrieval/geonet.py b/nzgmdb/data_retrieval/geonet.py index 2562f308..474f2175 100644 --- a/nzgmdb/data_retrieval/geonet.py +++ b/nzgmdb/data_retrieval/geonet.py @@ -106,21 +106,17 @@ def fetch_event_line(event_cat: Event, event_id: str): if preferred_origin.earth_model_id is None else str(preferred_origin.earth_model_id).split("/")[1] ) - ev_ndef = ( - None - if preferred_origin.quality.used_phase_count is None - else preferred_origin.quality.used_phase_count - ) - ev_nsta = ( - None - if preferred_origin.quality.used_station_count is None - else preferred_origin.quality.used_station_count - ) - std = ( - None - if preferred_origin.quality.standard_error is None - else preferred_origin.quality.standard_error - ) + quality = preferred_origin.quality + if quality is None: + ev_ndef = None + ev_nsta = None + std = None + else: + ev_ndef = None if quality.used_phase_count is None else quality.used_phase_count + ev_nsta = ( + None if quality.used_station_count is None else quality.used_station_count + ) + std = None if quality.standard_error is None else quality.standard_error pref_mag_type = preferred_magnitude.magnitude_type From 0c0f14f118025c9a84592601c39b5fdbfc2417de Mon Sep 17 00:00:00 2001 From: joelridden Date: Sun, 22 Feb 2026 19:36:31 +1300 Subject: [PATCH 33/72] pref mag issue --- nzgmdb/data_retrieval/geonet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nzgmdb/data_retrieval/geonet.py b/nzgmdb/data_retrieval/geonet.py index 474f2175..518e7fa7 100644 --- a/nzgmdb/data_retrieval/geonet.py +++ b/nzgmdb/data_retrieval/geonet.py @@ -87,8 +87,8 @@ def fetch_event_line(event_cat: Event, event_id: str): preferred_origin = event_cat.preferred_origin() preferred_magnitude = event_cat.preferred_magnitude() - # If the preferred origin is None, return None - if preferred_origin is None: + # If the preferred origin or magnitude is None, return None + if preferred_origin is None or preferred_magnitude is None: return None # Extract basic info from the catalogue From 07489bd88eed7061e015d6e60d82daa59e5c262c Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 23 Feb 2026 15:39:53 +1300 Subject: [PATCH 34/72] correct provider --- nzgmdb/config/config.yaml | 2 -- nzgmdb/data_retrieval/geonet.py | 23 ++++++++++++++++++++++- nzgmdb/scripts/run_nzgmdb.py | 2 +- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/nzgmdb/config/config.yaml b/nzgmdb/config/config.yaml index 7f54e993..6d7ecf4c 100644 --- a/nzgmdb/config/config.yaml +++ b/nzgmdb/config/config.yaml @@ -46,8 +46,6 @@ tmp_array_providers_networks: - "7S" - "9F" - "9G" - - "IU" - - "NZ" - "QC" - "X2" - "XB" diff --git a/nzgmdb/data_retrieval/geonet.py b/nzgmdb/data_retrieval/geonet.py index 518e7fa7..99574cba 100644 --- a/nzgmdb/data_retrieval/geonet.py +++ b/nzgmdb/data_retrieval/geonet.py @@ -286,6 +286,27 @@ def fetch_sta_extraction( """ config = cfg.Config() + # Get the provider and network codes + provider_networks = config.get_value("main_providers_networks") + provider_networks.update(config.get_value("tmp_array_providers_networks")) + + provider = next( + (prov for prov, nets in provider_networks.items() if network.code in nets), + None, + ) + + if provider is None: + print( + f"Warning: No provider found for network {network.code}. Skipping station {station.code}." + ) + skipped_reason = pd.DataFrame( + { + "record_id": [f"{event_id}_{station.code}"], + "skipped_reason": ["No provider found for network"], + } + ) + return pd.DataFrame(), skipped_reason + # Get the preferred_origin preferred_origin = event_cat.preferred_origin() ev_lat = preferred_origin.latitude @@ -356,7 +377,7 @@ def fetch_sta_extraction( # Create the station_extraction_table station_extraction_table = pd.DataFrame( { - "provider": ["GEONET"], + "provider": [provider], "net": [network.code], "sta": [station.code], "evid": [event_id], diff --git a/nzgmdb/scripts/run_nzgmdb.py b/nzgmdb/scripts/run_nzgmdb.py index 636cb19d..3b0afe3b 100644 --- a/nzgmdb/scripts/run_nzgmdb.py +++ b/nzgmdb/scripts/run_nzgmdb.py @@ -996,7 +996,7 @@ def run_full_nzgmdb( only_sites, only_record_ids_ffp, real_time, - add_tmp_arrays, + add_tmp_arrays=add_tmp_arrays, ) # Extract Waveforms From 9cdfefbc20e4778473a1dda2d83967a6a1e4e7fa Mon Sep 17 00:00:00 2001 From: Joel Ridden Date: Tue, 24 Feb 2026 16:49:08 +1300 Subject: [PATCH 35/72] if statement --- nzgmdb/data_retrieval/waveform_extraction.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nzgmdb/data_retrieval/waveform_extraction.py b/nzgmdb/data_retrieval/waveform_extraction.py index 295cc85e..2f8ee5c8 100644 --- a/nzgmdb/data_retrieval/waveform_extraction.py +++ b/nzgmdb/data_retrieval/waveform_extraction.py @@ -810,9 +810,13 @@ def extract_station_info( ) location = site_only_record_ids["record_id"].str.split("_").str[-1].values[0] - # Get the Stream - client = FDSN_Client("GEONET") - st = get_station_window(station_extraction_row, client, channel_codes, location) + if provider == "GEONET": + # Get the Stream + client = FDSN_Client("GEONET") + st = get_station_window(station_extraction_row, client, channel_codes, location) + else: + # Get the stream from the tmp array storage location + st = None # Check that data was found if st is None: From 244c4e35d95be5f53773a586ca4d4030fb55bcf4 Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 25 Feb 2026 09:40:14 +1300 Subject: [PATCH 36/72] multi-event inv fix --- nzgmdb/data_retrieval/waveform_extraction.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nzgmdb/data_retrieval/waveform_extraction.py b/nzgmdb/data_retrieval/waveform_extraction.py index 295cc85e..9fec5afd 100644 --- a/nzgmdb/data_retrieval/waveform_extraction.py +++ b/nzgmdb/data_retrieval/waveform_extraction.py @@ -15,7 +15,7 @@ import numpy as np import pandas as pd import scipy as sp -from obspy import Stream, Trace, UTCDateTime +from obspy import Stream, Trace, UTCDateTime, read_inventory from obspy.clients.fdsn import Client as FDSN_Client from obspy.clients.fdsn.header import ( FDSNNoDataException, @@ -829,6 +829,12 @@ def extract_station_info( multi_event_records=multi_event_records, ) + # Get the inventory xml file + xml_dir = file_structure.get_stationxml_dir(main_dir) + # Load the inventory information + inventory_file = xml_dir / f"{station}.xml" + inventory = read_inventory(inventory_file) if inventory_file.is_file() else None + # Get the unique channels (Using first 2 keys) and locations unique_channels = set([(tr.stats.channel[:2], tr.stats.location) for tr in st]) @@ -918,7 +924,7 @@ def extract_station_info( # Check for multi-event flagging start_time, end_time, stalta_score, sync_event = ( multi_event.compute_multi_event_scores( - mseed.copy(), sync_check_extraction_table + mseed.copy(), sync_check_extraction_table, inventory=inventory ) ) # Add to the multi_event_records list From 8ad985d5ab036876416b1b9876db0847605a8708 Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 25 Feb 2026 09:44:54 +1300 Subject: [PATCH 37/72] multi-event inv fix --- nzgmdb/data_retrieval/waveform_extraction.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nzgmdb/data_retrieval/waveform_extraction.py b/nzgmdb/data_retrieval/waveform_extraction.py index 295cc85e..9fec5afd 100644 --- a/nzgmdb/data_retrieval/waveform_extraction.py +++ b/nzgmdb/data_retrieval/waveform_extraction.py @@ -15,7 +15,7 @@ import numpy as np import pandas as pd import scipy as sp -from obspy import Stream, Trace, UTCDateTime +from obspy import Stream, Trace, UTCDateTime, read_inventory from obspy.clients.fdsn import Client as FDSN_Client from obspy.clients.fdsn.header import ( FDSNNoDataException, @@ -829,6 +829,12 @@ def extract_station_info( multi_event_records=multi_event_records, ) + # Get the inventory xml file + xml_dir = file_structure.get_stationxml_dir(main_dir) + # Load the inventory information + inventory_file = xml_dir / f"{station}.xml" + inventory = read_inventory(inventory_file) if inventory_file.is_file() else None + # Get the unique channels (Using first 2 keys) and locations unique_channels = set([(tr.stats.channel[:2], tr.stats.location) for tr in st]) @@ -918,7 +924,7 @@ def extract_station_info( # Check for multi-event flagging start_time, end_time, stalta_score, sync_event = ( multi_event.compute_multi_event_scores( - mseed.copy(), sync_check_extraction_table + mseed.copy(), sync_check_extraction_table, inventory=inventory ) ) # Add to the multi_event_records list From 5165d5bf1bce999e6dea48b72f679cc48d2e559f Mon Sep 17 00:00:00 2001 From: Joel Ridden Date: Mon, 2 Mar 2026 09:37:29 +1300 Subject: [PATCH 38/72] 3 component check --- nzgmdb/data_retrieval/waveform_extraction.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/nzgmdb/data_retrieval/waveform_extraction.py b/nzgmdb/data_retrieval/waveform_extraction.py index 295cc85e..53895897 100644 --- a/nzgmdb/data_retrieval/waveform_extraction.py +++ b/nzgmdb/data_retrieval/waveform_extraction.py @@ -888,6 +888,21 @@ def extract_station_info( ) ) + # Check for 3 component data, if not skip + if len(mseed) < 3: + stats = mseed[0].stats + skipped_records.append( + pd.DataFrame( + { + "record_id": [ + f"{event_id}_{stats.station}_{stats.channel}_{stats.location}" + ], + "reason": ["Less than 3 component traces"], + } + ) + ) + continue + # Calculate clip to determine if the record should be dropped clip = filtering.get_clip_probability(event_mag, r_hyp, mseed) From 048638f17d3578d374ac3b5f7b1e6c4c10121303 Mon Sep 17 00:00:00 2001 From: joelridden Date: Tue, 3 Mar 2026 10:31:03 +1300 Subject: [PATCH 39/72] tmp array start --- nzgmdb/data_retrieval/waveform_extraction.py | 52 ++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/nzgmdb/data_retrieval/waveform_extraction.py b/nzgmdb/data_retrieval/waveform_extraction.py index 9d725239..772e0941 100644 --- a/nzgmdb/data_retrieval/waveform_extraction.py +++ b/nzgmdb/data_retrieval/waveform_extraction.py @@ -739,6 +739,56 @@ def get_station_window( return get_inital_stream(start_time, end_time, channel_codes, loc, client, net, sta) +# def get_tmp_array_stream( +# net: str, +# sta: str, +# tmp_array_dir: Path, +# start_time: UTCDateTime, +# end_time: UTCDateTime, +# ): +# """ +# Get the initial stream of waveforms for a station from the temporary array storage location. +# +# Parameters +# ---------- +# net : str +# The network code to retrieve waveforms for. +# sta : str +# The station code to retrieve waveforms for. +# tmp_array_dir : Path +# The directory where the temporary array waveform files are stored. +# start_time : UTCDateTime +# The start time of the waveform data to retrieve. +# end_time : UTCDateTime +# The end time of the waveform data to retrieve. +# +# Returns +# ------- +# Stream +# An ObsPy Stream object containing the waveform data for the specified station. +# """ +# # Extract the parameters from the row +# event_id = station_extraction_row["evid"] +# station = station_extraction_row["sta"] +# +# # Get the tmp array directory +# tmp_array_dir = file_structure.get_tmp_array_dir(main_dir) +# # Get the mseed file path +# mseed_file = tmp_array_dir / f"{event_id}_{station}.mseed" +# +# if mseed_file.is_file(): +# try: +# st = Stream() +# st += Trace.read(str(mseed_file)) +# return st +# except Exception as e: # noqa: BLE001 +# print(f"Error reading mseed file for {event_id}_{station} from tmp array") +# print(e) +# return None +# else: +# return None + + def extract_station_info( station_extraction_row: pd.Series, main_dir: Path, @@ -776,6 +826,7 @@ def extract_station_info( multi_event_records, ) = ([], [], [], [], []) # Extract the parameters from the row + provider = station_extraction_row["provider"] event_id = station_extraction_row["evid"] station = station_extraction_row["sta"] network = station_extraction_row["net"] @@ -816,6 +867,7 @@ def extract_station_info( st = get_station_window(station_extraction_row, client, channel_codes, location) else: # Get the stream from the tmp array storage location + # st = get_tmp_array_stream st = None # Check that data was found From c4a3c4d462f326f877be3d46f5086496916ff527 Mon Sep 17 00:00:00 2001 From: joelridden Date: Tue, 3 Mar 2026 15:28:23 +1300 Subject: [PATCH 40/72] fmin fmax plot --- nzgmdb/data_retrieval/sites.py | 20 +- nzgmdb/management/data_registry.py | 2 + nzgmdb/scripts/generate_report.py | 378 ++++++++++++++++++----------- 3 files changed, 256 insertions(+), 144 deletions(-) diff --git a/nzgmdb/data_retrieval/sites.py b/nzgmdb/data_retrieval/sites.py index 1b5479a2..2e451be0 100644 --- a/nzgmdb/data_retrieval/sites.py +++ b/nzgmdb/data_retrieval/sites.py @@ -261,9 +261,11 @@ def create_site_table_response( # Only compute thresholds for stations where Z1.0 is missing mask_missing_z1 = tect_merged_df["Z1.0"].isna() - if mask_missing_z1.any(): + mask_q3 = tect_merged_df["Q_Z1.0"] == "Q3" + mask_to_compute = mask_missing_z1 | mask_q3 + if mask_to_compute.any(): # Prepare stations DataFrame for only missing rows, indexed by station code - stations = tect_merged_df.loc[mask_missing_z1, ["sta", "lon", "lat"]].set_index( + stations = tect_merged_df.loc[mask_to_compute, ["sta", "lon", "lat"]].set_index( "sta" )[["lon", "lat"]] try: @@ -295,15 +297,15 @@ def create_site_table_response( # Set extra ref / quality fields tect_merged_df.loc[ - mask_missing_z1, ["Z1.0_ref", "Z2.5_ref", "Q_Z1.0", "Q_Z2.5"] + mask_to_compute, ["Z1.0_ref", "Z2.5_ref", "Q_Z1.0", "Q_Z2.5"] ] = ["NZCVM (2026)", "NZCVM (2026)", "Q3", "Q3"] # Get the file path to the combined MVN GeoTIFF - NZGMDB_DATA.fetch("combined_mvn_wgs84.tif") - file_path = Path(NZGMDB_DATA.abspath) / "combined_mvn_wgs84.tif" + NZGMDB_DATA.fetch("nzcvm_v1.tif") + file_path = Path(NZGMDB_DATA.abspath) / "nzcvm_v1.tif" # Compute Vs30 for missing values - points = tect_merged_df.loc[mask_missing_z1, ["lat", "lon"]].to_numpy() + points = tect_merged_df.loc[mask_to_compute, ["lat", "lon"]].to_numpy() vs30_values = sample_points_from_geotiff(file_path, points).ravel() # Fill missing gaps in Vs30 using nearest-neighbour averaging @@ -312,11 +314,11 @@ def create_site_table_response( vs30_values_filled_rounded = np.round(vs30_values_filled) # Update Vs30 and related fields - tect_merged_df.loc[mask_missing_z1, "Vs30"] = vs30_values_filled_rounded + tect_merged_df.loc[mask_to_compute, "Vs30"] = vs30_values_filled_rounded # Ensure reference and quality fields are set for Vs30 where filled - vs30_mask = mask_missing_z1 & ~tect_merged_df["Vs30"].isna() - tect_merged_df.loc[vs30_mask, "Vs30_Ref"] = "Foster et al. (2019)" + vs30_mask = mask_to_compute & ~tect_merged_df["Vs30"].isna() + tect_merged_df.loc[vs30_mask, "Vs30_Ref"] = "Vs30 Map v1.0 (2026)" tect_merged_df.loc[vs30_mask, "Q_Vs30"] = "Q3" except (FileNotFoundError, ValueError, RuntimeError): diff --git a/nzgmdb/management/data_registry.py b/nzgmdb/management/data_registry.py index 6abc76ad..09ee8daf 100644 --- a/nzgmdb/management/data_registry.py +++ b/nzgmdb/management/data_registry.py @@ -6,6 +6,7 @@ "hik_kerm_fault_300km_wgs84_poslon.txt": "sha256:1a199978b6c9c608f8473539b639a8825c1091167da3d14b07c7268528320e03", "Geonet_Metadata_Summary_v1.4.csv": "sha256:7884422c3fcae0810c02948ba1a3bd39ba5793ba28189e90d730541be1c207c0", "combined_mvn_wgs84.tif": "sha256:b0aed1d441a3c441d784a5dd9016314ee74df164dcafee6e233211a43cbfba0f", + "nzcvm_v1.tif": "sha256:34e22f4065c4380b0a048a24b8bcf2282567183391037d76308f8e3dc60285f0", "puy_slab2_dep_02.26.18.xyz": "sha256:9ebe4feab4ee3b80e3fe403f2d873f94e4d7f06d937d721cc9e154ecee83e3c0", "reyners_relocations.csv": "sha256:7795c60dae67af14eb590b0d919fa850e2f2fe2fb3beb077cbac14d27eb8faf5", "focal_mech_tectonic_domain_v1.csv": "sha256:1f1e0c4b7f9ca1b87fb2ca4883e587f330fe82b5bbf9ebb6eb8f4d12aa2e1936", @@ -42,6 +43,7 @@ "focal_mech_tectonic_domain_v1.csv": "https://www.dropbox.com/scl/fi/zseg304cbjmti7gg5tdyv/focal_mech_tectonic_domain_v1.csv?rlkey=kfb9ttvnv9yi9zftixw6kmz4v&st=4j9pgpgj&dl=1", "Geonet_Metadata_Summary_v1.4.csv": "https://www.dropbox.com/scl/fi/iev7qmoqqzvc5quhf8mk8/Geonet-Metadata-Summary_v1.4.csv?rlkey=7twwwck5iy5zao7lwao6xodvm&st=6m3elzuu&dl=1", "combined_mvn_wgs84.tif": "https://www.dropbox.com/scl/fi/lzqijoivcg4wzybj06rsh/combined_mvn_wgs84.tif?rlkey=3g4milzk41c2lcsdggvy2ieql&st=re0izlz4&dl=1", + "nzcvm_v1.tif": "https://www.dropbox.com/scl/fi/b7qpyhd94vhii7z9lhmtg/nzcvm_v1.tif?rlkey=xc7k9f6gcx46ive0z91txn2tp&st=qqnvy1ib&dl=1", "GeoNet_CMT_solutions_20201129_PreferredNodalPlane_v1.csv": "https://www.dropbox.com/scl/fi/fq28jx8jlbozj0d1x5tnq/GeoNet_CMT_solutions_20201129_PreferredNodalPlane_v1.csv?rlkey=30xj6n7ara0vz4t8kg4pz8w5s&st=63x7nr3j&dl=1", "hik_kerm_fault_300km_wgs84_poslon.txt": "https://www.dropbox.com/scl/fi/ig3ajufpv4xg2qjfxxuup/hik_kerm_fault_300km_wgs84_poslon.txt?rlkey=9jajfkq2elrzwzzgh6px17k8e&st=6ham2oox&dl=1", "Mw_rrup.txt": "https://www.dropbox.com/scl/fi/e3o9v9ze9e4955xxtrl14/Mw_rrup.txt?rlkey=c663zntx7gaeyxt04i97r62nu&st=6ri3c620&dl=1", diff --git a/nzgmdb/scripts/generate_report.py b/nzgmdb/scripts/generate_report.py index 8a38d0c9..8aae12a1 100644 --- a/nzgmdb/scripts/generate_report.py +++ b/nzgmdb/scripts/generate_report.py @@ -11,6 +11,7 @@ from typing import Annotated, Optional import matplotlib.pyplot as plt +import matplotlib.ticker as mticker import numpy as np import pandas as pd import typer @@ -608,6 +609,139 @@ def important_set_figures( return bar_img_base64, pie_imgs_base64 +def plot_usable_period_records( + df_full: pd.DataFrame, + df_quality: pd.DataFrame, + title: Optional[str] = None, +) -> plt.Figure: + """ + Plot number of usable records vs period for Full and Quality datasets. + + Parameters + ---------- + df_full : pd.DataFrame + Full dataset DataFrame containing columns for fmin_X, fmin_Y, fmax_X, fmax_Y, and pSA_{period} columns. + df_quality : pd.DataFrame + Quality dataset DataFrame containing columns for fmin_X, fmin_Y, fmax_X, fmax_Y, and pSA_{period} columns. + title : Optional[str], optional + Title for the plot, by default None. + + Returns + ------- + str + Base64 encoded string of the plot image. + """ + + def extract_periods_from_columns(df: pd.DataFrame): + """ + Extracts periods from column names that start with "pSA_". + + Parameters + ---------- + df : pd.DataFrame + DataFrame containing columns with names like "pSA_{period}". + + Returns + ------- + np.ndarray + Sorted array of periods extracted from the column names. + """ + psa_cols = [c for c in df.columns if c.startswith("pSA_")] + periods = [] + for c in psa_cols: + periods.append(float(c.split("_")[1])) + return np.array(sorted(periods)) + + def count_usable_records(df: pd.DataFrame, periods: np.ndarray) -> np.ndarray: + """ + Count usable records per period using BOTH fmin and fmax. + + A record is usable if: + 1/fmax <= T <= 1/fmin + + Parameters + ---------- + df : pandas.DataFrame + Input dataframe containing `fmin_X`, `fmin_Y`, `fmax_X`, and `fmax_Y`. + periods : numpy.ndarray + Periods (s) to evaluate. + + Returns + ------- + numpy.ndarray + Array of counts for each period in `periods`. + """ + + fmin_max = np.maximum(df["fmin_X"].to_numpy(), df["fmin_Y"].to_numpy()) + fmax_min = np.minimum(df["fmax_X"].to_numpy(), df["fmax_Y"].to_numpy()) + + T_upper = 1.0 / fmin_max # long-period limit + T_lower = 1.0 / fmax_min # short-period limit + + # Vectorised evaluation + T_lower = T_lower[:, None] + T_upper = T_upper[:, None] + + usable = (periods >= T_lower) & (periods <= T_upper) + return usable.sum(axis=0) + + required_cols = ["fmin_X", "fmin_Y", "fmax_X", "fmax_Y"] + + df_full = df_full.dropna(subset=required_cols) + df_quality = df_quality.dropna(subset=required_cols) + + periods = extract_periods_from_columns(df_full) + + counts_full = count_usable_records(df_full, periods) + counts_quality = count_usable_records(df_quality, periods) + + fig, ax = plt.subplots(figsize=(8, 6)) + + ax.plot( + periods, + counts_full, + label="Full Database", + linewidth=2, + drawstyle="steps-post", + ) + + ax.plot( + periods, + counts_quality, + label="Quality Database", + linewidth=2, + drawstyle="steps-post", + ) + + ax.set_xscale("log") + ax.set_xlim(0.01, 10) + + ax.xaxis.set_major_locator(mticker.FixedLocator([0.01, 0.1, 1.0, 10.0])) + + ax.xaxis.set_major_formatter( + mticker.FuncFormatter( + lambda x, _: {0.01: "0.01", 0.1: "0.1", 1.0: "1", 10.0: "10"}.get(x, "") + ) + ) + + ax.xaxis.set_minor_formatter(mticker.NullFormatter()) + + ax.set_xlabel("Period (s)") + ax.set_ylabel("Number of Records") + if title: + ax.set_title(title) + ax.legend() + ax.grid(True, which="both", linestyle="--", alpha=0.4) + + fig.tight_layout() + + buf = BytesIO() + fig.savefig(buf, format="png", bbox_inches="tight", dpi=300) + plt.close(fig) + buf.seek(0) + return base64.b64encode(buf.read()).decode("utf-8") + + def skipped_reason_overlap_barplot(skipped_df: pd.DataFrame, title: str): """ Generate a bar plot showing the overlap of skipped reasons. @@ -1862,154 +1996,72 @@ def generate_report( html_parts.append("") # Add psa count Comparison - html_parts.append("

Quality pSA Record Count Comparison

") - html_parts.append('
') + html_parts.append("

pSA Record Count Comparison

") - fig, ax = plt.subplots(figsize=(16, 6), dpi=300) + img_base64_new = plot_usable_period_records(full_new, quality_new, "New NZGMDB") - # Generate fmin plots if compare_version_directory: - filtered_quality_old = apply_fmin_filter_df(quality_old, pre_4p3=False) - old_record_count = (~filtered_quality_old[PSA_KEYS].isna()).sum(axis=0) - ax.plot( - PERIODS, - old_record_count.loc[PSA_KEYS], - label="Old NZGMDB", - ) - - filtered_quality_new = apply_fmin_filter_df(quality_new) - new_record_count = (~filtered_quality_new[PSA_KEYS].isna()).sum(axis=0) - ax.plot( - PERIODS, - new_record_count.loc[PSA_KEYS], - label="New NZGMDB", - ) - - ax.set_xlabel("Period (s)") - ax.set_xscale("log") - ax.set_ylabel("Count") - ax.set_xlim(0.01, 10.0) - ax.legend() - ax.grid(linewidth=0.5, alpha=0.5, linestyle="--") - - fig.tight_layout() - - # Convert to base64 - buf = BytesIO() - fig.savefig(buf, format="png", bbox_inches="tight") - plt.close(fig) - buf.seek(0) - img_base64 = base64.b64encode(buf.read()).decode("utf-8") - # Embed in HTML - html_parts.append(f'') - html_parts.append("
") - - # Add IM Compare - html_parts.append("

IM Comparison

") - html_parts.append('
') - - quality_old_record_index = quality_old.set_index("record_id") - quality_new_record_index = quality_new.set_index("record_id") + img_base64_old = plot_usable_period_records(full_old, quality_old, "Old NZGMDB") + html_parts.append("
") + html_parts.append(f'') + html_parts.append(f'') + html_parts.append("
") + else: + html_parts.append('
') + html_parts.append(f'') + html_parts.append("
") - # IM Compare - shared_record_ids = np.intersect1d( - quality_old_record_index.index.values.astype(str), - quality_new_record_index.index.values.astype(str), - ) + if compare_version_directory: + # Add IM Compare + html_parts.append("

IM Comparison

") + html_parts.append('
') - plot_ims = [ - "PGV", - "PGA", - "pSA_0.01", - "pSA_0.1", - "pSA_0.5", - "pSA_1.0", - "pSA_3.0", - "pSA_10.0", - ] + quality_old_record_index = quality_old.set_index("record_id") + quality_new_record_index = quality_new.set_index("record_id") - fig, axs = get_fig_axes(len(plot_ims), 2, -1, ind_figsize=(8, 6)) + # IM Compare + shared_record_ids = np.intersect1d( + quality_old_record_index.index.values.astype(str), + quality_new_record_index.index.values.astype(str), + ) - for i, (cur_im, cur_ax) in enumerate(zip(plot_ims, axs)): - cur_old = quality_old_record_index.loc[shared_record_ids, cur_im] - cur_new = quality_new_record_index.loc[shared_record_ids, cur_im] + plot_ims = [ + "PGV", + "PGA", + "pSA_0.01", + "pSA_0.1", + "pSA_0.5", + "pSA_1.0", + "pSA_3.0", + "pSA_10.0", + ] - cur_max = max(cur_old.max(), cur_new.max()) + fig, axs = get_fig_axes(len(plot_ims), 2, -1, ind_figsize=(8, 6)) - cur_ax.scatter(cur_old, cur_new, s=1) - cur_ax.set_xlabel("Old NZGMDB") - cur_ax.set_ylabel("New NZGMDB") - cur_ax.set_title(cur_im) - cur_ax.plot( - [0, cur_max], [0, cur_max], color="black", linestyle="--", linewidth=0.5 - ) - cur_ax.set_xlim(0, cur_max) - cur_ax.set_ylim(0, cur_max) + for i, (cur_im, cur_ax) in enumerate(zip(plot_ims, axs)): + cur_old = quality_old_record_index.loc[shared_record_ids, cur_im] + cur_new = quality_new_record_index.loc[shared_record_ids, cur_im] - # If the cur_im is PGA or includes pSA, set the x and y scales to log - if cur_im == "PGA" or cur_im.startswith("pSA_"): - cur_ax.set_xscale("log") - cur_ax.set_yscale("log") - cur_ax.set_xlim(0.001, cur_max) - cur_ax.set_ylim(0.001, cur_max) + cur_max = max(cur_old.max(), cur_new.max()) - cur_ax.grid(which="both", linewidth=0.5, alpha=0.5, linestyle="--") + cur_ax.scatter(cur_old, cur_new, s=1) + cur_ax.set_xlabel("Old NZGMDB") + cur_ax.set_ylabel("New NZGMDB") + cur_ax.set_title(cur_im) + cur_ax.plot( + [0, cur_max], [0, cur_max], color="black", linestyle="--", linewidth=0.5 + ) + cur_ax.set_xlim(0, cur_max) + cur_ax.set_ylim(0, cur_max) - # Convert to base64 - buf = BytesIO() - fig.savefig(buf, format="png", bbox_inches="tight") - plt.close(fig) - buf.seek(0) - img_base64 = base64.b64encode(buf.read()).decode("utf-8") - # Embed in HTML - html_parts.append(f'') - html_parts.append("
") + # If the cur_im is PGA or includes pSA, set the x and y scales to log + if cur_im == "PGA" or cur_im.startswith("pSA_"): + cur_ax.set_xscale("log") + cur_ax.set_yscale("log") + cur_ax.set_xlim(0.001, cur_max) + cur_ax.set_ylim(0.001, cur_max) - html_parts.append( - """ -

GMM Input Comparison

-
- """ - ) - # GMM Input comparison - input_cols = [ - "ev_lat", - "ev_lon", - "ev_depth", - "mag", - "strike", - "dip", - "rake", - "z_tor", - "r_jb", - "r_rup", - "r_x", - "Vs30", - "Z1.0", - "Z2.5", - ] - # Generate figures and embed as base64 - for i, col in enumerate(input_cols): - cur_x_data = quality_old_record_index.loc[shared_record_ids, col].values - cur_y_data = quality_new_record_index.loc[shared_record_ids, col].values - nan_mask = np.isnan(cur_x_data) | np.isnan(cur_y_data) - cur_x_data = cur_x_data[~nan_mask] - cur_y_data = cur_y_data[~nan_mask] - lims = ( - np.quantile(cur_x_data, 0.01), - np.quantile(cur_x_data, 0.99), - ) - fig, ax = plt.subplots(figsize=(8, 6), dpi=300) - ax.scatter(cur_x_data, cur_y_data, alpha=0.5, s=1) - ax.plot(lims, lims, color="k") - ax.set_xlabel("Old NZGMDB") - ax.set_ylabel("New NZGMDB") - ax.grid(linewidth=0.5, alpha=0.5, linestyle="--") - ax.set_xlim(lims) - ax.set_ylim(lims) - ax.set_aspect("equal") - ax.set_title(f"{col} - N: {len(cur_x_data)}") - fig.tight_layout() + cur_ax.grid(which="both", linewidth=0.5, alpha=0.5, linestyle="--") # Convert to base64 buf = BytesIO() @@ -2019,6 +2071,62 @@ def generate_report( img_base64 = base64.b64encode(buf.read()).decode("utf-8") # Embed in HTML html_parts.append(f'') + html_parts.append("
") + + html_parts.append( + """ +

GMM Input Comparison

+
+ """ + ) + # GMM Input comparison + input_cols = [ + "ev_lat", + "ev_lon", + "ev_depth", + "mag", + "strike", + "dip", + "rake", + "z_tor", + "r_jb", + "r_rup", + "r_x", + "Vs30", + "Z1.0", + "Z2.5", + ] + # Generate figures and embed as base64 + for i, col in enumerate(input_cols): + cur_x_data = quality_old_record_index.loc[shared_record_ids, col].values + cur_y_data = quality_new_record_index.loc[shared_record_ids, col].values + nan_mask = np.isnan(cur_x_data) | np.isnan(cur_y_data) + cur_x_data = cur_x_data[~nan_mask] + cur_y_data = cur_y_data[~nan_mask] + lims = ( + np.quantile(cur_x_data, 0.01), + np.quantile(cur_x_data, 0.99), + ) + fig, ax = plt.subplots(figsize=(8, 6), dpi=300) + ax.scatter(cur_x_data, cur_y_data, alpha=0.5, s=1) + ax.plot(lims, lims, color="k") + ax.set_xlabel("Old NZGMDB") + ax.set_ylabel("New NZGMDB") + ax.grid(linewidth=0.5, alpha=0.5, linestyle="--") + ax.set_xlim(lims) + ax.set_ylim(lims) + ax.set_aspect("equal") + ax.set_title(f"{col} - N: {len(cur_x_data)}") + fig.tight_layout() + + # Convert to base64 + buf = BytesIO() + fig.savefig(buf, format="png", bbox_inches="tight") + plt.close(fig) + buf.seek(0) + img_base64 = base64.b64encode(buf.read()).decode("utf-8") + # Embed in HTML + html_parts.append(f'') # End of Section html_parts.append("
") From 948876310b5f401411f31575257b442ce0819f10 Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 6 Mar 2026 14:00:35 +1300 Subject: [PATCH 41/72] st from temp array --- nzgmdb/data_retrieval/waveform_extraction.py | 156 ++++++++++--------- nzgmdb/scripts/run_nzgmdb.py | 1 + 2 files changed, 87 insertions(+), 70 deletions(-) diff --git a/nzgmdb/data_retrieval/waveform_extraction.py b/nzgmdb/data_retrieval/waveform_extraction.py index 772e0941..02d7171d 100644 --- a/nzgmdb/data_retrieval/waveform_extraction.py +++ b/nzgmdb/data_retrieval/waveform_extraction.py @@ -15,7 +15,7 @@ import numpy as np import pandas as pd import scipy as sp -from obspy import Stream, Trace, UTCDateTime, read_inventory +from obspy import Stream, Trace, UTCDateTime, read_inventory, read from obspy.clients.fdsn import Client as FDSN_Client from obspy.clients.fdsn.header import ( FDSNNoDataException, @@ -691,33 +691,21 @@ def check_trace_issues(st: Stream, record_id: str, station_extraction_row: pd.Se def get_station_window( station_extraction_row: pd.Series, - client: FDSN_Client, - channel_codes: str, - loc: str, ): """ - Get the initial stream of waveforms for a station based on the extraction parameters. + Get the start and end time for the waveform extraction window for a station based on the parameters in the station extraction table. Parameters ---------- station_extraction_row : pd.Series A row from the station extraction table containing the parameters for waveform extraction. - client : FDSN_Client - The FDSN client to use for retrieving waveforms. - channel_codes : str - The channel codes to retrieve, formatted as a comma-separated string. - e.g. "HN?,BN?,HH?". - loc : str - The location code to retrieve waveforms for, typically "*". Returns ------- - Stream - An ObsPy Stream object containing the waveform data for the specified station. + tuple of UTCDateTime + A tuple containing the start time and end time for the waveform extraction window. """ # Extract the parameters from the row - net = station_extraction_row["net"] - sta = station_extraction_row["sta"] r_hyp = station_extraction_row["r_hyp"] ptime_est = UTCDateTime(station_extraction_row["ptime_est"]) ds_mean = station_extraction_row["ds_mean"] @@ -736,57 +724,73 @@ def get_station_window( # Note: both ds_mean and ds_std are in logspace end_time = ptime_est + np.exp(ds_mean) * np.exp(ds_std_multiplier * ds_std) - return get_inital_stream(start_time, end_time, channel_codes, loc, client, net, sta) - - -# def get_tmp_array_stream( -# net: str, -# sta: str, -# tmp_array_dir: Path, -# start_time: UTCDateTime, -# end_time: UTCDateTime, -# ): -# """ -# Get the initial stream of waveforms for a station from the temporary array storage location. -# -# Parameters -# ---------- -# net : str -# The network code to retrieve waveforms for. -# sta : str -# The station code to retrieve waveforms for. -# tmp_array_dir : Path -# The directory where the temporary array waveform files are stored. -# start_time : UTCDateTime -# The start time of the waveform data to retrieve. -# end_time : UTCDateTime -# The end time of the waveform data to retrieve. -# -# Returns -# ------- -# Stream -# An ObsPy Stream object containing the waveform data for the specified station. -# """ -# # Extract the parameters from the row -# event_id = station_extraction_row["evid"] -# station = station_extraction_row["sta"] -# -# # Get the tmp array directory -# tmp_array_dir = file_structure.get_tmp_array_dir(main_dir) -# # Get the mseed file path -# mseed_file = tmp_array_dir / f"{event_id}_{station}.mseed" -# -# if mseed_file.is_file(): -# try: -# st = Stream() -# st += Trace.read(str(mseed_file)) -# return st -# except Exception as e: # noqa: BLE001 -# print(f"Error reading mseed file for {event_id}_{station} from tmp array") -# print(e) -# return None -# else: -# return None + return start_time, end_time + + +def get_tmp_array_stream( + tmp_array_dir: Path, + net: str, + sta: str, + start_time: UTCDateTime, + end_time: UTCDateTime, +): + """ + Get the initial stream of waveforms for a station from the temporary array storage location. + + Parameters + ---------- + tmp_array_dir : Path + The directory where the temporary array waveform files are stored. + net : str + The network code to retrieve waveforms for. + sta : str + The station code to retrieve waveforms for. + start_time : UTCDateTime + The start time of the waveform data to retrieve. + end_time : UTCDateTime + The end time of the waveform data to retrieve. + + Returns + ------- + Stream + An ObsPy Stream object containing the waveform data for the specified station. + """ + net_dir = tmp_array_dir / net + if not net_dir.is_dir(): + return None + + st = Stream() + + pattern = f"{net}_{sta}_*" + selected_files = [] + for sta_dir in sorted(p for p in net_dir.glob(pattern) if p.is_dir()): + + for f in sta_dir.glob("*.mseed"): + # Example filename + # Y3.CASS..HHN__20090312T000000Z__20090411T000000Z.mseed + parts = f.name.split("__") + + file_start = UTCDateTime(parts[1]) + file_end = UTCDateTime(parts[2].replace(".mseed", "")) + + # Check overlap + if file_end >= start_time and file_start <= end_time: + selected_files.append(f) + + if not selected_files: + return None + + # Read files + for f in sorted(selected_files): + st += read(str(f)) + + # Merge overlapping / adjacent segments + st.merge(method=1, fill_value=None) + + # Trim to exact window + st.trim(start_time, end_time) + + return st def extract_station_info( @@ -795,6 +799,7 @@ def extract_station_info( event_catalogues: dict, extraction_table: pd.DataFrame, only_record_ids: pd.DataFrame = None, + tmp_array_dir: Path = None, ) -> StationExtractionResult: """ Extract the waveform data for a single station based on the extraction parameters. @@ -811,6 +816,8 @@ def extract_station_info( The full extraction table containing all extraction parameters. only_record_ids : pd.DataFrame, optional A DataFrame containing a subset of record IDs to use for extraction, if provided. + tmp_array_dir : Path, optional + The directory where the temporary array waveform files are stored, if using temporary array storage for waveforms. Returns ------- @@ -861,14 +868,19 @@ def extract_station_info( ) location = site_only_record_ids["record_id"].str.split("_").str[-1].values[0] + start_time, end_time = get_station_window(station_extraction_row) + net = station_extraction_row["net"] + sta = station_extraction_row["sta"] + if provider == "GEONET": # Get the Stream client = FDSN_Client("GEONET") - st = get_station_window(station_extraction_row, client, channel_codes, location) + st = get_inital_stream( + start_time, end_time, channel_codes, location, client, net, sta + ) else: # Get the stream from the tmp array storage location - # st = get_tmp_array_stream - st = None + st = get_tmp_array_stream(tmp_array_dir, net, sta, start_time, end_time) # Check that data was found if st is None: @@ -1067,6 +1079,7 @@ def extract_waveforms( n_procs: int = 1, only_record_ids_ffp: Path = None, batch_size: int = 1000, + tmp_array_dir: Path = None, ): """ Extract waveforms for each station in the station extraction table. @@ -1086,6 +1099,8 @@ def extract_waveforms( Full file path to the file containing a subset of record IDs to use for extraction, if provided. batch_size : int, optional The number of rows to process in each batch, by default 1000. + tmp_array_dir : Path, optional + The directory where the temporary array waveform files are stored, if using temporary array storage for waveforms. """ station_extraction_table = pd.read_csv( station_extraction_table_ffp, dtype={"evid": str} @@ -1165,6 +1180,7 @@ def extract_waveforms( event_catalogues=catalog_dict, extraction_table=station_extraction_table, only_record_ids=only_record_ids, + tmp_array_dir=tmp_array_dir, ), (row for _, row in batch_rows.iterrows()), ) diff --git a/nzgmdb/scripts/run_nzgmdb.py b/nzgmdb/scripts/run_nzgmdb.py index 3b0afe3b..d9b2eb82 100644 --- a/nzgmdb/scripts/run_nzgmdb.py +++ b/nzgmdb/scripts/run_nzgmdb.py @@ -1020,6 +1020,7 @@ def run_full_nzgmdb( extract_n_procs, only_record_ids_ffp, batch_size=geonet_batch_size, + tmp_array_dir=tmp_array_data_dir, ) # Merge the tectonic domains From 27f552919f42858fd0af258f03c3f9397e9bd651 Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 11 Mar 2026 12:44:56 +1300 Subject: [PATCH 42/72] tmp array working full --- nzgmdb/data_retrieval/sites.py | 3 ++ nzgmdb/mseed_management/creation.py | 5 +++ .../phase_arrival/gen_phase_arrival_table.py | 9 +++++- nzgmdb/phase_arrival/run_phasenet.py | 32 ++++++++++++++++--- nzgmdb/scripts/run_nzgmdb.py | 8 +++-- 5 files changed, 48 insertions(+), 9 deletions(-) diff --git a/nzgmdb/data_retrieval/sites.py b/nzgmdb/data_retrieval/sites.py index 2e451be0..c2825886 100644 --- a/nzgmdb/data_retrieval/sites.py +++ b/nzgmdb/data_retrieval/sites.py @@ -343,6 +343,9 @@ def create_site_table_response( "end_time", ], ] + # Adjust any "" loc codes to be "00" + # based on the FDSN Source Indentifiers documentation (https://docs.fdsn.org/projects/source-identifiers/en/latest/location-codes.html) + station_df = station_df.replace({"loc": {"": "00"}}) site_df = tect_merged_df.loc[ :, diff --git a/nzgmdb/mseed_management/creation.py b/nzgmdb/mseed_management/creation.py index e84b94c2..05837085 100644 --- a/nzgmdb/mseed_management/creation.py +++ b/nzgmdb/mseed_management/creation.py @@ -64,6 +64,11 @@ def write_mseed(mseed: Stream, event_id: str, station: str, output_directory: Pa channel = mseed[0].stats.channel[:2] location = mseed[0].stats.location + # If the location is empty, set it to "00" as the default value + # based on the FDSN Source Indentifiers documentation (https://docs.fdsn.org/projects/source-identifiers/en/latest/location-codes.html) + if location is None or location == "": + location = "00" + # Create the filename and add it to the output directory filename = f"{event_id}_{station}_{channel}_{location}.mseed" mseed_ffp = output_directory / filename diff --git a/nzgmdb/phase_arrival/gen_phase_arrival_table.py b/nzgmdb/phase_arrival/gen_phase_arrival_table.py index 7d50efa8..ba4e563d 100644 --- a/nzgmdb/phase_arrival/gen_phase_arrival_table.py +++ b/nzgmdb/phase_arrival/gen_phase_arrival_table.py @@ -20,6 +20,7 @@ def process_batch( conda_sh: Path, env_activate_command: str, bypass_records_ffp: Path | None = None, + xml_dir: Path | None = None, ): """ Process a single subfolder: run PhaseNet over mseeds. @@ -36,6 +37,8 @@ def process_batch( The command to activate the environment for running PhaseNet. bypass_records_ffp : Path The full file path to the bypass records file, which includes a custom p_wave_datetime and/or s_wave_datetime + xml_dir: Path + The path to the station xml files. Used for reducing FDSN calls that require station information. Raises ------ @@ -59,7 +62,7 @@ def process_batch( print(f"Skipping run_phasenet for Batch {batch_num} as results already exist") else: # Activate phaseNet environment and run over mseeds for the subfolder - phasenet_command = f"python {run_phasenet_script_ffp} {batch_txt} {output_dir} {bypass_records_ffp if bypass_records_ffp is not None else ''}" + phasenet_command = f"python {run_phasenet_script_ffp} {batch_txt} {output_dir} {bypass_records_ffp if bypass_records_ffp is not None else ''} {xml_dir if xml_dir is not None else ''}" shell_commands.run_command( phasenet_command, conda_sh, env_activate_command, log_file_path_phasenet ) @@ -78,6 +81,7 @@ def generate_phase_arrival_table( env_activate_command: str, n_procs: int, bypass_records_ffp: Path = None, + xml_dir: Path = None, ): """ Generate the phase arrival table utilizing phaseNet @@ -97,6 +101,8 @@ def generate_phase_arrival_table( The number of processes to use bypass_records_ffp : Path The full file path to the bypass records file, which includes a custom p_wave_ix + xml_dir: Path + The path to the station xml files. Used for reducing FDSN calls that require station information. """ # Get the Phase_arrival directory phase_dir = main_dir / "phase_arrival" @@ -126,6 +132,7 @@ def generate_phase_arrival_table( conda_sh=conda_sh, env_activate_command=env_activate_command, bypass_records_ffp=bypass_records_ffp, + xml_dir=xml_dir, ), batches, ) diff --git a/nzgmdb/phase_arrival/run_phasenet.py b/nzgmdb/phase_arrival/run_phasenet.py index 453790b2..b464655d 100644 --- a/nzgmdb/phase_arrival/run_phasenet.py +++ b/nzgmdb/phase_arrival/run_phasenet.py @@ -9,7 +9,7 @@ import mseedlib import numpy as np import pandas as pd -from obspy import Inventory, Stream, Trace, UTCDateTime +from obspy import Inventory, Stream, Trace, UTCDateTime, read_inventory from obspy.clients.fdsn import Client as FDSN_Client from obspy.clients.fdsn.header import FDSNNoDataException @@ -214,7 +214,8 @@ def process_mseed( inv = inventory try: - mseed = mseed.remove_response(inventory=inv, output="ACC") + output_type = "ACC" if channel[:2] in ["HN", "BN"] else "VEL" + mseed = mseed.remove_response(inventory=inv, output=output_type) except ValueError: skipped_record = pd.DataFrame( { @@ -317,7 +318,12 @@ def process_mseed( ) -def run_phasenet(mseed_files_ffp: Path, output_dir: Path, bypass_ffp: Path = None): +def run_phasenet( + mseed_files_ffp: Path, + output_dir: Path, + bypass_ffp: Path = None, + xml_dir: Path = None, +): """ Run PhaseNet on the mseed files. @@ -329,6 +335,8 @@ def run_phasenet(mseed_files_ffp: Path, output_dir: Path, bypass_ffp: Path = Non Output directory for skipped records and phase arrival information. bypass_ffp : Path, optional Optional bypass file path with known p and s wave datetimes, by default None + xml_dir : Path, optional + Optional directory containing station xml files to use for sensitivity removal, by default None (Will try extract from FDSN if not provided) """ # Read the .txt for the mseed files to process mseed_files = mseed_files_ffp.read_text().splitlines() @@ -350,7 +358,15 @@ def run_phasenet(mseed_files_ffp: Path, output_dir: Path, bypass_ffp: Path = Non bypass_row = bypass_df.loc[bypass_df["record_id"] == mseed_file.stem].iloc[ 0 ] - phase_arrival, skipped_record = process_mseed(mseed_file, h5_ffp, bypass_row) + inventory = None + if xml_dir is not None: + station = mseed_file.stem.split("_")[1] + xml_file = xml_dir / f"{station}.xml" + if xml_file.exists(): + inventory = read_inventory(xml_file) + phase_arrival, skipped_record = process_mseed( + mseed_file, h5_ffp, bypass_row, inventory=inventory + ) if phase_arrival is not None: phase_arrival_table.append(phase_arrival) if skipped_record is not None: @@ -395,5 +411,11 @@ def run_phasenet(mseed_files_ffp: Path, output_dir: Path, bypass_ffp: Path = Non help="Optional bypass file path with known p and s wave datetimes.", default=None, ) + parser.add_argument( + "--xml_dir", + type=Path, + help="Optional directory containing station xml files to use for sensitivity removal.", + default=None, + ) args = parser.parse_args() - run_phasenet(args.mseed_files_ffp, args.output_dir, args.bypass_ffp) + run_phasenet(args.mseed_files_ffp, args.output_dir, args.bypass_ffp, args.xml_dir) diff --git a/nzgmdb/scripts/run_nzgmdb.py b/nzgmdb/scripts/run_nzgmdb.py index d9b2eb82..cb2f8be1 100644 --- a/nzgmdb/scripts/run_nzgmdb.py +++ b/nzgmdb/scripts/run_nzgmdb.py @@ -235,6 +235,7 @@ def make_phase_arrival_table( env_activate_command, n_procs, bypass_records_ffp, + xml_dir=file_structure.get_stationxml_dir(main_dir), ) @@ -588,12 +589,12 @@ def generate_site_table_basin( site_df, station_df = sites.create_site_table_response(add_tmp_arrays) site_df = sites.add_site_basins(site_df, nzcvm_data_ffp) - site_df.to_csv( - flatfile_dir / file_structure.PreFlatfileNames.SITE_TABLE, index=False - ) station_df.to_csv( flatfile_dir / file_structure.PreFlatfileNames.STATION_TABLE, index=False ) + site_df.to_csv( + flatfile_dir / file_structure.PreFlatfileNames.SITE_TABLE, index=False + ) @cli.from_docstring(app) @@ -1070,6 +1071,7 @@ def run_full_nzgmdb( gmc_activate, phase_n_procs, bypass_records_ffp, + xml_dir=file_structure.get_stationxml_dir(main_dir), ) # Generate SNR From 6625a3508f71917b019a5b5d0e97312c6ffa0502 Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 11 Mar 2026 15:34:44 +1300 Subject: [PATCH 43/72] add extra networks --- nzgmdb/config/config.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/nzgmdb/config/config.yaml b/nzgmdb/config/config.yaml index 6d7ecf4c..8a498346 100644 --- a/nzgmdb/config/config.yaml +++ b/nzgmdb/config/config.yaml @@ -24,7 +24,7 @@ priority_phase_list: - Pn - Pg - Pb -channel_codes: "HN?,BN?,HH?,BH?" +channel_codes: "HN?,BN?,HH?,BH?,EH?,DH?" percentage_gap_allowed: 0.1 is_large_overlap: 0.5 # Provider / Network Filters @@ -33,9 +33,14 @@ main_providers_networks: - "IU" - "NZ" tmp_array_providers_networks: + AUSPASS: + - "2B" + - "2E" + - "6Y" IRIS: - "1U" - "2B" + - "2C" - "2L" - "2P" - "3C" @@ -48,11 +53,14 @@ tmp_array_providers_networks: - "9G" - "QC" - "X2" + - "XA" - "XB" - "XH" + - "XO" - "XQ" - "Y3" - "YA" + - "YG" - "YO" - "YR" - "Z1" @@ -60,6 +68,10 @@ tmp_array_providers_networks: - "ZP" - "ZT" - "ZX" + IRISPH5: + - "6B" + RASPISHAKE: + - "AM" # Mseed Variables vs30: 500 pre_event_time_difference: 15 From 17cda4e2b1257574244d26d621a669956cd6be63 Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 13 Mar 2026 12:00:57 +1300 Subject: [PATCH 44/72] detrend error --- nzgmdb/calculation/snr.py | 7 +++++++ nzgmdb/data_processing/multi_event.py | 1 + nzgmdb/data_processing/process_observed.py | 7 +++++++ .../data_processing/waveform_manipulation.py | 18 ++++++++++++++---- nzgmdb/management/custom_errors.py | 6 ++++++ 5 files changed, 35 insertions(+), 4 deletions(-) diff --git a/nzgmdb/calculation/snr.py b/nzgmdb/calculation/snr.py index 9441745c..01b272e0 100644 --- a/nzgmdb/calculation/snr.py +++ b/nzgmdb/calculation/snr.py @@ -121,6 +121,13 @@ def compute_snr_for_single_mseed( } skipped_record = pd.DataFrame([skipped_record_dict]) return None, skipped_record + except custom_errors.DetrendError: + skipped_record_dict = { + "record_id": mseed_file.stem, + "reason": "Unable to detrend record", + } + skipped_record = pd.DataFrame([skipped_record_dict]) + return None, skipped_record # Get the TP from the phase arrival table try: diff --git a/nzgmdb/data_processing/multi_event.py b/nzgmdb/data_processing/multi_event.py index 67a7cf4f..69ee4e39 100644 --- a/nzgmdb/data_processing/multi_event.py +++ b/nzgmdb/data_processing/multi_event.py @@ -153,6 +153,7 @@ def stalta_for_stream(stream: Stream, inventory: Inventory | None = None) -> flo custom_errors.InventoryNotFoundError, custom_errors.SensitivityRemovalError, custom_errors.RotationError, + custom_errors.DetrendError, ): return np.nan diff --git a/nzgmdb/data_processing/process_observed.py b/nzgmdb/data_processing/process_observed.py index 24421d2f..1e486c2a 100644 --- a/nzgmdb/data_processing/process_observed.py +++ b/nzgmdb/data_processing/process_observed.py @@ -107,6 +107,13 @@ def process_single_mseed( } skipped_record = pd.DataFrame([skipped_record_dict]) return skipped_record + except custom_errors.DetrendError: + skipped_record_dict = { + "record_id": mseed_file.stem, + "reason": "Unable to detrend record", + } + skipped_record = pd.DataFrame([skipped_record_dict]) + return skipped_record # Get the GMC fmin values fmin_h = ( diff --git a/nzgmdb/data_processing/waveform_manipulation.py b/nzgmdb/data_processing/waveform_manipulation.py index 4263fe0b..34fe1dfe 100644 --- a/nzgmdb/data_processing/waveform_manipulation.py +++ b/nzgmdb/data_processing/waveform_manipulation.py @@ -60,9 +60,15 @@ def initial_preprocessing( RotationError If the rotation fails """ - # Small Processing - mseed.detrend("demean") - mseed.detrend("linear") + try: + # Small Processing + mseed.detrend("demean") + mseed.detrend("linear") + except NotImplementedError: + # This is an issue with extracted waveforms where the trace has masked values. + raise custom_errors.DetrendError( + f"Failed to demean and detrend the data for station {mseed[0].stats.station} with location {mseed[0].stats.location}" + ) # Load config config = cfg.Config() @@ -90,7 +96,11 @@ def initial_preprocessing( try: client_NZ = FDSN_Client(provider) inv = client_NZ.get_stations( - level="response", network=network, station=station, location=location, channel=f"{channel}?" + level="response", + network=network, + station=station, + location=location, + channel=f"{channel}?", ) except FDSNNoDataException: raise custom_errors.InventoryNotFoundError( diff --git a/nzgmdb/management/custom_errors.py b/nzgmdb/management/custom_errors.py index ddf23e94..89aca6ef 100644 --- a/nzgmdb/management/custom_errors.py +++ b/nzgmdb/management/custom_errors.py @@ -15,6 +15,12 @@ class DiffrentiateError(Exception): pass +class DetrendError(Exception): + """Exception raised when detrend fails.""" + + pass + + class SensitivityRemovalError(Exception): """Exception raised when sensitivity removal fails.""" From f7d78a4d459cb7c421faf9ef1ec002b937ce9c28 Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 13 Mar 2026 12:12:41 +1300 Subject: [PATCH 45/72] type error fix --- nzgmdb/mseed_management/creation.py | 60 ++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/nzgmdb/mseed_management/creation.py b/nzgmdb/mseed_management/creation.py index 05837085..bb7f1be2 100644 --- a/nzgmdb/mseed_management/creation.py +++ b/nzgmdb/mseed_management/creation.py @@ -5,38 +5,62 @@ from pathlib import Path import mseedlib +import numpy as np from obspy import Stream -def write_stream_to_mseed(stream: Stream, output_file: Path): +def _coerce_mseed_samples(data: np.ndarray) -> tuple[np.ndarray, str]: """ - Write an ObsPy Stream object to a MiniSEED file using mseedlib. + Coerce trace samples to a MiniSEED-compatible dtype and return (samples, sample_type). - Parameters - ---------- - stream : obspy.core.stream.Stream - The Stream object to write to MiniSEED - output_file : Path - The path to the output MiniSEED file - - Raises - ------ - ValueError - If the sample type of the trace data is not supported + Uses: + - int32 -> "i" + - float32 -> "f" + """ + arr = np.asarray(data) + + # Ensure 1-D numeric + if arr.ndim != 1: + arr = arr.reshape(-1) + + if arr.dtype.kind in {"i", "u"}: + arr_i32 = np.ascontiguousarray(arr.astype(np.int32, copy=False)) + return arr_i32, "i" + + # Default to float32 for floats/others numeric-like + arr_f32 = np.ascontiguousarray(arr.astype(np.float32, copy=False)) + return arr_f32, "f" + + +def write_stream_to_mseed(stream: Stream, output_file: Path) -> None: + """ + Write an ObsPy Stream object to a MiniSEED file using mseedlib. """ mstl = mseedlib.MSTraceList() + for trace in stream: start_time = mseedlib.timestr2nstime(f"{trace.stats.starttime.isoformat()}Z") - sourceid = f"FDSN:{trace.stats.network}_{trace.stats.station}_{trace.stats.location}_{'_'.join(trace.stats.channel)}" + + # Prefer channel as a string; avoid join() on a string (it inserts underscores between characters). + channel = str(trace.stats.channel) + location = str(trace.stats.location or "00") + + sourceid = ( + f"FDSN:{trace.stats.network}_{trace.stats.station}_{location}_{channel}" + ) + + samples, sample_type = _coerce_mseed_samples(trace.data) + mstl.add_data( sourceid=sourceid, - data_samples=trace.data, - sample_type="i", - sample_rate=trace.stats.sampling_rate, + data_samples=samples, + sample_type=sample_type, + sample_rate=float(trace.stats.sampling_rate), start_time=start_time, ) - with open(output_file, "wb") as f: + output_file.parent.mkdir(exist_ok=True, parents=True) + with output_file.open("wb") as f: mstl.pack( lambda record, handler_data: handler_data["fh"].write(record), {"fh": f}, From d115eac77ee003f564036116ceae29366afe9a6e Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 13 Mar 2026 12:16:53 +1300 Subject: [PATCH 46/72] revert back type error --- nzgmdb/data_retrieval/waveform_extraction.py | 15 ++++- nzgmdb/mseed_management/creation.py | 60 ++++++-------------- 2 files changed, 31 insertions(+), 44 deletions(-) diff --git a/nzgmdb/data_retrieval/waveform_extraction.py b/nzgmdb/data_retrieval/waveform_extraction.py index a2738bd8..37ae19ca 100644 --- a/nzgmdb/data_retrieval/waveform_extraction.py +++ b/nzgmdb/data_retrieval/waveform_extraction.py @@ -1025,8 +1025,19 @@ def extract_station_info( year = event_cat.origins[0].time.year mseed_dir = file_structure.get_mseed_dir(main_dir, year, event_id) - # Write the mseed file - creation.write_mseed(mseed, event_id, station, mseed_dir) + try: + # Write the mseed file + creation.write_mseed(mseed, event_id, station, mseed_dir) + except Exception as e: + skipped_records.append( + pd.DataFrame( + { + "record_id": [record_id], + "reason": [f"Error writing mseed file: {str(e)}"], + } + ) + ) + continue for trace in mseed: chan = trace.stats.channel diff --git a/nzgmdb/mseed_management/creation.py b/nzgmdb/mseed_management/creation.py index bb7f1be2..05837085 100644 --- a/nzgmdb/mseed_management/creation.py +++ b/nzgmdb/mseed_management/creation.py @@ -5,62 +5,38 @@ from pathlib import Path import mseedlib -import numpy as np from obspy import Stream -def _coerce_mseed_samples(data: np.ndarray) -> tuple[np.ndarray, str]: - """ - Coerce trace samples to a MiniSEED-compatible dtype and return (samples, sample_type). - - Uses: - - int32 -> "i" - - float32 -> "f" - """ - arr = np.asarray(data) - - # Ensure 1-D numeric - if arr.ndim != 1: - arr = arr.reshape(-1) - - if arr.dtype.kind in {"i", "u"}: - arr_i32 = np.ascontiguousarray(arr.astype(np.int32, copy=False)) - return arr_i32, "i" - - # Default to float32 for floats/others numeric-like - arr_f32 = np.ascontiguousarray(arr.astype(np.float32, copy=False)) - return arr_f32, "f" - - -def write_stream_to_mseed(stream: Stream, output_file: Path) -> None: +def write_stream_to_mseed(stream: Stream, output_file: Path): """ Write an ObsPy Stream object to a MiniSEED file using mseedlib. + + Parameters + ---------- + stream : obspy.core.stream.Stream + The Stream object to write to MiniSEED + output_file : Path + The path to the output MiniSEED file + + Raises + ------ + ValueError + If the sample type of the trace data is not supported """ mstl = mseedlib.MSTraceList() - for trace in stream: start_time = mseedlib.timestr2nstime(f"{trace.stats.starttime.isoformat()}Z") - - # Prefer channel as a string; avoid join() on a string (it inserts underscores between characters). - channel = str(trace.stats.channel) - location = str(trace.stats.location or "00") - - sourceid = ( - f"FDSN:{trace.stats.network}_{trace.stats.station}_{location}_{channel}" - ) - - samples, sample_type = _coerce_mseed_samples(trace.data) - + sourceid = f"FDSN:{trace.stats.network}_{trace.stats.station}_{trace.stats.location}_{'_'.join(trace.stats.channel)}" mstl.add_data( sourceid=sourceid, - data_samples=samples, - sample_type=sample_type, - sample_rate=float(trace.stats.sampling_rate), + data_samples=trace.data, + sample_type="i", + sample_rate=trace.stats.sampling_rate, start_time=start_time, ) - output_file.parent.mkdir(exist_ok=True, parents=True) - with output_file.open("wb") as f: + with open(output_file, "wb") as f: mstl.pack( lambda record, handler_data: handler_data["fh"].write(record), {"fh": f}, From 737b183347422f790d8aa336a9329b1fd821fa5c Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 13 Mar 2026 18:09:40 +1300 Subject: [PATCH 47/72] remove the merge --- nzgmdb/data_retrieval/waveform_extraction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nzgmdb/data_retrieval/waveform_extraction.py b/nzgmdb/data_retrieval/waveform_extraction.py index 37ae19ca..b5fd4dff 100644 --- a/nzgmdb/data_retrieval/waveform_extraction.py +++ b/nzgmdb/data_retrieval/waveform_extraction.py @@ -785,7 +785,7 @@ def get_tmp_array_stream( st += read(str(f)) # Merge overlapping / adjacent segments - st.merge(method=1, fill_value=None) + # st.merge(method=1, fill_value=None) # Trim to exact window st.trim(start_time, end_time) From 4768e6419b782e1606693f5e857d81de70194cd2 Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 18 Mar 2026 17:23:51 +1300 Subject: [PATCH 48/72] phase arrival command fix --- nzgmdb/phase_arrival/gen_phase_arrival_table.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nzgmdb/phase_arrival/gen_phase_arrival_table.py b/nzgmdb/phase_arrival/gen_phase_arrival_table.py index ba4e563d..f55599d6 100644 --- a/nzgmdb/phase_arrival/gen_phase_arrival_table.py +++ b/nzgmdb/phase_arrival/gen_phase_arrival_table.py @@ -62,7 +62,11 @@ def process_batch( print(f"Skipping run_phasenet for Batch {batch_num} as results already exist") else: # Activate phaseNet environment and run over mseeds for the subfolder - phasenet_command = f"python {run_phasenet_script_ffp} {batch_txt} {output_dir} {bypass_records_ffp if bypass_records_ffp is not None else ''} {xml_dir if xml_dir is not None else ''}" + phasenet_command = f"python {run_phasenet_script_ffp} {batch_txt} {output_dir}" + if bypass_records_ffp is not None: + phasenet_command += f" --bypass_records_ffp {bypass_records_ffp}" + if xml_dir is not None: + phasenet_command += f" --xml_dir {xml_dir}" shell_commands.run_command( phasenet_command, conda_sh, env_activate_command, log_file_path_phasenet ) From 200aaeab2317db092f04d0dafcd4046c599ff795 Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 18 Mar 2026 17:39:07 +1300 Subject: [PATCH 49/72] tmp gmc 1 proc --- nzgmdb/config/machine_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nzgmdb/config/machine_config.yaml b/nzgmdb/config/machine_config.yaml index 3c57fd3b..10579187 100644 --- a/nzgmdb/config/machine_config.yaml +++ b/nzgmdb/config/machine_config.yaml @@ -41,7 +41,7 @@ rch: phase_table: 16 snr: 16 fmax: 64 - gmc: 16 + gmc: 1 process: 64 im: 32 distances: 64 From df8b76614ffea35d58a614a7143b5109a7b9887b Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 23 Mar 2026 14:05:00 +1300 Subject: [PATCH 50/72] fix xml for tmp --- nzgmdb/data_retrieval/inventory_xml.py | 50 +++++++++++++++----- nzgmdb/data_retrieval/waveform_extraction.py | 4 +- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/nzgmdb/data_retrieval/inventory_xml.py b/nzgmdb/data_retrieval/inventory_xml.py index 6e461f28..24a76a28 100644 --- a/nzgmdb/data_retrieval/inventory_xml.py +++ b/nzgmdb/data_retrieval/inventory_xml.py @@ -7,7 +7,7 @@ import pandas as pd from obspy.clients.fdsn import Client as FDSN_Client -from obspy.clients.fdsn.header import FDSNNoDataException +from obspy.clients.fdsn.header import FDSNException, FDSNNoDataException from nzgmdb.management import config as cfg from nzgmdb.management import file_structure @@ -63,18 +63,25 @@ def get_provider_inventory( raise ValueError("Provider must be specified if not using real-time data.") client = FDSN_Client(provider) networks = "*" if networks is None else ",".join(networks) - return client.get_stations( - network=networks, - station=stations, - channel=channel_codes, - level=level, - maxlatitude=max_lat, - minlatitude=min_lat, - maxlongitude=max_lon, - minlongitude=min_lon, - starttime=starttime, - endtime=endtime, - ) + try: + inv = client.get_stations( + network=networks, + station=stations, + channel=channel_codes, + level=level, + maxlatitude=max_lat, + minlatitude=min_lat, + maxlongitude=max_lon, + minlongitude=min_lon, + starttime=starttime, + endtime=endtime, + ) + except FDSNException: + print( + f"No inventory data found for provider {provider} with the specified parameters." + ) + inv = None + return inv def get_full_inventory( @@ -127,6 +134,8 @@ def get_full_inventory( starttime=starttime, endtime=endtime, ) + if inventory is None: + continue if return_inv is None: return_inv = inventory else: @@ -206,6 +215,7 @@ def get_full_inventory( def fetch_and_save_inventory( main_dir: Path, stations: list[str], + add_tmp_arrays: bool = False, starttime: str = "2000-01-01", endtime: str = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d"), ): @@ -218,6 +228,8 @@ def fetch_and_save_inventory( The main directory where the StationXML files will be saved. stations : list[str] A list of station codes to fetch the inventory data for. + add_tmp_arrays : bool, optional + Whether to include temporary array providers in the inventory fetch, by default False. starttime : str, optional The start time for the inventory data, by default "2000-01-01". endtime : str, optional @@ -230,6 +242,7 @@ def fetch_and_save_inventory( try: inv = get_full_inventory( + add_tmp_arrays=add_tmp_arrays, stations=all_stations, starttime=starttime, endtime=endtime, @@ -243,3 +256,14 @@ def fetch_and_save_inventory( sel.write(fname, format="STATIONXML") except FDSNNoDataException: print("No inventory data found for the specified stations and time range.") + + +df = pd.read_csv( + "/media/joel/data/nzgmdb/tmp_arrays/rch_run_template/flatfiles/station_table_all.csv" +) +df_chan = df[df["chan"].isin(["EH", "DH"])] +unique_sites = df_chan["sta"].unique() +main_dir = Path( + "/media/joel/data/nzgmdb/tmp_arrays/rch_run_template/flatfiles/dh_eh_xmls" +) +fetch_and_save_inventory(main_dir, unique_sites, add_tmp_arrays=True) diff --git a/nzgmdb/data_retrieval/waveform_extraction.py b/nzgmdb/data_retrieval/waveform_extraction.py index b5fd4dff..fa8a6520 100644 --- a/nzgmdb/data_retrieval/waveform_extraction.py +++ b/nzgmdb/data_retrieval/waveform_extraction.py @@ -1309,7 +1309,9 @@ def extract_waveforms( # Grab all the station xmls and write them as outputs unique_sites = station_extraction_table["sta"].unique() print(f"Fetching station XML metadata for {len(unique_sites)} unique sites") - inventory_xml.fetch_and_save_inventory(main_dir, unique_sites) + inventory_xml.fetch_and_save_inventory( + main_dir, unique_sites, add_tmp_arrays=tmp_array_dir is not None + ) print("Station XML metadata fetching complete.") # Combine all the event and sta_mag dataframes From 936bb115ffdaf7ad6a18d219dc7c40bba5b7d6f4 Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 23 Mar 2026 14:05:31 +1300 Subject: [PATCH 51/72] remove testing code --- nzgmdb/data_retrieval/inventory_xml.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/nzgmdb/data_retrieval/inventory_xml.py b/nzgmdb/data_retrieval/inventory_xml.py index 24a76a28..9492db7c 100644 --- a/nzgmdb/data_retrieval/inventory_xml.py +++ b/nzgmdb/data_retrieval/inventory_xml.py @@ -256,14 +256,3 @@ def fetch_and_save_inventory( sel.write(fname, format="STATIONXML") except FDSNNoDataException: print("No inventory data found for the specified stations and time range.") - - -df = pd.read_csv( - "/media/joel/data/nzgmdb/tmp_arrays/rch_run_template/flatfiles/station_table_all.csv" -) -df_chan = df[df["chan"].isin(["EH", "DH"])] -unique_sites = df_chan["sta"].unique() -main_dir = Path( - "/media/joel/data/nzgmdb/tmp_arrays/rch_run_template/flatfiles/dh_eh_xmls" -) -fetch_and_save_inventory(main_dir, unique_sites, add_tmp_arrays=True) From 15631a724c1929e6db2a6d99ec22884966333124 Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 25 Mar 2026 10:57:03 +1300 Subject: [PATCH 52/72] small fixes --- nzgmdb/data_retrieval/sites.py | 17 +++++++++++++-- nzgmdb/data_retrieval/waveform_extraction.py | 22 +++++++------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/nzgmdb/data_retrieval/sites.py b/nzgmdb/data_retrieval/sites.py index c2825886..2aa066bc 100644 --- a/nzgmdb/data_retrieval/sites.py +++ b/nzgmdb/data_retrieval/sites.py @@ -233,12 +233,25 @@ def create_site_table_response( } ) + # Remove the duplicated stations between different networks + all_info_df = all_info_df.drop_duplicates( + subset=[ + "sta", + "lat", + "lon", + "elev", + "chan", + "loc", + "loc_elev", + "start_time", + "end_time", + ] + ) + # separate into site and sta here to avoid merging issues exploding site_df = all_info_df[ ["provider", "net", "sta", "lat", "lon", "elev", "creation_date", "end_date"] ] - # Remove duplicate stations (keep first occurrence) - site_df = site_df.drop_duplicates(subset=["provider", "net", "sta"]) merged_df = site_df.merge( geo_meta_summary_df, diff --git a/nzgmdb/data_retrieval/waveform_extraction.py b/nzgmdb/data_retrieval/waveform_extraction.py index fa8a6520..125e49bf 100644 --- a/nzgmdb/data_retrieval/waveform_extraction.py +++ b/nzgmdb/data_retrieval/waveform_extraction.py @@ -911,7 +911,7 @@ def extract_station_info( for chan, loc in unique_channels: # Each unique channel and location pair is a new mseed file st_new = st.select(location=loc, channel=f"{chan}?") - record_id = f"{event_id}_{st_new[0].stats.station}_{st_new[0].stats.channel[:2]}_{st_new[0].stats.location}" + record_id = f"{event_id}_{st_new[0].stats.station}_{st_new[0].stats.channel[:2]}_{'00' if st_new[0].stats.location == '' else st_new[0].stats.location}" # Check trace issues st_revised, skipped, issues = check_trace_issues( @@ -934,29 +934,26 @@ def extract_station_info( ] for mseed in mseeds: + stats = mseed[0].stats + record_id = f"{event_id}_{stats.station}_{stats.channel[:2]}_{'00' if stats.location == '' else stats.location}" try: # Check the data is not all 0's if all([np.allclose(tr.data, 0) for tr in mseed]): - stats = mseed[0].stats + skipped_records.append( pd.DataFrame( { - "record_id": [ - f"{event_id}_{stats.station}_{stats.channel}_{stats.location}" - ], + "record_id": [record_id], "reason": ["All 0's"], } ) ) continue except TypeError: - stats = mseed[0].stats skipped_records.append( pd.DataFrame( { - "record_id": [ - f"{event_id}_{stats.station}_{stats.channel}_{stats.location}" - ], + "record_id": [record_id], "reason": ["TypeError when checking for all 0's"], } ) @@ -964,13 +961,10 @@ def extract_station_info( # Check for 3 component data, if not skip if len(mseed) < 3: - stats = mseed[0].stats skipped_records.append( pd.DataFrame( { - "record_id": [ - f"{event_id}_{stats.station}_{stats.channel}_{stats.location}" - ], + "record_id": [record_id], "reason": ["Less than 3 component traces"], } ) @@ -981,8 +975,6 @@ def extract_station_info( clip = filtering.get_clip_probability(event_mag, r_hyp, mseed) threshold = config.get_value("clip_threshold") - stats = mseed[0].stats - record_id = f"{event_id}_{stats.station}_{stats.channel[:2]}_{stats.location}" # Check if the record should be dropped if clip > threshold: From 4bf046048fbad9dbce6bec90111cc35192a7d257 Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 25 Mar 2026 12:05:54 +1300 Subject: [PATCH 53/72] report fixes --- nzgmdb/scripts/generate_report.py | 46 ++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/nzgmdb/scripts/generate_report.py b/nzgmdb/scripts/generate_report.py index 8aae12a1..0bbb40f0 100644 --- a/nzgmdb/scripts/generate_report.py +++ b/nzgmdb/scripts/generate_report.py @@ -1214,6 +1214,29 @@ def plot_site_table_image( ) +def parse_nzgmdb_version(value: str) -> NZGMDB_Versions: + """ + Parse a string input into an NZGMDB_Versions enum member. + + Parameters + ---------- + value : str + The input string representing the NZGMDB version (e.g., "4p3", "4.3", "v4p3"). + + Returns + ------- + NZGMDB_Versions + The corresponding NZGMDB_Versions enum member. + """ + try: + return NZGMDB_Versions(value.strip().lower()) + except ValueError as exc: + allowed = ", ".join(v.value for v in NZGMDB_Versions) + raise typer.BadParameter( + f"Invalid NZGMDB version: {value}. Allowed: {allowed}" + ) from exc + + @cli.from_docstring(app) def generate_report( new_version_directory: Annotated[ @@ -1235,21 +1258,17 @@ def generate_report( ), ] = None, new_version: Annotated[ - Optional[NZGMDB_Versions], + str, typer.Option( - None, - help="The version for the new database (choose from the enum).", case_sensitive=False, ), - ] = NZGMDB_Versions.V4p3, + ] = "4p3", old_version: Annotated[ - Optional[NZGMDB_Versions], + str, typer.Option( - None, - help="The version for the old database (choose from the enum).", case_sensitive=False, ), - ] = NZGMDB_Versions.V4p3, + ] = "4p3", ): """ Generate a HTML report comparing the new version of the database to a previous version. @@ -1265,10 +1284,13 @@ def generate_report( The Top Level directory containing the previous version of the database to compare against. If None, a summary of the new version will be generated instead and comparison plots will not be generated. new_version : NZGMDB_Versions | None - The version for the new database (choose from the enum). Default is NZGMDB_Versions.V4p3. + The version for the new database (e.g., "4p3", "4p4"). Used for labeling in the report. old_version : NZGMDB_Versions | None - The version for the old database (choose from the enum). Default is NZGMDB_Versions.V4p3. + The version for the old database (e.g., "4p3", "4p4"). Used for labeling in the report. """ + new_version = parse_nzgmdb_version(new_version) + old_version = parse_nzgmdb_version(old_version) + html_parts = [] # Start of HTML html_parts.append( @@ -2134,3 +2156,7 @@ def generate_report( # Save report with open(output_file, "w") as f: f.write("".join(html_parts)) + + +if __name__ == "__main__": + app() From c7dc06554fa5e7058d53fb0b72e093cafd31f13f Mon Sep 17 00:00:00 2001 From: joelridden Date: Fri, 27 Mar 2026 11:35:17 +1300 Subject: [PATCH 54/72] fix site dup --- nzgmdb/data_retrieval/sites.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nzgmdb/data_retrieval/sites.py b/nzgmdb/data_retrieval/sites.py index 2aa066bc..0ccda6e6 100644 --- a/nzgmdb/data_retrieval/sites.py +++ b/nzgmdb/data_retrieval/sites.py @@ -233,6 +233,9 @@ def create_site_table_response( } ) + for col in ("start_time", "end_time"): + all_info_df[col] = pd.to_datetime(all_info_df[col], format="ISO8601") + # Remove the duplicated stations between different networks all_info_df = all_info_df.drop_duplicates( subset=[ From d879df6fccfe6ee6c09684942d4448f4dc9f1747 Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 30 Mar 2026 12:00:36 +1300 Subject: [PATCH 55/72] incorrect bypass call --- nzgmdb/config/machine_config.yaml | 2 +- nzgmdb/phase_arrival/gen_phase_arrival_table.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nzgmdb/config/machine_config.yaml b/nzgmdb/config/machine_config.yaml index 10579187..3c57fd3b 100644 --- a/nzgmdb/config/machine_config.yaml +++ b/nzgmdb/config/machine_config.yaml @@ -41,7 +41,7 @@ rch: phase_table: 16 snr: 16 fmax: 64 - gmc: 1 + gmc: 16 process: 64 im: 32 distances: 64 diff --git a/nzgmdb/phase_arrival/gen_phase_arrival_table.py b/nzgmdb/phase_arrival/gen_phase_arrival_table.py index f55599d6..fd3403f3 100644 --- a/nzgmdb/phase_arrival/gen_phase_arrival_table.py +++ b/nzgmdb/phase_arrival/gen_phase_arrival_table.py @@ -64,7 +64,7 @@ def process_batch( # Activate phaseNet environment and run over mseeds for the subfolder phasenet_command = f"python {run_phasenet_script_ffp} {batch_txt} {output_dir}" if bypass_records_ffp is not None: - phasenet_command += f" --bypass_records_ffp {bypass_records_ffp}" + phasenet_command += f" --bypass_ffp {bypass_records_ffp}" if xml_dir is not None: phasenet_command += f" --xml_dir {xml_dir}" shell_commands.run_command( From 8f6477ed063020688688ba3eeb5ba3dbe5d7fbd2 Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 30 Mar 2026 13:25:15 +1300 Subject: [PATCH 56/72] bypass df fix --- nzgmdb/phase_arrival/run_phasenet.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nzgmdb/phase_arrival/run_phasenet.py b/nzgmdb/phase_arrival/run_phasenet.py index b464655d..b4d87a36 100644 --- a/nzgmdb/phase_arrival/run_phasenet.py +++ b/nzgmdb/phase_arrival/run_phasenet.py @@ -355,9 +355,10 @@ def run_phasenet( mseed_file = Path(mseed_file) bypass_row = None if bypass_ffp is not None: - bypass_row = bypass_df.loc[bypass_df["record_id"] == mseed_file.stem].iloc[ - 0 - ] + bypass_rows = bypass_df.loc[bypass_df["record_id"] == mseed_file.stem] + if len(bypass_rows) > 0: + bypass_row = bypass_rows.iloc[0] + inventory = None if xml_dir is not None: station = mseed_file.stem.split("_")[1] From a64a2510fc24c61a48ed768fe3081b277163774e Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 13 Apr 2026 10:19:43 +1200 Subject: [PATCH 57/72] phasenet checkpointing --- .../phase_arrival/gen_phase_arrival_table.py | 48 +++++++++++++------ nzgmdb/scripts/run_nzgmdb.py | 17 +++++++ 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/nzgmdb/phase_arrival/gen_phase_arrival_table.py b/nzgmdb/phase_arrival/gen_phase_arrival_table.py index fd3403f3..e2766683 100644 --- a/nzgmdb/phase_arrival/gen_phase_arrival_table.py +++ b/nzgmdb/phase_arrival/gen_phase_arrival_table.py @@ -84,6 +84,7 @@ def generate_phase_arrival_table( conda_sh: Path, env_activate_command: str, n_procs: int, + n_batches: int = None, bypass_records_ffp: Path = None, xml_dir: Path = None, ): @@ -103,6 +104,8 @@ def generate_phase_arrival_table( The command to activate the environment for running PhaseNet. n_procs : int The number of processes to use + n_batches : int, optional + The number of batches to split the mseed files into. If None, it will be set to the number of processes. (Default is None) bypass_records_ffp : Path The full file path to the bypass records file, which includes a custom p_wave_ix xml_dir: Path @@ -119,27 +122,42 @@ def generate_phase_arrival_table( mseed_files = list(main_dir.rglob("*.mseed")) # Split them into even batches based on number of mseeds and n_procs - # Ensure n_procs gets reduced if it is greater than the number of mseed files + # Ensure n_procs and n_batches gets reduced if it is greater than the number of mseed files n_procs = min(n_procs, len(mseed_files)) - mseed_batches = np.array_split(mseed_files, n_procs) + n_batches = n_batches or n_procs + n_batches = min(n_batches, len(mseed_files)) + mseed_batches = np.array_split(mseed_files, n_batches) batches = [ (batch, (phase_dir / f"batch_{idx}")) for idx, batch in enumerate(mseed_batches) ] - # Fetch results - with mp.Pool(n_procs) as p: - p.map( - functools.partial( - process_batch, - run_phasenet_script_ffp=run_phasenet_script_ffp, - conda_sh=conda_sh, - env_activate_command=env_activate_command, - bypass_records_ffp=bypass_records_ffp, - xml_dir=xml_dir, - ), - batches, - ) + # Checkpointing: only schedule batches that are missing outputs + pending_batches = [] + for batch, out_dir in batches: + phase_table_ffp = out_dir / file_structure.FlatfileNames.PHASE_ARRIVAL_TABLE + if phase_table_ffp.exists(): + batch_num = out_dir.name.split("_")[-1] + print(f"Skipping Batch {batch_num} (found existing phase arrival table)") + continue + pending_batches.append((batch, out_dir)) + + if not pending_batches: + print("All batches already have a phase arrival table; nothing to run.") + else: + # Fetch results (only for pending batches) + with mp.Pool(n_procs) as p: + p.map( + functools.partial( + process_batch, + run_phasenet_script_ffp=run_phasenet_script_ffp, + conda_sh=conda_sh, + env_activate_command=env_activate_command, + bypass_records_ffp=bypass_records_ffp, + xml_dir=xml_dir, + ), + pending_batches, + ) # For each subfolder combine the phase_arrival_table.csv and skipped_records.csv into a single file phase_results = [] diff --git a/nzgmdb/scripts/run_nzgmdb.py b/nzgmdb/scripts/run_nzgmdb.py index cb2f8be1..481db11a 100644 --- a/nzgmdb/scripts/run_nzgmdb.py +++ b/nzgmdb/scripts/run_nzgmdb.py @@ -197,6 +197,10 @@ def make_phase_arrival_table( typer.Argument(), ], n_procs: Annotated[int, typer.Option()] = 1, + n_batches: Annotated[ + int, + typer.Option(), + ] = None, bypass_records_ffp: Annotated[ Path, typer.Option( @@ -223,6 +227,9 @@ def make_phase_arrival_table( The command to activate the environment for running PhaseNet. n_procs : int, optional Number of processes to use (default is 1). + n_batches : int, optional + The number of batches to split the mseed files into for processing. + If not provided, it will be determined automatically based on the number of mseed files and n_procs. bypass_records_ffp : Path, optional The full file path to the bypass records file for custom P-wave index values. This allows you to specify custom P-wave index values for records that may not be @@ -234,6 +241,7 @@ def make_phase_arrival_table( conda_sh, env_activate_command, n_procs, + n_batches, bypass_records_ffp, xml_dir=file_structure.get_stationxml_dir(main_dir), ) @@ -847,6 +855,10 @@ def run_full_nzgmdb( int, typer.Option(), ] = 5000, + phase_arrival_n_batches: Annotated[ + int, + typer.Option(), + ] = None, real_time: Annotated[ bool, typer.Option(), @@ -891,6 +903,7 @@ def run_full_nzgmdb( Steps Included: - Fetch Geonet data + - Waveform extraction - Merge tectonic domains - Generate phase arrival table - Calculate SNR @@ -939,6 +952,9 @@ def run_full_nzgmdb( The batch size for Geonet data retrieval (default is 500). snr_batch_size : int, optional The batch size for the SNR calculation (default is 5000). + phase_arrival_n_batches : int, optional + The number of batches to split the phase arrival calculation into + (default is None, which will automatically determine the number of batches based on the number of records and n_procs). real_time : bool, optional If True, the function will run in real-time mode using a different client (default is False). upload : bool, optional @@ -1070,6 +1086,7 @@ def run_full_nzgmdb( conda_sh, gmc_activate, phase_n_procs, + phase_arrival_n_batches, bypass_records_ffp, xml_dir=file_structure.get_stationxml_dir(main_dir), ) From 2ef18090c85d930e5fbf7745371da8c8aa52a1e9 Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 13 Apr 2026 11:41:51 +1200 Subject: [PATCH 58/72] gmc checkpointing --- nzgmdb/scripts/run_gmc.py | 41 +++++++++++++++++++++++++----------- nzgmdb/scripts/run_nzgmdb.py | 8 +++++++ 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/nzgmdb/scripts/run_gmc.py b/nzgmdb/scripts/run_gmc.py index c8c96356..ec1e2f8f 100644 --- a/nzgmdb/scripts/run_gmc.py +++ b/nzgmdb/scripts/run_gmc.py @@ -198,6 +198,10 @@ def run_gmc_processing( int, typer.Option(), ] = 1, + gmc_n_batches: Annotated[ + int, + typer.Option(), + ] = None, waveform_dir: Annotated[ Path, typer.Option( @@ -239,6 +243,9 @@ def run_gmc_processing( Command to activate the GMC predict environment to run predictions. n_procs : int, optional Number of processes to use for multiprocessing (default is 1). + gmc_n_batches : int, optional + Number of batches to split the mseed files into for processing. + If None, it will be set to the number of processes (default is None). waveform_dir : Path, optional Directory containing all waveform files. output_dir : Path, optional @@ -273,11 +280,13 @@ def run_gmc_processing( # Get all the mseed files mseed_files = list(waveform_dir.rglob("*.mseed")) - # Ensure that n_procs is equal too or less than the number of mseed files + # Ensure that n_procs and n_batches is equal too or less than the number of mseed files n_procs = min(n_procs, len(mseed_files)) + n_batches = gmc_n_batches or n_procs + n_batches = min(n_batches, len(mseed_files)) - # Split them into even batches based on number of mseeds and n_procs - mseed_batches = np.array_split(mseed_files, n_procs) + # Split them into even batches based on number of mseeds and n_batches + mseed_batches = np.array_split(mseed_files, n_batches) # Create a partial function with common arguments pre-filled process_partial = functools.partial( @@ -293,15 +302,23 @@ def run_gmc_processing( xml_dir=file_structure.get_stationxml_dir(main_dir), ) - # Use multiprocessing with starmap and the partial function - with multiprocessing.Pool(n_procs) as p: - p.starmap( - process_partial, - [ - (batch, (gmc_dir / f"batch_{idx}")) - for idx, batch in enumerate(mseed_batches) - ], - ) + pending_jobs = [] + for idx, batch in enumerate(mseed_batches): + batch_dir = gmc_dir / f"batch_{idx}" + predictions_output = batch_dir / file_structure.FlatfileNames.GMC_PREDICTIONS + + if predictions_output.exists(): + print(f"Skipping Batch {idx} (found existing gmc_predictions.csv)") + continue + + pending_jobs.append((list(batch), batch_dir)) + + if not pending_jobs: + print("All batches already have gmc_predictions.csv; nothing to run.") + else: + # Use multiprocessing with starmap and the partial function (pending only) + with multiprocessing.Pool(n_procs) as p: + p.starmap(process_partial, pending_jobs) # For each subfolder combine the gmc_predictions.csv into a single file dfs = [] diff --git a/nzgmdb/scripts/run_nzgmdb.py b/nzgmdb/scripts/run_nzgmdb.py index 481db11a..493e3f28 100644 --- a/nzgmdb/scripts/run_nzgmdb.py +++ b/nzgmdb/scripts/run_nzgmdb.py @@ -859,6 +859,10 @@ def run_full_nzgmdb( int, typer.Option(), ] = None, + gmc_n_batches: Annotated[ + int, + typer.Option(), + ] = None, real_time: Annotated[ bool, typer.Option(), @@ -955,6 +959,9 @@ def run_full_nzgmdb( phase_arrival_n_batches : int, optional The number of batches to split the phase arrival calculation into (default is None, which will automatically determine the number of batches based on the number of records and n_procs). + gmc_n_batches : int, optional + The number of batches to split the GMC prediction step into + (default is None, which will automatically determine the number of batches based on the number of records and gmc_procs). real_time : bool, optional If True, the function will run in real-time mode using a different client (default is False). upload : bool, optional @@ -1154,6 +1161,7 @@ def run_full_nzgmdb( gmc_activate, gmc_predict_activate, gmc_n_procs, + gmc_n_batches, bypass_records_ffp=bypass_records_ffp, ) From 030bc0715a5d73a22c392b951b7ee71ae496946f Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 13 Apr 2026 13:10:14 +1200 Subject: [PATCH 59/72] remove gmc procs --- nzgmdb/scripts/run_nzgmdb.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/nzgmdb/scripts/run_nzgmdb.py b/nzgmdb/scripts/run_nzgmdb.py index 493e3f28..46f2281a 100644 --- a/nzgmdb/scripts/run_nzgmdb.py +++ b/nzgmdb/scripts/run_nzgmdb.py @@ -819,10 +819,6 @@ def run_full_nzgmdb( file_okay=False, ), ], - gmc_procs: Annotated[ - int, - typer.Option(), - ] = 1, n_procs: Annotated[int, typer.Option()] = 1, checkpoint: Annotated[ bool, @@ -906,6 +902,7 @@ def run_full_nzgmdb( This function orchestrates the full pipeline of NZGMDB, executing all necessary steps sequentially. Steps Included: + - Generate site table with basin information - Fetch Geonet data - Waveform extraction - Merge tectonic domains @@ -940,8 +937,6 @@ def run_full_nzgmdb( Command to activate gmc_predict environment to run the predictions. ko_matrix_path : Path Path to the ko matrix directory - gmc_procs : int, optional - Number of processes to use for GMC (default is 1). n_procs : int, optional The number of processes to use (default is 1). checkpoint : bool, optional @@ -1149,7 +1144,7 @@ def run_full_nzgmdb( ): print("Running GMC") gmc_n_procs = ( - gmc_procs + n_procs if machine is None else config.get_n_procs(machine, cfg.WorkflowStep.GMC) ) From c61dee4343c7790fae66d164184174db29ac082e Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 13 Apr 2026 14:15:55 +1200 Subject: [PATCH 60/72] decrease batch size --- nzgmdb/scripts/run_nzgmdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nzgmdb/scripts/run_nzgmdb.py b/nzgmdb/scripts/run_nzgmdb.py index 46f2281a..259a77b2 100644 --- a/nzgmdb/scripts/run_nzgmdb.py +++ b/nzgmdb/scripts/run_nzgmdb.py @@ -846,7 +846,7 @@ def run_full_nzgmdb( geonet_batch_size: Annotated[ int, typer.Option(), - ] = 500, + ] = 100, snr_batch_size: Annotated[ int, typer.Option(), From 13e78e2857d77e245e0664c634098cc2a88cd84f Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 13 Apr 2026 14:59:31 +1200 Subject: [PATCH 61/72] extraction script fix --- nzgmdb/scripts/run_nzgmdb.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/nzgmdb/scripts/run_nzgmdb.py b/nzgmdb/scripts/run_nzgmdb.py index 259a77b2..954e2972 100644 --- a/nzgmdb/scripts/run_nzgmdb.py +++ b/nzgmdb/scripts/run_nzgmdb.py @@ -117,6 +117,13 @@ def extract_waveforms( int, typer.Option(), ] = 1000, + tmp_array_dir: Annotated[ + Path, + typer.Option( + exists=True, + file_okay=False, + ), + ] = None, ): """ Extract waveforms using the station extraction table and save them as MiniSEED files. @@ -134,9 +141,16 @@ def extract_waveforms( The full file path to a set of record IDs to only run for. If provided, only these records will be processed. batch_size : int, optional The batch size for how many extracted waveforms to process before checkpointing (default is 1000). + tmp_array_dir : Path, optional + The directory the saved temporary array data is to be used for waveform extraction. """ waveform_extraction.extract_waveforms( - main_dir, station_extraction_table_ffp, n_procs, only_record_ids_ffp, batch_size + main_dir, + station_extraction_table_ffp, + n_procs, + only_record_ids_ffp, + batch_size, + tmp_array_dir, ) From 4dc52870529f7370591a544f11141000982c7c1f Mon Sep 17 00:00:00 2001 From: joelridden Date: Mon, 13 Apr 2026 15:25:54 +1200 Subject: [PATCH 62/72] adjust wiki --- wiki/Waveform-Extraction.md | 1 + 1 file changed, 1 insertion(+) diff --git a/wiki/Waveform-Extraction.md b/wiki/Waveform-Extraction.md index 8564bd17..30ad9a3c 100644 --- a/wiki/Waveform-Extraction.md +++ b/wiki/Waveform-Extraction.md @@ -21,6 +21,7 @@ python -m nzgmdb.scripts.run_nzgmdb extract-waveforms Date: Tue, 14 Apr 2026 13:43:52 +1200 Subject: [PATCH 63/72] wait 5 min --- nzgmdb/data_retrieval/waveform_extraction.py | 68 ++++++++++++++++---- 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/nzgmdb/data_retrieval/waveform_extraction.py b/nzgmdb/data_retrieval/waveform_extraction.py index 125e49bf..7301cd65 100644 --- a/nzgmdb/data_retrieval/waveform_extraction.py +++ b/nzgmdb/data_retrieval/waveform_extraction.py @@ -1190,18 +1190,62 @@ def extract_waveforms( print("Retrying in 120 seconds...") time.sleep(120) # Wait for 2 minutes before retrying - with mp.Pool(n_procs) as pool: - results = pool.map( - functools.partial( - extract_station_info, - main_dir=main_dir, - event_catalogues=catalog_dict, - extraction_table=station_extraction_table, - only_record_ids=only_record_ids, - tmp_array_dir=tmp_array_dir, - ), - (row for _, row in batch_rows.iterrows()), - ) + extract_fn = functools.partial( + extract_station_info, + main_dir=main_dir, + event_catalogues=catalog_dict, + extraction_table=station_extraction_table, + only_record_ids=only_record_ids, + tmp_array_dir=tmp_array_dir, + ) + + results = [] + pool = mp.Pool(processes=n_procs) + timeout_s = 60 * 5 # 5 min + try: + for _, row in batch_rows.iterrows(): + job = pool.apply_async(extract_fn, (row,)) + record_id = f"{row['evid']}_{row['sta']}" + try: + results.append(job.get(timeout=timeout_s)) + except mp.TimeoutError: + results.append( + StationExtractionResult( + sta_mag_line=[], + skipped_records=[ + pd.DataFrame( + { + "record_id": [record_id], + "reason": [ + f"Hung (> {timeout_s // 60} min)" + ], + } + ) + ], + clipped_records=[], + multi_trace_issues=[], + multi_event_records=[], + ) + ) + pool.terminate() + pool.join() + pool = mp.Pool(processes=n_procs) + finally: + pool.close() + pool.join() + + # with mp.Pool(n_procs) as pool: + # results = pool.map( + # functools.partial( + # extract_station_info, + # main_dir=main_dir, + # event_catalogues=catalog_dict, + # extraction_table=station_extraction_table, + # only_record_ids=only_record_ids, + # tmp_array_dir=tmp_array_dir, + # ), + # (row for _, row in batch_rows.iterrows()), + # ) # Extract the results ( From 3d73d3b6421c7f0855991bd80f313d8279fc9758 Mon Sep 17 00:00:00 2001 From: joelridden Date: Tue, 14 Apr 2026 16:15:48 +1200 Subject: [PATCH 64/72] type fixes --- nzgmdb/calculation/aftershocks.py | 6 +++--- nzgmdb/calculation/distances.py | 6 +++--- pyproject.toml | 7 +++++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/nzgmdb/calculation/aftershocks.py b/nzgmdb/calculation/aftershocks.py index 7b284210..111eda6e 100644 --- a/nzgmdb/calculation/aftershocks.py +++ b/nzgmdb/calculation/aftershocks.py @@ -185,7 +185,7 @@ def resample_polygon_1km(rupture_polygons: list[Polygon]) -> list[MultiPoint]: def calculate_crjb( rupture_poly: Polygon, boundary_points: MultiPoint, centroids: np.ndarray -) -> np.ndarray: +) -> np.ndarray | float: """ Calculates centroid Joyner-Boore (CRJB) distance for given earthquake centroids. @@ -200,8 +200,8 @@ def calculate_crjb( Returns ------- - numpy.ndarray - Array of min CRJB distances. + numpy.ndarray or float + Array of min CRJB distances for each centroid, or a single float if only one centroid is provided. """ # Gather the points of the boundary points = np.array([(p.x, p.y) for p in boundary_points.geoms]) diff --git a/nzgmdb/calculation/distances.py b/nzgmdb/calculation/distances.py index 24d343ce..1f3b91ff 100644 --- a/nzgmdb/calculation/distances.py +++ b/nzgmdb/calculation/distances.py @@ -107,9 +107,9 @@ def run_ccld_simulation( dip: float, rake: float, method: str, - strike2: float = None, - dip2: float = None, - rake2: float = None, + strike2: float | None = None, + dip2: float | None = None, + rake2: float | None = None, ) -> FocalMechanism: """ Run the CCLD simulation for an event. diff --git a/pyproject.toml b/pyproject.toml index 2dc56e84..87a4e60b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,4 +82,11 @@ checks = [ "RT03", "RT04", "YD01", +] + +[tool.ty] +ignore = ["reportGeneralTypeIssues"] +exclude = [ + "setup.py", + "nzgmdb/CCLD/ccldpy.py", ] \ No newline at end of file From eabd82c90b684dd64f420c4f9bb28d2acac33161 Mon Sep 17 00:00:00 2001 From: joelridden Date: Tue, 14 Apr 2026 16:18:27 +1200 Subject: [PATCH 65/72] remove exclude --- pyproject.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 87a4e60b..231dfa07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,8 +85,4 @@ checks = [ ] [tool.ty] -ignore = ["reportGeneralTypeIssues"] -exclude = [ - "setup.py", - "nzgmdb/CCLD/ccldpy.py", -] \ No newline at end of file +ignore = ["reportGeneralTypeIssues"] \ No newline at end of file From 1d453922b0bc5a65f90c9e2f8d11fdb69bd5b4de Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 15 Apr 2026 10:05:18 +1200 Subject: [PATCH 66/72] remove rule --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 231dfa07..03b94001 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,3 @@ checks = [ "RT04", "YD01", ] - -[tool.ty] -ignore = ["reportGeneralTypeIssues"] \ No newline at end of file From 67ab4a92c47dc444941db6140b5806ed323febda Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 15 Apr 2026 14:49:30 +1200 Subject: [PATCH 67/72] ty rules --- nzgmdb/calculation/distances.py | 22 ++++++++++++---------- nzgmdb/calculation/fmax.py | 2 +- nzgmdb/scripts/upload_to_dropbox.py | 2 +- pyproject.toml | 11 +++++++++++ 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/nzgmdb/calculation/distances.py b/nzgmdb/calculation/distances.py index 1f3b91ff..f3e94849 100644 --- a/nzgmdb/calculation/distances.py +++ b/nzgmdb/calculation/distances.py @@ -361,7 +361,7 @@ def get_nodal_plane_info( The focal type that determined the nodal plane (ff, geonet_rm, cmt, cmt_unc, domain) """ # Create the default return to be filled using defaultdict - nodal_plane_info = defaultdict(lambda: None) + nodal_plane_info: dict[str, object | None] = defaultdict(lambda: None) ccld_info = None # Split the cmt data into reviewed and unreviewed data @@ -565,7 +565,7 @@ def get_nodal_plane_info( ccld_info = run_ccld_simulation( event_id, event_row, strike, dip, rake, "D" ) - nodal_plane_info.update(ccld_info) + nodal_plane_info.update(ccld_info) # type: ignore[no-matching-overload] return nodal_plane_info # Find the closest point in the table @@ -596,7 +596,7 @@ def get_nodal_plane_info( if ccld_info is not None: # Update the nodal plane info with the ccld info - nodal_plane_info.update(ccld_info) + nodal_plane_info.update(ccld_info) # type: ignore[no-matching-overload] return nodal_plane_info @@ -613,7 +613,7 @@ def compute_distances_for_event( puy_objs: np.ndarray, nz_mech: dict, slab_faulting_geo: dict, -) -> tuple[Optional[pd.DataFrame], Optional[pd.DataFrame]]: +) -> tuple[Optional[pd.DataFrame], Optional[pd.DataFrame], Optional[pd.DataFrame]]: """ Compute the distances for a given event @@ -1012,7 +1012,7 @@ def perpendicular_height( point_vec = point - base_start cross = np.cross(base_vec, point_vec) base_len = np.linalg.norm(base_vec) - return np.linalg.norm(cross) / base_len if base_len else 0.0 + return float(np.linalg.norm(cross) / base_len) if base_len else 0.0 def inverse_square_integral( @@ -1133,11 +1133,12 @@ def distance_in_taupo( # Loop through all the stations for station_index, station in sta_df.iterrows(): + idx = int(station_index) # Create the line between the station and the event sta_transform = wgs2nztm.transform(station.lat, station.lon) line = LineString( [ - [rrups_transform[0][station_index], rrups_transform[1][station_index]], + [rrups_transform[0][idx], rrups_transform[1][idx]], [sta_transform[0], sta_transform[1]], ] ) @@ -1169,7 +1170,7 @@ def distance_in_taupo( ) line_points = line.intersection(taupo_polygon) - tvz_length = min(line_points.length / 1000 / r_epis[station_index], 1) + tvz_length = min(line_points.length / 1000 / r_epis[idx], 1) tvz_lengths.append(tvz_length) boundary_dists_rjb.append(boundary_dist_rjb) @@ -1242,9 +1243,10 @@ def calc_distances(main_dir: Path, n_procs: int = 1): ll_num = config.get_value("ll_num") nztm_num = config.get_value("nztm_num") wgs2nztm = Transformer.from_crs(ll_num, nztm_num) - taupo_transform = np.dstack( - np.array(wgs2nztm.transform(tvz_points.latitude, tvz_points.longitude)) - )[0] + x, y = wgs2nztm.transform( + tvz_points.latitude.to_numpy(), tvz_points.longitude.to_numpy() + ) + taupo_transform = np.column_stack((x, y)) taupo_polygon = Polygon(taupo_transform) # Go through the registry keys and check if they are .srf files to use diff --git a/nzgmdb/calculation/fmax.py b/nzgmdb/calculation/fmax.py index b4e46b3e..c172b942 100644 --- a/nzgmdb/calculation/fmax.py +++ b/nzgmdb/calculation/fmax.py @@ -20,7 +20,7 @@ def run_full_fmax_calc( waveform_dir: Path, snr_fas_output_dir: Path, n_procs: int = 1, - bypass_records_ffp: Path = None, + bypass_records_ffp: Path | None = None, ): """ Run the full procedure for each record to assess SNR produced from mseed files diff --git a/nzgmdb/scripts/upload_to_dropbox.py b/nzgmdb/scripts/upload_to_dropbox.py index c1aac9ad..424937df 100644 --- a/nzgmdb/scripts/upload_to_dropbox.py +++ b/nzgmdb/scripts/upload_to_dropbox.py @@ -443,7 +443,7 @@ def download_dropbox_archive( if not ignore_xml: xml_dir.mkdir(exist_ok=True) zips_to_download.append( - f"{dropbox_version_dir}/{xml_zip}", zip_dir / xml_zip, xml_dir + (f"{dropbox_version_dir}/{xml_zip}", zip_dir / xml_zip, xml_dir) ) # Ensure there is something to download diff --git a/pyproject.toml b/pyproject.toml index 03b94001..9ca6fe6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,3 +83,14 @@ checks = [ "RT04", "YD01", ] + +[tool.ty.rules] +default = "ignore" +invalid-parameter-default = "error" +invalid-argument-type = "error" +invalid-type-form = "error" +invalid-return-type = "error" +missing-argument = "error" +too-many-positional-arguments = "error" +missing-return = "error" +deprecated = "warn" From 245fcfed9f680a2a4c7b0e3d8ca2a985f6e1b6ba Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 15 Apr 2026 14:52:59 +1200 Subject: [PATCH 68/72] ignore test --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 9ca6fe6f..8133268e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,7 @@ checks = [ [tool.ty.rules] default = "ignore" +no-matching-overload = "ignore" invalid-parameter-default = "error" invalid-argument-type = "error" invalid-type-form = "error" From 5cb16b22d0088c5244157b30ab5fe768094ade77 Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 15 Apr 2026 15:25:50 +1200 Subject: [PATCH 69/72] proper ignores --- nzgmdb/data_retrieval/inventory_xml.py | 4 ++-- pyproject.toml | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/nzgmdb/data_retrieval/inventory_xml.py b/nzgmdb/data_retrieval/inventory_xml.py index 9492db7c..8d8c0430 100644 --- a/nzgmdb/data_retrieval/inventory_xml.py +++ b/nzgmdb/data_retrieval/inventory_xml.py @@ -62,10 +62,10 @@ def get_provider_inventory( if provider is None: raise ValueError("Provider must be specified if not using real-time data.") client = FDSN_Client(provider) - networks = "*" if networks is None else ",".join(networks) + networks_str = "*" if networks is None else ",".join(networks) try: inv = client.get_stations( - network=networks, + network=networks_str, station=stations, channel=channel_codes, level=level, diff --git a/pyproject.toml b/pyproject.toml index 8133268e..da2efdbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,13 +85,14 @@ checks = [ ] [tool.ty.rules] -default = "ignore" no-matching-overload = "ignore" -invalid-parameter-default = "error" -invalid-argument-type = "error" -invalid-type-form = "error" -invalid-return-type = "error" -missing-argument = "error" -too-many-positional-arguments = "error" -missing-return = "error" -deprecated = "warn" +unresolved-attribute = "ignore" +not-subscriptable = "ignore" +possibly-missing-submodule = "ignore" +invalid-type-arguments = "ignore" +unresolved-import = "ignore" +#invalid-parameter-default = "error" +#invalid-argument-type = "error" +#invalid-type-form = "error" +#invalid-return-type = "error" +#too-many-positional-arguments = "error" From ce98b0758a89e13ad6d5aca9f4e3617fc155ecfa Mon Sep 17 00:00:00 2001 From: joelridden Date: Tue, 21 Apr 2026 09:56:22 +1200 Subject: [PATCH 70/72] type fixing --- nzgmdb/calculation/distances.py | 2 +- nzgmdb/calculation/snr.py | 4 +-- nzgmdb/data_processing/merge_flatfiles.py | 4 +-- nzgmdb/data_processing/quality_db.py | 36 ++++++++++--------- .../data_processing/waveform_manipulation.py | 10 +++--- nzgmdb/data_retrieval/geonet.py | 22 ++++++------ nzgmdb/data_retrieval/inventory_xml.py | 4 +-- nzgmdb/data_retrieval/rupture_models.py | 4 +-- nzgmdb/data_retrieval/tect_domain.py | 16 ++++----- nzgmdb/data_retrieval/waveform_extraction.py | 8 ++--- pyproject.toml | 5 --- 11 files changed, 55 insertions(+), 60 deletions(-) diff --git a/nzgmdb/calculation/distances.py b/nzgmdb/calculation/distances.py index f3e94849..13d08051 100644 --- a/nzgmdb/calculation/distances.py +++ b/nzgmdb/calculation/distances.py @@ -1133,7 +1133,7 @@ def distance_in_taupo( # Loop through all the stations for station_index, station in sta_df.iterrows(): - idx = int(station_index) + idx = int(station_index) # type: ignore # Create the line between the station and the event sta_transform = wgs2nztm.transform(station.lat, station.lon) line = LineString( diff --git a/nzgmdb/calculation/snr.py b/nzgmdb/calculation/snr.py index 01b272e0..96dcdd01 100644 --- a/nzgmdb/calculation/snr.py +++ b/nzgmdb/calculation/snr.py @@ -229,9 +229,9 @@ def compute_snr_for_mseed_data( snr_fas_output_dir: Path, ko_directory: Path, n_procs: int = 1, - common_frequency_vector: np.ndarray = None, + common_frequency_vector: np.ndarray | None = None, batch_size: int = 5000, - bypass_records_ffp: Path = None, + bypass_records_ffp: Path | None = None, ): """ Compute the SNR for the data in the data_dir diff --git a/nzgmdb/data_processing/merge_flatfiles.py b/nzgmdb/data_processing/merge_flatfiles.py index 2446c16c..4bd10e7b 100644 --- a/nzgmdb/data_processing/merge_flatfiles.py +++ b/nzgmdb/data_processing/merge_flatfiles.py @@ -326,7 +326,7 @@ def custom_idxmin(group: pd.DataFrame): return gm_im_df_flat -def merge_flatfiles(main_dir: Path, bypass_records_ffp: Path = None): +def merge_flatfiles(main_dir: Path, bypass_records_ffp: Path | None = None): """ Merge the flatfiles into the final flatfiles, separating the components and ensuring that the data contains only the unique events and sites that made it to the IM calculation @@ -335,7 +335,7 @@ def merge_flatfiles(main_dir: Path, bypass_records_ffp: Path = None): ---------- main_dir : Path The main directory of the NZGMDB results (Highest level directory) - bypass_records_ffp : Path + bypass_records_ffp : Path, optional The full file path to the bypass records file, which includes a custom fmin, fmax, and p_wave_ix """ # Get the flatfile directory diff --git a/nzgmdb/data_processing/quality_db.py b/nzgmdb/data_processing/quality_db.py index bbec6288..fccc69b8 100644 --- a/nzgmdb/data_processing/quality_db.py +++ b/nzgmdb/data_processing/quality_db.py @@ -127,7 +127,9 @@ def filter_mag(catalogue: pd.DataFrame, mag_min: float): return skipped_records -def filter_has_score_mean(catalogue: pd.DataFrame, bypass_records: np.ndarray = None): +def filter_has_score_mean( + catalogue: pd.DataFrame, bypass_records: np.ndarray | None = None +): """ Filter the catalogue based on if there is a score from GMC. @@ -166,7 +168,7 @@ def filter_has_score_mean(catalogue: pd.DataFrame, bypass_records: np.ndarray = def filter_score_mean( catalogue: pd.DataFrame, score_min: float, - bypass_records: np.ndarray = None, + bypass_records: np.ndarray | None = None, include_z: bool = False, ): """ @@ -220,7 +222,7 @@ def filter_score_mean( def filter_multi_event( catalogue: pd.DataFrame, score_min: float, - bypass_records: np.ndarray = None, + bypass_records: np.ndarray | None = None, ): """ Filter the catalogue based on the multi-event STA/LTA and sync event check. @@ -262,7 +264,7 @@ def filter_multi_event( def filter_fmax( - catalogue: pd.DataFrame, fmax_min: float, bypass_records: np.ndarray = None + catalogue: pd.DataFrame, fmax_min: float, bypass_records: np.ndarray | None = None ): """ Filter the catalogue based on the fmax_min value for the fmax_X and fmax_Y. @@ -306,7 +308,7 @@ def filter_fmax( def filter_fmin( - catalogue: pd.DataFrame, fmin_max: float, bypass_records: np.ndarray = None + catalogue: pd.DataFrame, fmin_max: float, bypass_records: np.ndarray | None = None ): """ Filter the catalogue based on the fmin max value for the fmin_X and fmin_Y. @@ -385,7 +387,7 @@ def filter_missing_sta_info( def filter_ground_level_locations( - catalogue: pd.DataFrame, bypass_records: np.ndarray = None + catalogue: pd.DataFrame, bypass_records: np.ndarray | None = None ): """ Filter the catalogue based on the ground level locations @@ -425,7 +427,7 @@ def filter_ground_level_locations( def apply_clipNet_filter( clipped_records_ffp: Path, - bypass_records: np.ndarray = None, + bypass_records: np.ndarray | None = None, ): """ Apply the ClipNet filter to the catalogue @@ -471,7 +473,7 @@ def apply_clipNet_filter( def apply_jerk_filter( clipped_records_ffp: Path, - bypass_records: np.ndarray = None, + bypass_records: np.ndarray | None = None, ): """ Apply the Jerk filter from ClipNet to the catalogue @@ -516,7 +518,7 @@ def apply_jerk_filter( def filter_troublesome_sensitivity( - catalogue: pd.DataFrame, bypass_records: np.ndarray = None + catalogue: pd.DataFrame, bypass_records: np.ndarray | None = None ): """ Filter the catalogue by removing records that are known to be troublesome for sensitivity analysis. @@ -737,7 +739,7 @@ def filter_empirical_predictions( def filter_duplicate_channels( - catalogue: pd.DataFrame, bypass_records: np.ndarray = None + catalogue: pd.DataFrame, bypass_records: np.ndarray | None = None ): """ Filter the catalogue by removing lower-priority duplicate channel records. @@ -800,12 +802,12 @@ def filter_duplicate_channels( def apply_all_filters( catalogue: pd.DataFrame, clipped_records_ffp: Path, - bypass_records: np.ndarray = None, - score_min: float = None, - multi_score_min: float = None, - fmax_min: float = None, - fmin_max: float = None, - min_mag: float = None, + bypass_records: np.ndarray | None = None, + score_min: float | None = None, + multi_score_min: float | None = None, + fmax_min: float | None = None, + fmin_max: float | None = None, + min_mag: float | None = None, ): """ Apply all the quality filters to the catalogue. @@ -958,7 +960,7 @@ def apply_all_filters( def create_quality_db( main_dir: Path, - bypass_records_ffp: Path = None, + bypass_records_ffp: Path | None = None, ): """ Create the quality database by running the following checks: diff --git a/nzgmdb/data_processing/waveform_manipulation.py b/nzgmdb/data_processing/waveform_manipulation.py index 34fe1dfe..01134a05 100644 --- a/nzgmdb/data_processing/waveform_manipulation.py +++ b/nzgmdb/data_processing/waveform_manipulation.py @@ -17,7 +17,7 @@ def initial_preprocessing( mseed: Stream, apply_taper: bool = True, apply_zero_padding: bool = True, - inventory: Inventory = None, + inventory: Inventory | None = None, provider: str = "GEONET", network: str = "NZ", ) -> Stream: @@ -211,10 +211,10 @@ def butter_bandpass_filter( def high_and_low_cut_processing( mseed: Stream, dt: float, - fmin_h: float = None, - fmin_v: float = None, - fmax_h: float = None, - fmax_v: float = None, + fmin_h: float | None = None, + fmin_v: float | None = None, + fmax_h: float | None = None, + fmax_v: float | None = None, ): """ Process the waveform data by using the highcut and lowcut for the butter bandpass filter diff --git a/nzgmdb/data_retrieval/geonet.py b/nzgmdb/data_retrieval/geonet.py index 99574cba..acfcef21 100644 --- a/nzgmdb/data_retrieval/geonet.py +++ b/nzgmdb/data_retrieval/geonet.py @@ -413,8 +413,8 @@ def fetch_event_data( inventory: Inventory, site_table: pd.DataFrame, mw_rrup_data: np.ndarray, - only_sites: list[str] = None, - only_record_ids: pd.DataFrame = None, + only_sites: list[str] | None = None, + only_record_ids: pd.DataFrame | None = None, n_procs: int = 1, ): """ @@ -524,8 +524,8 @@ def process_batch( site_table: pd.DataFrame, mw_rrup_data: np.ndarray, n_procs: int = 1, - only_sites: list[str] = None, - only_record_ids: pd.DataFrame = None, + only_sites: list[str] | None = None, + only_record_ids: pd.DataFrame | None = None, mp_sites: bool = False, ): """ @@ -656,8 +656,8 @@ def process_batch( def download_earthquake_data( - start_date: datetime, - end_date: datetime, + start_date: datetime.datetime, + end_date: datetime.datetime, ): """ Download the earthquake data files from the geonet website @@ -740,13 +740,13 @@ def download_earthquake_data( def parse_geonet_information( main_dir: Path, - start_date: datetime, - end_date: datetime, + start_date: datetime.datetime, + end_date: datetime.datetime, n_procs: int = 1, batch_size: int = 500, - only_event_ids: list[str] = None, - only_sites: list[str] = None, - only_record_ids_ffp: Path = None, + only_event_ids: list[str] | None = None, + only_sites: list[str] | None = None, + only_record_ids_ffp: Path | None = None, real_time: bool = False, mp_sites: bool = False, add_tmp_arrays: bool = False, diff --git a/nzgmdb/data_retrieval/inventory_xml.py b/nzgmdb/data_retrieval/inventory_xml.py index 8d8c0430..10ed3512 100644 --- a/nzgmdb/data_retrieval/inventory_xml.py +++ b/nzgmdb/data_retrieval/inventory_xml.py @@ -14,8 +14,8 @@ def get_provider_inventory( - provider: str = None, - networks: list[str] = None, + provider: str | None = None, + networks: list[str] | None = None, channel_codes: str | None = None, stations: str = "*", level: str = "response", diff --git a/nzgmdb/data_retrieval/rupture_models.py b/nzgmdb/data_retrieval/rupture_models.py index 3078815d..f88cffc9 100644 --- a/nzgmdb/data_retrieval/rupture_models.py +++ b/nzgmdb/data_retrieval/rupture_models.py @@ -33,7 +33,7 @@ class RuptureModel(TypedDict): """The average width of the rupture model.""" -def get_seismic_data_from_url(url: str) -> dict: +def get_seismic_data_from_url(url: str) -> RuptureModel: """ Fetch and process the seismic data from a URL. @@ -51,7 +51,7 @@ def get_seismic_data_from_url(url: str) -> dict: df = pd.read_csv(url) # Define calculated data output variables - length = 0 + length = 0.0 widths = [] # Group by segment diff --git a/nzgmdb/data_retrieval/tect_domain.py b/nzgmdb/data_retrieval/tect_domain.py index 1c217b63..d51c7a7e 100644 --- a/nzgmdb/data_retrieval/tect_domain.py +++ b/nzgmdb/data_retrieval/tect_domain.py @@ -187,9 +187,9 @@ def create_regions( fault_file: Path, d_s: float, d_d: float, - region_a_offshore: dict = None, - region_b_on: dict = None, - region_c_downdip: dict = None, + region_a_offshore: dict | None = None, + region_b_on: dict | None = None, + region_c_downdip: dict | None = None, ): """ Determine an array of points on and offshore of a fault and divide them into regions. @@ -257,10 +257,9 @@ def create_regions( def ngasub2020_tectclass( row: pd.Series, - region_a_offshore: dict = False, - region_b_on: dict = False, - region_c_downdip: dict = False, - fault_label: str = np.nan, + region_a_offshore: dict, + region_b_on: dict, + region_c_downdip: dict, h_thresh: float = 10, v_thresh: float = 10, ): @@ -294,8 +293,6 @@ def ngasub2020_tectclass( Portion of faults in Region B (latitude, longitude, depth) as defined in NGA-SUB (2020). region_c_downdip : dict Portion of faults in Region C (latitude, longitude, depth) as defined in NGA-SUB (2020). - fault_label : str - The fault label associated with the event. h_thresh : float Horizontal distance threshold for classification. v_thresh : float @@ -310,6 +307,7 @@ def ngasub2020_tectclass( """ lat, lon, depth = row["lat"], row["lon"], row["depth"] + fault_label = np.nan # Initially classify as if farfield, correct later if neccessary if depth <= 30: diff --git a/nzgmdb/data_retrieval/waveform_extraction.py b/nzgmdb/data_retrieval/waveform_extraction.py index 7301cd65..16127e18 100644 --- a/nzgmdb/data_retrieval/waveform_extraction.py +++ b/nzgmdb/data_retrieval/waveform_extraction.py @@ -798,8 +798,8 @@ def extract_station_info( main_dir: Path, event_catalogues: dict, extraction_table: pd.DataFrame, - only_record_ids: pd.DataFrame = None, - tmp_array_dir: Path = None, + only_record_ids: pd.DataFrame | None = None, + tmp_array_dir: Path | None = None, ) -> StationExtractionResult: """ Extract the waveform data for a single station based on the extraction parameters. @@ -1095,9 +1095,9 @@ def extract_waveforms( main_dir: Path, station_extraction_table_ffp: Path, n_procs: int = 1, - only_record_ids_ffp: Path = None, + only_record_ids_ffp: Path | None = None, batch_size: int = 1000, - tmp_array_dir: Path = None, + tmp_array_dir: Path | None = None, ): """ Extract waveforms for each station in the station extraction table. diff --git a/pyproject.toml b/pyproject.toml index da2efdbb..25eeb5c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,8 +91,3 @@ not-subscriptable = "ignore" possibly-missing-submodule = "ignore" invalid-type-arguments = "ignore" unresolved-import = "ignore" -#invalid-parameter-default = "error" -#invalid-argument-type = "error" -#invalid-type-form = "error" -#invalid-return-type = "error" -#too-many-positional-arguments = "error" From 5a766601142eb3feb0667cdfc271288e8bc88780 Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 22 Apr 2026 13:52:05 +1200 Subject: [PATCH 71/72] type error fix --- nzgmdb/data_processing/quality_db.py | 2 +- nzgmdb/data_retrieval/waveform_extraction.py | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/nzgmdb/data_processing/quality_db.py b/nzgmdb/data_processing/quality_db.py index fccc69b8..44e29581 100644 --- a/nzgmdb/data_processing/quality_db.py +++ b/nzgmdb/data_processing/quality_db.py @@ -772,7 +772,7 @@ def filter_duplicate_channels( # Step 2: Create bypass flag using record_id if bypass_records is None: - bypass_records = [] + bypass_records = np.array([]) catalogue["bypass"] = catalogue["record_id"].isin(bypass_records) # Step 3: Define priority levels diff --git a/nzgmdb/data_retrieval/waveform_extraction.py b/nzgmdb/data_retrieval/waveform_extraction.py index 16127e18..665ce1f8 100644 --- a/nzgmdb/data_retrieval/waveform_extraction.py +++ b/nzgmdb/data_retrieval/waveform_extraction.py @@ -873,14 +873,23 @@ def extract_station_info( sta = station_extraction_row["sta"] if provider == "GEONET": - # Get the Stream client = FDSN_Client("GEONET") st = get_inital_stream( start_time, end_time, channel_codes, location, client, net, sta ) else: - # Get the stream from the tmp array storage location - st = get_tmp_array_stream(tmp_array_dir, net, sta, start_time, end_time) + if tmp_array_dir is not None: + st = get_tmp_array_stream(tmp_array_dir, net, sta, start_time, end_time) + else: + try: + client = FDSN_Client(provider) + st = get_inital_stream( + start_time, end_time, channel_codes, location, client, net, sta + ) + except Exception as e: + raise ValueError( + f"Failed to retrieve data for provider '{provider}': {e}" + ) # Check that data was found if st is None: From 83ff4f0e4422cb3b8fd67e38bc94f1121a721cc0 Mon Sep 17 00:00:00 2001 From: joelridden Date: Wed, 6 May 2026 15:53:22 +1200 Subject: [PATCH 72/72] type check fixes --- nzgmdb/data_retrieval/rupture_models.py | 2 +- .../phase_arrival/gen_phase_arrival_table.py | 6 +++--- nzgmdb/phase_arrival/run_phasenet.py | 10 +++++----- nzgmdb/scripts/generate_report.py | 18 +++++++++--------- nzgmdb/scripts/real_time_eq_runs.py | 10 +++++----- nzgmdb/scripts/run_gmc.py | 10 +++++----- nzgmdb/scripts/run_nzgmdb.py | 18 ++++++++++-------- 7 files changed, 38 insertions(+), 36 deletions(-) diff --git a/nzgmdb/data_retrieval/rupture_models.py b/nzgmdb/data_retrieval/rupture_models.py index f88cffc9..6837aaf5 100644 --- a/nzgmdb/data_retrieval/rupture_models.py +++ b/nzgmdb/data_retrieval/rupture_models.py @@ -100,7 +100,7 @@ def get_seismic_data_from_url(url: str) -> RuptureModel: "strike": strike, "dip": dip, "rake": rake, - "length": length, + "length": float(length), "width": width, } diff --git a/nzgmdb/phase_arrival/gen_phase_arrival_table.py b/nzgmdb/phase_arrival/gen_phase_arrival_table.py index e2766683..035cc530 100644 --- a/nzgmdb/phase_arrival/gen_phase_arrival_table.py +++ b/nzgmdb/phase_arrival/gen_phase_arrival_table.py @@ -84,9 +84,9 @@ def generate_phase_arrival_table( conda_sh: Path, env_activate_command: str, n_procs: int, - n_batches: int = None, - bypass_records_ffp: Path = None, - xml_dir: Path = None, + n_batches: int | None = None, + bypass_records_ffp: Path | None = None, + xml_dir: Path | None = None, ): """ Generate the phase arrival table utilizing phaseNet diff --git a/nzgmdb/phase_arrival/run_phasenet.py b/nzgmdb/phase_arrival/run_phasenet.py index b4d87a36..50321f0d 100644 --- a/nzgmdb/phase_arrival/run_phasenet.py +++ b/nzgmdb/phase_arrival/run_phasenet.py @@ -48,7 +48,7 @@ def create_empty_h5_file(h5_ffp: Path, group_name: str): def run_phase_net( input_data: np.ndarray, dt: float, - t: np.ndarray = None, + t: np.ndarray | None = None, return_prob_series: bool = False, ): """ @@ -113,8 +113,8 @@ def run_phase_net( def process_mseed( mseed_file: Path, h5_ffp: Path, - bypass_row: pd.Series = None, - inventory: Inventory = None, + bypass_row: pd.Series | None = None, + inventory: Inventory | None = None, ): """ Process an mseed file and return the phase arrival data. @@ -321,8 +321,8 @@ def process_mseed( def run_phasenet( mseed_files_ffp: Path, output_dir: Path, - bypass_ffp: Path = None, - xml_dir: Path = None, + bypass_ffp: Path | None = None, + xml_dir: Path | None = None, ): """ Run PhaseNet on the mseed files. diff --git a/nzgmdb/scripts/generate_report.py b/nzgmdb/scripts/generate_report.py index 0bbb40f0..66848bff 100644 --- a/nzgmdb/scripts/generate_report.py +++ b/nzgmdb/scripts/generate_report.py @@ -132,7 +132,7 @@ def apply_fmin_filter_df(df: pd.DataFrame, pre_4p3: bool = False) -> pd.DataFram return df -def format_percentage(pct: float, allvals: list[float]): +def format_percentage(pct: float, allvals: list[int | float]): """ Function to format pie chart labels with both percentage and absolute values. @@ -140,7 +140,7 @@ def format_percentage(pct: float, allvals: list[float]): ---------- pct : float The percentage value. - allvals : list[float] + allvals : list[int | float] The list of all values to compute the absolute value. Returns @@ -190,7 +190,7 @@ def plot_pie_chart(full_labels: list[str], full_sizes: list[int], title: str): colors=colors, autopct=lambda pct: format_percentage(pct, sizes), startangle=270, - ) + ) # type: ignore ax.set_title(f"{title} ({total_records} total records)") ax.axis("equal") @@ -236,7 +236,7 @@ def numpy_str_join(sep: str, *arrays: str | Sequence[str]) -> np.ndarray: numpy.ndarray The joined array. """ - result = arrays[0] + result = np.array(arrays[0]) for cur_array in arrays[1:]: result = np.char.add(result, sep) result = np.char.add(result, cur_array) @@ -458,7 +458,7 @@ def get_residuals( pred_im_keys = numpy_str_join("_", ims, pred_suffix) res_df = pd.DataFrame( data=results.loc[:, ims].values - results.loc[:, pred_im_keys].values, - columns=ims, + columns=list(ims), ) res_df.index = results.index @@ -613,7 +613,7 @@ def plot_usable_period_records( df_full: pd.DataFrame, df_quality: pd.DataFrame, title: Optional[str] = None, -) -> plt.Figure: +) -> str: """ Plot number of usable records vs period for Full and Quality datasets. @@ -1251,7 +1251,7 @@ def generate_report( typer.Argument(), ], compare_version_directory: Annotated[ - Path, + Optional[Path], typer.Option( exists=True, file_okay=False, @@ -1672,7 +1672,7 @@ def generate_report( else file_structure.PreFlatfileNames.STATION_MAGNITUDE_TABLE_EXTRACTION ) ) - / 3, + // 3, len( pd.read_csv( new_flatfiles_dir / file_structure.PreFlatfileNames.PHASE_ARRIVAL_TABLE @@ -1691,7 +1691,7 @@ def generate_report( else file_structure.PreFlatfileNames.STATION_MAGNITUDE_TABLE_EXTRACTION ) ) - / 3, + // 3, len( pd.read_csv( old_flatifles_dir diff --git a/nzgmdb/scripts/real_time_eq_runs.py b/nzgmdb/scripts/real_time_eq_runs.py index c7607a82..f81d7510 100644 --- a/nzgmdb/scripts/real_time_eq_runs.py +++ b/nzgmdb/scripts/real_time_eq_runs.py @@ -129,7 +129,7 @@ def reply_to_message_on_slack(thread_ts: str, reply_message: str): def download_earthquake_data( - start_date: datetime, end_date: datetime, mag_filter: float + start_date: datetime.datetime, end_date: datetime.datetime, mag_filter: float ) -> pd.DataFrame: """ Download the earthquake data from the GeoNet API @@ -237,9 +237,7 @@ def run_event( conda_sh: Annotated[Path, typer.Argument(exists=True, file_okay=True)], gmc_activate: Annotated[str, typer.Argument()], gmc_predict_activate: Annotated[str, typer.Argument()], - ko_matrix_path: Annotated[ - Path, typer.Argument(exists=True, file_okay=False) - ], + ko_matrix_path: Annotated[Path, typer.Argument(exists=True, file_okay=False)], add_seismic_now: Annotated[bool, typer.Option(is_flag=True)] = False, machine: Annotated[ cfg.MachineName, @@ -510,7 +508,9 @@ def poll_earthquake_data( init_start_date = None while True: # Get the last 10 minutes worth of data and check if there are any new events - end_date = datetime.datetime.utcnow() - datetime.timedelta(minutes=1) + end_date = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( + minutes=1 + ) # If an event was just executed, ensures we capture any events that may have been missed during the execution start_date = ( end_date - datetime.timedelta(minutes=10) diff --git a/nzgmdb/scripts/run_gmc.py b/nzgmdb/scripts/run_gmc.py index ec1e2f8f..781c3de8 100644 --- a/nzgmdb/scripts/run_gmc.py +++ b/nzgmdb/scripts/run_gmc.py @@ -5,7 +5,7 @@ import functools import multiprocessing from pathlib import Path -from typing import Annotated +from typing import Annotated, Optional import numpy as np import pandas as pd @@ -199,25 +199,25 @@ def run_gmc_processing( typer.Option(), ] = 1, gmc_n_batches: Annotated[ - int, + Optional[int], typer.Option(), ] = None, waveform_dir: Annotated[ - Path, + Optional[Path], typer.Option( exists=True, file_okay=False, ), ] = None, output_dir: Annotated[ - Path, + Optional[Path], typer.Option( exists=True, file_okay=False, ), ] = None, bypass_records_ffp: Annotated[ - Path, + Optional[Path], typer.Option(), ] = None, ): diff --git a/nzgmdb/scripts/run_nzgmdb.py b/nzgmdb/scripts/run_nzgmdb.py index 954e2972..31a7a6ff 100644 --- a/nzgmdb/scripts/run_nzgmdb.py +++ b/nzgmdb/scripts/run_nzgmdb.py @@ -4,7 +4,7 @@ from datetime import datetime from pathlib import Path -from typing import Annotated +from typing import Annotated, Optional import typer @@ -29,13 +29,15 @@ def fetch_geonet_data( n_procs: Annotated[int, typer.Option()] = 1, batch_size: Annotated[int, typer.Option()] = 500, only_event_ids: Annotated[ - list[str], typer.Option(callback=lambda x: [] if x is None else x[0].split(",")) + Optional[list[str]], + typer.Option(callback=lambda x: [] if x is None else x[0].split(",")), ] = None, only_sites: Annotated[ - list[str], typer.Option(callback=lambda x: [] if x is None else x[0].split(",")) + Optional[list[str]], + typer.Option(callback=lambda x: [] if x is None else x[0].split(",")), ] = None, only_record_ids_ffp: Annotated[ - Path, typer.Option(exists=True, dir_okay=False) + Optional[Path], typer.Option(exists=True, dir_okay=False) ] = None, real_time: Annotated[bool, typer.Option()] = False, mp_sites: Annotated[bool, typer.Option()] = False, @@ -107,7 +109,7 @@ def extract_waveforms( ], n_procs: Annotated[int, typer.Option()] = 1, only_record_ids_ffp: Annotated[ - Path, + Optional[Path], typer.Option( exists=True, dir_okay=False, @@ -118,7 +120,7 @@ def extract_waveforms( typer.Option(), ] = 1000, tmp_array_dir: Annotated[ - Path, + Optional[Path], typer.Option( exists=True, file_okay=False, @@ -212,7 +214,7 @@ def make_phase_arrival_table( ], n_procs: Annotated[int, typer.Option()] = 1, n_batches: Annotated[ - int, + Optional[int], typer.Option(), ] = None, bypass_records_ffp: Annotated[ @@ -839,7 +841,7 @@ def run_full_nzgmdb( typer.Option(), ] = False, only_event_ids: Annotated[ - list[str], + Optional[list[str]], typer.Option( callback=lambda x: [] if x is None else x[0].split(","), ),