diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml new file mode 100644 index 0000000..911fc41 --- /dev/null +++ b/.github/workflows/claude-review.yml @@ -0,0 +1,11 @@ +name: Claude PR Review + +on: + issue_comment: + types: [created] + +jobs: + review: + uses: ucgmsim/meta-ci-action/.github/workflows/claude-review.yml@main + secrets: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/git-extension.yml b/.github/workflows/git-extension.yml deleted file mode 100644 index f859604..0000000 --- a/.github/workflows/git-extension.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Check requirements.txt git URLs are properly formatted - -on: pull_request - -permissions: - contents: read - pull-requests: write - -jobs: - check-git-deps: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - - name: Set up Python - uses: actions/setup-python@v4 - - - name: Run Git dependency check - run: | - import tomllib - from pathlib import Path - import sys - - pyproject_path = Path("pyproject.toml") - if not pyproject_path.exists(): - print("Error: pyproject.toml not found in the current directory.") - sys.exit(1) - - with open(pyproject_path, "rb") as f: - config = tomllib.load(f) - - issues = [] - for section in ["project", "project.optional-dependencies"]: - if section in config and "dependencies" in config[section]: - deps = config[section]["dependencies"] - for dep in deps: - if "git+https://" in dep and not dep.endswith(".git"): - issues.append(dep) - - if issues: - print("The following git dependencies are missing the .git extension:") - for issue in issues: - print(f" - {issue}") - sys.exit(1) - else: - print("All git dependencies are correctly formatted.") - shell: python \ No newline at end of file diff --git a/.github/workflows/publish-PyPi.yml b/.github/workflows/publish-PyPi.yml new file mode 100644 index 0000000..4c0e432 --- /dev/null +++ b/.github/workflows/publish-PyPi.yml @@ -0,0 +1,62 @@ +name: Publish to PyPI +run-name: Publish Python Package to PyPI for release ${{ github.event.release.tag_name }} + +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag_name: + description: 'Git tag to checkout and publish' + required: false + type: string + +jobs: + build: + name: Build distribution 📦 + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag_name || github.ref }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + publish-to-pypi: + name: Publish to PyPI + needs: + - build + runs-on: ubuntu-latest + + environment: + name: pypi + url: https://pypi.org/p/nzgmdb + + permissions: + id-token: write + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index c3c85ff..60a84d1 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -27,4 +27,4 @@ jobs: - name: Run tests run: | source .venv/bin/activate - pytest tests + pytest -p no:black tests diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index bf9add3..a6515ac 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -5,4 +5,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: chartboost/ruff-action@v1 \ No newline at end of file + - uses: chartboost/ruff-action@v1 + with: + version: "0.15.22" \ No newline at end of file diff --git a/nzgmdb/data_retrieval/geonet.py b/nzgmdb/data_retrieval/geonet.py index 0c8e3fc..b70bea1 100644 --- a/nzgmdb/data_retrieval/geonet.py +++ b/nzgmdb/data_retrieval/geonet.py @@ -456,7 +456,7 @@ def fetch_sta_mag_line( def fetch_event_data( event_id: str, main_dir: Path, - client_NZ: FDSN_Client, + real_time: bool, inventory: Inventory, site_table: pd.DataFrame, mw_rrup_data: np.ndarray, @@ -473,8 +473,8 @@ def fetch_event_data( The event id to fetch the data for main_dir : Path The main directory of the NZGMDB results (Highest level directory) - client_NZ : FDSN_Client - The geonet client to fetch the data from New Zealand + real_time : bool + If the function is being used in real time use a different client inventory : Inventory The inventory of the stations from all networks to extract the stations from site_table : pd.DataFrame @@ -494,6 +494,13 @@ def fetch_event_data( The parsed event data. """ # Get the catalogue information + # Set constants + config = cfg.Config() + if real_time: + client_NZ = FDSN_Client(base_url=config.get_value("real_time_url")) + else: + # Get Station Information from geonet clients + client_NZ = FDSN_Client("GEONET") cat = client_NZ.get_events(eventid=event_id) event_cat = cat[0] @@ -579,7 +586,7 @@ def process_batch( batch_events: np.ndarray[str], batch_index: int, main_dir: Path, - client_NZ: FDSN_Client, + real_time: bool, inventory: Inventory, site_table: pd.DataFrame, mw_rrup_data: np.ndarray, @@ -599,8 +606,8 @@ def process_batch( The index of the current batch main_dir : Path The main directory of the NZGMDB results (Highest level directory) - client_NZ : FDSN_Client - The geonet client to fetch the data from New Zealand + real_time : bool + If the function is being used in real time use a different client inventory : Inventory The inventory of the stations from all networks to extract the stations from site_table : pd.DataFrame @@ -622,7 +629,7 @@ def process_batch( fetch_event_data( event_id, main_dir, - client_NZ, + real_time, inventory, site_table, mw_rrup_data, @@ -633,15 +640,12 @@ def process_batch( for event_id in batch_events ] else: - # This is a fix for multiprocessing issues with SSLContext - mp.set_start_method("spawn", force=True) - with mp.Pool(n_procs) as p: results = p.map( functools.partial( fetch_event_data, main_dir=main_dir, - client_NZ=client_NZ, + real_time=real_time, inventory=inventory, site_table=site_table, mw_rrup_data=mw_rrup_data, @@ -652,9 +656,6 @@ def process_batch( batch_events, ) - # Reset the start method to default - mp.set_start_method("fork", force=True) - # Extract the results event_data, sta_mag_data, skipped_records, clipped_records = [], [], [], [] for result in results: @@ -931,7 +932,7 @@ def parse_geonet_information( batch, index, main_dir, - client_NZ, + real_time, inventory, site_table, mw_rrup_data, diff --git a/nzgmdb/management/data_registry.py b/nzgmdb/management/data_registry.py index 9144365..6cbb7e8 100644 --- a/nzgmdb/management/data_registry.py +++ b/nzgmdb/management/data_registry.py @@ -7,7 +7,7 @@ "Geonet_Metadata_Summary_v1.4.csv": "sha256:7884422c3fcae0810c02948ba1a3bd39ba5793ba28189e90d730541be1c207c0", "puy_slab2_dep_02.26.18.xyz": "sha256:9ebe4feab4ee3b80e3fe403f2d873f94e4d7f06d937d721cc9e154ecee83e3c0", "reyners_relocations.csv": "sha256:7795c60dae67af14eb590b0d919fa850e2f2fe2fb3beb077cbac14d27eb8faf5", - "focal_mech_tectonic_domain_v1.csv": "sha256:1f1e0c4b7f9ca1b87fb2ca4883e587f330fe82b5bbf9ebb6eb8f4d12aa2e1936", + "focal_mech_tectonic_domain_v1.csv": "sha256:80d5f5d628cddf1be380bb793eb4e0620ac02ddf01b6cb3c47b8f88806b783e3", "GeoNet_CMT_solutions_20201129_PreferredNodalPlane_v1.csv": "sha256:16596bfdbd0019bad22eed31a0cbeabba7d58d9e569a450ae4139222a473c296", "Mw_rrup.txt": "sha256:016cf1bd6d99278a8cda917596cf8d817ef5b6249daad9cf3d6c4d9cc224f204", "3366146.srf": "sha256:21fa6c9f1f729167dcd78020e80f94a726d22ec784b90f338fb28cf830c034a0", @@ -27,7 +27,7 @@ } URLS = { - "focal_mech_tectonic_domain_v1.csv": "https://www.dropbox.com/scl/fi/zseg304cbjmti7gg5tdyv/focal_mech_tectonic_domain_v1.csv?rlkey=kfb9ttvnv9yi9zftixw6kmz4v&st=4j9pgpgj&dl=1", + "focal_mech_tectonic_domain_v1.csv": "https://www.dropbox.com/scl/fi/zseg304cbjmti7gg5tdyv/focal_mech_tectonic_domain_v1.csv?rlkey=kfb9ttvnv9yi9zftixw6kmz4v&st=22cqetno&dl=1", "Geonet_Metadata_Summary_v1.4.csv": "https://www.dropbox.com/scl/fi/iev7qmoqqzvc5quhf8mk8/Geonet-Metadata-Summary_v1.4.csv?rlkey=7twwwck5iy5zao7lwao6xodvm&st=6m3elzuu&dl=1", "GeoNet_CMT_solutions_20201129_PreferredNodalPlane_v1.csv": "https://www.dropbox.com/scl/fi/fq28jx8jlbozj0d1x5tnq/GeoNet_CMT_solutions_20201129_PreferredNodalPlane_v1.csv?rlkey=30xj6n7ara0vz4t8kg4pz8w5s&st=63x7nr3j&dl=1", "hik_kerm_fault_300km_wgs84_poslon.txt": "https://www.dropbox.com/scl/fi/ig3ajufpv4xg2qjfxxuup/hik_kerm_fault_300km_wgs84_poslon.txt?rlkey=9jajfkq2elrzwzzgh6px17k8e&st=6ham2oox&dl=1", diff --git a/requirements.txt b/requirements.txt index c2287d6..0677cec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,20 @@ -typer>0.12.3 -pandas -numpy<2.0.0 -obspy -mseedlib -scipy -requests -fiona -pyproj -pyyaml -shapely -gmprocess -h5py -pooch -oq_wrapper @ git+https://github.com/ucgmsim/Empirical_Engine.git#subdirectory=oq_wrapper +typer==0.27.1 +pandas==3.0.5 +numpy==1.26.4 +obspy==1.5.0 +mseedlib==0.0.10 +scipy==1.17.1 +requests==2.34.2 +fiona==1.10.1 +pyproj==3.7.2 +pyyaml==6.0.3 +shapely==2.1.2 +gmprocess==2.7.0 +h5py==3.16.0 +pooch==1.9.0 +oq_wrapper @ git+https://github.com/ucgmsim/Empirical_Engine.git@v2025.12.3#subdirectory=oq_wrapper openquake-engine @ git+https://github.com/gem/oq-engine.git@v3.23.2 -qcore @ git+https://github.com/ucgmsim/qcore.git -IM_calculation @ git+https://github.com/ucgmsim/IM_calculation.git -velocity_modelling @ git+https://github.com/ucgmsim/velocity_modelling.git@5d1d6ef -source-modelling @ git+https://github.com/ucgmsim/source_modelling.git \ No newline at end of file +qcore @ git+https://github.com/ucgmsim/qcore.git@2672a3b +IM_calculation @ git+https://github.com/ucgmsim/IM_calculation.git@v2025.12.1 +velocity_modelling @ git+https://github.com/ucgmsim/velocity_modelling.git@nzgmdb_4p3 +source-modelling @ git+https://github.com/ucgmsim/source_modelling.git@1671e58 \ No newline at end of file diff --git a/wiki/Parse-Geonet.md b/wiki/Parse-Geonet.md index a783e22..16b9cd8 100644 --- a/wiki/Parse-Geonet.md +++ b/wiki/Parse-Geonet.md @@ -87,7 +87,7 @@ The preferred magnitude selection follows a hierarchical approach: For each earthquake, stations are selected within a distance-dependant radius: - **Distance calculation:** Based on the `Mw_rrup.txt` lookup table -![](images/mw_rrup.png) +![](images/mw_rrup_cutoff_plot.png) - **Interpolation:** Uses cubic interpolation between magnitude and distance values - **Radius conversion:** Converts distance to degrees using ObsPy's geodetic functions - **Station filtering:** Uses ObsPy inventory selection with latitude, longitude, and maximum radius diff --git a/wiki/images/mw_rrup.png b/wiki/images/mw_rrup.png deleted file mode 100644 index a194165..0000000 Binary files a/wiki/images/mw_rrup.png and /dev/null differ diff --git a/wiki/images/mw_rrup_cutoff_plot.png b/wiki/images/mw_rrup_cutoff_plot.png new file mode 100644 index 0000000..3b54ccb Binary files /dev/null and b/wiki/images/mw_rrup_cutoff_plot.png differ