diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..13d84961 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,12 @@ +name: CI +on: [pull_request] + +jobs: + ci: + uses: ucgmsim/meta-ci-action/.github/workflows/ci.yml@v1 + with: + package-dir: workflow + system-packages: "gmt libgmt-dev ghostscript" + enable-coverage: true + cov-package: workflow + cov-fail-under: 95 diff --git a/.github/workflows/deptry.yml b/.github/workflows/deptry.yml deleted file mode 100644 index 835c3155..00000000 --- a/.github/workflows/deptry.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Deptry Dependency Check - -on: [pull_request] -jobs: - dependencies: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - # Install uv - - uses: astral-sh/setup-uv@v4 - with: - enable-cache: true - cache-dependency-glob: | - **/pyproject.toml - - # Install deptry - - run: uv venv - - run: uv sync --all-extras --dev - # Run deptry to check that all dependencies are present. - - run: uv run deptry . diff --git a/.github/workflows/git-extension.yml b/.github/workflows/git-extension.yml deleted file mode 100644 index 5060ce79..00000000 --- a/.github/workflows/git-extension.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Check pyproject.toml git URLs are properly formatted - -on: pull_request - -permissions: - contents: read - pull-requests: write - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - name: Check and fix git dependencies - run: | - python3 << 'EOF' - import tomllib - from urllib import parse - import re - - # Read pyproject.toml - with open('pyproject.toml', 'rb') as f: - data = tomllib.load(f) - - # Get project dependencies - dependencies = data.get('project', {}).get('dependencies', []) - - # Find git dependencies that need .git suffix - git_deps_to_fix = [] - - for dep in dependencies: - # Match git dependencies: git+https://... or git+http://... - # Does not include the PIP version specifier. - git_match = re.search(r'git\+(https?://[^@\s]+)', dep) - if git_match: - git_url = git_match.group(1) - # Extract base URL before any fragment (#subdirectory, etc.) - parse_result = parse.urlparse(git_url) - # Check if it needs .git suffix - if not parse_result.path.endswith('.git'): - fixed_path = parse_result.path + '.git' # add the git suffix - new_url = parse_result._replace(path=fixed_path).geturl() - new_dep = dep.replace(git_url, new_url) - git_deps_to_fix.append((dep, new_dep)) - - # Apply fixes to pyproject.toml - # This only applies the minimal changes (i.e. it doesn't strip comments, reformat the TOML, etc) - if git_deps_to_fix: - with open('pyproject.toml', 'r') as f: - content = f.read() - - for original, fixed in git_deps_to_fix: - content = content.replace(original, fixed) - - with open('pyproject.toml', 'w') as f: - f.write(content) - EOF - - uses: parkerbxyz/suggest-changes@v2 - with: - comment: 'Please change the following dependencies for consistency.' - event: 'REQUEST_CHANGES' diff --git a/.github/workflows/numpydoc.yml b/.github/workflows/numpydoc.yml deleted file mode 100644 index d3f04c7d..00000000 --- a/.github/workflows/numpydoc.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Numpydoc Lint - -on: pull_request - -jobs: - numpydoc-lint: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - uses: awalsh128/cache-apt-pkgs-action@latest - with: - packages: fd-find - version: 1.0 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.12" # Adjust version as needed - - - name: Install dependencies - run: | - pip install numpydoc - sudo apt-get install - - - name: Run Numpydoc Lint - run: | - fdfind . workflow/ -E "__init__.py" --extension py | xargs numpydoc lint diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml deleted file mode 100644 index e8a39e89..00000000 --- a/.github/workflows/pytest.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Pytest Check -on: [pull_request] -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Install GMT - run: sudo apt-get update && sudo apt-get install -y gmt libgmt-dev ghostscript - - name: Setup UV - uses: astral-sh/setup-uv@v5 - with: - enable-cache: true - cache-dependency-glob: | - **/pyproject.toml - - run: uv venv - - run: uv pip install -e ".[test,dev]" - - name: Run tests - run: uv run pytest --cov=workflow --cov-report=html tests - - name: Upload coverage data - uses: actions/upload-artifact@v4 - with: - name: coverage-data - path: .coverage - include-hidden-files: true - if-no-files-found: ignore - coverage: - needs: test - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v4 - - name: Setup UV - uses: astral-sh/setup-uv@v5 - - name: Install coverage tool - run: uv tool install coverage[toml] - - name: Download coverage data - uses: actions/download-artifact@v4 - with: - name: coverage-data - - name: Combine coverage and fail it it's under 100 % - run: | - uv tool run coverage html --skip-covered --skip-empty - # Report and write to summary. - uv tool run coverage report | sed 's/^/ /' >> $GITHUB_STEP_SUMMARY - # Report again and fail if under 95%. - uv tool run coverage report --fail-under=95 - - name: Upload HTML report if check failed - uses: actions/upload-artifact@v4 - with: - name: html-report - path: htmlcov - if: ${{ failure() }} diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml deleted file mode 100644 index 8e10796b..00000000 --- a/.github/workflows/ruff.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: Ruff -on: [pull_request] -jobs: - ruff: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: chartboost/ruff-action@v1 diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml deleted file mode 100644 index dc8179ff..00000000 --- a/.github/workflows/test-runner.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: Test Runner -on: - pull_request: - push: - -jobs: - test: - runs-on: [self-hosted, Linux, X64] - steps: - - run: echo "Runner is alive!" - diff --git a/.github/workflows/types.yml b/.github/workflows/types.yml deleted file mode 100644 index 9f5c0bc0..00000000 --- a/.github/workflows/types.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Type Check - -on: - push: - branches: [pegasus] - pull_request: - branches: [pegasus] - -jobs: - typecheck: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - - - name: Install uv - uses: astral-sh/setup-uv@v5 - with: - enable-cache: true - - - name: Install project with types - run: uv sync --all-extras --dev - - - name: Run type checking with ty - run: uv run ty check --exclude workflow/schemas.py --exclude setup.py diff --git a/.github/workflows/wiki.yml b/.github/workflows/wiki.yml deleted file mode 100644 index 1e127f39..00000000 --- a/.github/workflows/wiki.yml +++ /dev/null @@ -1,16 +0,0 @@ -# .github/workflows/deploy-wiki.yml -name: deploy-wiki -on: - push: - branches: "main" - paths: wiki/** - workflow_dispatch: -jobs: - deploy-wiki: - permissions: - contents: write - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - run: perl -i -pe 's;(\[.*?\])\((?!https:\/\/)(.*?)\.md\);\1(\2);g' wiki/*.md - - uses: actions4gh/deploy-wiki@v1 diff --git a/pyproject.toml b/pyproject.toml index db3da6e0..9c1b66e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,12 +42,18 @@ dependencies = [ test = [ "pytest>=6.0.0", # required for the tool.pytest section "hypothesis[numpy]>=6.0.0", + "pytest-cov", # CI runs pytest --cov + "coverage[toml]", # CI runs coverage report/html ] types = ["pandas-stubs", "types-geopandas", "types-requests", "scipy-stubs"] -dev = ["ruff", "deptry", "ty", "numpydoc"] +dev = ["ruff>=0.16.5", "deptry", "ty>=0.0.75", "numpydoc"] + +[tool.ty.src] +# Generated/dynamic schema definitions that ty cannot usefully analyse. +exclude = ["workflow/schemas.py"] [tool.deptry] -pep621_dev_dependency_groups = ["test", "dev", "types"] +optional_dependencies_dev_groups = ["test", "dev", "types"] [project.scripts] nshm2022-to-realisation = "workflow.scripts.nshm2022_to_realisation:app" diff --git a/tests/test_cli.py b/tests/test_cli.py index a36c2a1c..9cc91f95 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -49,7 +49,7 @@ def test_invocation_of_script(script_module: ModuleType) -> None: f"Module {script_module.__name__} is missing the 'app' attribute." ) - app = getattr(script_module, "app") + app = script_module.app assert isinstance(app, Typer), ( f"'app' in {script_module.__name__} should be a Typer instance, but got {type(app)}." diff --git a/tests/test_generate_domain.py b/tests/test_generate_domain.py index f4efe9a7..e38f5f9f 100644 --- a/tests/test_generate_domain.py +++ b/tests/test_generate_domain.py @@ -64,7 +64,7 @@ def test_estimate_domain_contains_fault_geometry() -> None: nz_outline = shapely.box(0, 0, 500000, 500000) result_domain = generate_domain.estimate_domain( - source_config=source_config, # type: ignore[invalid-argument-type] + source_config=source_config, # ty: ignore[invalid-argument-type] rrups=rrups, nz_outline=nz_outline, fault_buffer=2000.0, @@ -89,9 +89,9 @@ def test_generate_domain() -> None: dip=45.0, dip_dir=180.0, ) - source_config = SourceConfig(dict(source=source)) - magnitudes = Magnitudes(dict(source=magnitude_scaling.BoldM(6.0))) - rakes = Rakes(dict(source=180.0)) + source_config = SourceConfig({"source": source}) + magnitudes = Magnitudes({"source": magnitude_scaling.BoldM(6.0)}) + rakes = Rakes({"source": 180.0}) velocity_model_parameters = VelocityModelParameters( min_vs=500.0, diff --git a/tests/test_hf.py b/tests/test_hf.py index 6831a910..0a4a5358 100644 --- a/tests/test_hf.py +++ b/tests/test_hf.py @@ -64,7 +64,7 @@ def test_build_hf_input_serialisation() -> None: shallow_transition_range=1, deep_depth=2.0, deep_transition_range=1, - rvfrac_slip_sig=None + rvfrac_slip_sig=None, ) # Rather than create DomainParameters with a bounding box, we simplify with a mock object domain = SimpleNamespace(duration=100.0) @@ -75,7 +75,7 @@ def test_build_hf_input_serialisation() -> None: res, hf_config, rv, - domain, # type: ignore[invalid-argument-type] + domain, # ty: ignore[invalid-argument-type] ) lines = result.split("\n") diff --git a/tests/test_realisation.py b/tests/test_realisation.py index 5465f5f4..20c0efa8 100644 --- a/tests/test_realisation.py +++ b/tests/test_realisation.py @@ -3,7 +3,7 @@ # it stays consistent with the codebase. import json import struct -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path from unittest import mock @@ -630,10 +630,10 @@ def test_logtrail_init_empty() -> None: def test_logtrail_init_with_log_entries() -> None: """Test LogTrail initialization with a list of LogEntry objects.""" entry1 = realisations.LogEntry( - utility="util1", args=["a"], version="1", timestamp=datetime.now() + utility="util1", args=["a"], version="1", timestamp=datetime.now(tz=UTC) ) entry2 = realisations.LogEntry( - utility="util2", args=["b", "c"], timestamp=datetime.now(), version="1" + utility="util2", args=["b", "c"], timestamp=datetime.now(tz=UTC), version="1" ) trail = realisations.LogTrail(log=[entry1, entry2]) assert trail.log == [entry1, entry2] @@ -659,13 +659,13 @@ def test_logtrail_init_with_dicts_post_init() -> None: "utility": "util1", "args": ["a"], "version": "1", - "timestamp": datetime.now().isoformat(), + "timestamp": datetime.now(tz=UTC).isoformat(), }, { "utility": "util2", "args": ["b"], "version": "1", - "timestamp": datetime.now().isoformat(), + "timestamp": datetime.now(tz=UTC).isoformat(), }, ] # Pass raw list of dicts @@ -692,7 +692,7 @@ def test_logtrail_log_entry_method() -> None: def test_logtrail_to_dict() -> None: """Test converting LogTrail to a dictionary.""" - ts = datetime.now() + ts = datetime.now(tz=UTC) entry1 = realisations.LogEntry( utility="util1", args=["a"], version="1", timestamp=ts ) diff --git a/uv.lock b/uv.lock index af7694cb..f04799b2 100644 --- a/uv.lock +++ b/uv.lock @@ -8,11 +8,11 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.13.*' and platform_machine == 'ARM64' and sys_platform == 'win32'", "python_full_version == '3.13.*' and platform_machine != 'ARM64' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version < '3.13' and platform_machine == 'ARM64' and sys_platform == 'win32'", "python_full_version < '3.13' and platform_machine != 'ARM64' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] @@ -2841,27 +2841,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.16.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/8f/d8074b1f25e003164087a8bfe79a0f1a3945135764dbb6aaab04103dcaf9/ruff-0.16.4.tar.gz", hash = "sha256:13171aa9d9af2240ee3504e639de73122c67e74036de5ba2e1d01422cd17e3dc", size = 4899731, upload-time = "2026-08-20T17:43:59.196Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/80/779895ef584e089d22f2c6df0d0e99a65ec2df0805f1fffd439415b8c1f0/ruff-0.16.4-py3-none-linux_armv6l.whl", hash = "sha256:df4075f71ddac40b9934af60c3ec8a53047dd5a5fdc43224e6e4e8e9a27cb6f7", size = 10006909, upload-time = "2026-08-20T17:43:16.888Z" }, - { url = "https://files.pythonhosted.org/packages/a9/e6/f553199b5e8927a05cb5c422d921fd0656b29ab976e91c44802107c6b0da/ruff-0.16.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0c95538517af68004306b0fb3214ff2f2af67a65092aee77cd9eb86db6656604", size = 10240201, upload-time = "2026-08-20T17:43:19.337Z" }, - { url = "https://files.pythonhosted.org/packages/1c/70/4a6dc4bb34da4dee35e30f09bbd1bfbdd26f33b62fb9b8df31f08a199cd2/ruff-0.16.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:963f83df8e69e575b64d67dd447ebbc917db41a14bf38d4593a4183e7aaa8255", size = 9835122, upload-time = "2026-08-20T17:43:21.708Z" }, - { url = "https://files.pythonhosted.org/packages/24/12/c6e22d686372c15bcb7af99831f1a1be96df696491babf4f24e4f942c527/ruff-0.16.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32a5057c7ff3f6e6480a48fccfb3a412a690f48a3d03ac5cf08177d6c2da3ade", size = 9977162, upload-time = "2026-08-20T17:43:24.236Z" }, - { url = "https://files.pythonhosted.org/packages/46/49/72b10ec912f5ab5854992eaf7aa7cd36729b6937d9dc4e0fb41b3bf428ec/ruff-0.16.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b3dce8d9b0c57c265b91885a66a567d8ea1372e8eb4e250fa8e5e3f579e99cff", size = 9829789, upload-time = "2026-08-20T17:43:26.966Z" }, - { url = "https://files.pythonhosted.org/packages/fa/80/0f30e32e7f6ee26edc39075502db9d368d788a44a79b55f763eb4ab03796/ruff-0.16.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7dc651db49283c69f8e72c834eec4fe5573e4c646856aebece0ce385dceb2a80", size = 10527949, upload-time = "2026-08-20T17:43:29.384Z" }, - { url = "https://files.pythonhosted.org/packages/52/3d/86e8ad3542169e56cac3859a343afdb9df2ad54d35a59ce1e67baee83421/ruff-0.16.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3817b87dbcabc92f13b05019257c5b89b5b4d51b5fb20f56fb5235ceb723cd07", size = 11333695, upload-time = "2026-08-20T17:43:31.872Z" }, - { url = "https://files.pythonhosted.org/packages/d0/16/481c29b380c20a0054a8261066665e1b3488e23636c49d0a43e75975b9bb/ruff-0.16.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9fce1499134b2c8c68e5166f95705a5812062bb93aacc5f9873bb1a27084bc7", size = 10727741, upload-time = "2026-08-20T17:43:34.596Z" }, - { url = "https://files.pythonhosted.org/packages/5e/b6/56bc0b8cf45b54b28b3a5e6381c8945d51b5b18adf659454c32295209a31/ruff-0.16.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2d812e482f5a7e02eee26cd73d2a37ebbdf47d795ea63ba1b89110ae93e9fb3", size = 10286522, upload-time = "2026-08-20T17:43:37.288Z" }, - { url = "https://files.pythonhosted.org/packages/e8/8b/b345b4fb110f2fbe2bd31eabd271e5e8b3b7e4ee6c0e02f2dc6be78db000/ruff-0.16.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:6baaf984aa7976edf93d3b627fe2d1d22ee94bbca05fa6f90fc76d73924e3454", size = 10584182, upload-time = "2026-08-20T17:43:39.984Z" }, - { url = "https://files.pythonhosted.org/packages/29/e5/827b34041c35f58774a9681a4213994c164fc987800f4dddabcf451da0bf/ruff-0.16.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:bdfcf0b28662eb890372d50f92c283bb94e67e7635ed93c7fd533970acff7b2b", size = 10134195, upload-time = "2026-08-20T17:43:42.351Z" }, - { url = "https://files.pythonhosted.org/packages/0f/10/d0bffcdd6729b87afc82ba0ef377173356a7dc8e972f5179968cf2fdf98c/ruff-0.16.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b66b02cb9b04f537643cadf5768e5f98dc461890d530cb67113d71c8c76e605d", size = 9825821, upload-time = "2026-08-20T17:43:44.532Z" }, - { url = "https://files.pythonhosted.org/packages/f5/32/0db2a863b796ca62d83e92a07a3ccf00921b14db02059347576a2fda3d4b/ruff-0.16.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8528bf9a4b291a60bf02ea453511e8ce6215bd2b982ee80405b66b008b6c30a0", size = 10267658, upload-time = "2026-08-20T17:43:46.989Z" }, - { url = "https://files.pythonhosted.org/packages/b2/a0/fbdeb59e48c6261f523e56c8f12e9c08fbe693786595cc7e3959207a9232/ruff-0.16.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:fbd85d2875fdd67e833213a651f613bbf25303abf6aa822a5121f4531195678d", size = 10697071, upload-time = "2026-08-20T17:43:49.891Z" }, - { url = "https://files.pythonhosted.org/packages/aa/28/0c6dd865859c6d17bc8ccc34cb72b0e02d6c7eb25e8a1e22b5bea681e2c0/ruff-0.16.4-py3-none-win32.whl", hash = "sha256:312769988007aaeb8e189b443ccdd03c0e6374489e053467be6d96518ebff76e", size = 10021687, upload-time = "2026-08-20T17:43:52.281Z" }, - { url = "https://files.pythonhosted.org/packages/a3/03/e724450f621698117f9aa6dd241c94d0274ae96781378dc86745ae29f0e7/ruff-0.16.4-py3-none-win_amd64.whl", hash = "sha256:05d9d27a18c4bcbefada602480ec9e01e0bc949d432e0ced5df77edac195919c", size = 10567657, upload-time = "2026-08-20T17:43:54.78Z" }, - { url = "https://files.pythonhosted.org/packages/0e/fe/da8b9e1347696bb22120b77280ec5ce25d500ca5cb39d5ad6e5c18de19c1/ruff-0.16.4-py3-none-win_arm64.whl", hash = "sha256:a3a61621c9b6f6a89573e938a080e648f1695baa3f58570a3a707bc51ff65a21", size = 10451579, upload-time = "2026-08-20T17:43:57.135Z" }, +version = "0.16.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/85/c8e12473c93018f92d19dd988a294202e1c27426c47ec4de53ffb847b8d8/ruff-0.16.5.tar.gz", hash = "sha256:1b88500f9ffbcab3dedb0082c9f9492e91ec3d618aac1236a3e0189938f7040b", size = 4912003, upload-time = "2026-08-27T16:34:18.258Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/b6/77c90a970fe2dae17a723acbd011043ea97c98d7deacccefdc4ba74ec512/ruff-0.16.5-py3-none-linux_armv6l.whl", hash = "sha256:12e5f673e774c35fbb62f288809c7653b73445f8ecec6b6063fd6ea3521aa14b", size = 10011941, upload-time = "2026-08-27T16:33:41.287Z" }, + { url = "https://files.pythonhosted.org/packages/4b/46/6cf67cf6411885a1d6f7f6d801682f155536a85176d10b605e2ceffed8bd/ruff-0.16.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:eda58a5802de40e7ed5b32b64e0b32539338cc6fcd2c78f61e3ad6a0d79f51c3", size = 10204049, upload-time = "2026-08-27T16:33:44.056Z" }, + { url = "https://files.pythonhosted.org/packages/46/fd/c8720ca7a090abf0c2fef4abe8a5ef6e5127ed15196d8886ff75a2b370e2/ruff-0.16.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5ae9a7b9a8875131f40f8fe967cc86abf899779efd663cb7ce3d572d01da7eb", size = 9809037, upload-time = "2026-08-27T16:33:46.257Z" }, + { url = "https://files.pythonhosted.org/packages/43/45/a684caacdedaca180f52bacccc40bf0789d2c5a7c75f25324853e9eaedb5/ruff-0.16.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b719b0a1f4d59710d283ab2965f621684a108a9e41da622e3b23f0326cd0025", size = 9964129, upload-time = "2026-08-27T16:33:48.352Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f2/5d2bcdaca6b5b93d1b4dfc166cd2aebf7680143a1b38a28759df13a94d31/ruff-0.16.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2298f2780ed1be0c5cb1361e32ab7b1467f3cce7dabe101d2210a314f2fe42e9", size = 9821518, upload-time = "2026-08-27T16:33:50.57Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ff/011cce29accf9257d5974145b733fc653a37985ed6825413a3987cefbfe0/ruff-0.16.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:258f29035a2dd021e7861e631b227a5b3f14e50c1184c9a6a122c5f4576154d7", size = 10534835, upload-time = "2026-08-27T16:33:52.522Z" }, + { url = "https://files.pythonhosted.org/packages/d7/5a/f0cf109bada9bba0e96c90c21c9f9251803f57225c32d293327a03c710d6/ruff-0.16.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9a4f0432966834019c74d1b7e5c51224305d7713f3d7faf3e7451f1a3be3cde", size = 11252550, upload-time = "2026-08-27T16:33:54.521Z" }, + { url = "https://files.pythonhosted.org/packages/63/4d/1d481aaea2046c6a7ed7c291f9004c669cce3c087b6b376ed5b08271e3fe/ruff-0.16.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5eb3a8c3d0ade9cea42b591fd530368e8798380e30e0a308b85a5cf718f09ea", size = 10777949, upload-time = "2026-08-27T16:33:56.88Z" }, + { url = "https://files.pythonhosted.org/packages/ee/34/ee245ca55f64443233034b3d02b03236b19242004281247c079390b7facd/ruff-0.16.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef0f69e191a13a3c9816f63163c88790cb12cd157bbbb384e9c44745702ab105", size = 10311656, upload-time = "2026-08-27T16:33:59.12Z" }, + { url = "https://files.pythonhosted.org/packages/a7/4d/c33a333e341c0a2b96c715b52d89a606f5a34cd4ac493cd9b8d0187186b8/ruff-0.16.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:0eeab41fbea2c42f98dfb9822cdccda9d24ba38d49f6dc945b5c236d48f0ef29", size = 10532125, upload-time = "2026-08-27T16:34:01.166Z" }, + { url = "https://files.pythonhosted.org/packages/30/e1/a64cef78b40192497bb98a27a8aa8f2c98ee9ee15bc97f7712d94ef32937/ruff-0.16.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f0768e9df4300713fff30733c87575f68b6f1d8de41184e505b7fdd9c0c95eaf", size = 10097648, upload-time = "2026-08-27T16:34:03.16Z" }, + { url = "https://files.pythonhosted.org/packages/cc/4e/4cdc9ed3c3e109d2f71e62572a37457298d7bc7501ec3138babb7ed32bbd/ruff-0.16.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:95cc70cdc7aa80c338de356279d2adbeb2de0f520b9ecd8aba75b94e95e02f91", size = 9829344, upload-time = "2026-08-27T16:34:05.134Z" }, + { url = "https://files.pythonhosted.org/packages/39/4a/31ed35ce31729955fc583ee0d176d6e784c1290cb0b0a75cb2134c1ab72a/ruff-0.16.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d185c8398ded1bfd91c0c2cb258346307571eccc473a8490af8c3977399c384a", size = 10277117, upload-time = "2026-08-27T16:34:07.425Z" }, + { url = "https://files.pythonhosted.org/packages/a8/a0/60356d86687b4b666d593df213f4dc3041750d024cb7bf2cfa81cfd65c2e/ruff-0.16.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:fb8e3a3c4c6a784150a7ced53b015f4b253fc2bf97a610886419ead64b4756ef", size = 10711653, upload-time = "2026-08-27T16:34:09.712Z" }, + { url = "https://files.pythonhosted.org/packages/ed/20/656d67f5b25ca9bda4e02b1de25867b2954e1d19e03648060f167ad0f4cc/ruff-0.16.5-py3-none-win32.whl", hash = "sha256:288b0a5f080492fe5635db849f9e2e84aa3cce7b7f0e955997d416c507c76a26", size = 10034250, upload-time = "2026-08-27T16:34:11.8Z" }, + { url = "https://files.pythonhosted.org/packages/5b/42/ee8e68a207b9127fcde6c3d7e197def432f346cb1af159e1fa14ca0d1cdc/ruff-0.16.5-py3-none-win_amd64.whl", hash = "sha256:ddc6385fb2137f616357ca03d6c74f4be987f80fed4008566b754f6032b8546f", size = 10516714, upload-time = "2026-08-27T16:34:13.963Z" }, + { url = "https://files.pythonhosted.org/packages/73/e3/7df5a396e445b9ba49ce9a9437439a4d80042c61c0ade199abf8d16de1ac/ruff-0.16.5-py3-none-win_arm64.whl", hash = "sha256:a64abe90968719b851bb7cedffaa8753fbdbdadab483089682db623f3edc587e", size = 10391564, upload-time = "2026-08-27T16:34:16.064Z" }, ] [[package]] @@ -3301,27 +3301,27 @@ wheels = [ [[package]] name = "ty" -version = "0.0.74" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/88/0f/c767853e88567a2ec7e996dd95e3105b1bc62c95d103689311ef0f4a603c/ty-0.0.74.tar.gz", hash = "sha256:da14344fc8625fc9ff359bafb856ad575636ea86d9bb6a629b146bff27b380e6", size = 6786318, upload-time = "2026-08-22T15:05:54.054Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/95/6ded58bc97885c6d88fa1f9cd815031489200738f961cbf0466663213f80/ty-0.0.74-py3-none-linux_armv6l.whl", hash = "sha256:8969ef4e508debf00cf58f9ea85a539f799b1732c59cdfcecd037630b9755b30", size = 12790043, upload-time = "2026-08-22T15:05:05.015Z" }, - { url = "https://files.pythonhosted.org/packages/d9/8a/5e323603b6ab8731144421877ee8a0f8ac5a5511e67857127caa09f6730e/ty-0.0.74-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:51fb6cf5b98e1e1140825b2430943f78d744876a735231656eafbb4c3f7eca3c", size = 12371748, upload-time = "2026-08-22T15:05:08.609Z" }, - { url = "https://files.pythonhosted.org/packages/d0/44/ee72e08cb705281e8d8c42917dd577aa598a8a098008495fda5176ee3f6e/ty-0.0.74-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8ebe60b1f0a948c793d6c77fc9e9ddda599e4f023c04ab16e8e03bcb428c3fa0", size = 12282403, upload-time = "2026-08-22T15:05:11.448Z" }, - { url = "https://files.pythonhosted.org/packages/da/b3/fd935b694ff68bc278af50f7ad04770b36ce6306399baef7e1847b553a9d/ty-0.0.74-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa97f407a695c890a53615966a663c7d2167e2cabe88db7ca1a24d62635cdfc8", size = 12345164, upload-time = "2026-08-22T15:05:14.19Z" }, - { url = "https://files.pythonhosted.org/packages/54/5c/5b5825268e029ebb164c909780103dbbae367f069801410068bf1cef29b3/ty-0.0.74-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:673ddb733d4a0db31385ba1ed9ff1f6bd9dc5565413ce57b1ca5ac4c7803da5d", size = 12556646, upload-time = "2026-08-22T15:05:16.994Z" }, - { url = "https://files.pythonhosted.org/packages/56/e7/515914e571d62ce0101744fed3f881936eeb1b30dc37beb72b4f7ca1e289/ty-0.0.74-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1028e7c6b4f6145e9704552f43a5fffdcd51b42263ffdcd9c9677762bc395a4a", size = 13311653, upload-time = "2026-08-22T15:05:20.254Z" }, - { url = "https://files.pythonhosted.org/packages/b0/07/d1452babb6f9266c2122cabc095180b70ed306fb770b2996753814d2237d/ty-0.0.74-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79841a8890493021fb308772474983316eb91f7b56cb227a6a05a06b262a36f0", size = 13768284, upload-time = "2026-08-22T15:05:23.197Z" }, - { url = "https://files.pythonhosted.org/packages/b1/60/8d4a2fc7842a47210a1cb0a16a187d9de39ad5d509a00fb74c1c073afcde/ty-0.0.74-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94859d321f3c6a6c8f7bfc3f40e8319cda7e6e012e613440f3dfd145d5010e2e", size = 13422306, upload-time = "2026-08-22T15:05:26.248Z" }, - { url = "https://files.pythonhosted.org/packages/de/76/ebbc269a8c4efcc4d44624993bd188145f20d60ebda9680b15aaec42cc50/ty-0.0.74-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:970a8b2c09ff3be04c8a1c6767332d861be4fce85efe7bb205e4ade7c8655274", size = 12970637, upload-time = "2026-08-22T15:05:29.15Z" }, - { url = "https://files.pythonhosted.org/packages/9e/dd/b99f7236acbf856780ca1779a48143d2d9f2c24d7f531a0ce15a022b8a87/ty-0.0.74-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:795f763b3ded85574c2c2846a6fb8acf2aa76e9e83d761143e92b1f0c7ffa2cd", size = 13344891, upload-time = "2026-08-22T15:05:32.033Z" }, - { url = "https://files.pythonhosted.org/packages/0b/d7/9ff7449a4c7e6428f2c6f298e74cf24b70668f29d45c249507a723ff3782/ty-0.0.74-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:dc086db5367d912c31c0cc872deb7387290e779a4b9b54fcb944673a7cd52c7b", size = 12395272, upload-time = "2026-08-22T15:05:34.702Z" }, - { url = "https://files.pythonhosted.org/packages/b1/dd/b23a5b6b35d37df89dc8dc5daa09efd9245a668b50c4c81c25de21567dc1/ty-0.0.74-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:0314d7b391cf684e47c2fa093d2ce4c597cfc9b01d9a315fe204aed6359b271b", size = 12573079, upload-time = "2026-08-22T15:05:37.683Z" }, - { url = "https://files.pythonhosted.org/packages/23/c5/ccba16239d6129533c8b3603458d0f4dd2ba69478e47059073968e74261d/ty-0.0.74-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c4a45dd2e991e8bdae82ba78c8cd051b253f60bc71a6536598fa3ef580b4fc9b", size = 12832506, upload-time = "2026-08-22T15:05:40.505Z" }, - { url = "https://files.pythonhosted.org/packages/6d/1c/2390912634dff4f341f97b397f2aee341ff062be0a66cda37d59375454f2/ty-0.0.74-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:210e2eac6b018fb934e2b8dac3956a0ba076a3fb1fa6f135058c825e5b759b81", size = 13154752, upload-time = "2026-08-22T15:05:43.355Z" }, - { url = "https://files.pythonhosted.org/packages/c4/33/a8c12188227e6f74f91853a7374e01ed81d6ad21c16c8b70e92dbebfe46a/ty-0.0.74-py3-none-win32.whl", hash = "sha256:db0bb6a8f098ef9bd1be861f73b4f7c0320d40d4c05c7ae0a8677d4e7aa4f6e5", size = 12130002, upload-time = "2026-08-22T15:05:46.058Z" }, - { url = "https://files.pythonhosted.org/packages/21/5c/064f28ccb9c234cfce5a2f7aa69a256663d5ae5bb0290b3a9706cc4d1e4c/ty-0.0.74-py3-none-win_amd64.whl", hash = "sha256:bebff181515255b3c78bd2e7693ae66fab6064ad4feea2065c68bc01022aa678", size = 12771435, upload-time = "2026-08-22T15:05:48.811Z" }, - { url = "https://files.pythonhosted.org/packages/fe/06/d6becdaca0315346c26b6df97cb0eafa81de4f870945d6989e88704374ed/ty-0.0.74-py3-none-win_arm64.whl", hash = "sha256:1a3469eaaf8c85b1c0a15bede25d36daea4b09fce1d913e965b24e24b3f1d6c6", size = 12558299, upload-time = "2026-08-22T15:05:51.543Z" }, +version = "0.0.75" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/81/d0/d0c96f898d6974a4a3569ab3efdf9512c04ad99f9203effb55f72497fe97/ty-0.0.75.tar.gz", hash = "sha256:4c5eead33dfbf6e2ebb4f400f74b51ffc9bab702a6f23ddb648a1cbb740387e3", size = 6868326, upload-time = "2026-08-26T20:23:40.399Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/6c/b12d03505f17581f0cfa3c12273fe34c1d67b36dfda1bc561a6bdc16512b/ty-0.0.75-py3-none-linux_armv6l.whl", hash = "sha256:e5409f50db2246fd4bd039d93d261e0cfa1daa554a4fb77256f91072c570349a", size = 12972606, upload-time = "2026-08-26T20:22:59.716Z" }, + { url = "https://files.pythonhosted.org/packages/d1/aa/30f11eecd9215a9f87e8fe8baaf48f3ce905f5d75b8e4aac70f0091f130c/ty-0.0.75-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5e7b8b3472fb9bb2eeab314984b265df08a7a9d518867a9e6020eebc06570be2", size = 12527158, upload-time = "2026-08-26T20:23:02.767Z" }, + { url = "https://files.pythonhosted.org/packages/f2/11/7fd7001b0b5c6610bfbad7357e47d5fe6f82d4e84e94c53776a478f5e9f8/ty-0.0.75-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c6ccf34169821fe0d23e3360deeef981d217963412f1d087b9bdd32ec57f7a57", size = 12400533, upload-time = "2026-08-26T20:23:04.965Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7f/1e284ea3d348d7be02f12d83bc22ed9ef193033f863f05b64db99027f141/ty-0.0.75-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:842ebb41e9c6c334b40768704e20b1a69d5c6b08805b289d5e0e2565f49f2de1", size = 12420592, upload-time = "2026-08-26T20:23:07.427Z" }, + { url = "https://files.pythonhosted.org/packages/2d/ab/d813271543370c47fd74b5118f2066ab32b0983e907b1821f3f9a6d0fa7f/ty-0.0.75-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf7a5a723c5f1e0fab4ffbfe9bd95123a526ed48f206e5f25cb2161ca294007a", size = 12739219, upload-time = "2026-08-26T20:23:09.809Z" }, + { url = "https://files.pythonhosted.org/packages/31/5b/95b49cc5570fd92a7bf63732f649b31906158721e03c7fcb1b5be74ee3bf/ty-0.0.75-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54382f98e5da292fcd7104391afef5105c35bb2f312e29bea6f5fa419935255c", size = 13494046, upload-time = "2026-08-26T20:23:12.191Z" }, + { url = "https://files.pythonhosted.org/packages/2c/0d/502d2dd68173cf020e1ad2bdbab9544c86776de0b0e2ed15f8c2fe006e3d/ty-0.0.75-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac13b180dc2aade2cd243f56b01650e78bf091a2e522ad3bc947245d7837c613", size = 13938899, upload-time = "2026-08-26T20:23:14.764Z" }, + { url = "https://files.pythonhosted.org/packages/20/5b/f3b12a25c07224456219fc2bd20db0ad7e40b304be0ff6aad728da0135f9/ty-0.0.75-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:752df7951a443219d7f1ff817e3723c85d428565ff449e08a7a93ba821661526", size = 13656711, upload-time = "2026-08-26T20:23:17.145Z" }, + { url = "https://files.pythonhosted.org/packages/51/7b/f090ad306e2b15a07b332d647138c5264b89d9758855ecce8b8a10bcb153/ty-0.0.75-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fd399feedf7cee816563c1baec45fc1c0b3c89f1ea42364920b688004b5b7da", size = 13093499, upload-time = "2026-08-26T20:23:19.489Z" }, + { url = "https://files.pythonhosted.org/packages/f1/4b/f69b99aaaca0c7c65d5f114b186b26b21666f767b0c69eec99a2bdccc061/ty-0.0.75-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:d7625f6f56c7dc1e873579fdc9e432a0e21e302afe847ab60704d2303442a92e", size = 13520580, upload-time = "2026-08-26T20:23:21.789Z" }, + { url = "https://files.pythonhosted.org/packages/b6/e7/692c5f905c0345a15d2255fc74066d660f030254ae8dcdaf33f5a5c2f279/ty-0.0.75-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:89e7d527e95a2534b70cae29e94c104b84082760ea05927d23bb87280969c104", size = 12524095, upload-time = "2026-08-26T20:23:24.026Z" }, + { url = "https://files.pythonhosted.org/packages/ba/9a/f42b12cf265ea95344bf554764c4791cfb273bdd628aadd7c209af7cadc3/ty-0.0.75-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:0843f134440740706e01bee5f88f4cfc10e9b018bddb9e4ef4c12dc9fc0c9aef", size = 12756591, upload-time = "2026-08-26T20:23:26.126Z" }, + { url = "https://files.pythonhosted.org/packages/7c/5e/9b180c133cb9cce48179a7d2bf9e1802d992aa8176a918e0e05205760b42/ty-0.0.75-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1bd0ec0e50ee1376875c88891efe6f549c3560fa5b2ddad79a425cd5a6218b9c", size = 12998754, upload-time = "2026-08-26T20:23:28.353Z" }, + { url = "https://files.pythonhosted.org/packages/39/f6/3c6ef5dd550103e29905121c67fb96a374564f31a2f44c6faa1af98c2d61/ty-0.0.75-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1f9eafd561f90110d5e29f589ec3e956c4686e2f6631348d99276436f5cbe4d1", size = 13316474, upload-time = "2026-08-26T20:23:30.857Z" }, + { url = "https://files.pythonhosted.org/packages/bb/52/12776337874c821076bd5368e352ccd9e67174790abe3b856f749cb3524b/ty-0.0.75-py3-none-win32.whl", hash = "sha256:05063a6fafe2154b794a7f964515d148e51acd186d72d4a3acd347ee9fa19336", size = 12316315, upload-time = "2026-08-26T20:23:33.528Z" }, + { url = "https://files.pythonhosted.org/packages/53/e6/bb51e16af5c7138c9f52f8f3d0a401a371c6798d092e3b74926f186a9814/ty-0.0.75-py3-none-win_amd64.whl", hash = "sha256:81cf1ba5f6b7536ad56747865214255d9bc8e80533a689dbb9ddeaad464b09f1", size = 12917267, upload-time = "2026-08-26T20:23:35.978Z" }, + { url = "https://files.pythonhosted.org/packages/39/73/4542f829107468b5de4231af67f29927c093bfad11f3c1e5b2c08fb1206b/ty-0.0.75-py3-none-win_arm64.whl", hash = "sha256:541c9af5b7a0ad23d15ec315a7da81150833c359f48124ed3789ff25eacd6f42", size = 12711024, upload-time = "2026-08-26T20:23:38.159Z" }, ] [[package]] @@ -3501,8 +3501,10 @@ dev = [ { name = "ty" }, ] test = [ + { name = "coverage" }, { name = "hypothesis", extra = ["numpy"] }, { name = "pytest" }, + { name = "pytest-cov" }, ] types = [ { name = "pandas-stubs" }, @@ -3513,6 +3515,7 @@ types = [ [package.metadata] requires-dist = [ + { name = "coverage", extras = ["toml"], marker = "extra == 'test'" }, { name = "deptry", marker = "extra == 'dev'" }, { name = "geopandas" }, { name = "hypothesis", extras = ["numpy"], marker = "extra == 'test'", specifier = ">=6.0.0" }, @@ -3526,11 +3529,12 @@ requires-dist = [ { name = "parse", specifier = ">=1.21.0" }, { name = "psutil" }, { name = "pytest", marker = "extra == 'test'", specifier = ">=6.0.0" }, + { name = "pytest-cov", marker = "extra == 'test'" }, { name = "pyyaml" }, { name = "qcore-utils", specifier = ">=2025.12.2" }, { name = "requests" }, { name = "rich", specifier = ">=14.3.2" }, - { name = "ruff", marker = "extra == 'dev'" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.16.5" }, { name = "schema" }, { name = "scipy" }, { name = "scipy-stubs", marker = "extra == 'types'" }, @@ -3538,7 +3542,7 @@ requires-dist = [ { name = "source-modelling", specifier = ">=2026.8.1" }, { name = "structlog" }, { name = "tqdm" }, - { name = "ty", marker = "extra == 'dev'" }, + { name = "ty", marker = "extra == 'dev'", specifier = ">=0.0.75" }, { name = "typer" }, { name = "types-geopandas", marker = "extra == 'types'" }, { name = "types-requests", marker = "extra == 'types'" }, diff --git a/wiki/Custom-Workflows.md b/wiki/Custom-Workflows.md deleted file mode 100644 index 294c1935..00000000 --- a/wiki/Custom-Workflows.md +++ /dev/null @@ -1,474 +0,0 @@ -# Building Custom Workflows - -The workflow is designed to be extended for research workflows besides Cybershake. This tutorial will guide you through the steps of adding your own workflow stages to perform an experiment. - -The scenario we will test involves a simple experiment: Running two realisations side-by-side, where in one we shift the fault geometry down by 10km. We want to scale this workflow up to running dozens of simulations, so we want to automate the shifting by adding a workflow stage to do this for us. - -# Copying a Template Workflow - -The first stage to building any workflow is writing the `flow.cylc` file. To build this workflow file, we recommend either: - -1. Building the workflow file by hand or, -2. Using the workflow planning script to build a template workflow. - -For the sake of learning, we are going to build a workflow file by hand for a single realisation (the venerable NSHM rupture 0). The workflow planning script will make your life easier, see below for the tutorial on how to use this tool. - -The `cylc/flow.cylc` in the workflow repository is a good starting place. For brevity we only include the low-frequency workflow. An exercise for the reader would be to complete the workflow for high-frequency and intensity measure calculation. - -``` -[scheduling] - [[graph]] - R1 = """ - nshm_to_realisation => realisation_to_srf & generate_velocity_model_parameters - generate_velocity_model_parameters => generate_velocity_model & generate_station_coordinates & generate_model_coordinates - realisation_to_srf & generate_velocity_model & generate_station_coordinates & generate_model_coordinates => create_e3d_par - create_e3d_par => run_emod3d - """ -[runtime] - [[root]] - platform = mahuika-slurm - pre-script = """ - module load Apptainer - """ - [[[directives]]] - --account = nesi00213 - [[nshm_to_realisation]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif nshm2022-to-realisation /nshmdb.db 0 /share/realisation.json develop - [[realisation_to_srf]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif realisation-to-srf /share/realisation.json /share/realisation.srf - [[generate_velocity_model_parameters]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif generate-velocity-model-parameters /share/realisation.json - [[generate_velocity_model]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif sh -c 'generate-velocity-model /share/realisation.json /share/Velocity_Model --num-threads $(nproc)' - [[[directives]]] - --cpus-per-task = 32 - --time = 01:00:00 - [[generate_station_coordinates]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif generate-station-coordinates /share/realisation.json /share/stations - [[generate_model_coordinates]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif generate-model-coordinates /share/realisation.json /share/model - [[create_e3d_par]] - platform = localhost - script = apptainer exec /nesi/nobackup/nesi00213/containers/runner_latest.sif create-e3d-par $CYLC_WORKFLOW_SHARE_DIR/realisation.json $CYLC_WORKFLOW_SHARE_DIR/realisation.srf $CYLC_WORKFLOW_SHARE_DIR/Velocity_Model $CYLC_WORKFLOW_SHARE_DIR/stations $CYLC_WORKFLOW_SHARE_DIR/model $CYLC_WORKFLOW_SHARE_DIR/LF --emod3d-path /nesi/project/nesi00213/opt/maui/hybrid_sim_tools/emod3d-mpi_v3.0.8 --scratch-ffp $CYLC_WORKFLOW_SHARE_DIR/LF - [[run_emod3d]] - platform = maui-xc-slurm - pre-script = "" - script = srun /nesi/project/nesi00213/opt/maui/hybrid_sim_tools/emod3d-mpi_v3.0.8 -args "par=$CYLC_WORKFLOW_SHARE_DIR/LF/e3d.par" - [[[directives]]] - --ntasks = 80 - --hint = nomultithread - --time = 00:30:00 -``` - - -The shifting would be done by adjusting the source geometry (see the [Realisations](Realisations.md) documentation) of the realisation after generating it from the NSHM database. We could do this by writing our stage. - - -# Building the Shifting Stage - -This step requires two things: - -1. Writing our custom code to adjust the source geometry, -2. Adding the stage into our workflow. - -The first step is done by creating a Python file that reads the realisation and adjusts the source geometry, copying it into a new shifted realisation. The script is simple enough to be entirely contained here. - -``` python -from workflow.realisations import SourceConfig -import shutil -import sys - -input_realisation = Path(sys.argv[0]) -output_realisation = Path(sys.argv[1]) - -# copy the old realisation to the new realisation -shutil.copy(input_realisation, output_realisation) - -source_config = SourceConfig.read_from_realisation(input_realisation) - -for _, fault in source_config.source_geometries.items(): - # Add 10000m to the depth of every fault corner - fault.bounds[:, 2] += 10000 - -# write the changed source geometry back to the realisation -source_config.write_to_realisation(output_realisation) -``` - -Now we create a new stage for the workflow: - -``` -[[shift_fault]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/bin:/bin" /nesi/nobackup/nesi00213/containers/runner_latest.sif python /bin/shift_fault.py /share/realisation.json /share/realisation_shift.json -``` - -Let's break this down line-by-line: - -1. `[[shift_fault]]` declares the name of the new workflow stage -2. `platform = localhost` states that this stage runs on the login node (see [NeSI HPC Infrastructure](#nesi-hpc-infrastructure) for more details on platforms available). Broadly speaking, **you should run trivial tasks on localhost** to avoid using allocated core hours on the NeSI hardware. Examples of trivial tasks include moving or renaming files, or minor tweaks like the shifting script we just wrote. -3. `script =` starts the declaration of the script to execute in this stage. -4. `apptainer exec -c` invokes the `apptainer` executable telling the system to execute the following command inside a container (see [What is a Container?](#what-is-a-container)). **You should run tasks in a container** because the container has access to the latest versions of Python and has the environment set up already. -5. `--bind "..."` binds directories in the host system (that's where you execute Cylc from) into the container. The following table summarises the standard bindings we make. - -| Path in host | Path in container | Purpose | -|:------------------------------|:------------------|:----------------------------------------------------------------------------------------------------------------------------------------------| -| `~/cylc-run//share` | `/share` | Files shared between workflow jobs (like realisations, or velocity models). | -| `~/cylc-run//bin` | `/bin` | Extra binaries like user scripts | -| Auto-generated job directory | `/out` | Directory for output that isn't shared (like checkpoint files, or per station high-frequency output). This directory is unique for every job. | - -6. `/nesi/nobackup/nesi00213/containers/runner_latest.sif` is the path to the container to load. -7. `python /bin/shift_fault.py /share/realisation.json /share/realisation_shift.json` is the actual script we execute. Notice we use the path in the container rather than the host. - -You are encouraged to read the [Cylc tutorial](https://cylc.github.io/cylc-doc/latest/html/tutorial/index.html) to how to write your own stages. You can write stages without using containers, but then you must ensure the python packages and system libraries are available at the time of execution. - -We actually also need to copy some of the other stages and create `shift_` variants of these too, to use the geometry of the shifted realisations too. We will reuse the velocity model for each so those stages are not added. - -After all this, you should have a `flow.cylc` that looks like the following: - -``` -[scheduler] - allow implicit tasks = True -[scheduling] - [[graph]] - R1 = """ - nshm_to_realisation => realisation_to_srf & generate_velocity_model_parameters - generate_velocity_model_parameters => generate_velocity_model & generate_station_coordinates & generate_model_coordinates - realisation_to_srf & generate_velocity_model & generate_station_coordinates & generate_model_coordinates => create_e3d_par - create_e3d_par => run_emod3d - """ -[runtime] - [[root]] - platform = mahuika-slurm - pre-script = """ - module load Apptainer - """ - [[[directives]]] - --account = nesi00213 - [[nshm_to_realisation]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif nshm2022-to-realisation /nshmdb.db 0 /share/realisation.json develop - [[realisation_to_srf]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif realisation-to-srf /share/realisation.json /share/realisation.srf - [[generate_velocity_model_parameters]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif generate-velocity-model-parameters /share/realisation.json - [[generate_velocity_model]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif sh -c 'generate-velocity-model /share/realisation.json /share/Velocity_Model --num-threads $(nproc)' - [[[directives]]] - --cpus-per-task = 32 - --time = 01:00:00 - [[generate_station_coordinates]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif generate-station-coordinates /share/realisation.json /share/stations - [[generate_model_coordinates]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif generate-model-coordinates /share/realisation.json /share/model - [[create_e3d_par]] - platform = localhost - script = apptainer exec /nesi/nobackup/nesi00213/containers/runner_latest.sif create-e3d-par $CYLC_WORKFLOW_SHARE_DIR/realisation.json $CYLC_WORKFLOW_SHARE_DIR/realisation.srf $CYLC_WORKFLOW_SHARE_DIR/Velocity_Model $CYLC_WORKFLOW_SHARE_DIR/stations $CYLC_WORKFLOW_SHARE_DIR/model $CYLC_WORKFLOW_SHARE_DIR/LF --emod3d-path /nesi/project/nesi00213/opt/maui/hybrid_sim_tools/emod3d-mpi_v3.0.8 --scratch-ffp $CYLC_WORKFLOW_SHARE_DIR/LF - [[run_emod3d]] - platform = maui-xc-slurm - pre-script = "" - script = srun /nesi/project/nesi00213/opt/maui/hybrid_sim_tools/emod3d-mpi_v3.0.8 -args "par=$CYLC_WORKFLOW_SHARE_DIR/LF/e3d.par" - [[[directives]]] - --ntasks = 80 - --hint = nomultithread - --time = 00:30:00 - [[realisation_to_srf_shift]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif realisation-to-srf /share/realisation_shift.json /share/realisation_shift.srf - [[create_e3d_par_shift]] - platform = localhost - # NOTE: We change the realisation.json and realisation.srf here too! - script = apptainer exec /nesi/nobackup/nesi00213/containers/runner_latest.sif create-e3d-par $CYLC_WORKFLOW_SHARE_DIR/realisation_shift.json $CYLC_WORKFLOW_SHARE_DIR/realisation_shift.srf $CYLC_WORKFLOW_SHARE_DIR/Velocity_Model $CYLC_WORKFLOW_SHARE_DIR/stations $CYLC_WORKFLOW_SHARE_DIR/model $CYLC_WORKFLOW_SHARE_DIR/LF --emod3d-path /nesi/project/nesi00213/opt/maui/hybrid_sim_tools/emod3d-mpi_v3.0.8 --scratch-ffp $CYLC_WORKFLOW_SHARE_DIR/LF_shift - [[generate_velocity_model_parameters_shift]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif generate-velocity-model-parameters /share/realisation_shift.json - [[run_emod3d_shift]] - platform = maui-xc-slurm - pre-script = "" - script = srun /nesi/project/nesi00213/opt/maui/hybrid_sim_tools/emod3d-mpi_v3.0.8 -args "par=$CYLC_WORKFLOW_SHARE_DIR/LF_shift/e3d.par" - [[[directives]]] - --ntasks = 80 - --hint = nomultithread - --time = 00:30:00 - [[shift_fault]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/bin:/bin" /nesi/nobackup/nesi00213/containers/runner_latest.sif python /bin/shift_fault.py /share/realisation.json /share/realisation_shift.json -``` - -You are now ready to update the workflow graph. - -# Update the Workflow Graph - -The workflow graph controls the order in which jobs are executed. The old flow had the following diagram. - -``` -[scheduling] - [[graph]] - R1 = """ - nshm_to_realisation => realisation_to_srf & generate_velocity_model_parameters - generate_velocity_model_parameters => generate_velocity_model & generate_station_coordinates & generate_model_coordinates - realisation_to_srf & generate_velocity_model & generate_station_coordinates & generate_model_coordinates => create_e3d_par - create_e3d_par => run_emod3d - """ -``` - -```mermaid -flowchart LR - A[nshm_to_realisation] --> B[realisation_to_srf] - A --> C[generate_velocity_model_parameters] - C --> D[generate_velocity_model] - C --> E[generate_station_coordinates] - C --> F[generate_model_coordinates] - B[realisation_to_srf] --> G[create_e3d_par] - D --> G - E --> G - F --> G - G --> H[emod3d] -``` - -We should now introduce the shifted realisation generation after the `nshm_to_realisation` stage and create a workflow parallel to the first. - -```mermaid -flowchart LR - A[nshm_to_realisation] --> I[shift_fault] - I --> B[realisation_to_srf] - I --> J[realisation_to_srf_shift] - I --> K[generate_velocity_model_parameters_shift] - J --> L[create_e3d_par_shift] - K --> L - D --> L - E --> L - F --> L - L --> M[emod3d_shift] - A --> C[generate_velocity_model_parameters] - C --> D[generate_velocity_model] - C --> E[generate_station_coordinates] - C --> F[generate_model_coordinates] - B[realisation_to_srf] --> G[create_e3d_par] - D --> G - E --> G - F --> G - G --> H[emod3d] -``` - -Which looks like the following in the Cylc flow graph. - -``` -[scheduling] - [[graph]] - R1 = """ - nshm_to_realisation => realisation_to_srf & generate_velocity_model_parameters & shift_fault - shift_fault => realisation_to_srf_shift - realisation_to_srf_shift => generate_velocity_model_parameters_shift - realisation_to_srf_shift & generate_velocity_model_parameters_shift & generate_velocity_model & generate_station_coordinates & generate_model_coordinates => create_e3d_par_shift - create_e3d_par_shift => run_emod3d_shift - - generate_velocity_model_parameters => generate_velocity_model & generate_station_coordinates & generate_model_coordinates - realisation_to_srf & generate_velocity_model & generate_station_coordinates & generate_model_coordinates => create_e3d_par - create_e3d_par => run_emod3d - """ -``` - -So your flow.cylc should now look like: - -``` - -[scheduler] - allow implicit tasks = True -[scheduling] - [[graph]] - R1 = """ - nshm_to_realisation => realisation_to_srf & generate_velocity_model_parameters & shift_fault - shift_fault => realisation_to_srf_shift - realisation_to_srf_shift => generate_velocity_model_parameters_shift - realisation_to_srf_shift & generate_velocity_model_parameters_shift & generate_velocity_model & generate_station_coordinates & generate_model_coordinates => create_e3d_par_shift - create_e3d_par_shift => run_emod3d_shift - - generate_velocity_model_parameters => generate_velocity_model & generate_station_coordinates & generate_model_coordinates - realisation_to_srf & generate_velocity_model & generate_station_coordinates & generate_model_coordinates => create_e3d_par - create_e3d_par => run_emod3d - """ -[runtime] - [[root]] - platform = mahuika-slurm - pre-script = """ - module load Apptainer - """ - [[[directives]]] - --account = nesi00213 - [[nshm_to_realisation]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif nshm2022-to-realisation /nshmdb.db 0 /share/realisation.json develop - [[realisation_to_srf]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif realisation-to-srf /share/realisation.json /share/realisation.srf - [[generate_velocity_model_parameters]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif generate-velocity-model-parameters /share/realisation.json - [[generate_velocity_model]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif sh -c 'generate-velocity-model /share/realisation.json /share/Velocity_Model --num-threads $(nproc)' - [[[directives]]] - --cpus-per-task = 32 - --time = 01:00:00 - [[generate_station_coordinates]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif generate-station-coordinates /share/realisation.json /share/stations - [[generate_model_coordinates]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif generate-model-coordinates /share/realisation.json /share/model - [[create_e3d_par]] - platform = localhost - script = apptainer exec /nesi/nobackup/nesi00213/containers/runner_latest.sif create-e3d-par $CYLC_WORKFLOW_SHARE_DIR/realisation.json $CYLC_WORKFLOW_SHARE_DIR/realisation.srf $CYLC_WORKFLOW_SHARE_DIR/Velocity_Model $CYLC_WORKFLOW_SHARE_DIR/stations $CYLC_WORKFLOW_SHARE_DIR/model $CYLC_WORKFLOW_SHARE_DIR/LF --emod3d-path /nesi/project/nesi00213/opt/maui/hybrid_sim_tools/emod3d-mpi_v3.0.8 --scratch-ffp $CYLC_WORKFLOW_SHARE_DIR/LF - [[run_emod3d]] - platform = maui-xc-slurm - pre-script = "" - script = srun /nesi/project/nesi00213/opt/maui/hybrid_sim_tools/emod3d-mpi_v3.0.8 -args "par=$CYLC_WORKFLOW_SHARE_DIR/LF/e3d.par" - [[[directives]]] - --ntasks = 80 - --hint = nomultithread - --time = 00:30:00 - [[realisation_to_srf_shift]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif realisation-to-srf /share/realisation_shift.json /share/realisation_shift.srf - [[create_e3d_par_shift]] - platform = localhost - # NOTE: We change the realisation.json and realisation.srf here too! - script = apptainer exec /nesi/nobackup/nesi00213/containers/runner_latest.sif create-e3d-par $CYLC_WORKFLOW_SHARE_DIR/realisation_shift.json $CYLC_WORKFLOW_SHARE_DIR/realisation_shift.srf $CYLC_WORKFLOW_SHARE_DIR/Velocity_Model $CYLC_WORKFLOW_SHARE_DIR/stations $CYLC_WORKFLOW_SHARE_DIR/model $CYLC_WORKFLOW_SHARE_DIR/LF --emod3d-path /nesi/project/nesi00213/opt/maui/hybrid_sim_tools/emod3d-mpi_v3.0.8 --scratch-ffp $CYLC_WORKFLOW_SHARE_DIR/LF_shift - [[generate_velocity_model_parameters_shift]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/input:/input:ro" /nesi/nobackup/nesi00213/containers/runner_latest.sif generate-velocity-model-parameters /share/realisation_shift.json - [[run_emod3d_shift]] - platform = maui-xc-slurm - pre-script = "" - script = srun /nesi/project/nesi00213/opt/maui/hybrid_sim_tools/emod3d-mpi_v3.0.8 -args "par=$CYLC_WORKFLOW_SHARE_DIR/LF_shift/e3d.par" - [[[directives]]] - --ntasks = 80 - --hint = nomultithread - --time = 00:30:00 - [[shift_fault]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share,$CYLC_WORKFLOW_RUN_DIR/bin:/bin" /nesi/nobackup/nesi00213/containers/runner_latest.sif python /bin/shift_fault.py /share/realisation.json /share/realisation_shift.json -``` - -# Running the Simulation -Now that the hard work is done, running the simulation is simple. Move the `flow.cylc` to the `~/cylc-src/custom/` directory and then execute the steps `cylc install custom` and `cylc play custom`. Refer to the [workflow basic tutorial](Using-Workflow.md) for additional details on this step. - -# NeSI HPC Infrastructure - -The high performance computing (HPC) resources provided by NeSI consist of a cluster of computers that are available to use. NeSI provides a detailed [overview](https://docs.nesi.org.nz/Scientific_Computing/The_NeSI_High_Performance_Computers/) of the computers and what they can be used for. Here is the executive summary **for the new workflow**: - -| Platform | Purpose to QuakeCoRE | -|:--------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Mahuika | Big cluster of Linux computers (nodes) working in concert. All jobs except EMOD3D are run in the special Milan computers (nodes). | -| Maui | Specialised Cray supercomputer used for jobs that make use of [MPI](https://www.open-mpi.org/) to parallelise tasks across multiple computers (nodes) at once. Just used for EMOD3D | -| Mahuika Milan | Next-generation Linux cluster with container support. Our jobs are run on Milan wherever possible. | -| Lander02 | The computer which we use to access the other platforms. | - -When you connect to a NeSI host like maui you reach what is called the _login node_. This session is where you execute `cylc install` and where trivial tasks are executed. This information is summarised below. - -``` mermaid -flowchart LR - n1>"Lander"] -- SSH Proxy --> n2["Maui Login Node (your terminal is here)"] - n2 -- Interact via cylc --> n3["Maui"] & n4["Mahuika (Milan)"] - n5["Your Computer"] -->|ssh maui| n1 - style n1 stroke-width:2px,stroke-dasharray: 2 -``` - -# What is a Container? - -In simple terms, a container is a virtual computer that runs on top of another computer (called the host). The virtual computer inherits some of the host computer's infrastructure but crucially contains a different set of libraries and executables. We use containers at QuakeCoRE for a few purposes: - -1. To isolate the workflow stages, so that every workflow stage is run with the same state to minimise the surface area for bugs. -2. To simplify environment management, so that it is easier for researchers to build their own workflow stages. -3. To ensure the workflow is run with the latest software versions for things like Python and GMT. NeSI maintains its own versions but can be slow to keep them up to date. - -See [this video](https://www.youtube.com/watch?v=Gjnup-PuquQ) for a two-minute visual explainer on containers. Note that we use [Apptainer](https://apptainer.org/) containers as opposed to Docker containers, but the concepts are still the same. - -# Using the Workflow Planning Tool - -We often need to run the same jobs for a number of realisations. Building the Cylc workflow by hand for this can be tedious. Hence, we have built a workflow planning tool that can flexibly generate workflow scripts for the common case of: - -1. You want to simulate a number of realisations in the same way and, -2. you want to do this with a subset of the full Cybershake workflow. - -The tool is available if you install the workflow using pip: - -``` bash -pip install workflow @ https://github.com/ucgmsim/workflow -``` - -The script to plan workflow is called `plan-workflow`. If you want to plan workflows for two events identically to Cybershake you can execute - -``` bash -plan-workflow Fault_1 Fault_2 ~/cylc-src/template/flow.cylc --goal im_calc -``` - -This will create a workflow file `flow.cylc` to simulate two faults: Fault 1 and Fault 2, targeting intensity measure calculation. Running `cylc install template` and then `cylc graph template`, you should see an output like the following - -
-Cylc Graph - -![](images/planned_workflow.png) -
- -If you also want the `plot_ts` stage you can add it as a goal to the workflow plan. - -``` bash -plan-workflow Fault_1 Fault_2 ~/cylc-src/template/flow.cylc --goal im_calc --goal plot_ts -``` - -
-Cylc Graph with plot_ts - -![](images/planned_workflow_plot.png) -
- -The workflow planning tool is pretty flexible. You can exclude any task you want if you, for example, already have the output for a stage completed (like a custom SRF). To exclude a stage, use the `--excluding` option. Currently our generated workflows have assumed that `Fault_1` and `Fault_2` are sourced from the NSHM 2022 database. But if you have a custom realisation file to simulate, you can use the `--excluding` option to excluding the `nshm_to_realisation` stage. - -``` bash -plan-workflow Fault_1 Fault_2 ~/cylc-src/template/flow.cylc --goal im_calc --goal plot_ts --excluding nshm_to_realisation -``` - -
-Cylc Graph with plot_ts Stage and No Realisation Generation - -![](images/planned_workflow_no_realisation.png) -
- -You can use `--excluding-group` to exclude a predefined group of workflow stages. If you have completed all the pre-processing stages, for example, you can exclude the `preprocessing` group to plan simulation only. - -``` bash -plan-workflow Fault_1 Fault_2 ~/cylc-run/template/flow.cylc --goal im_calc --excluding-group preprocessing -``` - -
-Cylc Graph with No Preprocessing - -![](images/planned_workflow_no_prepro.png) -
- -Often times, we want to run a number of realisations of the same event: varying the magnitude, rupture propagation, and hypocentre of an event without changing the domain or velocity model. The workflow planner tool has support to generate workflows that reuse the velocity model for a number of realisations. - -``` bash -plan-workflow Event:2 ~/cylc-run/template/flow.cylc --goal create_e3d_par -``` - -
-Cylc Graph with Shared Velocity Model - -![](images/planned_workflow_shared.png) -
- -Notice that there is only `generate_velocity_model` job in this workflow, and the `create_e3d_par` jobs for Event and Event:1 (a different sample of Event) both use the velocity model from the main sample of the event. - -The `cylc graph` tool is hard to use for large workflows. The `--visualise` flag for the workflow planner opens the planned workflow in your browser. You can drag stages around and zoom in to understand the generated workflow execution graph. Adjust the physics settings in the visualisation to make the output easier to read, or disable it entirely to allow you to freely move the nodes. - -``` bash -plan-workflow Event:2 ~/cylc-src/template/flow.cylc --goal im_calc --goal plot_ts --visualise -``` - -
-HTML Visualisation of Complex Cylc Workflow - -![](images/planned_workflow_html_vis.png) -
diff --git a/wiki/EMOD3D.md b/wiki/EMOD3D.md deleted file mode 100644 index 7add3cfb..00000000 --- a/wiki/EMOD3D.md +++ /dev/null @@ -1,109 +0,0 @@ -# Compiling Tools in EMOD3D and NZVM for Workflow - -The workflow uses a few (and hopefully fewer in the future) C binaries built in EMOD3D. The Cybershake container has these prebuilt, so if you run your Python code in this container you do not need to build any binaries. The [EMOD3D Readme](https://github.com/ucgmsim/EMOD3D?tab=readme-ov-file#emod3d) has generic build instructions for tools in EMOD3D, but this page has more detailed instructions for each tool required by the workflow. - -If you are using a system with `gcc` version 14 or later, then you need to see the [extra steps](#extra-steps-for-gcc14-users). You can tell if you have a system with version 14 or later by executing `gcc --version` in the command line. You are using version 14 if you see something like the following. -``` -gcc (GCC) 14.2.1 20240910 -Copyright (C) 2024 Free Software Foundation, Inc. -This is free software; see the source for copying conditions. There is NO -warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -``` -**Hypocentre and NeSI are not using version 14 at this stage.** - - -> [!NOTE] -> Before you start! Clone the EMOD3D repository somewhere with the following command: `git clone https://github.com/ucgmsim/EMOD3D` - -## SRF Generation - -If you are generating a SRF with `realisation-to-srf`, you need to build genslip. **After cloning EMOD3D**, change directory (using `cd`) into the `EMOD3D` folder and execute the following in the command line (one line per command): - -``` shell -mkdir build -cd build -cmake .. -cmake --build . --target genslip_5.4.2 -``` - -You then need to pass the `--genslip-path` flag with the path to this binary when you use `realisation-to-srf` - -``` shell -realisation-to-srf realisation.json realisation.srf realisation.stoch --genslip-path /tools/srf2stoch --work-directory ... --velocity-model-ffp ... -``` - -Replace `` with the path to your cloned copy of EMOD3D where you built the tools. - - -## Stoch Generation -If you are generating a Stoch file with `generate-stoch`, you need to build srf2stoch. **After cloning EMOD3D**, change directory (using `cd`) into the `EMOD3D` folder and execute the following in the command line (one line per command): - -``` shell -mkdir build -cd build -cmake .. -cmake --build . --target srf2stoch -``` - -You then need to pass the `--srf2stoch-path` flag with the path to this binary when you use `generate-stoch` - -``` shell -generate-stoch realisation.json realisation.srf realisation.stoch --srf2stoch-path /tools/srf2stoch -``` - -Replace `` with the path to your cloned copy of EMOD3D where you built the tools. - - -## High Frequency Simulation - -If you are locally running high-frequency simulations using `hf-sim` then you need to build the high-frequency simulation binaries. -**After cloning EMOD3D**, change directory (using `cd`) into the `EMOD3D` folder and execute the following in the command line (one line per command): - -``` shell -mkdir build -cd build -cmake .. -cmake --build . --target hb_high_binmod_v6.0.3 -cmake --build . --target hb_high_binmod_v5.4.5.3 -``` - -You then need to pass the `--hf-sim-path` flag with the path to this binary when you use `hf-sim` - -``` shell -hf-sim realisation.json realisation.stoch stations.ll HF.bin --hf-sim-path /tools/hb_high_binmod_v6.0.3 --velocity-model ... --work-directory ... -``` - -Replace `` with the path to your cloned copy of EMOD3D where you built the tools. - -### My High-Frequency Simulation Fails! - -The high-frequency Fortran code is very brittle and old. Sometimes you need to use a different version to make it work. Try running the same `hf-sim` command with version `5.4.5.3`. - -``` shell -hf-sim realisation.json realisation.stoch stations.ll HF.bin --hf-sim-path /tools/hb_high_binmod_v5.4.5.3 --velocity-model ... --work-directory ... -``` - -## Velocity Modelling - -To create a velocity model you have to build `NZVM`. To build `NZVM`, run the following - -``` shell -git clone https://github.com/ucgmsim/Velocity-Model -cd Velocity-Model -make -``` - -Then when running the velocity model pass the path to the `NZVM` binary using the `--velocity-model-bin-path` flag. - -``` shell -generate-velocity-model realisation.json velocity_model_folder --velocity-model-bin-path /NZVM --work-directory ... -``` - -## Extra Steps for GCC14 Users - -After cloning the EMOD3D repository, running the following - -``` shell -cd EMOD3D -git checkout gcc14 -``` diff --git a/wiki/Home.md b/wiki/Home.md deleted file mode 100644 index 49f287a0..00000000 --- a/wiki/Home.md +++ /dev/null @@ -1 +0,0 @@ -Welcome to the wiki! diff --git a/wiki/Realisation-Proposal.md b/wiki/Realisation-Proposal.md deleted file mode 100644 index effbbbc6..00000000 --- a/wiki/Realisation-Proposal.md +++ /dev/null @@ -1,268 +0,0 @@ -# Realisation Specification -A realisation is the concrete specification of a single ground motion -simulation. We currently have multiple files that make up the -specification of a single realisation. - -1. A `root_params.yaml` file that specifies the parameters for a - single cybershake run. -2. A `fault_params.yaml` file that specifies the parameters for all - realisations involving a given fault in a single cybershake run. -3. A realisation file (CSV, or YAML for type-5) that specifies the - source model parameters (source geometry, magnitude, genslip and - srfgen seeds, etc) for a single realisation. -4. A `sim_params.yaml` file that specifies the other parameters for a - single realisation (HF, BB, IM parameters, EMOD3D parameters). -5. A separate YAML file with a filename pattern `RELNAME.yaml` that - mostly contains the same information as `sim_params.yaml`. - -This split has some advantages. The main advantage is that you can -modify the parameters in a hierarchical fashion. Changes to the EMOD3D -run parameters for all realisations in a single cybershake run can be -made at the level of `root_params.yaml` file. Almost certainly a -motivating factor for the `sim_params.yaml` and realisation file split -is that the old simulations were specified in CSV, which is not well -suited to the hierarchical data supplied in `sim_params.yaml`. - -However, the split has many disadvantages. It massively complicates -the workflow because we have to remember to read each of these files -(and in the correct order). These files are created independently of -each other, and hence can independently fail to be created or be -corrupted. The combinatorial explosion in realisation state makes the -workflow brittle. While the separation of simulation parameters and -realisation parameters made sense when these were in different -formats, type-5 realisation files could easily involve both -parameters. The split also means it is more difficult to reproduce the -results of a single simulation, because if we do not archive any one -of the four files we are missing some parameters determining the -simulation outcome. - -With an upcoming move to a Pegasus workflow, it's clear we need to -rethink realisation parameters. At the very least, whatever solution -we land on needs to be fully specified so we can develop tools against -it that verify the files. We need to document our methodology so that -in the future we can understand why certain decisions were made. This -page is a proposal of a fully specified realisation format and a scope -of what should be contained in a realisation file. - -# Design Aims of the New Realisation Format -The key design aims of the new realisation format are as follows: - -1. **Readability**. Researchers are domain experts in earthquake - research, not computer science. Realisations should be specified in - a plain-text and logical format. Ideally, the file(s) should be - self-documenting, i.e. attached to each variable is documentation - or comments explaining what it is, how it affects the workflow, and - the acceptable values. -2. **Writability**. Previously we have privileged creating - realisations via automated workflows with GCMT and NSHM. It is - frustrating to try and run a simulation with custom geometry or - simulation parameters because the specifications are too complicated - and undocumented to write ourselves. Any new specification should - be accompanied with tools that makes writing custom realisations easy. -3. **Referential Transparency**. Running a simulation twice with the - same realisation file(s) should produce the same simulation output. - This cannot be completely controlled by a realisation due to - reliance on external data files and differences in hardware, but we - can strive for the ideal here. -4. **Validation**. It should be possible to verify that every - parameter in the realisation file(s) are specified correctly. If - the values are not specified correctly, tools should exist that can - report the error in language that researchers can understand to fix - the problem. - - -# The Proposed Solution -We propose a simplified realisation specification that involves a -single file written in a JSON file format. This JSON file does the job -of the four files previously specifying realisations by including all -the source modelling parameters, and all the simulation parameters for -a single realisation. To ameliorate the loss of the hierarchical -modification in the old model, we will -write better tools to generate many different realisations so that -changes can be made in bulk. - -The main components of this solution will be: - -1. A library to read and write JSON realisation files. -2. A document produced that describes a valid JSON realisation file. -3. A tool to validate JSON realisation files. - -The main reason to choose JSON over YAML is that it allows us to -produce a [JSON Schema](https://json-schema.org/) for realisations. -JSON schemas are a documented, standardised, validated format for data -exchange used by many large companies like Zapier, Microsoft, Postman, -and GitHub. JSON schemas have extensive tooling built around them. -They allow, for example, text editors like VSCode to provide -documentation, auto-completion, and validation whilst you are writing -them (making custom simulations easier). There is [extensive library -support](https://json-schema.org/implementations) for validating -schemas in many programming languages. There are [CLI -tools](https://github.com/python-jsonschema/check-jsonschema) that can -validate files against JSON schemas, including even one built [built -into Microsoft -PowerShell](https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.utility/test-json). -Tools like Apple's [PKL](https://pkl-lang.org/index.html) language -format for configuration offer an (arguably superior) -alternative, but are probably too new for a mature scientific pipeline -to rely on. JSON schemas, on the other hand, are well-established. - -![An example of VS Code providing auto-completion and documentation for variables in a JSON file conforming to the VS Code settings schema.](images/json_schema.png) - -Using the schema, we can automatically generate a document -describing the realisation file, and can off-load the validation to -existing tools. Leaving us to produce only: A library for reading JSON -files and a schema. - -Currently realisations are generated in the automated pipeline for -GCMT and NSHM. There are no exposed tools to create custom -realisations except a broken library. For the new specification, we -propose an **inversion** of the original logic of realisation -generation: write a user-facing custom realisation building library -first. The GCMT and NSHM generation scripts would use this libary in -their automated pipeline. This change means that custom realisations -are first-class in the new framework and updated in lock-step with -simulation changes introduced to the automated workflow. Using -software design patterns like the Builder pattern, one could imagine -the custom interface to look something like - -```python -# Loop over a bunches of sources from: NSHMDB, GCMT, Custom outputs -realisation = RealisationBuilder('REALISATION NAME') - .with_seed(SEED) - # ^- used for randomised values - .with_sources({ - 'Some Fault Name': Plane(some_corners) - }) - .with_magnitude(7, proportioned_to_faults_automatically=True) - # or: with magnitude + custom magnitude proportions - .with_rupture_propagation(use_automatic_propogation=True) - # or: with a custom rupture propagation tree to specify fault jumping - .with_jump_locations(random=True) - # or: with custom jump locations - .with_hypocentre(random=True) - # or: with a fault + fault-local coordinates for the hypocentre - .with_simulation_parameters(defaults=True) - # and: with keyword overrides for any simulation parameters for this run - .build() - # Will throw a descriptive error if any of the above - # steps fail, to allow for validation. - -realisation.save('realisation_filepath.json') -``` - -Let's review how this tackles each of our design aims: - -1. **Readability**. JSON is a plain-text format with a logical layout. - The syntax is similar to Python dictionaries, which researchers - will be familiar with through exposure to Python programming. One - could argue it is less readable than YAML, however the addition of - the schema allows powerful documentation tools that more than make - up for the loss of syntax sugar. -2. **Writability**. The library proposed will allow researchers to - generate custom realisations in bulk in Python. The Builder pattern - is well-established in the Software industry as a solution for - building complex objects with a simple interface. -3. **Referential Transparency**. The realisation file specification is - now the one source of truth for simulation parameters. This one - file along with the data files now completely determines the - realisation up to hardware differences. -4. **Validation**. Realisation files now have a completely specified - schema using industry standard tooling to validate input files. We - immediately gain the benefits of all the lessons learned building - the JSON schema format, and we can catch errors in the realisation - specification before they end up in the workflow burning core - hours. - -# A High Level Example of Realisation Files -While the details of the realisation specification will come from the resulting schema, a high-level example of this format would look something like: - -```json -{ - "name": "Realisation Name", - "realisation_specification_version": 5, - "sources": { - "Acton": { - "type": "Fault", // or Plane, or Point (see Sources documentation). - "planes": [ - { - "corners": [ - { "lat": -45.4444, "lon": 168.3711, "depth": 0 }, - { "lat": -45.4761, "lon": 168.3553, "depth": 0 }, - { "lat": -45.4946, "lon": 168.5536, "depth": 27120 }, - { "lat": -45.4628, "lon": 168.5963, "depth": 27120 }, - ] - } - ] - } - }, - "rupture_propogation": { - "Acton": { - "parent": null, // specifies acton is the initial fault in the rupture - "hypocentre": [0.5, 0.5], // hypocentre in fault-local coordinates (see Sources docs) - "magnitude": 7, - "rake": 110 - } - }, // Total rupture moment magnitude is sum of powers of fault magnitudes. - "source_parameters": { - "genslip_dt": 0.05, // for genslip generation - "genslip_seed": 1, - "genslip_version": "5.4.2", - "srfgen_seed": 1 - }, - "domain_parameters": { - "resolution": 0.1, - "model_latitude": -45.5, - "model_longitude": 168.5, - "model_rotation": 170, - "model_width": 100, - "model_length": 100, - "model_depth": 40 - }, - "simulation_parameters" : { - "VM": {}, - "HF": {}, - "BB": {}, - "LF": {}, - // etc... - } -} -``` - -# Implementation Details - -The structure of the realisation implementation would consist of three components: - -1. A `Realisation` object containing all the details of the - realisation loaded from the JSON file. There may be other classes - within this object like `SourceParameters` etc. -2. A `RealisationBuilder` object that allows us to build realisations - efficiently. This may be elided and the `Realisation` object might - contain this functionality, but at least initially this separation - seems sensible. -3. A JSON schema specified by the [schema - library](https://pypi.org/project/schema/). The schema library - allows us to validate realisations with the composability of - standard Python values. This will make building a realisation - schema feasible. Schemas made with the schema library can be - compiled into JSON schemas that other tools like editors and CLI - tools can read. The schemas can then also be used for the - `RealisationBuilder` object for validation too. - -This will be updated as the implementation is completed and the details are ironed out. - -# Implementation Roadmap -Initially the simulation parameters for the realisation will -essentially be ignored. The Minimum Viable Product would be porting -the existing type-5 realisation spec for YAML to JSON with the schema -library validating the format. The new realisation format would be -used in the generation of SRFs and VM parameters for type-5 -realisations, but the existing code and workflow would never need to -know that this is using the new format. - -This will allow a drop in replacement for the existing slurm workflow -that doesn't require changing any of the existing code (except to -change references to YAML to references to JSON for type-5 -realisations). Then, as we migrate from slurm\_gm\_workflow to Pegasus -we can have the new scripts read parameters from the JSON realisation -file rather than YAML files incrementally. This allows for a gradual -transition to the new specification rather than an all-at-once shift. diff --git a/wiki/Realisations.md b/wiki/Realisations.md deleted file mode 100644 index dd00574b..00000000 --- a/wiki/Realisations.md +++ /dev/null @@ -1,271 +0,0 @@ -# Realisations - -This documentation outlines how to create, read and write realisations -using the new realisations workflow. To understand the rationale -behind this module, see [the proposal](Realisations-Proposal.md) on -realisations. Note that the proposal and the implementation differ -slightly as some components of the implementation have changed. - -# Creating and Writing Realisations - -Realisations are created by instantiating configuration objects, and -then reading or writing these to a realisation file. Realisations can -be incomplete in the sense that they may miss configuration -components at certain steps in realisation generation. To reflect this, the library interface is modular. Let's take a look at an example: - -```python -from workflow import realisations -from velocity_modelling import bounding_box -import numpy as np - -domain_parameters = realisations.DomainParameters( - resolution=0.1, # a 0.1km resolution - domain=bounding_box.BoundingBox.from_centroid_bearing_extents( - centroid=np.array([-43.53092, 172.63701]), - bearing=45, - extent_x=100, - extent_y=100 - ), - depth=40.0, - duration=60.0, - dt=0.005 -) - -domain_parameters.write_to_realisation('path/to/realisation.json') -``` - -Inside the `realisation.json` file you would find the following - -```json -{ - "domain": { - "resolution": 0.1, - "domain": [ - { - "latitude": -43.524793866326725, - "longitude": 171.76204128885567 - }, - { - "latitude": -42.894200350955856, - "longitude": 172.64076673694242 - }, - { - "latitude": -43.53034935969409, - "longitude": 173.51210368762364 - }, - { - "latitude": -44.16756820707226, - "longitude": 172.63312824122775 - } - ], - "depth": 40.0, - "duration": 60.0, - "dt": 0.005 - } -} -``` - -Everything under the `domain` keyword is the configuration for the domain parameters. Suppose after this we wished to add parameters for SRF generation. Then, we would write something like - -```python -srf_config = realisations.SRFConfig( - genslip_dt = 1.0, - genslip_seed=1, - genslip_version='5.4.2', - resolution=0.1, - srfgen_seed=1 -) - -srf_config.write_to_realisation('path/to/realisation.json') -``` - -And then, inside the `realisation.json` file you'll find - -```json -{ - "domain": { - "resolution": 0.1, - "domain": [ - { - "latitude": -43.524793866326725, - "longitude": 171.76204128885567 - }, - { - "latitude": -42.894200350955856, - "longitude": 172.64076673694242 - }, - { - "latitude": -43.53034935969409, - "longitude": 173.51210368762364 - }, - { - "latitude": -44.16756820707226, - "longitude": 172.63312824122775 - } - ], - "depth": 40.0, - "duration": 60.0, - "dt": 0.005 - }, - "srf": { - "genslip_dt": 1.0, - "genslip_seed": 1, - "genslip_version": "5.4.2", - "srfgen_seed": 1 - } -} -``` - -Notice that the srf config script did not need to read domain parameters to update the realisation file. This is what allows the realisations to be build bit-by-bit, in the same way that our workflow currently generates these files. If we were to have a central `Realisation` object, we'd end up needing to specify a lot of default dummy values everytime we loaded an incomplete realisation file. - -# Reading Realisations - -To read a realisation, you simply call the `read_from_realisation` classmethod with the filepath of the realisation and from the type of config you wish to read. For example, if the `realisations.json` is as in the previous section - -```json -{ - "domain": { - "resolution": 0.1, - "domain": [ - { - "latitude": -43.524793866326725, - "longitude": 171.76204128885567 - }, - { - "latitude": -42.894200350955856, - "longitude": 172.64076673694242 - }, - { - "latitude": -43.53034935969409, - "longitude": 173.51210368762364 - }, - { - "latitude": -44.16756820707226, - "longitude": 172.63312824122775 - } - ], - "depth": 40.0, - "duration": 60.0, - "dt": 0.005 - }, - "srf": { - "genslip_dt": 1.0, - "genslip_seed": 1, - "genslip_version": "5.4.2", - "srfgen_seed": 1 - } -} -``` - -Then, we may read the domain parameters with the following code - -```python ->>> realisations.DomainParameters.read_from_realisation('path/to/realisations.json') -DomainParameters(resolution=0.1, domain=..., depth=40.0, duration=60.0, dt=0.005) -``` - -At read time, basic input validation checks are made. If we made the `resolution` parameter 0, then we get an error -```python ->>> realisations.DomainParameters.read_from_realisation('path/to/realisations.json') -Traceback (most recent call last): -... -schema.SchemaError: Key 'resolution' error: -is_positive(0.0) should evaluate to True -``` - -# Creating Your Own Configuration Section - -To add your own realisation config object, you need to create a configuration object inheriting from `RealisationConfiguration` and assign the object a matching schema. - -We will now walk through an example adding the `DomainParameters` object to the realisation specification. - -## Creating a Configuration Object -We first create the class with the all the domain parameters we need for our simulation - -```python -@dataclasses.dataclass -class DomainParameters(RealisationConfiguration): - """ - Parameters defining the spatial and temporal domain for simulation. - - Attributes - ---------- - resolution : float - The simulation resolution in kilometres. - domain : BoundingBox - The bounding box for the domain. - depth : float - The depth of the domain (in metres). - duration : float - The simulation duration (in seconds). - dt : float - The resolution of the domain in time (in seconds). - """ - - _config_key: ClassVar[str] = "domain" - _schema: ClassVar[Schema] = schemas.DOMAIN_SCHEMA - - resolution: float - domain: BoundingBox - depth: float - duration: float - dt: float - - @property - def nx(self) -> int: - """int: The number of x coordinate positions in the discretised domain.""" - return int(np.round(self.domain.extent_x / self.resolution)) - - @property - def ny(self) -> int: - """int: The number of y coordinate positions in the discretised domain.""" - return int(np.round(self.domain.extent_y / self.resolution)) - - @property - def nz(self) -> int: - """int: The number of z coordinate positions in the discretised domain.""" - return int(np.round(self.depth / self.resolution)) - - def to_dict(self) -> dict: - """ - Convert the object to a dictionary representation. - - Returns - ------- - dict - Dictionary representation of the object. - """ - param_dict = dataclasses.asdict(self) - param_dict["domain"] = to_name_coordinate_dictionary( - self.domain.corners, - ) - return param_dict -``` - -We have to write a `to_dict` method for any configuration object we create. This method specifies how to serialise the configuration into a dictionary. The keys and values must be JSON-serialisable python objects. Most of the time, you are fine to just have this method return the output of `dataclasses.asdict`. If you are writing numpy values you will need to write a custom serialisation function. The helper function `to_name_coordinate_dictionary` converts numpy arrays of varying shapes into lists of keyword dictionaries specifying coordinates. - -The value of `_config_key` is the key the configuration is read and written to from the realisation. The `_schema` class variable points to the schema to validate input with when loading the realisation. -## Creating the Schema -The next step is specifying the configuration schema. Schemas should validate the types and general bounds of their input variables. They should not perform complex input validation. You should describe each keyword using the `description=` keyword argument and a schema `Literal`. There are a number of prewritten helper functions like `is_positive` to validate certain inputs. - -```python -DOMAIN_SCHEMA = Schema( - { - Literal("resolution", description="The simulation resolution (in km)"): And( - float, is_positive - ), - Literal("domain", description="The corners of the simulation domain."): And( - Use(corners_to_array), Use(BoundingBox.from_wgs84_coordinates) - ), - Literal("depth", description="The depth of the model (in km)"): And( - float, is_positive - ), - Literal( - "duration", description="The duration of the simulation (in seconds)" - ): And(float, is_positive), - Literal("dt", "The resolution of the domain in time (in seconds)."): And( - float, is_positive - ), - } -) -``` diff --git a/wiki/Stages.md b/wiki/Stages.md deleted file mode 100644 index 49a54d40..00000000 --- a/wiki/Stages.md +++ /dev/null @@ -1,254 +0,0 @@ -# Workflow Stages - -The workflow repository contains all the tools necessary to build a -workflow for ground motion model simulations. We build and maintain a -Cybershake workflow, but researchers may wish to build their own. For -this reason, the workflow is build out of composable parts we intend -for anyone to reuse to build their own workflow based on the -Cybershake workflow. Below you will find documentation for all the -workflow stages, their inputs, outputs and environments. - -```mermaid -flowchart LR - A[NSHM To Realisation] --> B[SRF Generation] - A --> C[Domain Generation] - B --> D[Stoch Generation] - C --> E[Velocity Model Generation] - C --> F[Station Selection] - C --> G[Write Model Coordinates] - B --> H[Create EMOD3D Parameters] - E --> H - F --> H - G --> H - H --> I[EMOD3D] - D --> J[High Frequency Simulation] - I -->|Optionally| K[Merge Timeslices] - K --> L[Create Simulation Video] -``` - - -Many of the stages will run in a _container_. A container is a self-contained execution environment with all the system and Python libraries required to execute workflow stages. It is also isolated from the host system and may not be able to access certain directories. We maintain a [cybershake container](https://hub.docker.com/r/earthquakesuc/runner) that has a copy of the latest validated workflow, and environment. Use this container as much as possible in your own scripts to run your code. - -## NSHM To Realisation - -### Description -Construct a realisation from a rupture in the [NSHM 2022](https://nshm.gns.cri.nz/RuptureMap). -### Inputs - - 1. A copy of the [NSHM 2022 database](https://www.dropbox.com/scl/fi/50kww45wpsnmtf3pn2okz/nshmdb.db?rlkey=4mjuomuevl1963fjwfximgldm&st=50ax73gl&dl=0). - 2. A rupture id to simulate. You can find a rupture id from the [rupture explorer](https://nshm.gns.cri.nz/RuptureMap). Alternatively, you can use the visualisation tools to find one. - 3. The version of the [scientific defaults](https://github.com/ucgmsim/workflow/blob/pegasus/workflow/default_parameters/README.md#L1) to use. If you don't know what version to use, choose the latest version. Versions are specified as `YY.M.D.R`, where `R` is the resolution of the simulation (1 = 100m). For example `24.2.2.1`. The special `develop` version is for testing workflow iterations and not to be used for accurate scientific simulation. -### Outputs - A realisation file containing: - 1. The definition of all the faults in the the rupture, - 2. A rupture propagation plan (i.e. how the rupture jumps between faults, and where), - 3. The estimated rupture magnitude and apportionment to the involved faults. - 4. The definition of the rakes. -### Environment - Can be run in the cybershake container. Can also be run from your own computer using the `nshm2022-to-realisation` command which is installed after running `pip install workflow@git+https://github.com/ucgmsim/workflow`. -### Usage - `nshm2022-to-realisation [OPTIONS] NSHM_DB_FILE RUPTURE_ID REALISATION_FFP DEFAULTS_VERSION` -### For More Help - See the output of `nshm2022-to-realisation --help` or [nshm2022\_to\_realisation.py](https://github.com/ucgmsim/workflow/blob/pegasus/workflow/scripts/nshm2022_to_realisation.py). - - -## SRF Generation -### Description - Produce an SRF from a realisation. -### Inputs - A realisation file containing: - 1. A source configuration, - 2. A rupture propagation configuration, - 3. A metadata configuration. - - Typically, this information comes from a stage like [NSHM To Realisation](#nshm-to-realisation). - -### Outputs - - 1. An [SRF](https://wiki.canterbury.ac.nz/display/QuakeCore/File+Formats+Used+In+Ground+Motion+Simulation#FileFormatsUsedInGroundMotionSimulation-SRFFormat) file containing the source slip definition for the realisation, - 2. An updated realisation file containing the parameters used for SRF generation copied from the scientific defaults. - -### Environment - Can be run in the cybershake container. Can also be run from your own computer using the `realisation-to-srf` command which is installed after running `pip install workflow@git+https://github.com/ucgmsim/workflow`. If you are executing on your own computer you also need to specify the work directory (with the `--work-directory` flag), a 1D velocity model (`--velocity-model-ffp`), and the path to a genslip binary (`--genslip-path`). -### Usage - `realisation-to-srf [OPTIONS] REALISATION_FFP OUTPUT_SRF_FILEPATH` -### For More Help - See the output of `realisation-to-srf --help` or [realisation_to_srf.py](https://github.com/ucgmsim/workflow/blob/pegasus/workflow/scripts/realisation_to_srf.py) -### Visualisation - You can visualise the output of this stage using the SRF plotting tools in the [source modelling](https://github.com/ucgmsim/source_modelling/blob/plots/wiki/Plotting-Tools.md) repository. Many of the tools take realisations as optional arguments to enhance the plot output. - -## Stoch Generation -### Description - Generate Stoch file for HF simulation. This file is just a down-sampled version of the SRF. -### Inputs - A realisation file containing a metadata configuration, and a generated SRF file. -### Outputs - A [Stoch](https://wiki.canterbury.ac.nz/display/QuakeCore/File+Formats+Used+In+Ground+Motion+Simulation#FileFormatsUsedInGroundMotionSimulation-Stochformat) file containing a down-sampled version of the SRF. -### Usage - `generate-stoch [OPTIONS] REALISATION_FFP SRF_FFP STOCH_FFP` -### Environment - Can be run in the cybershake container. Can also be run from your own computer using the `generate-stoch` command which is installed after running `pip install workflow@git+https://github.com/ucgmsim/workflow`. If you are executing on your own computer you also need to specify the `srf2stoch` path (`--srf2stoch-path`). -### For More Help -See the output of `generate-stoch --help` or [generate_stoch.py](https://github.com/ucgmsim/workflow/blob/pegasus/workflow/scripts/generate_stoch.py). -## Domain Generation -### Description - Find a suitable simulation domain, estimating a rupture radius that captures significant ground motion, and the time the simulation should run for to capture this ground motion. -### Inputs - A realisation file containing a metadata configuration, source definitions and rupture propagation information. - -### Outputs - A realisation file containing velocity model and domain extent parameters. - -### Environment - Can be run in the cybershake container. Can also be run from your own computer using the `generate-velocity-model-parameters` command which is installed after running `pip install workflow@git+https://github.com/ucgmsim/workflow`. -### Usage - `generate-velocity-model-parameters [OPTIONS] REALISATION_FFP` - -### For More Help - See the output of `generate-velocity-model-parameters --help` or [generate_velocity_model_parameters.py](https://github.com/ucgmsim/workflow/blob/pegasus/workflow/scripts/generate_velocity_model_parameters.py). - -## Velocity Model Generation -### Description -Generate a velocity model for a domain. -### Inputs -A realisation file containing: - -1. Domain parameters, -2. Velocity model parameters. - -### Outputs -A directory consisting of [velocity model files](https://wiki.canterbury.ac.nz/display/QuakeCore/File+Formats+Used+In+Ground+Motion+Simulation#FileFormatsUsedInGroundMotionSimulation-VelocityModelFiles). - -### Environment - Can be run in the cybershake container. Can also be run from your own computer using the `generate-velocity-model` command which is installed after running `pip install workflow@git+https://github.com/ucgmsim/workflow`. If you are executing on your own computer you also need to specify the `NZVM` path (`--velocity-model-bin-path`) and the work directory (`--work-directory`). - -### Usage -`generate-velocity-model [OPTIONS] REALISATION_FFP VELOCITY_MODEL_OUTPUT` - -### For More Help -See the output of `generate-velocity-model --help` or [generate_velocity_model.py](https://github.com/ucgmsim/workflow/blob/pegasus/workflow/scripts/generate_velocity_model.py) - -## Station Selection - -### Description -Filter a station list for in-domain stations to simulate high frequency and broadband output for. -### Inputs -1. A station list and, -2. A realisation file containing domain parameters. -### Outputs -1. A station list containing only stations in-domain and with unique discretised coordinate positions in two formats: - - Stations in the format "longitude latitude name" format in "stations.ll", - - Stations in the format "x y name" format in "stations.statcord". The x and y are the discretised positions of each station in the domain. -### Environment -Can be run in the cybershake container. Can also be run from your own computer using the `generate-station-coordinates` command which is installed after running `pip install workflow@git+https://github.com/ucgmsim/workflow`. If you do run this on your own computer, you need a version of `ll2gp` installed. -### Usage -`generate-station-coordinates [OPTIONS] REALISATIONS_FFP OUTPUT_PATH` -### For More Help -See the output of `generate-station-coordinates --help` or [generate_station_coordinates.py](https://github.com/ucgmsim/workflow/blob/pegasus/workflow/scripts/generate_station_coordinates.py) for more help. - -## Write Model Coordinates -### Description -Write out model parameters for EMOD3D. -### Inputs -1. A realisation file containing domain parameters. -### Outputs -1. A model parameters file describing the location of the domain in latitude, longitude, -2. A grid parameters file describing the discretisation of the domain. -### Environment -Can be run in the cybershake container. Can also be run from your own computer using the `generate-model-coordinates` command which is installed after running `pip install workflow@git+https://github.com/ucgmsim/workflow`. -### Usage -`generate-station-coordinates [OPTIONS] REALISATIONS_FFP OUTPUT_PATH` -### For More Help -See the output of `generate-model-coordinates --help` or [generate_model_coordinates.py](https://github.com/ucgmsim/workflow/blob/pegasus/workflow/scripts/generate_model_coordinates.py) for more help. - -## EMOD3D -### Description -Run a low frequency ground motion simulation using EMOD3D. -### Inputs -1. A parameter file in "key=value" format, -2. An SRF file, -3. A station file list (latitude, longitude, and x, y), see . -### Outputs -1. Ground acceleration timeslice files, one per core. -2. Seismograms, one per station. -### Environment -This stage must be run on a system with MPI installed. Typically, we run this stage in Maui on NeSI HPCs or Kisti. Due to high computational requirements, this stage usually cannot be run locally. -### Usage -On an HPC with slurm enabled `srun emod3d-mpi_v3.0.8 -args "par=$CYLC_WORKFLOW_SHARE_DIR/LF/e3d.par"` will run EMOD3D. Depending on the number of cores rerequested, this may invoke multiple proesses on different compute nodes (for Maui, this will occur when the number of cores exceeds 40). EMOD3D has support for checkpointing, so repeat invocations will continue from their previous checkpointed stage. -### For More Help -See Graves, 1996[^1] for a description of the mathematical and technical details of EMOD3D's implementation. - -[^1]: Graves, Robert W. "Simulating seismic wave propagation in 3D elastic media using staggered-grid finite differences." Bulletin of the seismological society of America 86.4 (1996): 1091-1106. - -## High Frequency Simulation -### Description -Generate stochastic high frequency ground acceleration data for a number of stations. -### Inputs -1. A station list (in the "latitude longitude name" format), -2. A 1D velocity model, -3. A stoch file, -4. A realisation with domain parameters and metadata. -### Outputs -1. A combined HF simulation output containing ground acceleration data for each station. -### Environment -Can be run in the cybershake container. Can also be run from your own computer using the `hf-sim` command which is installed after running `pip install workflow@git+https://github.com/ucgmsim/workflow`. If you do run this on your own computer, you need a version of `hb_high_binmod` installed. - -> [!NOTE] -> The high-frequency code is very brittle. It is recommended you have both versions 6.0.3 and 5.4.5 built to run with. Sometimes it is necessary to switch between versions if one does not work. -### Usage -`hf-sim [OPTIONS] REALISATION_FFP STOCH_FFP STATION_FILE OUT_FILE` -### For More Help -See the output of `hf-sim --help` or [hf_sim.py](https://github.com/ucgmsim/workflow/blob/pegasus/workflow/scripts/hf_sim.py). - -## Create EMOD3D Parameters - -### Description -Write parameters for EMOD3D simulation. -### Inputs -1. A realisation file containing domain parameters, velocity model parameters, and realisation metadata, -2. An SRF file, -3. A generated velocity model, -4. Station coordinates. -### Outputs -An EMOD3D parameter file containing a mixture of simulations parameters. Parameters source values from the defaults specified the realisation defaults version. The `emod3d` section of the realisation file overrides default values. -### Environment -Can be run in the cybershake container. Can also be run from your own computer using the `create-e3d-par` command which is installed after running `pip install workflow@git+https://github.com/ucgmsim/workflow`. -### Usage -`create-e3d-par [OPTIONS] REALISATION_FFP SRF_FILE_FFP VELOCITY_MODEL_FFP STATIONS_FFP GRID_FFP OUTPUT_FFP` -### For More Help -See the output of `create-e3d-par --help` or [create_e3d_par.py](https://github.com/ucgmsim/workflow/blob/pegasus/workflow/scripts/create_e3d_par.py). -See our description of the [EMOD3D Parameters](https://wiki.canterbury.ac.nz/pages/viewpage.action?pageId=100794983) for documentation on the EMOD3D parameter file format. - -## Create Simulation Video -### Description -Create a simulation video from the low frequency simulation output. -### Inputs -1. A merged timeslice file. -### Outputs -1. An animation of the low frequency simulation output. See [youtube](https://www.youtube.com/watch?v=Crdk3k0Prew) for an example of these videos. -### Environment -Can be run in the cybershake container. Can also be run from your own computer using the `plot-ts` command which is installed after running `pip install workflow@git+https://github.com/ucgmsim/workflow`. If running on your own computer, you need to install [gmt](https://www.generic-mapping-tools.org/) and [ffmpeg](https://www.ffmpeg.org/). This stage does not run well on Windows, and is very dependent on the gmt version installed. Hypocentre is already setup to run `plot_ts.py` without installing anything. -### Usage -`plot-ts [OPTIONS] SRF_FFP XYTS_INPUT_DIRECTORY OUTPUT_FFP` -### For More Help -See the output of `plot-ts --help` or [plot_ts.py](https://github.com/ucgmsim/workflow/blob/pegasus/workflow/scripts/plot_ts.py) -## Broadband Simulation -### Description -Combine high-frequency and low-frequency simulation waveforms for each station into a broadband simulation file. -### Inputs -1. A realisation file containing: - - Realisation metadata, - - Domain parameters. -2. Station list (latitude, longitude, name), -3. Stations VS30 reference values, -4. Low frequency waveform directory, -5. High frequency output file, -6. Velocity model directory. -### Outputs -An output [broadband file](https://wiki.canterbury.ac.nz/display/QuakeCore/File+Formats+Used+In+Ground+Motion+Simulation#FileFormatsUsedInGroundMotionSimulation-LF/HF/BBbinaryformat). -### Environment -Can be run in the cybershake container. Can also be run from your own computer using the `bb-sim` command which is installed after running `pip install workflow@git+https://github.com/ucgmsim/workflow`. If running on your own computer, you need to configure a work directory (`--work-directory`). -### Usage -`bb-sim REALISATION_FFP STATION_FFP STATION_VS30_FFP LOW_FREQUENCY_WAVEFORM_DIRECTORY HIGH_FREQUENCY_WAVEFORM_FILE VELOCITY_MODEL_DIRECTORY OUTPUT_FFP` -### For More Help -See the output of `bb-sim --help` or [bb_sim.py](https://github.com/ucgmsim/workflow/blob/pegasus/workflow/scripts/bb_sim.py) for more help. diff --git a/wiki/Using-Workflow.md b/wiki/Using-Workflow.md deleted file mode 100644 index 6c44adf7..00000000 --- a/wiki/Using-Workflow.md +++ /dev/null @@ -1,402 +0,0 @@ -# What Is a Workflow? -When we talk about "running a ground motion simulation", we really mean this as shorthand for a whole process of tasks that must be completed. Just a simple ground motion simulation of only low-frequency waveforms requires: - -1. Defining the fault geometry for the event, -2. Determining the simulation domain (how much of New Zealand to simulate, and how long the simulation should last) -3. Generating the initial conditions (what is the slip on the fault, how fast can waves propagate through the ground at various locations and depths) -4. Running the simulation with these inputs using a finite-difference PDE solver - -This is, of course, very high-level. In practice, each step also has sub-steps. Moreover, the jobs have dependencies and inputs; you can't run the finite-difference solver until you've defined the domain to simulate and the slip on the fault. - -We define a _workflow_ to be all the following: - -1. The definition of the steps in the process, -1. the dependencies between these steps, -1. the code used to execute the steps, and -1. the inputs to these steps. - -The outputs of the workflow are what we think of when we think of a "ground motion simulation". That is, the broadband seismic waveforms, the intensity measures, or the fault slip definitions, or anything else produced as a product of the steps defined in the workflow. - -# NeSI and Workflow Infrastructure - -[NeSI](https://www.nesi.org.nz/) is the New Zealand eScience Infrastructure organisation. NeSI provides a national platform of shared high-performance computing tools and eResearch services. NeSI owns two platforms we make extensive use of: - -- **The Maui Supercomputer:** This is a specialised Cray supercomputer. Initially targeted for [NIWA's](https://niwa.co.nz/) weather modelling needs, it is also available for researchers to run large-scale simulations. - -- **The Mahuika Computing Cluster:** This a bunch of Linux computers running together. They appear to you as one computer, but in the background distribute tasks across hundreds of smaller computers. Mahuika is the main platform we run simulations on. - -Neither Mahuika nor Maui works like a typical computer. Normally, you can execute any task you want at any time. However, because the supercomputers are shared resources, you can only _request_ to run a job on the supercomputer. It is up to the _scheduler_ to decide when to run your job. Managing jobs in our workflows is a difficult problem (some workflows have thousands of individual steps). - -This tutorial will guide you through running workflows on NeSI's platforms and manage these workflows using _Cylc_. - -## Gaining Access to NeSI - -Gaining access to NeSI is not automatic. Sung in the software team can set you up with an account. - -> [!NOTE] -> You cannot follow this tutorial without having an account on NeSI first and setup `ssh` to connect to NeSI's computers. The software team can set this up for you! - -# Hypocentre - -In addition to NeSI, QuakeCoRE has access to an in-house computer called Hypocentre. This computer cannot scale to the size of the NeSI supercomputers, but it is still very powerful. The computer has 48 cores and 250GB of RAM available. For context, a high-end desktop computer might have around 32GB of RAM and 16 cores, while the computer you're reading this tutorial on likely has at most half of that. - -The advantage of Hypocentre is that we are not subject to resource limits on our simulations, and accessing Hypocentre is much easier than accessing NeSI. The software team administers Hypocentre and they can install additional software for simulations as required. - -For many small workflows (e.g. running a single simulation), Hypocentre is a better platform for your needs than NeSI.Talk with the software team before running simulations to get a better understanding of whether you can use Hypocentre for your experiments. - -# What is Cylc? - -[Cylc](https://cylc.github.io/cylc-doc/stable/html/index.html) is a workflow orchestration tool. We define the workflow, and Cylc's job is to monitor the steps, queue the jobs with NeSI's scheduler, and respond to your requests to stop jobs, restart jobs or inspect their run logs. - -Cylc can also manage workflows run on Hypocentre, but we will not cover the details of that process here. - -# Using Workflow - -This document is a tutorial for the new workflow. This tutorial highlights a contrast between how the old and new workflows works, for those familiar with the old workflow. - -The tutorial simulates a simple rupture from the National Seismic Hazard Model 2022. - -## Useful Resources - -Your primary resource for help with workflow is, of course, the in-house software team at QuakeCoRE. However, some queries are easily answered by the additional resources we list here. - -Cylc is well-supported on NeSI. See the [Cylc on NeSI](https://docs.nesi.org.nz/Scientific_Computing/Supported_Applications/Cylc/) documentation for a more detailed description of using Cylc. The [official Cylc documentation](https://cylc.github.io/cylc-doc/stable/html/index.html) is another helpful resource. You might find support on the [Cylc forums](https://cylc.discourse.group/). Finally, since Cylc is maintained in-house at NeSI, you may also find help at [NeSI's office hours](https://docs.nesi.org.nz/Getting_Started/Getting_Help/Weekly_Online_Office_Hours/). - -## Setup - -We'll assume that you are running the workflow on NeSI. If you want to run this example in a different environment (such as Hypocentre), refer to the [extra steps](#extra-steps) at the end of this tutorial. First, login into Mahuika using your favourite SSH client (like [MobaXterm](https://mobaxterm.mobatek.net/)): - -``` shell -ssh mahuika -``` - -> [!NOTE] -> This requires you have setup SSH according to the Software Team's instructions. You *cannot run this tutorial from Maui*. - -All that is required for NeSI users to set up the workflow is to execute the setup script. Run the following: - -``` shell -/nesi/nobackup/nesi00213/workflow-setup -source ~/.bashrc -``` - -Then, copy the tutorial workflow: - -``` shell -cp -r /nesi/nobackup/nesi00213/tutorial ~/cylc-src -``` - -## An Aside: Cylc Source and Run Directories - -The two most important directories you need to know are created by the setup script: `cylc-src` and `cylc-run`. - -### The Cylc Source Directory - -The `~/cylc-src/` directory defines your workflows. Every folder in this directory is a separate workflow definition. For example, when we copied the tutorial into the `~/cylc-src/` directory, we were really defining a new workflow called `tutorial`. - -### The Cylc Run Directory - -When you instruct Cylc to run a workflow, Cylc copies the workflow definition from the source directory to the `~/cylc-run/` directory. For example, Cylc will later create a directory `~/cylc-run/tutorial`, which will contain all the runs of the tutorial workflow. - -## Viewing Your Workflow Definition - -At this point you will have a folder `~/cylc-src/tutorial` containing the following directory structure - -``` -cylc-src/tutorial -├── flow.cylc -└── input - └── stations.ll -``` - -Let's go over each of the files and explain their purpose. - -### The Workflow Definition File - -The `flow.cylc` file defines our workflow. - -
-flow.cylc - -``` cylc -[scheduler] - allow implicit tasks = True -[scheduling] - [[graph]] - R1 = """ - copy_input => nshm_to_realisation - nshm_to_realisation => realisation_to_srf & generate_velocity_model_parameters - generate_velocity_model_parameters => generate_velocity_model & generate_station_coordinates & generate_model_coordinates - realisation_to_srf & generate_velocity_model & generate_station_coordinates & generate_model_coordinates => create_e3d_par - create_e3d_par => run_emod3d - """ - -[runtime] - [[root]] - platform = mahuika-slurm - pre-script = """ - module load Apptainer - """ - [[[directives]]] - --account = nesi00213 - - [[copy_input]] - platform = localhost - script = cp -r $CYLC_WORKFLOW_RUN_DIR/input/* $CYLC_WORKFLOW_SHARE_DIR - [[nshm_to_realisation]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share" /nesi/nobackup/nesi00213/containers/runner_latest.sif nshm2022-to-realisation /nshmdb.db 0 /share/realisation.json 24.2.2.4 - [[realisation_to_srf]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share" /nesi/nobackup/nesi00213/containers/runner_latest.sif realisation-to-srf /share/realisation.json /share/realisation.srf - [[generate_velocity_model_parameters]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share" /nesi/nobackup/nesi00213/containers/runner_latest.sif generate-velocity-model-parameters /share/realisation.json - [[generate_velocity_model]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share" /nesi/nobackup/nesi00213/containers/runner_latest.sif sh -c 'generate-velocity-model /share/realisation.json /share/Velocity_Model --num-threads $(nproc)' - [[[directives]]] - --cpus-per-task = 32 - --time = 01:00:00 - [[generate_station_coordinates]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share" /nesi/nobackup/nesi00213/containers/runner_latest.sif generate-station-coordinates /share/realisation.json /share/stations --stat-file /share/stations.ll - [[generate_model_coordinates]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share" /nesi/nobackup/nesi00213/containers/runner_latest.sif generate-model-coordinates /share/realisation.json /share/model - [[create_e3d_par]] - platform = localhost - script = apptainer exec /nesi/nobackup/nesi00213/containers/runner_latest.sif create-e3d-par $CYLC_WORKFLOW_SHARE_DIR/realisation.json $CYLC_WORKFLOW_SHARE_DIR/realisation.srf $CYLC_WORKFLOW_SHARE_DIR/Velocity_Model $CYLC_WORKFLOW_SHARE_DIR/stations $CYLC_WORKFLOW_SHARE_DIR/model $CYLC_WORKFLOW_SHARE_DIR/LF --emod3d-path /nesi/project/nesi00213/opt/maui/hybrid_sim_tools/emod3d-mpi_v3.0.8 --scratch-ffp $CYLC_WORKFLOW_SHARE_DIR/LF - [[run_emod3d]] - platform = maui-xc-slurm - pre-script = "" - script = srun /nesi/project/nesi00213/opt/maui/hybrid_sim_tools/emod3d-mpi_v3.0.8 -args "par=$CYLC_WORKFLOW_SHARE_DIR/LF/e3d.par" - [[[directives]]] - --ntasks = 80 - --hint = nomultithread - --time = 01:00:00 -``` -
- -The workflow file defines the jobs to run (which are the sections like `[[create_e3d_par]]` and `[[copy_input]]`), as well as the order to run the jobs in (which are the contents of the `[[graph]]` section). Workflows are constructed as flow diagrams, and you can ask Cylc to visualise the flow diagram defining a workflow. Here is the visualisation of the above `flow.cylc` - -``` mermaid -flowchart LR - A[nshm_to_realisation] --> B[realisation_to_srf] - I[copy_input] --> A - A --> C[generate_velocity_model_parameters] - C --> D[generate_velocity_model] - C --> E[generate_station_coordinates] - C --> F[generate_model_coordinates] - B[realisation_to_srf] --> G[create_e3d_par] - D --> G - E --> G - F --> G - G --> H[run_emod3d] -``` - -This workflow is enough to simulate the low-frequency ground motion. - -### The input directory - -The `input` directory contains input files that are copied into the workflow. The copying is done by the `copy_input` job in the workflow file. In many of your custom workflows you'll add your own files to copy here. Our input directory contains a file `stations.ll`, which defines a list of seismic stations across the whole country. - -## Installing the Workflow - -To install the workflow invoke: - -``` shell -cylc install tutorial -``` - -This will copy the source files in `~/cylc-src/tutorial` to `~/cylc-run/tutorial/run1`. Every subsequent install of this workflow will create a new folder under `tutorial`, so running `cylc install tutorial` again creates `~/cylc-run/tutorial/run2`, and so on. - -> [!NOTE] -> To access the latest run of a given workflow, you can visit `~/cylc-run/tutorial/runN`. - -Let's look at the tutorial workflow directory. - -``` shell -~/cylc-run/tutorial/runN@mahuika $ ls -log share work flow.cylc -``` - -Besides the `flow.cylc` file we have three additional directories: - -1. The `log` directory which contains the log files for the run. You never need to read this file directly, because Cylc provides convenience commands to read logs. -2. The `share` directory. This directory contains files that are shared between jobs, like the `realisation.json` file, but also the final outputs like our animation. -3. The `work` directory. Some jobs, like EMOD3D, produce many files that other jobs don't care about. These are saved in the work directory `work//...` to promote job isolation. - -When the workflow is done, your output will live in the `share` directory. - -## Running the Workflow - -This step is simple! - -``` shell -cylc play tutorial -``` - -The above command instructs Cylc to run your workflow. You can now monitor it in two places. The first place is the Cylc logs accessed via `cylc log tutorial`. The second logging location is the NeSI slurm queue. The slurm queue is the queue all jobs running on HPC must wait in. You can access the slurm queue for your jobs with `squeue -u $USER`. - -Cylc has other ways to monitor your workflow, including a GUI. See NeSI's [documentation](https://docs.nesi.org.nz/Scientific_Computing/Supported_Applications/Cylc/#different-ways-to-interact-with-cylc) on the different ways they support interacting with cylc including the terminal user interface, GUI and Jupyter notebooks. - -## Inspecting and Interrupting Workflow Runs - -Once your workflow is running you can monitor it using logs, the TUI or the GUI, as discussed in the prior section. A complete description of the tools is out-of-scope for this tutorial but here is a the quick cheatsheet. - -### To Get an Updating Console Log of Job Debug Logs - -Execute the following on the command line: - -``` bash -~@mahuika $ watch cylc cat-log tutorial -f e -``` - -This will output all of the logs produced by the workflow run every two seconds. This is the logging the jobs themselves produce, as opposed to Cylc's logs (which you access via `cylc log tutorial`) -### To Get a TUI Interface for the Workflow - -A _terminal user interface_ (hereafter, TUI) is a portable graphical interface you can access inside the terminal. Execute `cylc tui tutorial` to get a window showing all of the jobs currently planned for the workflow. Using your keyboard, you can stop, restart, and view the status of all jobs in the workflow. See the [interventions](https://cylc.github.io/cylc-doc/stable/html/user-guide/interventions/index.html) documentation to see how to interrupt a workflow and change its behaviour. For a description of the status icons visible next to each job, see [task and job states](https://cylc.github.io/cylc-doc/stable/html/user-guide/running-workflows/tasks-jobs-ui.html#id1). - -### - -## Inspecting the Output - -Once the tutorial workflow has completed, let's look at the output. Inside `~/cylc-run/tutorial/runN/` you should see a directory structure like the following: - -
-~/cylc-run/tutorial/runN/ - -``` -. -│── flow.cylc -│── input -│ ╰── stations.ll -│── log -│ ... -│── share -│ │── LF -│ │ │── Log -│ │ │ │── Rupture\ 0─00000.rlog -│ │ │ ... -│ │ │── OutBin -│ │ │ │── Rupture\ 0_seis─00000.e3d -│ │ │ ... -│ │ │── Restart -│ │ │── SeismoBin -│ │ │ │── Rupture\ 0_seis─00000.e3d -│ │ │ ... -│ │ │── SlipOut -│ │ │── SlipOut─00000 -│ │ │ ... -│ │ │── TSFiles -│ │ ╰── e3d.par -│ │── Velocity_Model -│ │ │── in_basin_mask.b -│ │ │── rho3dfile.d -│ │ │── vp3dfile.p -│ │ ╰── vs3dfile.s -│ │── model -│ │ │── grid_file -│ │ ╰── model_params -│ │── realisation.json -│ │── realisation.json~ -│ │── realisation.srf -│ │── stations -│ │ │── stations.ll -│ │ ╰── stations.statcords -│ ╰── stations.ll -╰── work - ╰── 1 - │── generate_velocity_model - │ │ ... - │ ╰── nzvm.cfg - ╰── realisation_to_srf - │── gsf - │ ╰── acton.gsf - │── rupture_0.srf - ╰── srf - ╰── acton.srf -``` -
- -As described earlier, the `share` directory contains data shared between jobs, and is where your final outputs usually reside. Some of the jobs produced intermediate output in the `work/1/` directory. These files are not required These files can be useful for debugging. - -We have built a number of tools to inspect the output of these runs. - -### Inspecting the Source Model -When we talk about _source modelling_, we refer to the modelling of the rupture on the fault. Source modelling is performed by the `realisation_to_srf` stage, and the output of this stage is a Source Rupture Format file (SRF) stored in `/share/realisation.srf`. - -You can view the SRF using the tools in the [source_modelling](https://github.com/ucgmsim/source_modelling) repository. The `plot-srf` tool will produce an output something like the following. - -![](images/rupture_1.png) - -### Inspecting the Velocity Model - -The velocity model refers to the modelling of the density, P-wave and S-wave velocities measured and estimated for various parts of the country. You can view the velocity model output using the tools in the [velocity_modelling](https://github.com/ucgmsim/velocity_modelling/) repository. The velocity model also defines the simulation domain. - -![](images/rupture_1_vm.png) -### Viewing the Seismic Waveforms - -The low-frequency simulation waveforms live in the `share/LF` subdirectory. We can produce an animation of these waveforms with the `plot-ts` utility. First, [create and activate a new virtual environment](https://docs.python.org/3/library/venv.html#creating-virtual-environments) and then execute the following: - -``` shell -cd ~/cylc-run/tutorial/runN -pip install git+https://github.com/ucgmsim/workflow.git -plot-ts share/realisation.srf share/LF/OutBin output.mp4 --work-directory /tmp -``` - -This will produce a video (see [here](images/simulation.mp4) for the video this simulation produced). - - -## Extra Steps - -If you are running the workflow on Hypocentre, your environment looks a little different: - -1. The environment container is located at `/nesi/hypo_data/runner.sif` -2. EMOD3D is run with `mpirun` instead of `srun` -3. You must [build EMOD3D yourself](EMOD3D.md) -4. The slurm directives are not available on Hypocentre - -You can change the tutorial Cylc workflow to the following to accommodate these changes. - -
-flow.cylc - -``` cylc -[scheduler] - allow implicit tasks = True -[scheduling] - [[graph]] - R1 = """ - copy_input => nshm_to_realisation - nshm_to_realisation => realisation_to_srf & generate_velocity_model_parameters - generate_velocity_model_parameters => generate_velocity_model & generate_station_coordinates & generate_model_coordinates - realisation_to_srf & generate_velocity_model & generate_station_coordinates & generate_model_coordinates => create_e3d_par - create_e3d_par => run_emod3d - """ - -[runtime] - [[copy_input]] - platform = localhost - script = cp -r $CYLC_WORKFLOW_RUN_DIR/input/* $CYLC_WORKFLOW_SHARE_DIR - [[nshm_to_realisation]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share" /mnt/hypo_data/runner.sif nshm2022-to-realisation /nshmdb.db 0 /share/realisation.json 24.2.2.4 - [[realisation_to_srf]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share" /mnt/hypo_data/runner.sif realisation-to-srf /share/realisation.json /share/realisation.srf - [[generate_velocity_model_parameters]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share" /mnt/hypo_data/runner.sif generate-velocity-model-parameters /share/realisation.json - [[generate_velocity_model]] - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share" /mnt/hypo_data/runner.sif sh -c 'generate-velocity-model /share/realisation.json /share/Velocity_Model --num-threads $(nproc)' - [[generate_station_coordinates]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share" /mnt/hypo_data/runner.sif generate-station-coordinates /share/realisation.json /share/stations --stat-file /share/stations.ll - [[generate_model_coordinates]] - platform = localhost - script = apptainer exec -c --bind "$PWD:/out,$CYLC_WORKFLOW_SHARE_DIR:/share" /mnt/hypo_data/runner.sif generate-model-coordinates /share/realisation.json /share/model - [[create_e3d_par]] - platform = localhost - script = apptainer exec /mnt/hypo_data/runner.sif create-e3d-par $CYLC_WORKFLOW_SHARE_DIR/realisation.json $CYLC_WORKFLOW_SHARE_DIR/realisation.srf $CYLC_WORKFLOW_SHARE_DIR/Velocity_Model $CYLC_WORKFLOW_SHARE_DIR/stations $CYLC_WORKFLOW_SHARE_DIR/model $CYLC_WORKFLOW_SHARE_DIR/LF --emod3d-path ~/EMOD3D/tools/emod3d-mpi_v3.0.8 --scratch-ffp $CYLC_WORKFLOW_SHARE_DIR/LF - [[run_emod3d]] - platform = localhost - script = mpirun ~/EMOD3D/tools/emod3d-mpi_v3.0.8 -args "par=$CYLC_WORKFLOW_SHARE_DIR/LF/e3d.par" -``` -
diff --git a/wiki/images/json_schema.png b/wiki/images/json_schema.png deleted file mode 100644 index c37e062d..00000000 Binary files a/wiki/images/json_schema.png and /dev/null differ diff --git a/wiki/images/planned_workflow.png b/wiki/images/planned_workflow.png deleted file mode 100644 index 1f746cf3..00000000 Binary files a/wiki/images/planned_workflow.png and /dev/null differ diff --git a/wiki/images/planned_workflow_html_vis.png b/wiki/images/planned_workflow_html_vis.png deleted file mode 100644 index 2162c7a3..00000000 Binary files a/wiki/images/planned_workflow_html_vis.png and /dev/null differ diff --git a/wiki/images/planned_workflow_no_prepro.png b/wiki/images/planned_workflow_no_prepro.png deleted file mode 100644 index e177ef3a..00000000 Binary files a/wiki/images/planned_workflow_no_prepro.png and /dev/null differ diff --git a/wiki/images/planned_workflow_no_realisation.png b/wiki/images/planned_workflow_no_realisation.png deleted file mode 100644 index 6390fc65..00000000 Binary files a/wiki/images/planned_workflow_no_realisation.png and /dev/null differ diff --git a/wiki/images/planned_workflow_plot.png b/wiki/images/planned_workflow_plot.png deleted file mode 100644 index a81bdd66..00000000 Binary files a/wiki/images/planned_workflow_plot.png and /dev/null differ diff --git a/wiki/images/planned_workflow_shared.png b/wiki/images/planned_workflow_shared.png deleted file mode 100644 index 1696a3e9..00000000 Binary files a/wiki/images/planned_workflow_shared.png and /dev/null differ diff --git a/wiki/images/rupture_1.png b/wiki/images/rupture_1.png deleted file mode 100644 index cf15a6fb..00000000 Binary files a/wiki/images/rupture_1.png and /dev/null differ diff --git a/wiki/images/rupture_1_vm.png b/wiki/images/rupture_1_vm.png deleted file mode 100644 index 082b3bf1..00000000 Binary files a/wiki/images/rupture_1_vm.png and /dev/null differ diff --git a/wiki/images/simulation.mp4 b/wiki/images/simulation.mp4 deleted file mode 100644 index f604e897..00000000 Binary files a/wiki/images/simulation.mp4 and /dev/null differ diff --git a/workflow/defaults.py b/workflow/defaults.py index fa783c25..023c0926 100644 --- a/workflow/defaults.py +++ b/workflow/defaults.py @@ -6,8 +6,8 @@ import yaml -import workflow.default_parameters.root as root from workflow import utils +from workflow.default_parameters import root class DefaultsVersion(StrEnum): diff --git a/workflow/realisations.py b/workflow/realisations.py index 0cb26307..ebb4ec75 100644 --- a/workflow/realisations.py +++ b/workflow/realisations.py @@ -15,9 +15,10 @@ import struct import sys from abc import ABC +from collections.abc import Sequence from importlib import metadata from pathlib import Path -from typing import Any, ClassVar, Literal, Self, Union +from typing import Any, ClassVar, Literal, Self import numpy as np import numpy.typing as npt @@ -36,16 +37,16 @@ def to_name_coordinate_dictionary( coordinate_array: npt.NDArray[np.float64], - coordinate_names: list[str] = ["latitude", "longitude", "depth"], -) -> Union[dict[str, float], list[dict[str, float]]]: + coordinate_names: Sequence[str] = ("latitude", "longitude", "depth"), +) -> dict[str, float] | list[dict[str, float]]: """Convert an array of coordinates values into a (list of) dictionaries tagged with coordinate names. Parameters ---------- coordinate_array : np.ndarray The array of values. Should have shape (k,), (m, k) where k is at most the length of `coordinate_names`. - coordinate_names : list[str] - The names of the coordinates. Defaults to ['latitude', 'longitude', 'depth']. + coordinate_names : Sequence[str] + The names of the coordinates. Defaults to ('latitude', 'longitude', 'depth'). Returns ------- @@ -75,8 +76,6 @@ def to_name_coordinate_dictionary( class RealisationParseError(Exception): """Realisation JSON parse error.""" - pass - @dataclasses.dataclass class RealisationConfiguration(ABC): @@ -1270,7 +1269,7 @@ def from_utility(cls, utility: str, args: list[str]) -> Self: return cls( utility=utility, version=version, - timestamp=datetime.datetime.now(), + timestamp=datetime.datetime.now(tz=datetime.UTC), args=args, ) diff --git a/workflow/schemas.py b/workflow/schemas.py index 5df560cf..6f5e54cf 100644 --- a/workflow/schemas.py +++ b/workflow/schemas.py @@ -106,43 +106,43 @@ def from_dict(cls, params_dict: dict) -> "PointSourceParams": # Accordingly, the most trivial of these functions lack docstrings. -def _is_positive(x: float) -> bool: # noqa: D103 # numpydoc ignore=GL08 +def _is_positive(x: float) -> bool: # numpydoc ignore=GL08 return x > 0 -def _is_non_negative(x: float) -> bool: # noqa: D103 # numpydoc ignore=GL08 +def _is_non_negative(x: float) -> bool: # numpydoc ignore=GL08 return x >= 0 -def _is_valid_latitude(latitude: float) -> bool: # noqa: D103 # numpydoc ignore=GL08 +def _is_valid_latitude(latitude: float) -> bool: # numpydoc ignore=GL08 return -90 <= latitude <= 90 -def _is_valid_longitude(longitude: float) -> bool: # noqa: D103 # numpydoc ignore=GL08 +def _is_valid_longitude(longitude: float) -> bool: # numpydoc ignore=GL08 return -180 <= longitude <= 180 def _is_plausible_magnitude( magnitude: float, -) -> bool: # noqa: D103 # numpydoc ignore=GL08 +) -> bool: # numpydoc ignore=GL08 return magnitude < 11 -def _is_valid_degrees(degrees: float) -> bool: # noqa: D103 # numpydoc ignore=GL08 +def _is_valid_degrees(degrees: float) -> bool: # numpydoc ignore=GL08 return -360 <= degrees <= 360 def _is_valid_local_coordinate( coordinate: float, -) -> bool: # noqa: D103 # numpydoc ignore=GL08 +) -> bool: # numpydoc ignore=GL08 return 0 <= coordinate <= 1 -def _is_valid_bearing(bearing: float) -> bool: # noqa: D103 # numpydoc ignore=GL08 +def _is_valid_bearing(bearing: float) -> bool: # numpydoc ignore=GL08 return 0 <= bearing <= 360 -def _is_proportion(x: float | int) -> bool: # noqa: D103 # numpydoc ignore=GL08 +def _is_proportion(x: float) -> bool: # numpydoc ignore=GL08 return 0 <= x <= 1 diff --git a/workflow/scripts/create_e3d_par.py b/workflow/scripts/create_e3d_par.py index 36fe9492..e0d838c9 100644 --- a/workflow/scripts/create_e3d_par.py +++ b/workflow/scripts/create_e3d_par.py @@ -212,7 +212,7 @@ def emod3d_metadata( } -def format_as_emod3d_value(value: int | float | str | Path) -> str: +def format_as_emod3d_value(value: float | str | Path) -> str: """Format a value in a format valid for an e3d.par file. Parameters diff --git a/workflow/scripts/gcmt_auto_simulate.py b/workflow/scripts/gcmt_auto_simulate.py old mode 100755 new mode 100644 index 6a3ca80b..b8c603e0 --- a/workflow/scripts/gcmt_auto_simulate.py +++ b/workflow/scripts/gcmt_auto_simulate.py @@ -68,7 +68,7 @@ def gcmt_auto_simulate( with open(old_gcmt_solutions_path) as old_gcmt_solutions_handle: old_gcmt_solutions = json.load(old_gcmt_solutions_handle) else: - old_gcmt_solutions = dict() + old_gcmt_solutions = {} nz_polygon = utils.get_nz_outline_polygon() solutions_to_simulate = [ gcmt_id @@ -93,7 +93,9 @@ def gcmt_auto_simulate( ] if not solutions_to_simulate: raise typer.Exit(code=0) - now = datetime.datetime.now() + # Local time (via astimezone) so the workflow id reads naturally for the + # operator; the tz-aware now() keeps it unambiguous. + now = datetime.datetime.now(tz=datetime.UTC).astimezone() workflow_id = f"gcmt_{now.strftime('%Y%m%d_%H%M%S')}" cylc_directory = Path.home() / "cylc-src" / workflow_id cylc_directory.mkdir(exist_ok=True, parents=True) diff --git a/workflow/scripts/gcmt_to_realisation.py b/workflow/scripts/gcmt_to_realisation.py index 3b266d12..0afd5a06 100644 --- a/workflow/scripts/gcmt_to_realisation.py +++ b/workflow/scripts/gcmt_to_realisation.py @@ -31,7 +31,7 @@ import warnings from enum import StrEnum, auto from pathlib import Path -from typing import Annotated, Optional +from typing import Annotated import numpy as np import pandas as pd @@ -96,14 +96,14 @@ def gcmt_to_realisation( hypocentre_strategy: Annotated[ SamplingStrategy, typer.Option() ] = SamplingStrategy.CENTROID, - shypo: Annotated[Optional[float], typer.Option(min=0, max=1)] = None, - dhypo: Annotated[Optional[float], typer.Option(min=0, max=1)] = None, + shypo: Annotated[float | None, typer.Option(min=0, max=1)] = None, + dhypo: Annotated[float | None, typer.Option(min=0, max=1)] = None, lat_hypo: Annotated[ - Optional[float], + float | None, typer.Option(min=-90, max=90), ] = None, lon_hypo: Annotated[ - Optional[float], + float | None, typer.Option(min=-180, max=180), ] = None, scaling_relation: Annotated[ @@ -174,18 +174,18 @@ def gcmt_to_realisation( if gcmt_event_id in gcmt_solutions.index: row = gcmt_solutions.loc[gcmt_event_id] - latitude = float(row["Latitude"]) # type: ignore[invalid-argument-type] - longitude = float(gcmt_solutions.at[gcmt_event_id, "Longitude"]) # type: ignore[invalid-argument-type] - centroid_depth = float(gcmt_solutions.at[gcmt_event_id, "CD"]) # type: ignore[invalid-argument-type] - solution_moment = float(gcmt_solutions.at[gcmt_event_id, "Mo"]) # type: ignore[invalid-argument-type] - - strike1 = float(gcmt_solutions.at[gcmt_event_id, "strike1"]) # type: ignore[invalid-argument-type] - dip1 = float(gcmt_solutions.at[gcmt_event_id, "dip1"]) # type: ignore[invalid-argument-type] - rake1 = float(gcmt_solutions.at[gcmt_event_id, "rake1"]) # type: ignore[invalid-argument-type] - - strike2 = float(gcmt_solutions.at[gcmt_event_id, "strike2"]) # type: ignore[invalid-argument-type] - dip2 = float(gcmt_solutions.at[gcmt_event_id, "dip2"]) # type: ignore[invalid-argument-type] - rake2 = float(gcmt_solutions.at[gcmt_event_id, "rake2"]) # type: ignore[invalid-argument-type] + latitude = float(row["Latitude"]) # ty: ignore[invalid-argument-type] + longitude = float(gcmt_solutions.at[gcmt_event_id, "Longitude"]) # ty: ignore[invalid-argument-type] + centroid_depth = float(gcmt_solutions.at[gcmt_event_id, "CD"]) # ty: ignore[invalid-argument-type] + solution_moment = float(gcmt_solutions.at[gcmt_event_id, "Mo"]) # ty: ignore[invalid-argument-type] + + strike1 = float(gcmt_solutions.at[gcmt_event_id, "strike1"]) # ty: ignore[invalid-argument-type] + dip1 = float(gcmt_solutions.at[gcmt_event_id, "dip1"]) # ty: ignore[invalid-argument-type] + rake1 = float(gcmt_solutions.at[gcmt_event_id, "rake1"]) # ty: ignore[invalid-argument-type] + + strike2 = float(gcmt_solutions.at[gcmt_event_id, "strike2"]) # ty: ignore[invalid-argument-type] + dip2 = float(gcmt_solutions.at[gcmt_event_id, "dip2"]) # ty: ignore[invalid-argument-type] + rake2 = float(gcmt_solutions.at[gcmt_event_id, "rake2"]) # ty: ignore[invalid-argument-type] nodal_plane_1 = NodalPlane(strike1, dip1, rake1) nodal_plane_2 = NodalPlane(strike2, dip2, rake2) diff --git a/workflow/scripts/generate_domain.py b/workflow/scripts/generate_domain.py index 8263b9db..84f769cf 100644 --- a/workflow/scripts/generate_domain.py +++ b/workflow/scripts/generate_domain.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Domain Generation. Description @@ -237,7 +236,7 @@ def average_rake(rakes: Rakes, magnitudes: Magnitudes) -> float: moments = {k: moment / max_moment for k, moment in moments.items()} weighted_rakes = list(utils.dict_zip(rakes.rakes, moments).values()) - return geo.avg_wbearing(weighted_rakes) # type: ignore[invalid-argument-type] + return geo.avg_wbearing(weighted_rakes) # ty: ignore[invalid-argument-type] def estimate_simulation_duration( @@ -486,7 +485,7 @@ def estimate_domain( model_domain = bounding_box.minimum_area_bounding_box_for_polygons_masked( must_include=fault_buffer_polygons, may_include=rrup_bounding_polygons, - mask=nz_outline, # type: ignore[invalid-argument-type] + mask=nz_outline, # ty: ignore[invalid-argument-type] ) return model_domain diff --git a/workflow/scripts/generate_rupture_propagation.py b/workflow/scripts/generate_rupture_propagation.py index a4dab6fc..b4df16d1 100644 --- a/workflow/scripts/generate_rupture_propagation.py +++ b/workflow/scripts/generate_rupture_propagation.py @@ -35,7 +35,7 @@ import random from enum import StrEnum, auto from pathlib import Path -from typing import Annotated, Optional +from typing import Annotated import numpy as np import typer diff --git a/workflow/scripts/generate_velocity_model.py b/workflow/scripts/generate_velocity_model.py index 9aa2f5c7..fcb4c774 100644 --- a/workflow/scripts/generate_velocity_model.py +++ b/workflow/scripts/generate_velocity_model.py @@ -37,7 +37,7 @@ import shutil import subprocess from pathlib import Path -from typing import Annotated, Optional +from typing import Annotated import typer @@ -165,7 +165,7 @@ def generate_velocity_model( Path, typer.Argument(readable=True, exists=True, dir_okay=False) ], velocity_model_output: Annotated[ - Optional[Path], typer.Argument(writable=True, file_okay=False) + Path | None, typer.Argument(writable=True, file_okay=False) ] = None, velocity_model_bin_path: Annotated[ Path | None, typer.Option(exists=True, readable=True) @@ -174,7 +174,7 @@ def generate_velocity_model( Path, typer.Option(exists=False, writable=True, file_okay=False) ] = Path("/out"), use_nzcvm: Annotated[bool, typer.Option()] = False, - num_threads: Annotated[Optional[int], typer.Option(min=1)] = None, + num_threads: Annotated[int | None, typer.Option(min=1)] = None, ) -> None: """ Generate a velocity model for a seismic realisation using NZVM. diff --git a/workflow/scripts/hf_sim.py b/workflow/scripts/hf_sim.py index 99048001..2ffd495f 100644 --- a/workflow/scripts/hf_sim.py +++ b/workflow/scripts/hf_sim.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """High Frequency Simulation. Description @@ -123,8 +122,10 @@ def build_hf_input( substituted to yield a high-frequency input in for each station. """ - shallow_min, shallow_max, deep_min, deep_max = rupture_velocity_hf_transition_bands( - rupture_velocity + # Underscore-prefixed because the line consuming them is commented out + # below, pending the EMOD3D PR noted there. + _shallow_min, _shallow_max, _deep_min, _deep_max = ( + rupture_velocity_hf_transition_bands(rupture_velocity) ) hf_sim_input = [ "", @@ -139,7 +140,7 @@ def build_hf_input( f"{domain_parameters.duration} {resolution.dt} {hf_config.fmax} {hf_config.kappa} {hf_config.qfexp}", f"{rupture_velocity.rvfrac} {rupture_velocity.rvfrac_shal} {rupture_velocity.rvfrac_deep} {hf_config.czero} {hf_config.calpha}", # TODO: This requires PR from EMOD3D to merge before we can do this! - # f"{shallow_min} {shallow_max} {deep_min} {deep_max}", + # f"{_shallow_min} {_shallow_max} {_deep_min} {_deep_max}", f"{hf_config.mom or -1} {hf_config.rupv or -1}", stoch_ffp, velocity_model, @@ -152,9 +153,11 @@ def build_hf_input( # If running v5.4.5 it stops reading input here and so # these parameters are unused. It is harmless to add them # regardless of version - f"{hf_config.stress_parameter_adjustment_fault_area or -1} " - f"{hf_config.stress_parameter_adjustment_target_magnitude or -1} " - f"{hf_config.stress_parameter_adjustment_tect_type or -1}", + ( + f"{hf_config.stress_parameter_adjustment_fault_area or -1} " + f"{hf_config.stress_parameter_adjustment_target_magnitude or -1} " + f"{hf_config.stress_parameter_adjustment_tect_type or -1}" + ), 0, # seek bytes to 0 (no binary offset for this output) "", ] diff --git a/workflow/scripts/im_calc.py b/workflow/scripts/im_calc.py index a1e5b19f..6d16ccfc 100644 --- a/workflow/scripts/im_calc.py +++ b/workflow/scripts/im_calc.py @@ -176,11 +176,6 @@ def calculate_intensity_measures( intensity_measures = override_ims or intensity_measure_parameters.ims - if IM.FAS in intensity_measures and not ko_directory: - raise ValueError( - "FAS calculation requires KO directory. Please provide a valid KO directory." - ) - nyquist_frequency = 1 / (2 * resolution.dt) im_function_map = { @@ -202,11 +197,20 @@ def calculate_intensity_measures( periods=np.array( intensity_measure_parameters.valid_periods, dtype=np.float64 ), - dt=resolution.dt, + dt=np.float64(resolution.dt), step=psa_step, cores=cores, ), - IM.FAS: functools.partial( + } + + # Built separately from the literal above so the ko_directory check narrows + # away None: FAS is the only measure that needs it. + if IM.FAS in intensity_measures: + if ko_directory is None: + raise ValueError( + "FAS calculation requires KO directory. Please provide a valid KO directory." + ) + im_function_map[IM.FAS] = functools.partial( ims.fourier_amplitude_spectra, dt=resolution.dt, freqs=intensity_measure_parameters.fas_frequencies[ @@ -214,8 +218,8 @@ def calculate_intensity_measures( ], ko_directory=ko_directory, cores=cores, - ), - } + ) + latitude = broadband.latitude.values longitude = broadband.longitude.values station_locations = np.stack((latitude, longitude), axis=-1) @@ -304,8 +308,8 @@ def calculate_intensity_measures( list(source_geometries.source_geometries.values()), # ty: ignore[invalid-argument-type] station_locations, ) - dataset["rx"] = xr.DataArray(rx, dims="station", coords=dict(station=stations)) - dataset["ry"] = xr.DataArray(ry, dims="station", coords=dict(station=stations)) + dataset["rx"] = xr.DataArray(rx, dims="station", coords={"station": stations}) + dataset["ry"] = xr.DataArray(ry, dims="station", coords={"station": stations}) waveform = broadband.waveform.values.astype(np.float64) diff --git a/workflow/scripts/lf_to_xarray.py b/workflow/scripts/lf_to_xarray.py index 4386dc00..2a69bb3b 100644 --- a/workflow/scripts/lf_to_xarray.py +++ b/workflow/scripts/lf_to_xarray.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Low-frequency output merger. Description diff --git a/workflow/scripts/merge_ts.py b/workflow/scripts/merge_ts.py index 63d2ebc9..a694fb85 100644 --- a/workflow/scripts/merge_ts.py +++ b/workflow/scripts/merge_ts.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Merge EMOD3D Timeslices. Description diff --git a/workflow/scripts/migrate.py b/workflow/scripts/migrate.py index b3ddc8a5..1895324e 100644 --- a/workflow/scripts/migrate.py +++ b/workflow/scripts/migrate.py @@ -9,7 +9,7 @@ from collections.abc import MutableMapping from enum import Enum, auto from pathlib import Path -from typing import Annotated, TypeVar +from typing import Annotated, TypeGuard import parse import schema @@ -25,13 +25,18 @@ console = Console() -def is_realisation_configuration(cls: type) -> bool: +# Every use site refers to a RealisationConfiguration *subclass* (the classes +# returned by realisation_configurations), not an instance of one. +type ConfigType = type[realisations.RealisationConfiguration] + + +def is_realisation_configuration(cls: object) -> TypeGuard[ConfigType]: """Returns True if the class is a subclass of realisation configuration. Parameters ---------- - cls : type - Type to check. + cls : object + Object to check. Returns ------- @@ -45,9 +50,6 @@ def is_realisation_configuration(cls: type) -> bool: ) -ConfigType = TypeVar("ConfigType", bound=realisations.RealisationConfiguration) - - def realisation_configurations() -> list[ConfigType]: """Return a list of all realisation configurations. @@ -64,7 +66,7 @@ def realisation_configurations() -> list[ConfigType]: def loadable_defaults( - configurations: list[type], defaults: DefaultsVersion + configurations: list[ConfigType], defaults: DefaultsVersion ) -> dict[ConfigType, realisations.RealisationConfiguration]: """Filter a list of realisation configurations for those with loadable defaults. @@ -72,7 +74,7 @@ def loadable_defaults( Parameters ---------- - configurations : list[type] + configurations : list[ConfigType] Configurations to filter. defaults : defaults.DefaultsVersion Defaults to try and load. @@ -91,7 +93,7 @@ def loadable_defaults( realisation configuration. """ - config_defaults = {} + config_defaults: dict[ConfigType, realisations.RealisationConfiguration] = {} for config in configurations: if not is_realisation_configuration(config): raise TypeError( @@ -99,7 +101,7 @@ def loadable_defaults( ) else: try: - default_config = config.read_from_defaults(defaults) # type: ignore[unresolved-attribute] + default_config = config.read_from_defaults(defaults) config_defaults[config] = default_config except realisations.RealisationParseError: continue @@ -370,10 +372,10 @@ def migrate( if not default_config: continue default_config_dict = default_config.to_dict() - current_config = json_data.get(config._config_key, dict()) + current_config = json_data.get(config._config_key, {}) if current_config != default_config_dict: print_diff(current_config, default_config_dict) - print("") + print() response = auto_response.get((config, Action.UPDATE)) or should_update( config ) @@ -430,7 +432,7 @@ def migrate( console.print(str(e)) -@cli.from_docstring(app, name="migrate") # type: ignore[invalid-argument-type] +@cli.from_docstring(app, name="migrate") def migrate_all( realisation_directory: Annotated[ Path, typer.Argument(exists=True, file_okay=False) @@ -458,7 +460,7 @@ def migrate_all( If given, print instead of writing. Useful to check what would be migrated. """ - auto_response = dict() + auto_response = {} configs = realisation_configurations() defaults = loadable_defaults(configs, defaults_version) diff --git a/workflow/scripts/nshm2022_to_realisation.py b/workflow/scripts/nshm2022_to_realisation.py index 2415cd85..18fdd7fa 100755 --- a/workflow/scripts/nshm2022_to_realisation.py +++ b/workflow/scripts/nshm2022_to_realisation.py @@ -308,14 +308,17 @@ def generate_realisation( faults_info = db.get_rupture_fault_info(rupture_id) seeds = Seeds.read_from_realisation_or_random(realisation_ffp) np.random.seed(seed=seeds.nshm_to_realisation_seed) - source_config = SourceConfig(faults) + # `faults` is a dict[str, Fault], but these APIs declare invariant + # dict[str, IsSource] / dict[str, Fault | Plane] parameters that they only + # read from. Passing a copy widens the value type without upstream changes. + source_config = SourceConfig(dict(faults)) rakes = { fault_name: fault_info.rake for fault_name, fault_info in faults_info.items() } avg_rake = np.mean(list(rakes.values())) components = moment.find_connected_faults( - faults, separation_distance, dip_delta, min_connected_depth + dict(faults), separation_distance, dip_delta, min_connected_depth ) magnitudes = default_magnitude_estimation(faults, components, float(avg_rake)) if lat_hypo is not None and lon_hypo is not None: @@ -334,7 +337,11 @@ def generate_realisation( else: # The ty ignore below can be removed once NSHM2022DB merges the change to # accept dict[str, BoldM] in most_likely_fault (branch support-BoldM-in-workflow). - mfds_rates = db.most_likely_fault(rupture_id, magnitudes) # ty: ignore[invalid-argument-type] + mfds_rates = db.most_likely_fault( + nshmdb.FaultSystem.Crustal, + rupture_id, + magnitudes, # ty: ignore[invalid-argument-type] + ) mfds_probabilities = np.array(list(mfds_rates.values())) if np.allclose(mfds_probabilities, 0): mfds_probabilities = np.ones_like(mfds_probabilities) @@ -350,7 +357,7 @@ def generate_realisation( ) rupture_causality_tree = rupture_propagation.sample_rupture_propagation( - faults, + dict(faults), initial_source=initial_fault, strategy=str(strategy), # type: ignore jump_impossibility_limit_distance=round(jump_cutoff * 1000), @@ -360,7 +367,7 @@ def generate_realisation( rupture_propagation_config = RupturePropagationConfig( rupture_causality_tree=rupture_causality_tree, jump_points=rupture_propagation.jump_points_from_rupture_tree( - faults, rupture_causality_tree, min_depth=min_connected_depth + dict(faults), rupture_causality_tree, min_depth=min_connected_depth ), hypocentre=hypocentre, ) diff --git a/workflow/scripts/realisation_to_srf.py b/workflow/scripts/realisation_to_srf.py index 978ee623..fb9038e9 100644 --- a/workflow/scripts/realisation_to_srf.py +++ b/workflow/scripts/realisation_to_srf.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """SRF Generation. Description @@ -529,7 +528,7 @@ def generate_fault_srf( if isinstance(fault, Point): generate_point_source_srf(name, params, environment) # Return None here as no other code in this function should be run if generating a point source SRF. - return None + return resolution = params.srf_config.resolution @@ -891,7 +890,7 @@ def generate_srf( srf_name, ) else: - source_name = list(source_config.source_geometries)[0] + source_name = next(iter(source_config.source_geometries)) input_srf_path = work_directory / "srf" / f"{normalise_name(source_name)}.srf" output_srf_path = work_directory / f"{srf_name}.srf" shutil.move(input_srf_path, output_srf_path) diff --git a/workflow/utils.py b/workflow/utils.py index 69ecbd1c..c63358af 100644 --- a/workflow/utils.py +++ b/workflow/utils.py @@ -5,7 +5,7 @@ import tempfile import urllib.request from collections.abc import Mapping -from typing import Any, TypeVar, overload +from typing import Any, overload import geopandas as gpd import numpy as np @@ -106,42 +106,39 @@ def get_available_cores() -> int: raise RuntimeError("Cannot determine CPU count.") -K = TypeVar("K") -V1 = TypeVar("V1") -V2 = TypeVar("V2") -V3 = TypeVar("V3") - - # These overloads provide better type inference in the common case @overload -def dict_zip( - __d1: Mapping[K, V1], *, strict: bool = ... +def dict_zip[K, V1]( + d1: Mapping[K, V1], /, *, strict: bool = ... ) -> dict[K, tuple[V1]]: ... # numpydoc ignore=GL08 @overload -def dict_zip( - __d1: Mapping[K, V1], __d2: Mapping[K, V2], *, strict: bool = ... +def dict_zip[K, V1, V2]( + d1: Mapping[K, V1], d2: Mapping[K, V2], /, *, strict: bool = ... ) -> dict[K, tuple[V1, V2]]: ... # numpydoc ignore=GL08 @overload -def dict_zip( - __d1: Mapping[K, V1], - __d2: Mapping[K, V2], - __d3: Mapping[K, V3], +def dict_zip[K, V1, V2, V3]( + d1: Mapping[K, V1], + d2: Mapping[K, V2], + d3: Mapping[K, V3], + /, *, strict: bool = ..., ) -> dict[K, tuple[V1, V2, V3]]: ... # numpydoc ignore=GL08 @overload -def dict_zip( +def dict_zip[K]( *dicts: Mapping[K, Any], strict: bool = ... ) -> dict[K, tuple[Any, ...]]: ... # numpydoc ignore=GL08 -def dict_zip(*dicts: Mapping[K, Any], strict: bool = True) -> dict[K, tuple[Any, ...]]: +def dict_zip[K]( + *dicts: Mapping[K, Any], strict: bool = True +) -> dict[K, tuple[Any, ...]]: """ Takes the product of one or more dictionaries. @@ -192,7 +189,7 @@ def merge_dictionaries(dict_a: dict[str, Any], dict_b: dict[str, Any]) -> None: for key, value in dict_b.items(): if key in dict_a and isinstance(dict_a[key], dict) and isinstance(value, dict): - merge_dictionaries(dict_a[key], dict_b[key]) + merge_dictionaries(dict_a[key], value) else: dict_a[key] = value