integrates rinex v4.02 useage

pyproject.toml

@@ -100,7 +100,9 @@ docs = { cmd = "myst start", description = "Serve docs locally with live reload"
 docs-build = { cmd = "myst build --html", description = "Build docs as static HTML" }
 
 # Clone and compile garpos from source (importable via PYTHONPATH set in activation)
-clone-garpos = { cmd = "bash -c 'if [ -d .pixi/garpos ]; then git -C .pixi/garpos pull; else git clone https://github.com/s-watanabe-jhod/garpos.git .pixi/garpos; fi'" }
+# Pinned to v1.0.2: compile-garpos hardcodes the bin/garpos_v102/f90lib path,
+# which does not exist in newer upstream releases.
+clone-garpos = { cmd = "bash -c 'if [ -d .pixi/garpos ]; then git -C .pixi/garpos fetch --tags && git -C .pixi/garpos checkout v1.0.2; else git clone --branch v1.0.2 --depth 1 https://github.com/s-watanabe-jhod/garpos.git .pixi/garpos; fi'" }
 compile-garpos = { cmd = "gfortran -shared -fPIC -fopenmp -O3 -o lib_raytrace.so sub_raytrace.f90 lib_raytrace.f90", cwd = ".pixi/garpos/bin/garpos_v102/f90lib", depends-on = ["clone-garpos"] }
 test-garpos = { cmd = "pytest tests/test_garpos.py -v", depends-on = ["compile-garpos"] }
 

src/earthscope_sfg_workflows/data_mgmt/adapters/memory.py

@@ -11,6 +11,7 @@
 from collections import defaultdict
 from dataclasses import replace
 from itertools import count
+from pathlib import Path
 from upath import UPath
 
 from ..model import ArchiveFile, AssetEntry, AssetKind, SFGScope, FileInfo

src/earthscope_sfg_workflows/data_mgmt/archives/archive_pull.py

@@ -16,7 +16,7 @@
 from earthscope_sfg_workflows.data_mgmt.ports import ArchiveAuthError
 from earthscope_sfg_workflows.logging import ProcessLogger as logger
 from earthscope_sfg_tools.datamodels.metadata import Site, Vessel, import_site, import_vessel
-from ..model import AssetKind, SFGScope
+from ..model import AssetKind
 
 _detector = FileTypeDetector()
 

src/earthscope_sfg_workflows/data_mgmt/core.py

@@ -18,12 +18,12 @@
     CampaignLayout,
     NetworkLayout,
     DirectoryTree,
+    SFGScope,
     StationLayout,
     GARPOSLayout,
     SurveyLayout,
-    TileDBLayout,
 )
-from .ports import AssetCatalogPort, FileStorePort
+from .ports import FileStorePort
 
 
 # ---------------------------------------------------------------------------
@@ -32,8 +32,14 @@
 
 
 DEFAULT_PATTERNS: tuple[tuple[re.Pattern[str], AssetKind], ...] = (
+    # Legacy RINEX v2 short names (e.g. "SLT11540.26o" / "BRDC1540.26n").
     (re.compile(r"\.\d{2}o$", re.IGNORECASE), AssetKind.RINEX2),
     (re.compile(r"\.\d{2}n$", re.IGNORECASE), AssetKind.RINEX3),
+    # RINEX v3/v4 long names (e.g. "SLT100USA_R_20261541758_01D_20C_MO.rnx"
+    # for observation data, "BRDC00IGS_R_20261540000_01D_MN.rnx" for nav).
+    # Both share the ".rnx" extension; disambiguate on the data-type suffix.
+    (re.compile(r"_[A-Z]O\.rnx$", re.IGNORECASE), AssetKind.RINEX2),
+    (re.compile(r"_[A-Z]N\.rnx$", re.IGNORECASE), AssetKind.RINEX3),
     (re.compile(r"sonardyne", re.IGNORECASE), AssetKind.SONARDYNE),
     (re.compile(r"NOV000"), AssetKind.NOVATEL000),
     (re.compile(r"NOV770"), AssetKind.NOVATEL770),

src/earthscope_sfg_workflows/data_mgmt/model.py

@@ -11,7 +11,6 @@
 from dataclasses import dataclass, field, replace
 from datetime import datetime
 from enum import Enum
-from earthscope_sfg_tools.datamodels import Campaign, Site, Survey
 from upath import UPath
 
 # ---------------------------------------------------------------------------

src/earthscope_sfg_workflows/data_mgmt/ports.py

@@ -16,6 +16,7 @@
 
 from __future__ import annotations
 
+from pathlib import Path
 from typing import Protocol, runtime_checkable
 from upath import UPath
 

src/earthscope_sfg_workflows/modeling/garpos_tools/garpos_handler.py

@@ -17,10 +17,8 @@
 
 sns.set_theme(style="whitegrid")
 
-from earthscope_sfg_tools.datamodels.metadata import Campaign, Site, Survey  # noqa: E402
+from earthscope_sfg_tools.datamodels.metadata import Survey  # noqa: E402
 from earthscope_sfg_tools.tiledb_integration import (  # noqa: E402
-    TDBIMUPositionArray,
-    TDBKinPositionArray,
     TDBShotDataArray,
 )
 

src/earthscope_sfg_workflows/pipelines/plotting.py

@@ -50,22 +50,30 @@ def get_rinex_timelast(rinex_asset: AssetEntry) -> datetime.datetime:
     ref_date = datetime.datetime(1970, 1, 1, 0, 0, 0)
     with open(rinex_asset.local_path) as f:
         for line in f:
-            # line sample: 23  6 24 23 59 59.5000000  0  9G21G27G32G08G10G23G24G02G18
-            if line.strip().startswith(year):
-                date_line = line.strip().split()
-                try:
-                    current_date = datetime.datetime(
-                        year=2000 + int(date_line[0]),
-                        month=int(date_line[1]),
-                        day=int(date_line[2]),
-                        hour=int(date_line[3]),
-                        minute=int(date_line[4]),
-                        second=int(float(date_line[5])),
-                    )
-                    if current_date > ref_date:
-                        ref_date = current_date
-                except Exception:
-                    pass
+            stripped = line.strip()
+            if stripped.startswith(">"):
+                # RINEX v3/v4 epoch record: "> 2026 06 03 17 58 25.5000000  0 36"
+                date_line = stripped[1:].split()
+                full_year = True
+            elif stripped.startswith(year):
+                # RINEX v2 epoch record: "23  6 24 23 59 59.5000000  0  9G21..."
+                date_line = stripped.split()
+                full_year = False
+            else:
+                continue
+            try:
+                current_date = datetime.datetime(
+                    year=int(date_line[0]) if full_year else 2000 + int(date_line[0]),
+                    month=int(date_line[1]),
+                    day=int(date_line[2]),
+                    hour=int(date_line[3]),
+                    minute=int(date_line[4]),
+                    second=int(float(date_line[5])),
+                )
+                if current_date > ref_date:
+                    ref_date = current_date
+            except Exception:
+                pass
     return ref_date
 
 

src/earthscope_sfg_workflows/pipelines/qc_pipeline.py

@@ -5,7 +5,6 @@
 import datetime
 import json
 import os
-import sys
 import threading
 from collections import deque
 from dataclasses import replace
@@ -433,9 +432,12 @@ def get_rinex_files(self) -> None:
         if rinex_cfg.override or not self.catalog.is_merge_complete(**merge_signature):
             try:
                 # tdb2rnx writes RINEX files to CWD; run from rinex_dest.
-                # Remove any pre-existing .rnx files so the post-run glob is clean.
+                # Remove any pre-existing RINEX output so the post-run glob is
+                # clean. Matches both the v3/v4 long name (*.rnx, current
+                # output format) and the legacy v2 short name (*.??o, in case
+                # a directory still has files from before the naming switch).
                 rinex_dest.mkdir(parents=True, exist_ok=True)
-                for _stale in rinex_dest.glob("*.rnx"):
+                for _stale in [*rinex_dest.glob("*.rnx"), *rinex_dest.glob("*.??o")]:
                     _stale.unlink()
                 old_cwd = Path.cwd()
                 try:

src/earthscope_sfg_workflows/pipelines/shotdata_gnss_refinement.py

@@ -453,9 +453,7 @@ def _interp_positions(query_times) -> np.ndarray:
     shotdata.loc[mask_ping, "isUpdated"] = True
 
     mask_return = ~np.isnan(predicted_return_pos[:, 0])
-    shotdata.loc[mask_return, ["east1", "north1", "up1"]] = predicted_return_pos[
-        mask_return, :
-    ]
+    shotdata.loc[mask_return, ["east1", "north1", "up1"]] = predicted_return_pos[mask_return, :]
     shotdata.loc[mask_return, "isUpdated"] = True
 
     nan_pings = np.isnan(predicted_ping_pos).any(axis=1).sum()

src/earthscope_sfg_workflows/pipelines/sv3_pipeline.py

@@ -14,7 +14,24 @@
 
 # third-party — monkey-patch targets must be imported before the patches below
 import tiledb as _tiledb
+from earthscope_sfg_tools import tiledb_integration as novb_ops
+from earthscope_sfg_tools.novatel_tools.utils import get_metadata, get_metadatav2
+from earthscope_sfg_tools.seafloor_site_tools.soundspeed_operations import (
+    CTD_to_svp_v1,
+    CTD_to_svp_v2,
+    seabird_to_soundvelocity,
+)
+from earthscope_sfg_tools.sonardyne_tools import sv3_operations as sv3_ops
+from earthscope_sfg_tools.tiledb_integration import (
+    TDBIMUPositionArray,
+    TDBKinPositionArray,
+    TDBShotDataArray,
+    rinex_qc,
+    tdb2rnx,
+)
 from earthscope_sfg_tools.tiledb_integration.arrays import TBDArray as _TBDArray
+from earthscope_sfg_workflows.data_mgmt.ports import AssetCatalogPort
+from earthscope_sfg_workflows.logging import ProcessLogger
 from pride_ppp import (
     ProcessingMode,
     PrideProcessor,
@@ -23,6 +40,29 @@
 )
 from pride_ppp.factories.processor import PrideProcessor as _PrideProcessorCls
 from pride_ppp.specifications.config import PRIDEPPPFileConfig as _PRIDEPPPFileConfig
+from rich.progress import track
+
+# local
+from ..data_mgmt.model import (
+    RINEX_KINDS,
+    AssetEntry,
+    AssetKind,
+    CampaignLayout,
+    SFGScope,
+    TileDBLayout,
+    rinex_kind_for_version,
+)
+from ..data_mgmt.utils import get_merge_signature_shotdata
+from .config import PrideConfig, RinexConfig, SV3PipelineConfig
+from .exceptions import (
+    NoDFOP00Found,
+    NoKinFound,
+    NoNovatelFound,
+    NoRinexBuilt,
+    NoRinexFound,
+    NoSVPFound,
+)
+from .shotdata_gnss_refinement import merge_shotdata_kinposition
 
 # pride_ppp <= current version omits `ISB model` from generated config_files;
 # pdp3 >= 3.2.7 requires it.  Patch write_config_file to inject the line.
@@ -77,48 +117,6 @@ def _tbd_write_df_patched(self, df, validate: bool = True):
 
 _TBDArray.write_df = _tbd_write_df_patched
 
-# third-party
-from earthscope_sfg_tools import tiledb_integration as novb_ops
-from earthscope_sfg_tools.novatel_tools.utils import get_metadata, get_metadatav2
-from earthscope_sfg_tools.seafloor_site_tools.soundspeed_operations import (
-    CTD_to_svp_v1,
-    CTD_to_svp_v2,
-    seabird_to_soundvelocity,
-)
-from earthscope_sfg_tools.sonardyne_tools import sv3_operations as sv3_ops
-from earthscope_sfg_tools.tiledb_integration import (
-    TDBIMUPositionArray,
-    TDBKinPositionArray,
-    TDBShotDataArray,
-    rinex_qc,
-    tdb2rnx,
-)
-from earthscope_sfg_workflows.data_mgmt.ports import AssetCatalogPort
-from earthscope_sfg_workflows.logging import ProcessLogger
-from rich.progress import track
-
-# local
-from ..data_mgmt.model import (
-    RINEX_KINDS,
-    AssetEntry,
-    AssetKind,
-    CampaignLayout,
-    SFGScope,
-    TileDBLayout,
-    rinex_kind_for_version,
-)
-from ..data_mgmt.utils import get_merge_signature_shotdata
-from .config import PrideConfig, RinexConfig, SV3PipelineConfig
-from .exceptions import (
-    NoDFOP00Found,
-    NoKinFound,
-    NoNovatelFound,
-    NoRinexBuilt,
-    NoRinexFound,
-    NoSVPFound,
-)
-from .shotdata_gnss_refinement import merge_shotdata_kinposition
-
 
 def _pipeline_method(fn):
     """Wrap a pipeline method so only one runs at a time per instance."""
@@ -685,9 +683,12 @@ def get_rinex_files(self) -> None:
             )
             try:
                 # tdb2rnx writes RINEX files to CWD; run from rinex_dest.
-                # Remove any pre-existing .rnx files so the post-run glob is clean.
+                # Remove any pre-existing RINEX output so the post-run glob is
+                # clean. Matches both the v3/v4 long name (*.rnx, current
+                # output format) and the legacy v2 short name (*.??o, in case
+                # a directory still has files from before the naming switch).
                 rinex_dest.mkdir(parents=True, exist_ok=True)
-                for _stale in rinex_dest.glob("*.rnx"):
+                for _stale in [*rinex_dest.glob("*.rnx"), *rinex_dest.glob("*.??o")]:
                     _stale.unlink()
                 old_cwd = Path.cwd()
                 try:
@@ -807,7 +808,12 @@ def process_rinex(self) -> None:
             override=pride_cfg.override,
         )
         rinex_entries = [
-            e for e in rinex_entries if e.local_path is not None and e.kind in RINEX_KINDS
+            e
+            for e in rinex_entries
+            if e.local_path is not None
+            and e.kind in RINEX_KINDS
+            and e.local_path.exists()
+            and e.local_path.stat().st_size > 0
         ]
 
         if not rinex_entries:

src/earthscope_sfg_workflows/services/ingest_service.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import concurrent.futures
-import re
 import tarfile
 import threading
 from datetime import datetime, timezone
@@ -15,7 +14,7 @@
 from rich.progress import track
 from upath import UPath
 
-from earthscope_sfg_workflows.data_mgmt.core import DEFAULT_PATTERNS, FileTypeDetector
+from earthscope_sfg_workflows.data_mgmt.core import FileTypeDetector
 from earthscope_sfg_workflows.data_mgmt.model import AssetEntry, AssetKind, IngestReport
 from earthscope_sfg_workflows.data_mgmt.ports import ArchiveError
 

src/earthscope_sfg_workflows/services/processing_service.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import json
-from pathlib import Path
 from typing import TYPE_CHECKING, Literal, Optional
 
 from pride_ppp import PrideCLIConfig

src/earthscope_sfg_workflows/services/sync_service.py

@@ -224,7 +224,14 @@ def _compress_rinex(self, rinex_dir: UPath) -> None:
                 continue
             if any(ext in rinex_file.suffix for ext in ["S", "d", ".gz"]):
                 continue
-            new_suffix = rinex_file.suffix[:-1] + "d"
+            if rinex_file.suffix.lower() == ".rnx":
+                # RINEX v3/v4 long name: swap the whole extension for the
+                # Hatanaka-compact one (e.g. "..._MO.rnx" -> "..._MO.crx").
+                new_suffix = ".crx"
+            else:
+                # Legacy RINEX v2 short name: swap the trailing "o" for "d"
+                # (e.g. ".26o" -> ".26d").
+                new_suffix = rinex_file.suffix[:-1] + "d"
             compressed = rinex_file.with_suffix(new_suffix + ".gz")
             if not compressed.exists():
                 try:

src/earthscope_sfg_workflows/utils/model_update.py

@@ -40,7 +40,7 @@ def validate_keys_recursively(config_dict: dict, model_class: BaseModel, path: s
                 suggestion_text = "No similar keys found."
 
             errors.append(
-                f"Invalid key '{current_path}' in {model_class.__repr_name__()}. {suggestion_text}"
+                f"Invalid key '{current_path}' in {model_class.__name__}. {suggestion_text}"
             )
 
         # If the value is a dict and the field exists, check nested structure

src/earthscope_sfg_workflows/workflows/session.py

@@ -40,22 +40,23 @@
 )
 
 from earthscope_sfg_tools.datamodels.metadata import Campaign, Site
-
-_Site = Site  # alias kept for the .from_json() classmethod call below
-
 from earthscope_sfg_workflows.data_mgmt.filestore.disk_filestore import FsspecFileStore
 from earthscope_sfg_workflows.data_mgmt.ports import (
     ArchiveSourcePort,
     AssetCatalogPort,
-    FileStorePort,
 )
-from earthscope_sfg_workflows.logging import GarposLogger as logger
+
+_Site = Site  # alias kept for the .from_json() classmethod call below
 
 
 if TYPE_CHECKING:  # pragma: no cover
+    from earthscope_sfg_tools.datamodels.metadata import Survey
     from earthscope_sfg_tools.tiledb_integration import (
         TDBAcousticArray,
         TDBGNSSObsArray,
+        TDBIMUPositionArray,
+        TDBKinPositionArray,
+        TDBShotDataArray,
     )
     from earthscope_sfg_workflows.services.ingest_service import IngestService
     from earthscope_sfg_workflows.services.processing_service import ProcessingService

src/earthscope_sfg_workflows/workflows/workflow_handler.py

@@ -11,7 +11,6 @@
 
 import os
 import re
-import warnings
 from pathlib import Path
 from typing import Literal, Optional
 
@@ -22,7 +21,7 @@
 from earthscope_sfg_workflows.logging import ProcessLogger as logger
 from earthscope_sfg_workflows.logging import change_all_logger_dirs
 
-from ..data_mgmt.model import DEFAULT_PREPROCESS_KINDS, DEFAULT_INTERMEDIATE_KINDS
+from ..data_mgmt.model import DEFAULT_PREPROCESS_KINDS
 from earthscope_sfg_tools.datamodels.metadata import Site
 from ..modeling.garpos_tools.schemas import InversionParams
 from ..modeling.garpos_tools.garpos_handler import GarposHandler