Feature/retire fiona

.github/workflows/tests.yml

@@ -2,12 +2,9 @@ name: CI Tests
 
 on:
   push:
-    branches:
-      - master
+    branches: [master]
   pull_request:
-    branches:
-      - master
-    types: [opened, synchronize, reopened]
+  workflow_dispatch:
 
 jobs:
   test:

dggrid4py/dggrid_runner.py

@@ -25,7 +25,7 @@
 
 import numpy as np
 import pandas as pd
-import fiona
+import pyogrio
 
 import geopandas as gpd
 from .interrupt import crosses_interruption, interrupt_cell, get_geom_coords
@@ -36,7 +36,8 @@
 AnyGeometry = BaseGeometry | GeometryArray
 
 
-fiona_drivers = fiona.supported_drivers
+# Cache pyogrio drivers for get_geo_out() function
+_geo_drivers = pyogrio.list_drivers()
 
 # SHAPEFILE is included more natively in DGGRID than GeoJSON (esp. as clip_subset_type)
 def get_geo_out(legacy=True, has_gdal=True):
@@ -51,7 +52,7 @@ def get_geo_out(legacy=True, has_gdal=True):
         return { "driver": "ESRI Shapefile", "legacy_driver": "Shapefile", "ext": "shp"}
 
     if legacy is False and has_gdal is True:
-        if "FlatGeobuf" in fiona_drivers.keys() and "w" in fiona_drivers["FlatGeobuf"]:
+        if "FlatGeobuf" in _geo_drivers.keys() and "w" in _geo_drivers["FlatGeobuf"]:
             return { "driver": "FlatGeobuf", "legacy_driver": "FlatGeobuf", "ext": "fgb"}
 
         return { "driver": "GPKG", "legacy_driver": "GPKG", "ext": "gpgk"}
@@ -701,7 +702,7 @@ def check_gdal_support(self):
         if self.has_gdal:
             print(f"GDAL types should be possible: has GDAL={self.has_gdal}")
             print("check your dggrid binary | mac: otool -L | Linux ldd ")
-            print(fiona_drivers)
+            print(_geo_drivers)
 
         else:
             print(f"GDAL types should not be used: has GDAL={self.has_gdal}")
@@ -1362,7 +1363,7 @@ def grid_cell_polygons_for_extent(
             print(dggs_ops)
 
         path = Path(tmp_dir) / f"temp_{dggs_type}_{resolution}_out_{tmp_id}.{self.tmp_geo_out['ext']}"
-        gdf = gpd.read_file(path.resolve(), driver=self.tmp_geo_out['driver'])
+        gdf = gpd.read_file(path.resolve(), engine="pyogrio")
 
         if not self.debug:
             try:
@@ -1446,7 +1447,7 @@ def grid_cell_centroids_for_extent(
         if self.debug:
             print(dggs_ops)
 
-        gdf = gpd.read_file( Path(tmp_dir) / f"temp_{dggs_type}_{resolution}_out_{tmp_id}.{self.tmp_geo_out['ext']}", driver=self.tmp_geo_out['driver'] )
+        gdf = gpd.read_file( Path(tmp_dir) / f"temp_{dggs_type}_{resolution}_out_{tmp_id}.{self.tmp_geo_out['ext']}", engine="pyogrio" )
 
         if not self.debug:
             try:
@@ -1533,7 +1534,7 @@ def grid_cell_polygons_from_cellids(
         if self.debug:
             print(dggs_ops)
 
-        gdf = gpd.read_file( Path(tmp_dir) / f"temp_{dggs_type}_{resolution}_out_{tmp_id}.{self.tmp_geo_out['ext']}", driver=self.tmp_geo_out['driver'] )
+        gdf = gpd.read_file( Path(tmp_dir) / f"temp_{dggs_type}_{resolution}_out_{tmp_id}.{self.tmp_geo_out['ext']}", engine="pyogrio" )
 
         if not cell_id_list is None and len(cell_id_list) > 0 and not seq_df is None:
             # we have to adjust the columns formats for the IDs/Seqnums/Name field to ensure they are comparable for the join
@@ -1643,7 +1644,7 @@ def grid_cell_centroids_from_cellids(
         if self.debug:
             print(dggs_ops)
 
-        gdf = gpd.read_file( Path(tmp_dir) / f"temp_{dggs_type}_{resolution}_out_{tmp_id}.{self.tmp_geo_out['ext']}", driver=self.tmp_geo_out['driver'] )
+        gdf = gpd.read_file( Path(tmp_dir) / f"temp_{dggs_type}_{resolution}_out_{tmp_id}.{self.tmp_geo_out['ext']}", engine="pyogrio" )
 
         if not cell_id_list is None and len(cell_id_list) > 0 and not seq_df is None:
             # we have to adjust the columns formats for the IDs/Seqnums/Name field to ensure they are comparable for the join
@@ -2177,7 +2178,7 @@ def specify_clip_settings(
             clip_metafile_settings.update({'clip_subset_type': cast(DggsClipSubsetTypeT, tmp_geo_out['legacy_driver'].upper())})
         clip_gdf = gpd.GeoDataFrame(pd.DataFrame({'id': [1], 'geometry': [clip_geom]}), geometry='geometry', crs=4326)
         clip_path = Path(tmp_dir) / f"temp_clip_{tmp_id}.{tmp_geo_out['ext']}"
-        clip_gdf.to_file(str(clip_path), driver=tmp_geo_out['driver'])
+        clip_gdf.to_file(str(clip_path), engine="pyogrio")
         clip_metafile_settings.update({'clip_region_files':
                                        str((Path(tmp_dir) / f"temp_clip_{tmp_id}.{tmp_geo_out['ext']}").resolve())})
         if "clipper_scale_factor" in conf_extra:

docs/requirements.txt

@@ -3,8 +3,8 @@ sphinx-rtd-theme==3
 myst-parser==4
 numpy
 pandas
-fiona
 shapely
 geopandas
+pyogrio
 dggrid4py
 

pyproject.toml

@@ -32,7 +32,6 @@ requires-python = ">=3.10, <4.0.0"
 python = ">=3.10, <4.0.0"
 numpy = ">=1.22.0, <3.0.0"
 pandas = ">=1.4.0"
-fiona = ">=1.8.0, <2.0.0"
 shapely = ">=1.8.0, <3.0.0"
 geopandas = ">=0.9.0, <2.0.0"
 pygeodesy = "^25.10.10"

tests/test_clipper_scale_factor.py

@@ -45,8 +45,8 @@ def mock_dggrid_grid_gen_run(__metafile):
 
     # pre-check temp file paths to ignore in check of specific values
     meta_args = dict([line.split(" ") for line in metafile])
-    assert meta_args["clip_region_files"].startswith("/tmp/dggrid")
-    assert meta_args["cell_output_file_name"].startswith("/tmp/dggrid")
+    assert "dggrid" in meta_args["clip_region_files"]
+    assert "dggrid" in meta_args["cell_output_file_name"]
     meta_args.pop("clip_region_files")
     meta_args.pop("cell_output_file_name")
     metafile_patched = [f"{key} {val}" for key, val in meta_args.items()]
@@ -112,8 +112,8 @@ def mock_dggrid_grid_gen_run(__metafile):
 
     # pre-check temp file paths to ignore in check of specific values
     meta_args = dict([line.split(" ") for line in metafile])
-    assert meta_args["clip_region_files"].startswith("/tmp/dggrid")
-    assert meta_args["cell_output_file_name"].startswith("/tmp/dggrid")
+    assert "dggrid" in meta_args["clip_region_files"]
+    assert "dggrid" in meta_args["cell_output_file_name"]
     meta_args.pop("clip_region_files")
     meta_args.pop("cell_output_file_name")
     metafile_patched = [f"{key} {val}" for key, val in meta_args.items()]

tests/test_dggrid.py

@@ -274,8 +274,8 @@ def mock_dggrid_grid_gen_run(__metafile):
 
     # pre-check temp file paths to ignore in check of specific values
     meta_args = dict([line.split(" ") for line in metafile])
-    assert meta_args["clip_region_files"].startswith("/tmp/dggrid")
-    assert meta_args["cell_output_file_name"].startswith("/tmp/dggrid")
+    assert "dggrid" in meta_args["clip_region_files"]
+    assert "dggrid" in meta_args["cell_output_file_name"]
     meta_args.pop("clip_region_files")
     meta_args.pop("cell_output_file_name")
     metafile_patched = [f"{key} {val}" for key, val in meta_args.items()]
@@ -341,8 +341,8 @@ def mock_dggrid_grid_gen_run(__metafile):
 
     # pre-check temp file paths to ignore in check of specific values
     meta_args = dict([line.split(" ") for line in metafile])
-    assert meta_args["clip_region_files"].startswith("/tmp/dggrid")
-    assert meta_args["cell_output_file_name"].startswith("/tmp/dggrid")
+    assert "dggrid" in meta_args["clip_region_files"]
+    assert "dggrid" in meta_args["cell_output_file_name"]
     meta_args.pop("clip_region_files")
     meta_args.pop("cell_output_file_name")
     metafile_patched = [f"{key} {val}" for key, val in meta_args.items()]
@@ -422,15 +422,15 @@ def mock_dgapi_grid_gen(*args, **kwargs):
     meta_args = dict([line.split(" ", 1) for line in dgapi_grid_transform_metafile])
     dgapi_grid_transform_input_file_name = meta_args.pop("input_file_name")
     dgapi_grid_transform_output_file_name = meta_args.pop("output_file_name")
-    assert dgapi_grid_transform_input_file_name.startswith("/tmp/dggrid")
-    assert dgapi_grid_transform_output_file_name.startswith("/tmp/dggrid")
+    assert "dggrid" in dgapi_grid_transform_input_file_name
+    assert "dggrid" in dgapi_grid_transform_output_file_name
     dgapi_grid_transform_metafile_patched = [f"{key} {val}" for key, val in meta_args.items()]
 
     meta_args = dict([line.split(" ", 1) for line in dgapi_grid_gen_metafile])
     dgapi_grid_gen_cell_output_file_name = meta_args.pop("cell_output_file_name")
     dgapi_grid_gen_clip_region_files = meta_args.pop("clip_region_files")
-    assert dgapi_grid_gen_cell_output_file_name.startswith("/tmp/dggrid")
-    assert dgapi_grid_gen_clip_region_files.startswith("/tmp/dggrid")
+    assert "dggrid" in dgapi_grid_gen_cell_output_file_name
+    assert "dggrid" in dgapi_grid_gen_clip_region_files
     dgapi_grid_gen_metafile_patched = [f"{key} {val}" for key, val in meta_args.items()]
 
     assert dgapi_grid_transform_output_file_name != dgapi_grid_gen_cell_output_file_name
@@ -524,4 +524,4 @@ def mock_dgapi_grid_gen(*args, **kwargs):
         ],
         columns=result.columns,  # ensure ordering matches to allow compare
     )
-    assert_geodataframe_equal(result, expect)
+    assert_geodataframe_equal(result, expect, check_less_precise=True)