diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..015823f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +name: CI + +on: + push: + branches: ["**"] + pull_request: + branches: ["**"] + +jobs: + lint-and-test: + name: Lint & Tests check + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: "pip" + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install --upgrade build + pip install ".[dev]" + + - name: Run Ruff linter + run: ruff check . + + - name: Run Ruff formatter check + run: ruff format --check . + + - name: Run tests + run: pytest diff --git a/doc/conf.py b/doc/conf.py index 6c96055..053812f 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -6,51 +6,56 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = 'pixcdust' -copyright = '2025, Zawadzki Lionel' -author = 'Zawadzki Lionel' -release = '0.1.0' +project = "pixcdust" +copyright = "2025, Zawadzki Lionel" +author = "Zawadzki Lionel" +release = "0.1.0" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = ['sphinx.ext.viewcode','autoapi.extension','nbsphinx',"sphinxcontrib.collections"] +extensions = [ + "sphinx.ext.viewcode", + "autoapi.extension", + "nbsphinx", + "sphinxcontrib.collections", +] html_show_sourcelink = False set_type_checking_flag = True nbsphinx_allow_errors = True -templates_path = ['_templates'] +templates_path = ["_templates"] exclude_patterns = [] -autoapi_dirs = ['../pixcdust'] -source_suffix = ['.rst'] +autoapi_dirs = ["../pixcdust"] +source_suffix = [".rst"] # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_theme = 'alabaster' -html_static_path = ['_static'] +html_theme = "alabaster" +html_static_path = ["_static"] # manual build path collections = { - 'notebooks': { - 'driver': 'copy_folder', - 'source': 'pixcdust/notebooks', - 'target': 'notebooks/', - 'ignore': ['*.py', '.sh'], - 'safe': False, + "notebooks": { + "driver": "copy_folder", + "source": "pixcdust/notebooks", + "target": "notebooks/", + "ignore": ["*.py", ".sh"], + "safe": False, } } # ReadTheDoc build path collections = { - 'notebooks': { - 'driver': 'copy_folder', - 'source': '../pixcdust/notebooks', - 'target': 'notebooks/', - 'ignore': ['*.py', '.sh'], - 'safe': False, + "notebooks": { + "driver": "copy_folder", + "source": "../pixcdust/notebooks", + "target": "notebooks/", + "ignore": ["*.py", ".sh"], + "safe": False, } } diff --git a/pixcdust/converters/core.py b/pixcdust/converters/core.py index 14107bd..4a8113b 100644 --- a/pixcdust/converters/core.py +++ b/pixcdust/converters/core.py @@ -16,12 +16,11 @@ """Interface used by all Pixcdust Converters.""" import copy -from dataclasses import dataclass import operator +from collections.abc import Iterable +from dataclasses import dataclass from pathlib import Path -from typing import Optional, Union, Iterable - import geopandas as gpd @@ -44,9 +43,9 @@ class Converter: def __init__( self, path_in: str | Iterable[str] | Path | Iterable[Path], - variables: Optional[list[str]] = None, - area_of_interest: Optional[gpd.GeoDataFrame] = None, - conditions: Optional[dict[str, dict[str, Union[str, float]]]] = None, + variables: list[str] | None = None, + area_of_interest: gpd.GeoDataFrame | None = None, + conditions: dict[str, dict[str, str | float]] | None = None, ): """Basic initialisation of a pixcdust converter. @@ -91,8 +90,10 @@ class ConverterWSE(Converter): variables: Optionally only read these variables. area_of_interest: Optionally only read points in area_of_interest. """ - def database_from_nc(self, path_out: str | Path, mode: str = "w", compute_wse: bool = True) \ - -> None: + + def database_from_nc( + self, path_out: str | Path, mode: str = "w", compute_wse: bool = True + ) -> None: """Convert the path_in files to path_out. Args: path_out: Output path of the convertion. @@ -109,19 +110,21 @@ def _append_wse_vars(self): self.variables.append(var) def _compute_wse(self, gdf): - gdf[self._get_name_wse_var()] = \ - gdf[self._get_vars_wse_computation()[0]] - \ - gdf[self._get_vars_wse_computation()[1]] + gdf[self._get_name_wse_var()] = ( + gdf[self._get_vars_wse_computation()[0]] + - gdf[self._get_vars_wse_computation()[1]] + ) @staticmethod def _get_vars_wse_computation() -> list[str]: """Names of fields used to compute wse.""" - return ['height', 'geoid'] + return ["height", "geoid"] @staticmethod def _get_name_wse_var() -> str: """Output name for wse.""" - return 'wse' + return "wse" + @dataclass class GeoLayerH3Projecter: @@ -132,10 +135,11 @@ class GeoLayerH3Projecter: resolution: Resolution """ + data: gpd.GeoDataFrame resolution: int - def filter_variable(self, conditions: dict[str,dict[str, Union[str, float]]]) -> None: + def filter_variable(self, conditions: dict[str, dict[str, str | float]]) -> None: """filters from xarray dataset based on operator and threshold on specific variables @@ -154,36 +158,40 @@ def filter_variable(self, conditions: dict[str,dict[str, Union[str, float]]]) -> AttributeError: if operator is not the function name of\ the operator module """ - _k_operator = 'operator' - _k_to = 'threshold' + _k_operator = "operator" + _k_to = "threshold" # Test if conditions dict meets specifications print(conditions) - for k in conditions.keys(): - if k not in self.data.columns: - raise IOError( - f'dict conditions expected existing\ + for var, condition in conditions.items(): + if var not in self.data.columns: + raise OSError( + f"dict conditions expected existing\ variables (in {self.data.columns}),\ - received {k}' + received {var}" ) - for instructions in conditions[k].keys(): + for instructions in condition: if instructions not in [_k_operator, _k_to]: raise ValueError( - f'dict conditions expected {_k_to} and {_k_operator}\ + f"dict conditions expected {_k_to} and {_k_operator}\ keys in dict {conditions},\ - received {instructions}' + received {instructions}" ) - print(f"operator.{conditions[k][_k_operator]}") - ope = getattr(operator, conditions[k][_k_operator]) + op_name = condition[_k_operator] + # Typing issue, improvement would be to use TypedDict for conditions + assert isinstance(op_name, str) + print(f"operator.{op_name}") + ope = getattr(operator, op_name) self.data = self.data[ ope( - self.data[k], - conditions[k][_k_to], + self.data[var], + condition[_k_to], ) ] def compute_h3_layer(self) -> None: """Project data to h3.""" from pixcdust.dggs import h3_tools + self.data = h3_tools.gdf_to_h3_gdf( self.data, self.resolution, @@ -192,6 +200,7 @@ def compute_h3_layer(self) -> None: def compute_healpix_layer(self) -> None: """Project data to Healpix.""" from pixcdust.dggs import h3_tools + self.data = h3_tools.gdf_to_healpix_gdf( self.data, self.resolution, diff --git a/pixcdust/converters/geo_utils.py b/pixcdust/converters/geo_utils.py index 5a80ab2..b959ee3 100644 --- a/pixcdust/converters/geo_utils.py +++ b/pixcdust/converters/geo_utils.py @@ -15,13 +15,11 @@ # """Converters utility""" -import xarray as xr import geopandas as gpd +import xarray as xr -def geoxarray_to_geodataframe( - ds: xr.Dataset, - *args, **kwargs) -> gpd.GeoDataFrame: +def geoxarray_to_geodataframe(ds: xr.Dataset, *args, **kwargs) -> gpd.GeoDataFrame: """Converts an xarray.Dataset with points coordinates into\ a geopandas.GeodataFrame with xvec diff --git a/pixcdust/converters/gpkg.py b/pixcdust/converters/gpkg.py index f323a47..7258d9e 100644 --- a/pixcdust/converters/gpkg.py +++ b/pixcdust/converters/gpkg.py @@ -16,18 +16,17 @@ """Geopackage converters.""" import os -from pathlib import Path -from typing import Optional, Union from dataclasses import dataclass +from pathlib import Path -from tqdm import tqdm import fiona import geopandas as gpd +from tqdm import tqdm from pixcdust.converters.core import ConverterWSE, GeoLayerH3Projecter +from pixcdust.readers.gpkg import GpkgReader from pixcdust.readers.netcdf import NcSimpleReader from pixcdust.readers.zarr import ZarrReader -from pixcdust.readers.gpkg import GpkgReader class Nc2GpkgConverter(ConverterWSE): @@ -45,15 +44,16 @@ class Nc2GpkgConverter(ConverterWSE): """ - def database_from_nc(self, path_out: str | Path, mode: str = "w", compute_wse: bool = True) \ - -> None: + def database_from_nc( + self, path_out: str | Path, mode: str = "w", compute_wse: bool = True + ) -> None: path_out = str(path_out) if compute_wse: self._append_wse_vars() for path in tqdm(self.path_in): ncsimple = NcSimpleReader( path, - variables= self.variables, + variables=self.variables, area_of_interest=self.area_of_interest, conditions=self.conditions, ) @@ -62,23 +62,26 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w", compute_wse: b _, dt_time_start, cycle_number, pass_number, tile_number, swath_side = ( ncsimple.extract_info_from_nc_attrs(path) ) - time_start = dt_time_start.strftime('%Y%m%d') + time_start = dt_time_start.strftime("%Y%m%d") - layer_name = f"{time_start}_{cycle_number}_\ -{pass_number}_{tile_number}{swath_side}" + layer_name = ( + f"{time_start}_{cycle_number}_{pass_number}_{tile_number}{swath_side}" + ) # cheking if output file and layer already exist - if os.path.exists(path_out) and mode == "w": - if layer_name in fiona.listlayers(path_out): - tqdm.write( - f"skipping layer {layer_name} \ + if ( + os.path.exists(path_out) + and mode == "w" + and layer_name in fiona.listlayers(path_out) + ): + tqdm.write( + f"skipping layer {layer_name} \ (already in geopackage {path_out})" - ) - continue + ) + continue # converting data from xarray to geodataframe ncsimple.open_dataset() - gdf = ncsimple.to_geodataframe( - ) + gdf = ncsimple.to_geodataframe() if gdf.size == 0: tqdm.write( @@ -109,18 +112,19 @@ class GpkgDGGSProjecter: path: str dggs_res: int - conditions: Optional[dict[str,dict[str, Union[str, float]]]] = None + conditions: dict[str, dict[str, str | float]] | None = None healpix: bool = False - dggs_layer_pattern: str = '_h3' - path_out: Optional[str] = None + dggs_layer_pattern: str = "_h3" + path_out: str | None = None # database: GpkgReader def __post_init__(self) -> None: self.database = GpkgReader(self.path) self.database.layers = [ - layer for layer in fiona.listlayers(self.path) + layer + for layer in fiona.listlayers(self.path) if not layer.endswith(self.dggs_layer_pattern) - ] + ] if self.path_out is None: self.path_out = self.path @@ -165,6 +169,7 @@ class Zarr2GpkgConverter: Attributes: path: Gpkg pixelcloud to convert. """ + path: str data: gpd.GeoDataFrame = None diff --git a/pixcdust/converters/shapefile.py b/pixcdust/converters/shapefile.py index 0f5f9f4..fab4c42 100644 --- a/pixcdust/converters/shapefile.py +++ b/pixcdust/converters/shapefile.py @@ -45,14 +45,18 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: except FileExistsError: pass for path in tqdm(self.path_in): - ncsimple = NcSimpleReader(path, - variables=self.variables, - area_of_interest=self.area_of_interest, - conditions=self.conditions, - ) + ncsimple = NcSimpleReader( + path, + variables=self.variables, + area_of_interest=self.area_of_interest, + conditions=self.conditions, + ) filename_out = os.path.splitext(os.path.basename(path))[0] - path_shp = os.path.join(path_out, filename_out + '.shp', ) + path_shp = os.path.join( + path_out, + filename_out + ".shp", + ) # cheking if output file and layer already exist if os.path.exists(path_shp) and mode == "w": continue diff --git a/pixcdust/converters/zarr.py b/pixcdust/converters/zarr.py index b0fe07f..d69a5d2 100644 --- a/pixcdust/converters/zarr.py +++ b/pixcdust/converters/zarr.py @@ -17,23 +17,20 @@ import os import shutil +from collections.abc import Iterable from pathlib import Path -from typing import Optional, Iterable +import dask +import dask.utils import fsspec -from typing import Tuple, List, Union - import geopandas as gpd import zcollection import zcollection.indexing -import dask -import dask.utils - from pixcdust.converters.core import Converter -from pixcdust.readers.netcdf import NcSimpleReader, NcSimpleConstants +from pixcdust.readers.netcdf import NcSimpleConstants, NcSimpleReader -TIME_VARNAME = 'time' +TIME_VARNAME = "time" class Nc2ZarrConverter(Converter): @@ -53,9 +50,9 @@ class Nc2ZarrConverter(Converter): def __init__( self, path_in: str | Iterable[str] | Path | Iterable[Path], - variables: Optional[list[str]] = None, - area_of_interest: Optional[gpd.GeoDataFrame] = None, - conditions: Optional[dict[str, dict[str, Union[str, float]]]] = None, + variables: list[str] | None = None, + area_of_interest: gpd.GeoDataFrame | None = None, + conditions: dict[str, dict[str, str | float]] | None = None, ): """Basic initialisation of a pixcdust converter. @@ -71,29 +68,32 @@ def __init__( "classification":{'operator': "ge", 'threshold': 3},\ } """ - super().__init__(path_in=path_in, - variables=variables, - area_of_interest=area_of_interest, - conditions=conditions) + super().__init__( + path_in=path_in, + variables=variables, + area_of_interest=area_of_interest, + conditions=conditions, + ) self.collection: zcollection.collection.Collection = None self.__time_varname: str = TIME_VARNAME self.__fs = fsspec.filesystem("file") - self.__chunk_size = dask.utils.parse_bytes('2MiB') + self.__chunk_size = dask.utils.parse_bytes("2MiB") self.__cst = NcSimpleConstants() def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: - if mode in ['o', 'overwrite'] and os.path.exists(path_out): + if mode in ["o", "overwrite"] and os.path.exists(path_out): shutil.rmtree(path_out) - with dask.distributed.LocalCluster(processes=True) as cluster, \ - dask.distributed.Client(cluster) as client: - + with ( + dask.distributed.LocalCluster(processes=True) as cluster, + dask.distributed.Client(cluster) as _, + ): xr_ds = NcSimpleReader( path=self.path_in, variables=self.variables, area_of_interest=self.area_of_interest, - conditions=self.conditions + conditions=self.conditions, ) xr_ds.open_mfdataset( @@ -102,18 +102,15 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: zc_ds = zcollection.Dataset.from_xarray( xr_ds.to_xarray().drop_vars(self.__cst.default_added_points_name), - ) + ) zc_ds.block_size_limit = self.__chunk_size - zc_ds.chunks = { - list(zc_ds.dimensions.keys())[0]: self.__chunk_size - } + zc_ds.chunks = {next(iter(zc_ds.dimensions.keys())): self.__chunk_size} init = True if not os.path.exists(path_out) and init: - partition_handler = zcollection.partitioning.Date( - (xr_ds.cst.default_added_time_name, ), - 's', + (xr_ds.cst.default_added_time_name,), + "s", ) self.collection = zcollection.create_collection( @@ -129,9 +126,8 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: self.collection = zcollection.open_collection( path_out, filesystem=self.__fs, - mode='w', - ) + mode="w", + ) self.collection.insert( - zc_ds, - merge_callable=zcollection.collection.merging.merge_time_series + zc_ds, merge_callable=zcollection.collection.merging.merge_time_series ) diff --git a/pixcdust/dggs/dggs_converter.py b/pixcdust/dggs/dggs_converter.py index b3064d3..812d49f 100644 --- a/pixcdust/dggs/dggs_converter.py +++ b/pixcdust/dggs/dggs_converter.py @@ -18,14 +18,16 @@ import h3 import numpy as np import xarray as xr -from xarray import Dataset import xdggs -from scipy.interpolate import griddata -from astropy_healpix import HEALPix from astropy import units as u +from astropy_healpix import HEALPix +from scipy.interpolate import griddata +from xarray import Dataset -def prepare_dataset_h3(ds: Dataset, resolution: int, interp: bool=False, method: str = 'linear') -> Dataset: +def prepare_dataset_h3( + ds: Dataset, resolution: int, interp: bool = False, method: str = "linear" +) -> Dataset: """ Convert a Dataset with latitude and longitude coordinates into an H3-indexed grid. @@ -53,8 +55,14 @@ def prepare_dataset_h3(ds: Dataset, resolution: int, interp: bool=False, method: if interp: # Compute the bounding box for the dataset in lat/lon - lon_min, lon_max = ds.longitude.min().values.item(), ds.longitude.max().values.item() - lat_min, lat_max = ds.latitude.min().values.item(), ds.latitude.max().values.item() + lon_min, lon_max = ( + ds.longitude.min().values.item(), + ds.longitude.max().values.item(), + ) + lat_min, lat_max = ( + ds.latitude.min().values.item(), + ds.latitude.max().values.item(), + ) # Define the bounding box coordinates bbox_coords = [ @@ -85,17 +93,21 @@ def prepare_dataset_h3(ds: Dataset, resolution: int, interp: bool=False, method: # Interpolate the values onto the H3 grid interpolated_values = griddata( - points=(lat, lon), - values=values, - xi=ll_points, - method=method + points=(lat, lon), values=values, xi=ll_points, method=method ) data[var] = interpolated_values else: # Compute H3 index for each point in the dataset - h3_indices = np.array([h3.api.basic_int.geo_to_h3(lat_, lon_, resolution) for lat_, lon_ in zip(lat.values, lon.values)]) - ll_points = np.array([h3.api.basic_int.h3_to_geo(i) for i in np.unique(h3_indices)]) + h3_indices = np.array( + [ + h3.api.basic_int.geo_to_h3(lat_, lon_, resolution) + for lat_, lon_ in zip(lat.values, lon.values) + ] + ) + ll_points = np.array( + [h3.api.basic_int.h3_to_geo(i) for i in np.unique(h3_indices)] + ) # Create a dictionary to store values by H3 cell h3_data = {pix_id: [] for pix_id in np.unique(h3_indices)} @@ -107,18 +119,19 @@ def prepare_dataset_h3(ds: Dataset, resolution: int, interp: bool=False, method: h3_data[h3_id].append(values[idx]) # Compute the mean value for each variable in each H3 cell - data = {var: np.array([np.mean(np.array(h3_data[h3_id])) for h3_id in h3_data]) for var in ds.data_vars} + data = { + var: np.array([np.mean(np.array(h3_data[h3_id])) for h3_id in h3_data]) + for var in ds.data_vars + } coords = { "cell_ids": np.unique(h3_indices), - 'h3_lon': ('cell_ids', ll_points[:, 1]), - 'h3_lat': ('cell_ids', ll_points[:, 0]) + "h3_lon": ("cell_ids", ll_points[:, 1]), + "h3_lat": ("cell_ids", ll_points[:, 0]), } ds_h3 = xr.Dataset( - {var: (('cell_ids',), data[var]) for var in data}, - coords=coords, - attrs=ds.attrs + {var: (("cell_ids",), data[var]) for var in data}, coords=coords, attrs=ds.attrs ) ds_h3.cell_ids.attrs = {"grid_name": "h3", "resolution": resolution} @@ -126,7 +139,9 @@ def prepare_dataset_h3(ds: Dataset, resolution: int, interp: bool=False, method: return ds_h3 -def prepare_dataset_healpix(ds: Dataset, resolution: int = 8, interp: bool=False, method: str = 'linear') -> Dataset: +def prepare_dataset_healpix( + ds: Dataset, resolution: int = 8, interp: bool = False, method: str = "linear" +) -> Dataset: """ Convert a Dataset with latitude and longitude coordinates into an HEALPix-indexed grid. @@ -154,8 +169,8 @@ def prepare_dataset_healpix(ds: Dataset, resolution: int = 8, interp: bool=False healpix = HEALPix(nside=nside, order="nested") # Get HEALPix pixel centers - lats = ds['latitude'].values - lons = ds['longitude'].values + lats = ds["latitude"].values + lons = ds["longitude"].values # pix_indices = np.array(hp.ang2pix(nside, lons, lats, nest=nest, lonlat=True)) # healpix_lon, healpix_lat = hp.pix2ang(nside, np.unique(pix_indices), nest=nest, lonlat=True) pix_indices = healpix.lonlat_to_healpix(lons * u.deg, lats * u.deg) @@ -178,10 +193,7 @@ def prepare_dataset_healpix(ds: Dataset, resolution: int = 8, interp: bool=False # Interpolate the values onto the HEALPix grid interpolated_values = griddata( - points=(lons, lats), - values=values, - xi=interp_points, - method=method + points=(lons, lats), values=values, xi=interp_points, method=method ) data[var] = interpolated_values @@ -196,19 +208,22 @@ def prepare_dataset_healpix(ds: Dataset, resolution: int = 8, interp: bool=False healpix_data[h3_id].append(values[idx]) # Compute the mean value for each variable in each HEALPix cell - data = {var: np.array([np.mean(np.array(healpix_data[h3_id])) for h3_id in healpix_data]) for var in ds.data_vars} + data = { + var: np.array( + [np.mean(np.array(healpix_data[h3_id])) for h3_id in healpix_data] + ) + for var in ds.data_vars + } coords = { - 'cell_ids': np.unique(pix_indices), - 'healpix_lon': ('cell_ids', healpix_lon), - 'healpix_lat': ('cell_ids', healpix_lat) + "cell_ids": np.unique(pix_indices), + "healpix_lon": ("cell_ids", healpix_lon), + "healpix_lat": ("cell_ids", healpix_lat), } # Create the new dataset with the aggregated or interpolated data ds_healpix = xr.Dataset( - {var: (('cell_ids',), data[var]) for var in data}, - coords=coords, - attrs=ds.attrs + {var: (("cell_ids",), data[var]) for var in data}, coords=coords, attrs=ds.attrs ) ds_healpix.cell_ids.attrs = { diff --git a/pixcdust/dggs/h3_tools.py b/pixcdust/dggs/h3_tools.py index e2fad98..93d2e77 100644 --- a/pixcdust/dggs/h3_tools.py +++ b/pixcdust/dggs/h3_tools.py @@ -16,11 +16,11 @@ import geopandas as gpd import h3 -from astropy_healpix import HEALPix from astropy import units as u - +from astropy_healpix import HEALPix from shapely.geometry import Polygon + def h3_to_polygon(h3_index): boundary = h3.api.basic_int.h3_to_geo_boundary(h3_index, geo_json=True) return Polygon(boundary) @@ -33,30 +33,34 @@ def get_h3_res_name(res: int) -> str: def gdf_to_h3_gdf( gdf: gpd.GeoDataFrame, resolution: int, - ) -> gpd.GeoDataFrame: +) -> gpd.GeoDataFrame: """ - Convert a GeoDataFrame with latitude and longitude columns to a GeoDataFrame - where rows are aggregated into H3 hexagons based on a given resolution. + Convert a GeoDataFrame with latitude and longitude columns to a GeoDataFrame + where rows are aggregated into H3 hexagons based on a given resolution. - Args: - gdf (gpd.GeoDataFrame): The input GeoDataFrame containing 'latitude' and 'longitude' columns. - resolution (int): The H3 resolution level for hexagon indexing. + Args: + gdf (gpd.GeoDataFrame): The input GeoDataFrame containing 'latitude' and 'longitude' columns. + resolution (int): The H3 resolution level for hexagon indexing. - Returns: - gpd.GeoDataFrame: A new GeoDataFrame where the rows are grouped by H3 hexagons, - containing the mean of the grouped variables, and a geometry column with polygons - representing the H3 hexagons. - """ + Returns: + gpd.GeoDataFrame: A new GeoDataFrame where the rows are grouped by H3 hexagons, + containing the mean of the grouped variables, and a geometry column with polygons + representing the H3 hexagons. + """ # Get the column name for H3 index based on the resolution h3_col = get_h3_res_name(resolution) # Apply the H3 function to each row to calculate the H3 index based on latitude, longitude, and resolution - gdf[h3_col] = gdf.apply(lambda row: h3.api.basic_int.geo_to_h3(row['latitude'], row['longitude'], resolution), - axis=1) + gdf[h3_col] = gdf.apply( + lambda row: h3.api.basic_int.geo_to_h3( + row["latitude"], row["longitude"], resolution + ), + axis=1, + ) # Drop the latitude, longitude, and geometry columns as they are no longer needed - h3_df = gdf.drop(columns=['latitude', 'longitude', 'geometry']) + h3_df = gdf.drop(columns=["latitude", "longitude", "geometry"]) # Group by the H3 index column, and compute the mean of all other columns in each group h3_df = h3_df.groupby(h3_col).mean().reset_index() @@ -82,40 +86,45 @@ def get_healpix_res_name(res: int) -> str: def gdf_to_healpix_gdf( gdf: gpd.GeoDataFrame, resolution: int, - ) -> gpd.GeoDataFrame: +) -> gpd.GeoDataFrame: """ - Convert a GeoDataFrame with latitude and longitude columns to a GeoDataFrame - where rows are aggregated into Healpix geometry based on a given resolution. + Convert a GeoDataFrame with latitude and longitude columns to a GeoDataFrame + where rows are aggregated into Healpix geometry based on a given resolution. - Args: - gdf (gpd.GeoDataFrame): The input GeoDataFrame containing 'latitude' and 'longitude' columns. - resolution (int): The Healpix resolution level for indexing. + Args: + gdf (gpd.GeoDataFrame): The input GeoDataFrame containing 'latitude' and 'longitude' columns. + resolution (int): The Healpix resolution level for indexing. - Returns: - gpd.GeoDataFrame: A new GeoDataFrame where the rows are grouped by H3 hexagons, - containing the mean of the grouped variables, and a geometry column with polygons - representing the Healpix geometry. - """ + Returns: + gpd.GeoDataFrame: A new GeoDataFrame where the rows are grouped by H3 hexagons, + containing the mean of the grouped variables, and a geometry column with polygons + representing the Healpix geometry. + """ nside = 2**resolution # Init heaplix grid - healpix = HEALPix(nside=nside, order='nested') + healpix = HEALPix(nside=nside, order="nested") # Get the column name for Healpix index based on the resolution healpix_col = get_healpix_res_name(resolution) # Apply the Healpix function to each row to calculate the Healpix index based on latitude, longitude, and resolution gdf[healpix_col] = gdf.apply( - lambda row: healpix.lonlat_to_healpix(row['longitude'] * u.deg, row['latitude'] * u.deg), axis=1 + lambda row: healpix.lonlat_to_healpix( + row["longitude"] * u.deg, row["latitude"] * u.deg + ), + axis=1, ) # Drop the latitude, longitude, and geometry columns as they are no longer needed - healpix_df = gdf.drop(columns=['latitude', 'longitude', 'geometry']) + healpix_df = gdf.drop(columns=["latitude", "longitude", "geometry"]) # Group by the Healpix index column, and compute the mean of all other columns in each group healpix_df = healpix_df.groupby(healpix_col).mean().reset_index() # Convert each Healpix index into a polygon geometry representing its boundaries - geometry = healpix_df[healpix_col].apply(lambda pix: healpix_to_polygon(healpix, pix)) + geometry = healpix_df[healpix_col].apply( + lambda pix: healpix_to_polygon(healpix, pix) + ) return gpd.GeoDataFrame(data=healpix_df, geometry=geometry, crs=4326) diff --git a/pixcdust/downloaders/hydroweb_next.py b/pixcdust/downloaders/hydroweb_next.py index 65d953b..7b65cbe 100644 --- a/pixcdust/downloaders/hydroweb_next.py +++ b/pixcdust/downloaders/hydroweb_next.py @@ -15,19 +15,17 @@ # """Downloaders for hydroweb.next. Require an API-Key see HELP_MESSAGE.""" -from abc import ABC, abstractmethod -import os -from pathlib import Path -from typing import Optional, Union, Tuple, List import datetime +import os import zipfile +from abc import ABC, abstractmethod +from pathlib import Path import geopandas +import py_hydroweb import shapely import tqdm -import py_hydroweb -from eodag import EODataAccessGateway, SearchResult -from eodag import setup_logging +from eodag import EODataAccessGateway, SearchResult, setup_logging HELP_MESSAGE = """ Download products from hydroweb.next (https://hydroweb.next.theia-land.fr) @@ -60,15 +58,16 @@ class Downloader(ABC): """ + PROVIDER = "hydroweb_next" def __init__( self, collection_name: str, - geometry: Union[str, list[str], geopandas.GeoDataFrame, None] = (None,), - dates: Optional[Tuple[datetime.date, datetime.date]] = None, + geometry: str | list[str] | geopandas.GeoDataFrame | None = (None,), + dates: tuple[datetime.date, datetime.date] | None = None, path_download: str | Path = "/tmp/hydroweb_next", - verbose: Optional[int] = 0, + verbose: int | None = 0, ): """Downloader for hydroweb.next STAC API initialization. @@ -93,15 +92,13 @@ def __init__( self.verbose = verbose self.query_args = {} - self.search_results: List[SearchResult] = [] + self.search_results: list[SearchResult] = [] if not os.path.isdir(self.path_download): os.mkdir(self.path_download) self.setup() self.query_args = self.define_query() - - @staticmethod def _explode_simplify_geometry( @@ -141,16 +138,15 @@ def _explode_simplify_geometry( axis=1, ) if (geom["nodes_count"] > 200).any(): - raise AttributeError(( + raise AttributeError( "One or several of your search polygons have too many nodes," "consider using the tolerance parameter" "in order to simplify the polygons." - )) + ) return geom - - def search_download(self, tolerance: Optional[float] = None) -> None: + def search_download(self, tolerance: float | None = None) -> None: """Search files according to the query and download them. Args: @@ -171,10 +167,10 @@ def search_download(self, tolerance: Optional[float] = None) -> None: for geom in geometries.geometry.values: self._search(geom.__geo_interface__) else: - raise AttributeError(( + raise AttributeError( "geometry should string (WKT) or geopandas.GeoDataFrame, " f"received {type(self.geometry)} instead" - )) + ) # This command actually downloads the matching products downloaded_paths = self._download() @@ -191,17 +187,19 @@ def setup(self) -> None: @abstractmethod def define_query(self) -> dict: pass - + @abstractmethod - def _search(self, geom:Optional[str] = None) -> None: + def _search(self, geom: str | None = None) -> None: pass @abstractmethod - def _download(self) -> List: + def _download(self) -> list: pass + class EODownloader(Downloader): """Downloader for SWOT Pixel Cloud files from hydroweb.next.""" + def __init__(self, *args, **kwargs): """Downloader for SWOT Pixel Cloud files from hydroweb.next initialization. @@ -220,7 +218,6 @@ def __init__(self, *args, **kwargs): """ super().__init__(*args, **kwargs) - def setup(self) -> None: self.dag = EODataAccessGateway() @@ -233,19 +230,18 @@ def setup(self) -> None: self.__check_collection_name() - def __check_collection_name(self) -> None: list_collections = [ d.id for d in self.dag.list_collections(provider=self.PROVIDER) ] if self.collection_name not in list_collections: - raise ValueError(( + raise ValueError( "Did not find collection_name in " f"list of available collections in {self.PROVIDER}." f"\nAvailable collections are: {list_collections}" - )) - + ) + def define_query(self) -> dict: # Default search criteria when iterating over collection pages default_search_criteria = { @@ -258,35 +254,33 @@ def define_query(self) -> dict: } if self.dates is not None: - self.query_args["start"] = \ - self.dates[0].strftime("%Y-%m-%dT%H:%M:%SZ") - self.query_args["end"] = \ - self.dates[1].strftime("%Y-%m-%dT%H:%M:%SZ") + self.query_args["start"] = self.dates[0].strftime("%Y-%m-%dT%H:%M:%SZ") + self.query_args["end"] = self.dates[1].strftime("%Y-%m-%dT%H:%M:%SZ") self.query_args.update(default_search_criteria) return self.query_args - def _search(self, geom:Optional[str] = None) -> None: + def _search(self, geom: str | None = None) -> None: if geom is not None: self.query_args["geom"] = geom self.search_results = self.dag.search_all(**self.query_args) # Iterate over all pages to find all products - #for page_results in self.dag.search_iter_page(**self.query_args): + # for page_results in self.dag.search_iter_page(**self.query_args): # self.search_results.extend(page_results) - def _download(self) -> List: - #donwload only .nc asset + def _download(self) -> list: + # donwload only .nc asset downloaded_paths = self.dag.download_all( - self.search_results, asset=r".*\.nc$", - output_dir=self.path_download + self.search_results, asset=r".*\.nc$", output_dir=self.path_download ) return downloaded_paths class DefaultDownloader(Downloader): """Downloader for SWOT Pixel Cloud files from hydroweb.next.""" + def __init__(self, *args, **kwargs): """Downloader for SWOT Pixel Cloud files from hydroweb.next initialization. @@ -306,38 +300,45 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def setup(self) -> None: - apikey = os.environ['HYDROWEB_API_KEY'] + apikey = os.environ["HYDROWEB_API_KEY"] self.client = py_hydroweb.Client(api_key=apikey) self.downloaded_paths = [] def define_query(self) -> dict: if self.dates is not None: - self.query_args["start_datetime"] = {'gte': self.dates[0].isoformat(timespec='milliseconds')+"Z"} - self.query_args["end_datetime"] = {'lte':self.dates[1].isoformat(timespec='milliseconds')+"Z"} + self.query_args["start_datetime"] = { + "gte": self.dates[0].isoformat(timespec="milliseconds") + "Z" + } + self.query_args["end_datetime"] = { + "lte": self.dates[1].isoformat(timespec="milliseconds") + "Z" + } return self.query_args - def _search(self, geom:Optional[str] = None) -> None: + def _search(self, geom: str | None = None) -> None: # This command actually downloads the matching products basket = py_hydroweb.DownloadBasket("pixcdust_basket") - - kwargs= {"collection_id": self.collection_name, "query": self.query_args,"folder": self.collection_name} + + kwargs = { + "collection_id": self.collection_name, + "query": self.query_args, + "folder": self.collection_name, + } if geom is not None: kwargs.update({"intersects": geom}) basket.add_collection(**kwargs) - self.download_id = self.client.submit_download(download_basket=basket) - - - def _download(self) -> List: - downloaded_zip_path = self.client.download_zip(download_id=self.download_id, output_folder=self.path_download) + def _download(self) -> list: + downloaded_zip_path = self.client.download_zip( + download_id=self.download_id, output_folder=self.path_download + ) with zipfile.ZipFile(downloaded_zip_path, "r") as zf: # Liste des chemins à extraire files = [name for name in zf.namelist() if name.lower().endswith(".nc")] - for member in tqdm.tqdm(files, desc = "Extracting zip"): + for member in tqdm.tqdm(files, desc="Extracting zip"): zf.extract(member, path=self.path_download) downloaded_path = os.path.join(self.path_download, member) self.downloaded_paths.append(downloaded_path) @@ -345,31 +346,29 @@ def _download(self) -> List: os.remove(downloaded_zip_path) self.client.delete_download(download_id=self.download_id) - - + return self.downloaded_paths -def PixCDownloader(*args, backend='default',**kwargs): +def PixCDownloader(*args, backend="default", **kwargs): """Downloader for SWOT Pixel Cloud files from hydroweb.next initialization. - Keyword Args: - geometry: A geometry used as search criteria. Defaults to None. - dates: Minimum and maximum dates to be used as search criteria. - Defaults to None. - path_download: - download path. Defaults to "/tmp/hydroweb_next". - verbose: Verbose level (0: nothing, 1: only progress bars, 2: INFO, 3: DEBUG). - Defaults to 0. - - Raises: - AttributeError: if the geometry is not one - of (str, tuple, list, geopandas.GeoDataFrame) + Keyword Args: + geometry: A geometry used as search criteria. Defaults to None. + dates: Minimum and maximum dates to be used as search criteria. + Defaults to None. + path_download: + download path. Defaults to "/tmp/hydroweb_next". + verbose: Verbose level (0: nothing, 1: only progress bars, 2: INFO, 3: DEBUG). + Defaults to 0. + + Raises: + AttributeError: if the geometry is not one + of (str, tuple, list, geopandas.GeoDataFrame) """ if backend == "eodag": - print(f'using backend: {backend}') - return EODownloader("SWOT_L2_HR_PIXC",*args, **kwargs) + print(f"using backend: {backend}") + return EODownloader("SWOT_L2_HR_PIXC", *args, **kwargs) else: - print('using default backend: py-hydroweb') - return DefaultDownloader("SWOT_L2_HR_PIXC",*args, **kwargs) - + print("using default backend: py-hydroweb") + return DefaultDownloader("SWOT_L2_HR_PIXC", *args, **kwargs) diff --git a/pixcdust/notebooks/convert/convert_to_zcollection.ipynb b/pixcdust/notebooks/convert/convert_to_zcollection.ipynb index c590c3d..9e35c80 100644 --- a/pixcdust/notebooks/convert/convert_to_zcollection.ipynb +++ b/pixcdust/notebooks/convert/convert_to_zcollection.ipynb @@ -6,8 +6,9 @@ "metadata": {}, "outputs": [], "source": [ - "from pixcdust.converters.zarr import Nc2ZarrConverter\n", - "from glob import glob" + "from glob import glob\n", + "\n", + "from pixcdust.converters.zarr import Nc2ZarrConverter" ] }, { @@ -53,10 +54,10 @@ ], "source": [ "pixc = Nc2ZarrConverter(\n", - " path_in = sorted(glob('/tmp/pixc/**/*.nc',recursive=True)),\n", - " variables=['height', 'sig0', 'classification'],\n", - " )\n", - "pixc.database_from_nc(path_out= \"/tmp/pixc_zarr\")" + " path_in=sorted(glob(\"/tmp/pixc/**/*.nc\", recursive=True)),\n", + " variables=[\"height\", \"sig0\", \"classification\"],\n", + ")\n", + "pixc.database_from_nc(path_out=\"/tmp/pixc_zarr\")" ] }, { @@ -72,13 +73,14 @@ "metadata": {}, "outputs": [], "source": [ - "from pixcdust.readers.zarr import ZarrReader\n", - "import datetime" + "import datetime\n", + "\n", + "from pixcdust.readers.zarr import ZarrReader" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -104,7 +106,16 @@ ], "source": [ "pixcr = ZarrReader(\"/tmp/pixc_zarr\")\n", - "pixcr.read((datetime.datetime(2023,4,6), datetime.datetime(2023,4,8)))\n", + "pixcr.read(\n", + " (\n", + " datetime.datetime(\n", + " 2023, 4, 6, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo\n", + " ),\n", + " datetime.datetime(\n", + " 2023, 4, 8, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo\n", + " ),\n", + " )\n", + ")\n", "pixc" ] }, diff --git a/pixcdust/notebooks/convert/dggs_tuto.ipynb b/pixcdust/notebooks/convert/dggs_tuto.ipynb index d8a38ba..f85730b 100644 --- a/pixcdust/notebooks/convert/dggs_tuto.ipynb +++ b/pixcdust/notebooks/convert/dggs_tuto.ipynb @@ -13,18 +13,19 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "5a3343de-dc8e-492b-b5e6-9f6cf87f55c8", "metadata": {}, "outputs": [], "source": [ "# imports\n", - "from pixcdust.downloaders.hydroweb_next import PixCDownloader\n", - "from pixcdust.readers import NcSimpleReader\n", + "import glob\n", + "from datetime import UTC, datetime\n", + "\n", "import geopandas as gpd\n", - "from datetime import datetime\n", - "import xarray as xr\n", - "import glob" + "\n", + "from pixcdust.downloaders.hydroweb_next import PixCDownloader\n", + "from pixcdust.readers import NcSimpleReader" ] }, { @@ -37,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "1bc7ccfd-8edb-4170-b933-c0fb33822521", "metadata": {}, "outputs": [], @@ -47,8 +48,8 @@ "\n", "# Limiting time period\n", "dates = (\n", - " datetime(2023,4,6),\n", - " datetime(2023,4,8),\n", + " datetime(2023, 4, 6, tzinfo=datetime.now(UTC).astimezone().tzinfo),\n", + " datetime(2023, 4, 8, tzinfo=datetime.now(UTC).astimezone().tzinfo),\n", ")" ] }, @@ -106,8 +107,8 @@ " gdf_geom,\n", " dates,\n", " verbose=1,\n", - " path_download='/tmp/pixc',\n", - " )\n", + " path_download=\"/tmp/pixc\",\n", + ")\n", "pixcdownloader.search_download()" ] }, @@ -141,7 +142,7 @@ ], "source": [ "# Search swot nc files\n", - "swot_nc_files = glob.glob('/tmp/pixc/**/*.nc',recursive=True)\n", + "swot_nc_files = glob.glob(\"/tmp/pixc/**/*.nc\", recursive=True)\n", "swot_nc_files" ] }, @@ -847,7 +848,7 @@ ], "source": [ "# Chose one or more variables\n", - "reader.data['height']" + "reader.data[\"height\"]" ] }, { @@ -1451,7 +1452,9 @@ ], "source": [ "# Reproject variables into h3 grid\n", - "ds_h3 = reader.to_h3(variables = 'height', resolution = 8) # Modify the resolution if needeed\n", + "ds_h3 = reader.to_h3(\n", + " variables=\"height\", resolution=8\n", + ") # Modify the resolution if needeed\n", "ds_h3" ] }, @@ -1479,7 +1482,7 @@ ], "source": [ "# Show\n", - "ds_h3['height'].dggs.explore()" + "ds_h3[\"height\"].dggs.explore()" ] }, { @@ -2082,7 +2085,9 @@ ], "source": [ "# Reproject variables into healpix grid\n", - "ds_healpix = reader.to_healpix(variables = 'height', resolution = 13) # modify resolution if needed\n", + "ds_healpix = reader.to_healpix(\n", + " variables=\"height\", resolution=13\n", + ") # modify resolution if needed\n", "ds_healpix" ] }, @@ -2110,7 +2115,7 @@ ], "source": [ "# Show\n", - "ds_healpix['height'].dggs.explore()" + "ds_healpix[\"height\"].dggs.explore()" ] }, { diff --git a/pixcdust/notebooks/convert/download_pixc_to_gpkg.ipynb b/pixcdust/notebooks/convert/download_pixc_to_gpkg.ipynb index e4d9361..e69255a 100644 --- a/pixcdust/notebooks/convert/download_pixc_to_gpkg.ipynb +++ b/pixcdust/notebooks/convert/download_pixc_to_gpkg.ipynb @@ -21,18 +21,20 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "from pixcdust.downloaders.hydroweb_next import PixCDownloader\n", + "from datetime import UTC, datetime\n", + "\n", "import geopandas as gpd\n", - "from datetime import datetime" + "\n", + "from pixcdust.downloaders.hydroweb_next import PixCDownloader" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -41,8 +43,8 @@ "\n", "# Limiting time period\n", "dates = (\n", - " datetime(2023,4,6),\n", - " datetime(2023,4,8),\n", + " datetime(2023, 4, 6, tzinfo=datetime.now(UTC).astimezone().tzinfo),\n", + " datetime(2023, 4, 8, tzinfo=datetime.now(UTC).astimezone().tzinfo),\n", ")" ] }, @@ -80,8 +82,8 @@ " gdf_geom,\n", " dates,\n", " verbose=1,\n", - " path_download='/tmp/pixc1',\n", - " )\n", + " path_download=\"/tmp/pixc1\",\n", + ")\n", "pixcdownloader.search_download()" ] }, @@ -169,8 +171,9 @@ "metadata": {}, "outputs": [], "source": [ - "from pixcdust.converters.gpkg import Nc2GpkgConverter\n", - "from glob import glob" + "from glob import glob\n", + "\n", + "from pixcdust.converters.gpkg import Nc2GpkgConverter" ] }, { @@ -216,16 +219,17 @@ ], "source": [ "# You can specify conditions on variables to filter data\n", - "conditions= {\"sig0\":{'operator': \"gt\", 'threshold': 20}, # sig0 > 20\n", - " \"classification\":{'operator': \"ge\", 'threshold': 3}, # classification >= 3\n", - " }\n", + "conditions = {\n", + " \"sig0\": {\"operator\": \"gt\", \"threshold\": 20}, # sig0 > 20\n", + " \"classification\": {\"operator\": \"ge\", \"threshold\": 3}, # classification >= 3\n", + "}\n", "\n", "pixc = Nc2GpkgConverter(\n", - " path_in = glob(pixcdownloader.path_download+'/*/*.nc'),\n", - " variables=['height', 'sig0', 'classification'],\n", - " area_of_interest=gdf_geom,\n", - " conditions=conditions\n", - " )\n", + " path_in=glob(pixcdownloader.path_download + \"/*/*.nc\"),\n", + " variables=[\"height\", \"sig0\", \"classification\"],\n", + " area_of_interest=gdf_geom,\n", + " conditions=conditions,\n", + ")\n", "pixc.database_from_nc(path_out=\"/tmp/pixc_gpkg.gpkg\")" ] }, @@ -867,10 +871,8 @@ "source": [ "from pixcdust.readers.gpkg import GpkgReader\n", "\n", - "# nb: you may specify \n", - "pixc_read = GpkgReader(\n", - " \"/tmp/pixc_gpkg.gpkg\"\n", - ")\n", + "# nb: you may specify\n", + "pixc_read = GpkgReader(\"/tmp/pixc_gpkg.gpkg\")\n", "pixc_read.read()\n", "pixc_read.data" ] @@ -897,7 +899,10 @@ ], "source": [ "from pixcdust.converters.gpkg import GpkgDGGSProjecter\n", - "h3_grid = GpkgDGGSProjecter(\"/tmp/pixc_gpkg.gpkg\", 10, path_out = '../data/h3_gpkg.gpkg', healpix=False) # True for healpix projection\n", + "\n", + "h3_grid = GpkgDGGSProjecter(\n", + " \"/tmp/pixc_gpkg.gpkg\", 10, path_out=\"../data/h3_gpkg.gpkg\", healpix=False\n", + ") # True for healpix projection\n", "h3_grid.compute_layers()" ] }, @@ -1521,9 +1526,7 @@ } ], "source": [ - "pixc_read = GpkgReader(\n", - " '../data/h3_gpkg.gpkg'\n", - ")\n", + "pixc_read = GpkgReader(\"../data/h3_gpkg.gpkg\")\n", "pixc_read.read()\n", "pixc_read.data" ] @@ -2704,30 +2707,30 @@ } ], "source": [ - "import folium\n", "import branca.colormap as cmp\n", + "import folium\n", "\n", - "layer_h3 = pixc_read.read_single_layer('20230407_483_16_78L_10__h3')\n", + "layer_h3 = pixc_read.read_single_layer(\"20230407_483_16_78L_10__h3\")\n", "\n", "# creating a colormap\n", "linear = cmp.LinearColormap(\n", - " ['blue', 'purple', 'orange', 'yellow'],\n", - " vmin=layer_h3['wse'].min(), # minimum wse value\n", - " vmax=layer_h3['wse'].max(), # maximum wse value\n", - " caption='Water Surface Elevation (m)' #Caption for Color scale or Legend\n", + " [\"blue\", \"purple\", \"orange\", \"yellow\"],\n", + " vmin=layer_h3[\"wse\"].min(), # minimum wse value\n", + " vmax=layer_h3[\"wse\"].max(), # maximum wse value\n", + " caption=\"Water Surface Elevation (m)\", # Caption for Color scale or Legend\n", ")\n", "# Initiating map\n", "m = folium.Map([43.6, 1.43], zoom_start=12, tiles=\"cartodbpositron\")\n", "\n", "folium.GeoJson(\n", " layer_h3,\n", - " style_function = lambda row: {\n", - " 'fillColor': linear(row['properties'][\"wse\"]),\n", - " 'weight': 0, #how thick the border has to be\n", - " 'fillOpacity': 1\n", + " style_function=lambda row: {\n", + " \"fillColor\": linear(row[\"properties\"][\"wse\"]),\n", + " \"weight\": 0, # how thick the border has to be\n", + " \"fillOpacity\": 1,\n", " },\n", ").add_to(m)\n", - "linear.add_to(m) #adds colorscale and legend\n", + "linear.add_to(m) # adds colorscale and legend\n", "m" ] }, diff --git a/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb b/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb index 74e8f96..f41cb0f 100644 --- a/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb +++ b/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb @@ -20,19 +20,20 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "import pixcdust\n", - "from pixcdust.downloaders.hydroweb_next import PixCDownloader\n", + "from datetime import UTC, datetime\n", + "\n", "import geopandas as gpd\n", - "from datetime import datetime" + "\n", + "from pixcdust.downloaders.hydroweb_next import PixCDownloader" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -40,8 +41,8 @@ "gdf_geom = gpd.read_file(\"../data/aoi.gpkg\")\n", "\n", "dates = (\n", - " datetime(2023,4,6),\n", - " datetime(2023,4,8),\n", + " datetime(2023, 4, 6, tzinfo=datetime.now(UTC).astimezone().tzinfo),\n", + " datetime(2023, 4, 8, tzinfo=datetime.now(UTC).astimezone().tzinfo),\n", ")" ] }, @@ -106,8 +107,8 @@ " gdf_geom,\n", " dates,\n", " verbose=1,\n", - " path_download='/tmp/pixc',\n", - " )\n", + " path_download=\"/tmp/pixc\",\n", + ")\n", "pixcdownloader.search_download()" ] }, @@ -127,8 +128,9 @@ "metadata": {}, "outputs": [], "source": [ - "from pixcdust.converters.zarr import Nc2ZarrConverter\n", - "from glob import glob" + "from glob import glob\n", + "\n", + "from pixcdust.converters.zarr import Nc2ZarrConverter" ] }, { @@ -138,17 +140,18 @@ "outputs": [], "source": [ "# You can specify conditions on variables to filter data\n", - "conditions= {\"sig0\":{'operator': \"gt\", 'threshold': 20}, # sig0 > 20\n", - " \"classification\":{'operator': \"ge\", 'threshold': 3}, # classification >= 3\n", - " }\n", + "conditions = {\n", + " \"sig0\": {\"operator\": \"gt\", \"threshold\": 20}, # sig0 > 20\n", + " \"classification\": {\"operator\": \"ge\", \"threshold\": 3}, # classification >= 3\n", + "}\n", "\n", "pixc = Nc2ZarrConverter(\n", - " path_in = glob(pixcdownloader.path_download+'/**/*.nc', recursive=True),\n", - " variables=['height', 'sig0', 'classification'],\n", - " area_of_interest=gdf_geom,\n", - " conditions=conditions,\n", - " )\n", - "pixc.database_from_nc(path_out='/tmp/pixc_zarr')" + " path_in=glob(pixcdownloader.path_download + \"/**/*.nc\", recursive=True),\n", + " variables=[\"height\", \"sig0\", \"classification\"],\n", + " area_of_interest=gdf_geom,\n", + " conditions=conditions,\n", + ")\n", + "pixc.database_from_nc(path_out=\"/tmp/pixc_zarr\")" ] }, { @@ -180,7 +183,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -1292,13 +1295,21 @@ } ], "source": [ - "from pixcdust.readers.zarr import ZarrReader\n", "import datetime\n", "\n", - "pixc_read = ZarrReader(\n", - " \"/tmp/pixc_zarr\"\n", + "from pixcdust.readers.zarr import ZarrReader\n", + "\n", + "pixc_read = ZarrReader(\"/tmp/pixc_zarr\")\n", + "pixc_read.read(\n", + " (\n", + " datetime.datetime(\n", + " 2023, 4, 6, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo\n", + " ),\n", + " datetime.datetime(\n", + " 2023, 4, 8, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo\n", + " ),\n", + " )\n", ")\n", - "pixc_read.read((datetime.datetime(2023,4,6), datetime.datetime(2023,4,8)))\n", "pixc_read.data" ] }, diff --git a/pixcdust/notebooks/read_and_use/read_netcdf.ipynb b/pixcdust/notebooks/read_and_use/read_netcdf.ipynb index f0379f6..3647e52 100644 --- a/pixcdust/notebooks/read_and_use/read_netcdf.ipynb +++ b/pixcdust/notebooks/read_and_use/read_netcdf.ipynb @@ -13,8 +13,9 @@ "metadata": {}, "outputs": [], "source": [ - "from pixcdust.readers.netcdf import NcSimpleReader\n", - "import glob" + "import glob\n", + "\n", + "from pixcdust.readers.netcdf import NcSimpleReader" ] }, { @@ -23,7 +24,7 @@ "metadata": {}, "outputs": [], "source": [ - "swot_nc_files = glob.glob('/tmp/pixc/**/*.nc',recursive=True)" + "swot_nc_files = glob.glob(\"/tmp/pixc/**/*.nc\", recursive=True)" ] }, { @@ -35,11 +36,14 @@ "path = swot_nc_files[0]\n", "\n", "# You can specify conditions on variables to filter data\n", - "conditions= {\"sig0\":{'operator': \"gt\", 'threshold': 20}, # sig0 > 20\n", - " \"classification\":{'operator': \"ge\", 'threshold': 3}, # classification >= 3\n", - " }\n", + "conditions = {\n", + " \"sig0\": {\"operator\": \"gt\", \"threshold\": 20}, # sig0 > 20\n", + " \"classification\": {\"operator\": \"ge\", \"threshold\": 3}, # classification >= 3\n", + "}\n", "\n", - "ncsimple = NcSimpleReader(path, variables=['height', 'sig0', 'classification'], conditions=conditions)" + "ncsimple = NcSimpleReader(\n", + " path, variables=[\"height\", \"sig0\", \"classification\"], conditions=conditions\n", + ")" ] }, { @@ -1318,7 +1322,7 @@ } ], "source": [ - "ncsimple.to_xarray() # same thing as ncsimple.data" + "ncsimple.to_xarray() # same thing as ncsimple.data" ] }, { diff --git a/pixcdust/readers/__init__.py b/pixcdust/readers/__init__.py index c869fa5..f16e27d 100644 --- a/pixcdust/readers/__init__.py +++ b/pixcdust/readers/__init__.py @@ -18,6 +18,12 @@ They support the Netcdf official format and converted Zarr or Geopackage database. """ -from pixcdust.readers.netcdf import NcSimpleReader from pixcdust.readers.gpkg import GpkgReader -from pixcdust.readers.zarr import ZarrReader \ No newline at end of file +from pixcdust.readers.netcdf import NcSimpleReader +from pixcdust.readers.zarr import ZarrReader + +__all__ = [ + "GpkgReader", + "NcSimpleReader", + "ZarrReader", +] diff --git a/pixcdust/readers/base_reader.py b/pixcdust/readers/base_reader.py index 203b7e5..6c1b87d 100644 --- a/pixcdust/readers/base_reader.py +++ b/pixcdust/readers/base_reader.py @@ -16,19 +16,20 @@ """Interface used by all Pixcdust Readers.""" import re -from typing import Optional, Iterable, Union, List +from collections.abc import Iterable from pathlib import Path -import xarray as xr -import pandas as pd -import geopandas as gpd +import geopandas as gpd +import pandas as pd +import xarray as xr -PIXC_DATE_RE=re.compile(r'_\d{8}T\d{6}_\d{8}T\d{6}_') +PIXC_DATE_RE = re.compile(r"_\d{8}T\d{6}_\d{8}T\d{6}_") """Regex patern used to extract the date (daystartThourstart_dayxendThoursend) from a pixc file name. """ -def sorted_by_date(file_list: Iterable[Union[str, Path]]) -> List[Union[str, Path]]: + +def sorted_by_date(file_list: Iterable[str | Path]) -> list[str | Path]: """Sort the filenames by date as some converters need monotonic dates. The date is parsed from the filename according to PIXC_DATE_RE. Args: @@ -37,12 +38,14 @@ def sorted_by_date(file_list: Iterable[Union[str, Path]]) -> List[Union[str, Pat Returns: Sorted file_list. """ - def file_name_to_date(file_name: Union[str, Path]): + + def file_name_to_date(file_name: str | Path): date_founds = PIXC_DATE_RE.findall(str(file_name)) if date_founds: return date_founds[-1] return file_name - return sorted(file_list, key = file_name_to_date) # sort by date + + return sorted(file_list, key=file_name_to_date) # sort by date class BaseReader: @@ -63,13 +66,16 @@ class BaseReader: "classification":{'operator': "ge", 'threshold': 3},\ } """ - MULTI_FILE_SUPPORT=False - def __init__(self, - path: str | Iterable[str] | Path | Iterable[Path], - variables: Optional[list[str]] = None, - area_of_interest: Optional[gpd.GeoDataFrame] = None, - conditions: Optional[dict[str, dict[str, Union[str, float]]]] = None, - ): + + MULTI_FILE_SUPPORT = False + + def __init__( + self, + path: str | Iterable[str] | Path | Iterable[Path], + variables: list[str] | None = None, + area_of_interest: gpd.GeoDataFrame | None = None, + conditions: dict[str, dict[str, str | float]] | None = None, + ): """Basic pixcdust database reader configuration. Args: @@ -83,21 +89,21 @@ def __init__(self, } """ if isinstance(path, str | Path): - self.path: str | Iterable[str] = str(path) + self.path: str | Iterable[str] = str(path) self.multi_file_db = False else: if not self.MULTI_FILE_SUPPORT: raise ValueError("This reader does not support opening multiple files.") self.multi_file_db = True # sort the filenames by date as some converters need monotonic dates. - self.path = [str(p) for p in sorted_by_date(path)] + self.path = [str(p) for p in sorted_by_date(path)] self.area_of_interest = area_of_interest - self._data: Optional[xr.Dataset] = None + self._data: xr.Dataset | None = None self.variables = variables self.conditions = conditions @property - def data(self) -> xr.Dataset: + def data(self) -> xr.Dataset: """Return an xarray.Dataset view from the database loaded. Equivalent to to_xarray. @@ -128,7 +134,6 @@ def to_dataframe(self) -> pd.DataFrame: """ return self.data.to_dataframe() - def to_geodataframe( self, ) -> gpd.GeoDataFrame: diff --git a/pixcdust/readers/gpkg.py b/pixcdust/readers/gpkg.py index e6c6b29..af5228a 100644 --- a/pixcdust/readers/gpkg.py +++ b/pixcdust/readers/gpkg.py @@ -16,13 +16,12 @@ """Converted Pixcdust GeoPackage Reader.""" from pathlib import Path -from typing import Optional, List -from tqdm import tqdm import fiona -import xarray as xr -import pandas as pd import geopandas as gpd +import pandas as pd +import xarray as xr +from tqdm import tqdm from pixcdust.readers.base_reader import BaseReader @@ -41,10 +40,9 @@ class GpkgReader(BaseReader): MULTI_FILE_SUPPORT: False, only support one file. """ - def __init__(self, - path: str | Path, - area_of_interest: Optional[gpd.GeoDataFrame] = None - ): + def __init__( + self, path: str | Path, area_of_interest: gpd.GeoDataFrame | None = None + ): """Gpkg pixcdust database reader configuration. Read the list of layers from path. @@ -53,17 +51,18 @@ def __init__(self, area_of_interest: Optionally only read points in area_of_interest. """ super().__init__(path, area_of_interest=area_of_interest) - self._gdf_data: Optional[gpd.GeoDataFrame] = None - self.layers: list[str] = fiona.listlayers(self.path) + self._gdf_data: gpd.GeoDataFrame | None = None + self.layers: list[str] = fiona.listlayers(self.path) @property - def data(self) -> xr.Dataset: + def data(self) -> xr.Dataset: return self._gdf_data.to_xarray() @data.setter def data(self, obj: xr.Dataset) -> None: - raise NotImplementedError("PixCGpkgReader internal data representation is a GeoDataFrame.") - + raise NotImplementedError( + "PixCGpkgReader internal data representation is a GeoDataFrame." + ) def to_geodataframe( self, @@ -99,7 +98,7 @@ def read_single_layer(self, layer: str) -> gpd.GeoDataFrame: return layer_data - def read(self, layers: Optional[List[str]] = None) -> None: + def read(self, layers: list[str] | None = None) -> None: """Load all layers, or subset of layers, from geopackage database. You can then access from data or with methods like to_xarray, to_dataframe or to_geodataframe. @@ -114,7 +113,6 @@ def read(self, layers: Optional[List[str]] = None) -> None: layers = self.layers for layer in tqdm(layers): - layer_data = self.read_single_layer( layer, ) diff --git a/pixcdust/readers/netcdf.py b/pixcdust/readers/netcdf.py index d5a58ae..59bf527 100644 --- a/pixcdust/readers/netcdf.py +++ b/pixcdust/readers/netcdf.py @@ -15,20 +15,19 @@ # """Pre-conversion SWOT Pixel Cloud Netcdf reader.""" +import operator +from collections.abc import Iterable from dataclasses import dataclass, field -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path -from typing import Tuple, Optional, Iterable, Union +import dask.array as da +import geopandas import numpy as np -import xvec # noqa # pylint: disable=unused-import # xvec provide xvec accessor to xarray. - import xarray as xr -import geopandas -import operator -import dask.array as da +import xvec # noqa # pylint: disable=unused-import from pixcdust.dggs.dggs_converter import prepare_dataset_h3, prepare_dataset_healpix from pixcdust.readers.base_reader import BaseReader @@ -56,11 +55,9 @@ class NcSimpleConstants: @dataclass class NcFormatCfg: - """Class configuring how a SWOT pixel cloud files is expected to be structured. - """ - constants: NcSimpleConstants = field( - default_factory=NcSimpleConstants - ) + """Class configuring how a SWOT pixel cloud files is expected to be structured.""" + + constants: NcSimpleConstants = field(default_factory=NcSimpleConstants) trusted_group: str = "pixel_cloud" forbidden_variables: list[str] = field( default_factory=lambda: [ @@ -90,15 +87,17 @@ class NcSimpleReader(BaseReader): "classification":{'operator': "ge", 'threshold': 3},\ } """ + MULTI_FILE_SUPPORT = True - def __init__(self, - path: str | Iterable[str] | Path | Iterable[Path], - variables: Optional[list[str]] = None, - area_of_interest: Optional[geopandas.GeoDataFrame] = None, - format_cfg : Optional[NcFormatCfg] = None, - conditions: Optional[dict[str, dict[str, Union[str, float]]]] = None, - ): + def __init__( + self, + path: str | Iterable[str] | Path | Iterable[Path], + variables: list[str] | None = None, + area_of_interest: geopandas.GeoDataFrame | None = None, + format_cfg: NcFormatCfg | None = None, + conditions: dict[str, dict[str, str | float]] | None = None, + ): """Netcdf pixcdust reader configuration. Args: @@ -122,7 +121,9 @@ def __init__(self, self.conditions = conditions @staticmethod - def extract_info_from_nc_attrs(filename: str) -> Tuple[str, datetime, int, int, int, str]: + def extract_info_from_nc_attrs( + filename: str, + ) -> tuple[str, datetime, int, int, int, str]: """Extracts orbit information from global attributes\ in a SWOT pixel cloud netcdf. @@ -145,9 +146,11 @@ def extract_info_from_nc_attrs(filename: str) -> Tuple[str, datetime, int, int, pass_number = np.uint16(ds_glob.attrs[cst.default_pass_num_name]) cycle_number = np.uint16(ds_glob.attrs[cst.default_cyc_num_name]) time_granule_start = ds_glob.attrs[cst.default_time_start_name] - dt_time_start = datetime.strptime( - time_granule_start, cst.default_time_format_attrs - ).replace(microsecond=0) + dt_time_start = ( + datetime.strptime(time_granule_start, cst.default_time_format_attrs) + .replace(microsecond=0) + .astimezone(UTC) + ) return ( time_granule_start, @@ -166,38 +169,44 @@ def filter_variable(self) -> None: ValueError: If 'operator' or 'threshold' keys are not in conditions. AttributeError: If operator is not the function name of the operator module. """ - _k_operator = 'operator' - _k_to = 'threshold' + _k_operator = "operator" + _k_to = "threshold" # Loop through each condition and apply the filter - for var, condition in self.conditions.items(): - if var not in self.data.variables: - raise IOError( - f"Variable '{var}' not found in dataset variables (available: {list(self.data.variables)})" + if self.conditions: + for var, condition in self.conditions.items(): + if var not in self.data.variables: + raise OSError( + f"Variable '{var}' not found in dataset variables (available: {list(self.data.variables)})" + ) + + # Ensure the condition dictionary has the correct keys + if _k_operator not in condition or _k_to not in condition: + raise ValueError( + f"Condition for variable '{var}' must include '{_k_operator}' and '{_k_to}'" + ) + + # Get the operator function dynamically from the operator module + try: + operator_func = getattr(operator, condition[_k_operator]) + except AttributeError: + raise AttributeError( + f"Operator '{condition[_k_operator]}' is not a valid operator in the operator module" + ) + + threshold = condition[_k_to] + + # Compute the boolean condition if it's a Dask array + if isinstance(self.data[var].data, da.Array): + self.data[var] = self.data[var].compute() + + # Apply the filter using .where() on the dataset + self.data = self.data.where( + operator_func(self.data[var], threshold), drop=True ) - # Ensure the condition dictionary has the correct keys - if _k_operator not in condition or _k_to not in condition: - raise ValueError(f"Condition for variable '{var}' must include '{_k_operator}' and '{_k_to}'") - - # Get the operator function dynamically from the operator module - try: - operator_func = getattr(operator, condition[_k_operator]) - except AttributeError: - raise AttributeError( - f"Operator '{condition[_k_operator]}' is not a valid operator in the operator module") - - threshold = condition[_k_to] - - # Compute the boolean condition if it's a Dask array - if isinstance(self.data[var].data, da.Array): - self.data[var] = self.data[var].compute() - - # Apply the filter using .where() on the dataset - self.data = self.data.where(operator_func(self.data[var], threshold), drop=True) - def read(self, orbit_info: bool = False) -> None: - """ Load self.path file(s). + """Load self.path file(s). You can then access from data or with methods like to_xarray, to_dataframe or to_geodataframe. @@ -212,7 +221,7 @@ def read(self, orbit_info: bool = False) -> None: return self.open_dataset() def open_dataset(self) -> None: - """ Load the self.path file (need only one file in self.path). + """Load the self.path file (need only one file in self.path). You can then access from data or with methods like to_xarray, to_dataframe or to_geodataframe. """ @@ -230,10 +239,10 @@ def open_dataset(self) -> None: self.__postprocess_points() def open_mfdataset( - self, - orbit_info: bool = False, + self, + orbit_info: bool = False, ) -> None: - """ Load self.path file(s) as a nested array. + """Load self.path file(s) as a nested array. You can then access from data or with methods like to_xarray, to_dataframe or to_geodataframe. @@ -267,7 +276,7 @@ def open_mfdataset( if self.variables: # check if variables in forbidden variables before loading if len(set(self.variables).intersection(set(self.forbidden_variables))) > 0: - raise IOError( + raise OSError( f"variables from {self.forbidden_variables} \ cannot be extracted" ) @@ -288,11 +297,13 @@ def open_mfdataset( self.__postprocess_points() - def to_h3(self, - variables: str | list[str] | None=None, - resolution: int = 8, - interp: bool=False, - method: str = 'linear') -> xr.Dataset: + def to_h3( + self, + variables: str | list[str] | None = None, + resolution: int = 8, + interp: bool = False, + method: str = "linear", + ) -> xr.Dataset: """ Convert a Dataset with latitude and longitude coordinates into an H3-indexed grid. @@ -311,12 +322,17 @@ def to_h3(self, data = self.to_xarray()[variables] else: data = self.to_xarray() - return prepare_dataset_h3(data, resolution=resolution, interp=interp, method=method) + return prepare_dataset_h3( + data, resolution=resolution, interp=interp, method=method + ) - def to_healpix(self, variables: str | list[str] | None=None, - resolution: int = 8, - interp: bool= False, - method: str = 'linear') -> xr.Dataset: + def to_healpix( + self, + variables: str | list[str] | None = None, + resolution: int = 8, + interp: bool = False, + method: str = "linear", + ) -> xr.Dataset: """ Convert a Dataset with latitude and longitude coordinates into an HEALPix-indexed grid. @@ -335,7 +351,9 @@ def to_healpix(self, variables: str | list[str] | None=None, data = self.to_xarray()[variables] else: data = self.to_xarray() - return prepare_dataset_healpix(data, resolution=resolution, interp=interp, method=method) + return prepare_dataset_healpix( + data, resolution=resolution, interp=interp, method=method + ) def __postprocess_points(self) -> None: """Adds a points coordinates containing shapely.Points (longitude, latitude) diff --git a/pixcdust/readers/zarr.py b/pixcdust/readers/zarr.py index 3172d9e..ee8fd66 100644 --- a/pixcdust/readers/zarr.py +++ b/pixcdust/readers/zarr.py @@ -15,9 +15,8 @@ # """Converted Pixcdust zarr database Reader.""" -from typing import Optional, Tuple - import datetime + import xarray as xr import zcollection @@ -38,13 +37,10 @@ class ZarrReader(BaseReader): MULTI_FILE_SUPPORT: False, only support one file. """ - def read( self, - date_interval: Optional[ - Tuple[datetime.datetime, datetime.datetime] - ] | None = None, - ) -> None: + date_interval: tuple[datetime.datetime, datetime.datetime] | None = None, + ) -> None: """Load a zarr database. You can then access from data or with methods like to_xarray, to_dataframe or to_geodataframe. @@ -56,17 +52,27 @@ def read( collection = zcollection.open_collection( self.path, - mode='r', + mode="r", ) if date_interval: date_min = date_interval[0] date_max = date_interval[1] + # LOCALE timezone data_z = collection.load( - filters=lambda keys: date_min <= datetime.datetime( - keys['year'], keys['month'], keys['day'], - keys['hour'], keys['minute'], keys['second'], - ) <= date_max + filters=lambda keys: ( + date_min + <= datetime.datetime( + keys["year"], + keys["month"], + keys["day"], + keys["hour"], + keys["minute"], + keys["second"], + tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo, + ) + <= date_max + ) ) else: data_z = collection.load() diff --git a/pixcdust/tests/conftest.py b/pixcdust/tests/conftest.py index 9d53fee..fe36819 100644 --- a/pixcdust/tests/conftest.py +++ b/pixcdust/tests/conftest.py @@ -1,50 +1,78 @@ -import os from pathlib import Path -from typing import List import pytest from pixcdust.tests.init_tests import JsonTestsSettings, init_hydroweb_env +MOCK_DATA_DIR = Path(__file__).parent / "mock_data" +SAMPLE_NC = MOCK_DATA_DIR / "swot_pixc.nc" + def pytest_addoption(parser): - parser.addoption("--dl", action="store_true", default=False, help="run dowloaders tests") + parser.addoption( + "--dl", action="store_true", default=False, help="run dowloaders tests" + ) + parser.addoption( + "--realdata", + action="store_true", + default=False, + help="run tests on real SWOT data (requires API key)", + ) def pytest_configure(config): config.addinivalue_line("markers", "downloader: mark test as testing downloads") + config.addinivalue_line( + "markers", "realdata: mark test as requireing real SWOT data" + ) def pytest_collection_modifyitems(config, items): - if config.getoption("--dl"): - # option given: do not skip tests - return skip_dl = pytest.mark.skip(reason="need --dl option to run") + skip_realdata = pytest.mark.skip( + reason="need --realdata option and a configured input_folder" + ) for item in items: - if "downloader" in item.keywords: + if "downloader" in item.keywords and not config.getoption("--dl"): item.add_marker(skip_dl) + if "realdata" in item.keywords and not config.getoption("--realdata"): + item.add_marker(skip_realdata) @pytest.fixture(scope="session") def tests_settings() -> JsonTestsSettings: return JsonTestsSettings() + +@pytest.fixture(scope="session") +def sample_nc() -> Path: + assert SAMPLE_NC.is_file() + print(SAMPLE_NC) + return SAMPLE_NC + + @pytest.fixture(scope="session") def input_folder(tests_settings) -> Path: - return tests_settings.input_folder + try: + return tests_settings.input_folder + except KeyError: + return MOCK_DATA_DIR + @pytest.fixture(scope="session") -def input_files(input_folder) -> List[Path]: - return list(input_folder.glob("**/*nc")) +def input_files(input_folder) -> list[Path]: + return list(input_folder.glob("**/*nc")) + @pytest.fixture(scope="session") -def first_file(input_folder) -> Path: - return next(input_folder.glob("**/*_20240803T*nc")) +def first_file(input_folder, sample_nc) -> Path: + return next(iter(input_folder.glob("**/*_20240803T*nc")), sample_nc) + @pytest.fixture(scope="session") -def tmp_folder(tests_settings) -> Path: - tests_settings.tmp_folder.mkdir(exist_ok=True) - return tests_settings.tmp_folder +def tmp_folder(tmp_path_factory) -> Path: + return tmp_path_factory.mktemp("pixcdust-test") + @pytest.fixture() def hydroweb_env(tests_settings) -> None: diff --git a/pixcdust/tests/init_tests.py b/pixcdust/tests/init_tests.py index 2a34b9a..dfd2e44 100644 --- a/pixcdust/tests/init_tests.py +++ b/pixcdust/tests/init_tests.py @@ -1,50 +1,48 @@ import argparse -import os -from typing import Union - -from datetime import datetime import json +import os +from datetime import UTC, datetime from pathlib import Path + from pixcdust.downloaders.hydroweb_next import PixCDownloader + class JsonTestsSettings: - """ Reader-writer for the test configuration. - """ - CONFIG_FILE_NAME = 'conftest.json' + """Reader-writer for the test configuration.""" + + CONFIG_FILE_NAME = "conftest.json" + def __init__(self): try: with open(self._config_path) as f: self._settings = json.load(f) - except FileNotFoundError : + except FileNotFoundError: self._settings = {} @property def input_folder(self) -> Path: - """ Path to folder where test input data are downloaded and stored. - """ + """Path to folder where test input data are downloaded and stored.""" try: return Path(self._settings["input_folder"]) except KeyError: raise KeyError("Test input folder not set. Configure it with init_tests.py") @input_folder.setter - def input_folder(self, value: Union[Path, str]) -> None: + def input_folder(self, value: Path | str) -> None: self._settings["input_folder"] = str(value) @property def tmp_folder(self) -> Path: - """ Path to folder where test outputs data are written. - """ + """Path to folder where test outputs data are written.""" return Path(self._settings.get("tmp_folder", "/tmp/pixcdust-test")) @tmp_folder.setter - def tmp_folder(self, value: Union[Path, str]) -> None: + def tmp_folder(self, value: Path | str) -> None: self._settings["tmp_folder"] = str(value) @property def hydroweb_auth(self) -> str: - """Hydroweb.next personal API key. - """ + """Hydroweb.next personal API key.""" return self._settings.get("hydroweb_auth", "") @hydroweb_auth.setter @@ -52,11 +50,10 @@ def hydroweb_auth(self, value: str) -> None: self._settings["hydroweb_auth"] = value def write(self) -> None: - """Write the config in JSON to self._config_path. - """ - with open(self._config_path, mode='w') as f: + """Write the config in JSON to self._config_path.""" + with open(self._config_path, mode="w") as f: json.dump(self._settings, f) - os.chmod(self._config_path,0o600) + os.chmod(self._config_path, 0o600) @property def _config_path(self) -> Path: @@ -64,7 +61,8 @@ def _config_path(self) -> Path: Should be the absolute path of tests/conftest.json """ - return Path(__file__).parent.absolute()/self.CONFIG_FILE_NAME + return Path(__file__).parent.absolute() / self.CONFIG_FILE_NAME + def init_hydroweb_env(test_settings: JsonTestsSettings) -> None: """Configure the Hydroweb.next API key of the current environment. @@ -85,21 +83,21 @@ def download_test_data(path_download: Path, backend: str) -> None: path_download: where to store the test data. """ dates = ( - datetime(2024,8,1), - datetime(2024,8,15), + # LOCALE timezone + datetime(2024, 8, 1, tzinfo=datetime.now(UTC).astimezone().tzinfo), + datetime(2024, 8, 15, tzinfo=datetime.now(UTC).astimezone().tzinfo), ) geometry = "POLYGON((-1.50580 43.39543,-1.36597 43.39543,-1.36597 43.56471,-1.50580 43.56471,-1.50580 43.39543))" pixcdownloader = PixCDownloader( - geometry, - dates, - backend=backend, - verbose=0, - path_download=str(path_download) - ) + geometry, dates, backend=backend, verbose=0, path_download=str(path_download) + ) pixcdownloader.search_download() + TEST_DATA_COUNT = 2 + + def check_test_data(path_download: Path) -> bool: data_list = list(path_download.glob("**/*.nc")) return len(data_list) == TEST_DATA_COUNT @@ -107,27 +105,38 @@ def check_test_data(path_download: Path) -> bool: if __name__ == "__main__": parser = argparse.ArgumentParser( - prog='init_tests', - description='Configure the tests and if missing download the test data' + prog="init_tests", + description="Configure the tests and if missing download the test data", + ) + parser.add_argument( + "-I", "--input_folder", help="path where is downloaded the test input data" + ) + parser.add_argument( + "-T", "--tmp_folder", help="path where is writen the temporary data" + ) + parser.add_argument( + "-H", "--hydroweb_auth", help="api key to download from hydroweb" + ) + parser.add_argument( + "-D", + "--download", + help="force download of the test data", + nargs="?", + const="true", ) - parser.add_argument("-I","--input_folder", help="path where is downloaded the test input data") - parser.add_argument("-T","--tmp_folder", help="path where is writen the temporary data") - parser.add_argument("-H","--hydroweb_auth", help="api key to download from hydroweb") - parser.add_argument("-D","--download", help="force download of the test data", nargs='?', const='true') args = parser.parse_args() settings = JsonTestsSettings() - dl_cfg_changed=False + dl_cfg_changed = False if args.input_folder: - dl_cfg_changed=True + dl_cfg_changed = True settings.input_folder = args.input_folder if args.tmp_folder: settings.tmp_folder = args.tmp_folder if args.hydroweb_auth: - dl_cfg_changed=True + dl_cfg_changed = True settings.hydroweb_auth = args.hydroweb_auth - settings.write() path_download = settings.input_folder @@ -135,9 +144,8 @@ def check_test_data(path_download: Path) -> bool: if args.download is None: if not check_test_data(path_download): # config changed and the data is missing. - download_test_data(path_download,'default') + download_test_data(path_download, "default") else: if args.download.lower() == "true": # download requested by user - download_test_data(path_download,'default') - + download_test_data(path_download, "default") diff --git a/pixcdust/tests/mock.py b/pixcdust/tests/mock.py index d191ab0..a52ea23 100644 --- a/pixcdust/tests/mock.py +++ b/pixcdust/tests/mock.py @@ -1,8 +1,8 @@ -from datetime import datetime -import numpy as np +from datetime import UTC, datetime -import xarray as xr import geopandas as gpd +import numpy as np +import xarray as xr from pixcdust.readers.netcdf import NcSimpleConstants @@ -42,7 +42,10 @@ def mock_xarray(length: int = 10000) -> xr.Dataset: # mocking data x = coords[cst.default_lat_name] cst_time_array = np.ones(len(x)).astype(datetime) - cst_time_array[:] = datetime(2024, 6, 5, 11, 40, 12) + # LOCALE timezone + cst_time_array[:] = datetime( + 2024, 6, 5, 11, 40, 12, tzinfo=datetime.now(UTC).astimezone().tzinfo + ) data_vars = { "height": (dims, np.sin(x) + np.random.normal(scale=10, size=len(x))), @@ -79,4 +82,4 @@ def mock_xarray(length: int = 10000) -> xr.Dataset: def mock_area_of_interest() -> gpd.GeoDataFrame: - raise NotImplementedError \ No newline at end of file + raise NotImplementedError diff --git a/pixcdust/tests/test_converters.py b/pixcdust/tests/test_converters.py index 5633df7..95211f1 100644 --- a/pixcdust/tests/test_converters.py +++ b/pixcdust/tests/test_converters.py @@ -1,36 +1,43 @@ import random -from pathlib import PosixPath, Path -from typing import List, Union +from pathlib import Path import fiona -import numpy as np import geopandas as gpd +import numpy as np import pytest -from shapely.geometry import Polygon import xarray as xr +from shapely.geometry import Polygon -from pixcdust.converters.gpkg import Nc2GpkgConverter, GpkgDGGSProjecter +from pixcdust.converters.gpkg import GpkgDGGSProjecter, Nc2GpkgConverter from pixcdust.converters.shapefile import Nc2ShpConverter from pixcdust.converters.zarr import Nc2ZarrConverter from pixcdust.readers import GpkgReader -from pixcdust.readers.zarr import ZarrReader from pixcdust.readers.netcdf import NcSimpleReader +from pixcdust.readers.zarr import ZarrReader LIM_AREA_POL = Polygon( - [(-1.50580, 43.39543), (-1.36597, 43.39543), (-1.36597, 43.56471), (-1.50580, 43.56471), (-1.50580, 43.39543)]) -LIM_AREA_GEOM = gpd.GeoDataFrame(index=[0], crs='epsg:4326', geometry=[LIM_AREA_POL]) + [ + (-1.50580, 43.39543), + (-1.36597, 43.39543), + (-1.36597, 43.56471), + (-1.50580, 43.56471), + (-1.50580, 43.39543), + ] +) +LIM_AREA_GEOM = gpd.GeoDataFrame(index=[0], crs="epsg:4326", geometry=[LIM_AREA_POL]) """Geometry used as area of interest of limited area tests.""" def test_nc_simple_reader_conditions(input_files): """Test NcSimpleReader with conditions on variables.""" # Define conditions - conditions = {"classification": {'operator': "ge", 'threshold': 4}, # classification >= 4 - "classification": {'operator': "le", 'threshold': 3}, # classification <= 3 - "sig0": {'operator': "gt", 'threshold': 15} # sig0 > 15 - } + conditions = { + "height": {"operator": "ge", "threshold": 4}, # height >= 4 + "classification": {"operator": "le", "threshold": 3}, # classification <= 3 + "sig0": {"operator": "gt", "threshold": 15}, # sig0 > 15 + } - converted_vars = ['height', 'sig0', 'classification'] + converted_vars = ["height", "sig0", "classification"] # Instantiate the NcSimpleReader with conditions reader = NcSimpleReader( @@ -46,18 +53,19 @@ def test_nc_simple_reader_conditions(input_files): for var, condition in conditions.items(): op = condition.get("operator") val = condition.get("threshold") - if op == 'ge': + if op == "ge": assert (reader.data[var] >= val).all(), f"{var} not >= {val}" - elif op == 'le': + elif op == "le": assert (reader.data[var] <= val).all(), f"{var} not <= {val}" - elif op == 'gt': + elif op == "gt": assert (reader.data[var] > val).all(), f"{var} not > {val}" - elif op == 'lt': + elif op == "lt": assert (reader.data[var] < val).all(), f"{var} not < {val}" -def validate_conversion_to_nc(read_data: xr.Dataset, converted_vars:List[str], first_file: Union[str, Path])\ - -> None: +def validate_conversion_to_nc( + read_data: xr.Dataset, converted_vars: list[str], first_file: str | Path +) -> None: """Compare the start of a converted database to the first original netcdf file. Args: @@ -69,15 +77,16 @@ def validate_conversion_to_nc(read_data: xr.Dataset, converted_vars:List[str], f """ ncsimple = NcSimpleReader(str(first_file)) ncsimple.open_dataset() - validate_conversion(read_data, converted_vars, ncsimple.data,is_longer=True) + validate_conversion(read_data, converted_vars, ncsimple.data, is_longer=True) + def validate_conversion( - read_data: xr.Dataset, - converted_vars:List[str], - expected_data: xr.Dataset, - is_longer: bool, - len_tol: int = 0, - sort_var: bool = False + read_data: xr.Dataset, + converted_vars: list[str], + expected_data: xr.Dataset, + is_longer: bool, + len_tol: int = 0, + sort_var: bool = False, ) -> None: """Compare the read data to the expected data. @@ -108,27 +117,33 @@ def validate_conversion( else: last = len(read_var) - np.testing.assert_allclose(read_var[last-30:last-1], expected_var[expected_last-30:expected_last-1]) - r = random.randrange(30,last) + np.testing.assert_allclose( + read_var[last - 30 : last - 1], + expected_var[expected_last - 30 : expected_last - 1], + ) + r = random.randrange(30, last) if len_tol == 0: - np.testing.assert_allclose(read_var[r-30:r-1], expected_var[r-30:r-1]) + np.testing.assert_allclose( + read_var[r - 30 : r - 1], expected_var[r - 30 : r - 1] + ) if is_longer: assert len(read_var) > expected_last else: assert expected_last + len_tol >= len(read_var) >= expected_last - len_tol +@pytest.mark.realdata def test_convert_zarr_full_area(input_files, first_file, tmp_folder): """Test zarr conversion without area_of_interest. It is compared to the input data. """ # Conversion - output = str(tmp_folder / "zarr_conv_test_full") - converted_vars = ['height', 'sig0', 'classification'] + output = str(tmp_folder / "zarr_conv_test_full") + converted_vars = ["height", "sig0", "classification"] pixc = Nc2ZarrConverter( - input_files, - variables=converted_vars, + input_files, + variables=converted_vars, ) pixc.database_from_nc(output, mode="o") @@ -137,19 +152,23 @@ def test_convert_zarr_full_area(input_files, first_file, tmp_folder): pixc_read.read() validate_conversion_to_nc(pixc_read.data, converted_vars, first_file) + @pytest.fixture(scope="session") def converted_lim_gpkg(input_files, tmp_folder): output_gpkg = str(tmp_folder / "gpkg_conv_test_lim") - converted_vars = ['height', 'sig0', 'classification'] + converted_vars = ["height", "sig0", "classification"] Nc2GpkgConverter( - input_files, - variables=converted_vars, - area_of_interest=LIM_AREA_GEOM, + input_files, + variables=converted_vars, + area_of_interest=LIM_AREA_GEOM, ).database_from_nc(output_gpkg, mode="o") return output_gpkg -def test_convert_gpkg_and_zarr_limited_area(input_files, first_file, tmp_folder, converted_lim_gpkg): +@pytest.mark.realdata +def test_convert_gpkg_and_zarr_limited_area( + input_files, first_file, tmp_folder, converted_lim_gpkg +): """Test geopackage and zarr conversion with area_of_interest. They are compared to each other. @@ -157,13 +176,13 @@ def test_convert_gpkg_and_zarr_limited_area(input_files, first_file, tmp_folder, """ # Conversion output_zarr = str(tmp_folder / "zarr_conv_test_lim") - converted_vars = ['height', 'sig0', 'classification'] + converted_vars = ["height", "sig0", "classification"] output_gpkg = converted_lim_gpkg Nc2ZarrConverter( - input_files, - variables=converted_vars, - area_of_interest=LIM_AREA_GEOM, + input_files, + variables=converted_vars, + area_of_interest=LIM_AREA_GEOM, ).database_from_nc(output_zarr, mode="o") # Validation @@ -171,20 +190,27 @@ def test_convert_gpkg_and_zarr_limited_area(input_files, first_file, tmp_folder, gpkg_read.read() zarr_read = ZarrReader(output_zarr) zarr_read.read() - validate_conversion(gpkg_read.data, converted_vars, zarr_read.data, is_longer=False, len_tol=2, sort_var="longitude") + validate_conversion( + gpkg_read.data, + converted_vars, + zarr_read.data, + is_longer=False, + len_tol=2, + sort_var="longitude", + ) +@pytest.mark.realdata def test_convert_shape_limited_area(input_files, first_file, tmp_folder): - """Test shapefile conversion with area_of_interest. - """ + """Test shapefile conversion with area_of_interest.""" # Conversion - output = str(tmp_folder / "shp_conv_test_full") - converted_vars = ['height', 'sig0', 'classification'] + output = str(tmp_folder / "shp_conv_test_full") + converted_vars = ["height", "sig0", "classification"] pixc = Nc2ShpConverter( - input_files, - variables=converted_vars, - area_of_interest=LIM_AREA_GEOM, + input_files, + variables=converted_vars, + area_of_interest=LIM_AREA_GEOM, ) pixc.database_from_nc(output, mode="o") @@ -197,6 +223,7 @@ def test_convert_shape_limited_area(input_files, first_file, tmp_folder): # Test for GpkgDGGSProjecter +@pytest.mark.realdata def test_gpkg_dggs_projecter(converted_lim_gpkg, tmp_folder): """Test the GpkgDGGSProjecter class by converting a sample Gpkg to a DGGS projection.""" @@ -204,7 +231,7 @@ def test_gpkg_dggs_projecter(converted_lim_gpkg, tmp_folder): # Define parameters dggs_res = 10 healpix = False - dggs_layer_pattern = '_h3' + dggs_layer_pattern = "_h3" output_path = str(tmp_folder / "gpkg_dggs_output") # Create an instance of GpkgDGGSProjecter @@ -213,7 +240,7 @@ def test_gpkg_dggs_projecter(converted_lim_gpkg, tmp_folder): dggs_res=dggs_res, healpix=healpix, dggs_layer_pattern=dggs_layer_pattern, - path_out=output_path + path_out=output_path, ) # Validate initialization diff --git a/pixcdust/tests/test_converters_mock.py b/pixcdust/tests/test_converters_mock.py index 64bba3d..2c54310 100644 --- a/pixcdust/tests/test_converters_mock.py +++ b/pixcdust/tests/test_converters_mock.py @@ -1,25 +1,20 @@ import unittest -from pixcdust.tests.mock import mock_xarray from pixcdust.converters.gpkg import Nc2GpkgConverter from pixcdust.converters.zarr import Nc2ZarrConverter +from pixcdust.tests.mock import mock_xarray class TestConverters(unittest.TestCase): - """Class for testing Converters, to be implemented - """ + """Class for testing Converters, to be implemented""" def setUp(self): - """function to set up the test environment - """ - self.list_vars = [ - "height", "sig0", "classification", "geoid", "cross_track" - ] + """function to set up the test environment""" + self.list_vars = ["height", "sig0", "classification", "geoid", "cross_track"] self.data = mock_xarray() def test_convert_ds_to_gpkg(self): - """function for testing the conversion to geopackage - """ + """function for testing the conversion to geopackage""" pixc = Nc2GpkgConverter( "/tmp", variables=self.list_vars, @@ -31,7 +26,7 @@ def test_convert_ds_to_gpkg(self): # TODO: add relevant tests def test_convert_ds_to_zarr(self): - """function for testing the conversion from + """function for testing the conversion from netcdf to zarr with zcollection """ diff --git a/pixcdust/tests/test_dggs.py b/pixcdust/tests/test_dggs.py index d45141c..56df9c0 100644 --- a/pixcdust/tests/test_dggs.py +++ b/pixcdust/tests/test_dggs.py @@ -1,7 +1,10 @@ +import pytest import xarray as xr + from pixcdust.readers import NcSimpleReader +@pytest.mark.realdata def test_h3_conversion(first_file): """ Test the conversion of the dataset to the H3 grid. @@ -12,18 +15,19 @@ def test_h3_conversion(first_file): reader = NcSimpleReader(first_file) reader.read() # Run the H3 conversion function - ds_h3 = reader.to_h3(variables='height', resolution=resolution) + ds_h3 = reader.to_h3(variables="height", resolution=resolution) # Assertions to verify correct output assert isinstance(ds_h3, xr.Dataset), "The result should be an xarray Dataset" # Verify that H3 cell IDs exist and have expected properties - assert 'cell_ids' in ds_h3.coords, "H3 cell IDs should be present in the output" + assert "cell_ids" in ds_h3.coords, "H3 cell IDs should be present in the output" # Ensure the data is not empty after conversion - assert len(ds_h3['cell_ids']) > 0, "The output dataset should have H3 cell IDs" + assert len(ds_h3["cell_ids"]) > 0, "The output dataset should have H3 cell IDs" +@pytest.mark.realdata def test_healpix_conversion(first_file): """ Test the conversion of the dataset to the HEALPix grid. @@ -33,13 +37,15 @@ def test_healpix_conversion(first_file): reader = NcSimpleReader(first_file) reader.read() # Run the HEALPix conversion function - ds_healpix = reader.to_healpix(variables='height', resolution=resolution) + ds_healpix = reader.to_healpix(variables="height", resolution=resolution) # Assertions to verify correct output assert isinstance(ds_healpix, xr.Dataset), "The result should be an xarray Dataset" # Verify that HEALPix cell IDs and coordinates exist - assert 'cell_ids' in ds_healpix.coords, "HEALPix cell IDs should be present" + assert "cell_ids" in ds_healpix.coords, "HEALPix cell IDs should be present" # Ensure the data is not empty after conversion - assert len(ds_healpix['cell_ids']) > 0, "The output dataset should have HEALPix cell IDs" + assert len(ds_healpix["cell_ids"]) > 0, ( + "The output dataset should have HEALPix cell IDs" + ) diff --git a/pixcdust/tests/test_downloaders.py b/pixcdust/tests/test_downloaders.py index 22de58b..9a2a8d0 100644 --- a/pixcdust/tests/test_downloaders.py +++ b/pixcdust/tests/test_downloaders.py @@ -11,7 +11,7 @@ def test_hydroweb_next(hydroweb_env, input_folder, tmp_folder): Only run with the option --ddl. """ dl_dir = tmp_folder / "download_test" - download_test_data(dl_dir,'default') + download_test_data(dl_dir, "default") dl_files = sorted(dl_dir.glob("**/*.nc")) all_input_files = sorted(input_folder.glob("**/*.nc")) @@ -20,6 +20,7 @@ def test_hydroweb_next(hydroweb_env, input_folder, tmp_folder): for dl_f, input_f in zip(dl_files, all_input_files): assert dl_f.stat().st_size == input_f.stat().st_size + @pytest.mark.downloader def test_hydroweb_next_eodag(hydroweb_env, input_folder, tmp_folder): """Test hydroweb.next eodag downloader. @@ -28,7 +29,7 @@ def test_hydroweb_next_eodag(hydroweb_env, input_folder, tmp_folder): Only run with the option --ddl. """ dl_dir = tmp_folder / "download_test_eodag" - download_test_data(dl_dir,'eodag') + download_test_data(dl_dir, "eodag") # fix: avoid listing eodag's .download repo dl_files = sorted(dl_dir.glob("**/[!.]*.nc")) diff --git a/pixcdust/tools/convert_pixc.py b/pixcdust/tools/convert_pixc.py index 9ea7761..59245fa 100644 --- a/pixcdust/tools/convert_pixc.py +++ b/pixcdust/tools/convert_pixc.py @@ -15,13 +15,13 @@ # import click - import geopandas as gpd +from pixcdust.converters.core import Converter from pixcdust.converters.gpkg import Nc2GpkgConverter -from pixcdust.converters.zarr import Nc2ZarrConverter from pixcdust.converters.shapefile import Nc2ShpConverter -from pixcdust.converters.core import Converter +from pixcdust.converters.zarr import Nc2ZarrConverter + def paths_glob(ctx, param, paths): return list(paths) @@ -36,26 +36,24 @@ def paths_glob(ctx, param, paths): help="list of variables of interest to extract from SWOT PIXC files,\ separated with commas ','", ) -@click.option("--aoi", type=click.File(mode='r'), default=None) +@click.option("--aoi", type=click.File(mode="r"), default=None) @click.option( - "-m", "--mode", - type=click.Choice(['w', 'o']), + "-m", + "--mode", + type=click.Choice(["w", "o"]), help="Mode for writing in database", - default=('w'), + default=("w"), ) @click.argument( - 'format_out', - type=click.Choice( - ['gpkg', 'zarr', 'shp'], - case_sensitive=False - ), + "format_out", + type=click.Choice(["gpkg", "zarr", "shp"], case_sensitive=False), ) @click.argument( - 'path_out', + "path_out", type=click.Path(), ) @click.argument( - 'paths_in', + "paths_in", nargs=-1, callback=paths_glob, ) @@ -66,7 +64,7 @@ def cli( variables: str, aoi: str, mode: str, - ): +): """_summary_ Args: @@ -84,13 +82,13 @@ def cli( NotImplementedError: _description_ """ if variables is not None: - variables.strip('()') - variables.strip('[]') - list_vars = variables.split(',') + variables.strip("()") + variables.strip("[]") + list_vars = variables.split(",") for var in list_vars: if any(not c.isalnum() for c in var): raise click.BadOptionUsage( - 'variables', + "variables", "apart from the commas, no special caracter may be used", ) @@ -102,19 +100,19 @@ def cli( else: gdf_aoi = None - if format_out.lower() == 'gpkg': - pixc : Converter = Nc2GpkgConverter( + if format_out.lower() == "gpkg": + pixc: Converter = Nc2GpkgConverter( paths_in, variables=list_vars, area_of_interest=gdf_aoi, ) - elif format_out.lower() == 'zarr': + elif format_out.lower() == "zarr": pixc = Nc2ZarrConverter( sorted(paths_in), variables=list_vars, area_of_interest=gdf_aoi, ) - elif format_out.lower() == 'shp': + elif format_out.lower() == "shp": pixc = Nc2ShpConverter( paths_in, variables=list_vars, @@ -122,8 +120,8 @@ def cli( ) else: raise NotImplementedError( - f'the conversion format {format_out} has not been implemented yet', - ) + f"the conversion format {format_out} has not been implemented yet", + ) pixc.database_from_nc(path_out, mode=mode) diff --git a/pyproject.toml b/pyproject.toml index f1eeebf..364e36c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,68 +1,68 @@ -[tool.poetry] +[project] name = "pixcdust" version = "0.2.0" description = "" -authors = ["zawadzl "] +authors = [ + {name = "zawadzl",email = "lionel.zawadzki@cnes.fr"} +] readme = "README.md" -packages = [{include = "pixcdust"}] - -license = "Apache 2.0" +license = "Apache-2.0" keywords = ["SWOT", "dggs", "hydroweb", "water", "hydrology", "inland water", "coastal", "scientific data", "search", "download", "cnes"] +requires-python = ">=3.12" +dependencies = [ + "numpy>=2.0,<3.0", + "fiona>=1.9.5,<2.0.0", + "pandas>=2.2.0,<3.0.0", + "geopandas>=1.1.3,<2.0.0", + "h3>=3.7.6,<4.0.0", + "astropy>=8.0.0,<9.0.0", + "astropy-healpix>=1.1.0,<2.0.0", + "xarray>=2024.2.0,<2025.0.0", + "shapely>=2.0.3,<3.0.0", + "click>=8.1.7,<9.0.0", + "zcollection>=2026.6.0,<2027.0.0", + "netcdf4>=1.6.5,<2.0.0", + "tqdm>=4.66.4,<5.0.0", + "pyogrio>=0.8.0,<0.9.0", + "pyarrow>=16.1.0,<17.0.0", + "py-hydroweb>=1.2.0,<2.0.0", + "folium>=0.16.0,<0.17.0", + "branca>=0.7.2,<0.8.0", + "xvec>=0.3.0,<0.4.0", + "eodag>=4.4.0,<5.0.0", + "xdggs>=0.6.0,<0.7.0", + "scipy>=1.18.0,<2.0.0", +] -[tool.poetry.urls] -Documentation = "https://pixcdust.readthedocs.io/en/latest/index.html" -#Changelog = -Source = "https://github.com/SWOT-community/PixCDust/tree/master" - - -[tool.poetry.dependencies] -python = "~3.12" -numpy = ">=2.0, <3.0" -fiona = "^1.9.5" -pandas = "^2.2.0" -geopandas = "^1.1.3" -h3 = "^3.7.6" -astropy = "^8.0.0" -astropy-healpix = "^1.1.0" -xarray = "^2024.2.0" -shapely = "^2.0.3" -click = "^8.1.7" -zcollection = "^2026.6.0" -netcdf4 = "^1.6.5" -tqdm = "^4.66.4" -pyogrio = "^0.8.0" -pyarrow = "^16.1.0" -py-hydroweb = "^1.2.0" -folium = "^0.16.0" -branca = "^0.7.2" -xvec = "^0.3.0" -eodag = "^4.4.0" -xdggs = "^0.6.0" -scipy = "^1.18.0" +[project.urls] +documentation = "https://pixcdust.readthedocs.io/en/latest/index.html" +repository = "https://github.com/SWOT-community/PixCDust" +[project.optional-dependencies] +dev = [ + "decorator>=5.1.1,<6.0.0", + "ipykernel>=6.29.4,<7.0.0", + "matplotlib>=3.9.0,<4.0.0", + "pytest>=9.1.1,<10.0.0", + "black>=26.5.1,<27.0.0", + "flake8>=7.3.0,<8.0.0", + "ruff>=0.16.5,<0.17.5", +] +doc = [ + "gitpython", + "sphinx", + "sphinx-rtd-theme", + "pydata-sphinx-theme", + "sphinx-autodoc-typehints", + "nbsphinx", + "sphinx-autoapi", + "ipykernel", + "sphinx-collections", +] -[tool.poetry.group.dev.dependencies] -decorator = "^5.1.1" -ipykernel = "^6.29.4" -matplotlib = "^3.9.0" -pytest = "^9.1.1" -black = "^26.5.1" -flake8 = "^7.3.0" - -[tool.poetry.group.docs] -optional = true - -[tool.poetry.group.docs.dependencies] -gitpython = "*" -sphinx = "*" -sphinx-rtd-theme = "*" -pydata-sphinx-theme = "*" -sphinx-autodoc-typehints = "*" -nbsphinx = "*" -sphinx-autoapi = "*" -ipykernel = "*" -sphinx-collections = "*" +[tool.poetry] +packages = [{include = "pixcdust"}] [build-system] -requires = ["poetry-core"] +requires = ["poetry-core>=2.0.0,<3.0.0"] build-backend = "poetry.core.masonry.api"