From a437bd8c4775b99b2b0c1a50afefc9ac8f10be91 Mon Sep 17 00:00:00 2001 From: Pierre Tysebaert-Plagne Date: Fri, 21 Aug 2026 17:45:57 +0200 Subject: [PATCH 1/8] Start test revamp Added marker for test requiring real data Added fixture far the sample data --- pixcdust/tests/conftest.py | 55 ++++++++++--- pixcdust/tests/test_converters.py | 131 ++++++++++++++++++------------ pixcdust/tests/test_dggs.py | 17 ++-- 3 files changed, 132 insertions(+), 71 deletions(-) diff --git a/pixcdust/tests/conftest.py b/pixcdust/tests/conftest.py index 9d53fee..e6f2162 100644 --- a/pixcdust/tests/conftest.py +++ b/pixcdust/tests/conftest.py @@ -3,48 +3,77 @@ from typing import List import pytest - from pixcdust.tests.init_tests import JsonTestsSettings, init_hydroweb_env +MOCK_DATA_DIR = Path(__file__).parent / "mock_data" +SAMPLE_NC = MOCK_DATA_DIR / "swot_pixc.nc" + def pytest_addoption(parser): - parser.addoption("--dl", action="store_true", default=False, help="run dowloaders tests") + parser.addoption( + "--dl", action="store_true", default=False, help="run dowloaders tests" + ) + parser.addoption( + "--realdata", + action="store_true", + default=False, + help="run tests on real SWOT data (requires API key)", + ) def pytest_configure(config): config.addinivalue_line("markers", "downloader: mark test as testing downloads") + config.addinivalue_line( + "markers", "realdata: mark test as requireing real SWOT data" + ) def pytest_collection_modifyitems(config, items): - if config.getoption("--dl"): - # option given: do not skip tests - return skip_dl = pytest.mark.skip(reason="need --dl option to run") + skip_realdata = pytest.mark.skip( + reason="need --realdata option and a configured input_folder" + ) for item in items: - if "downloader" in item.keywords: + if "downloader" in item.keywords and not config.getoption("--dl"): item.add_marker(skip_dl) + if "realdata" in item.keywords and not config.getoption("--realdata"): + item.add_marker(skip_realdata) @pytest.fixture(scope="session") def tests_settings() -> JsonTestsSettings: return JsonTestsSettings() + +@pytest.fixture(scope="session") +def sample_nc() -> Path: + assert SAMPLE_NC.is_file() + print(SAMPLE_NC) + return SAMPLE_NC + + @pytest.fixture(scope="session") def input_folder(tests_settings) -> Path: - return tests_settings.input_folder + try: + return tests_settings.input_folder + except KeyError: + return MOCK_DATA_DIR + @pytest.fixture(scope="session") def input_files(input_folder) -> List[Path]: - return list(input_folder.glob("**/*nc")) + return list(input_folder.glob("**/*nc")) + @pytest.fixture(scope="session") -def first_file(input_folder) -> Path: - return next(input_folder.glob("**/*_20240803T*nc")) +def first_file(input_folder, sample_nc) -> Path: + return next(iter(input_folder.glob("**/*_20240803T*nc")), sample_nc) + @pytest.fixture(scope="session") -def tmp_folder(tests_settings) -> Path: - tests_settings.tmp_folder.mkdir(exist_ok=True) - return tests_settings.tmp_folder +def tmp_folder(tmp_path_factory) -> Path: + return tmp_path_factory.mktemp("pixcdust-test") + @pytest.fixture() def hydroweb_env(tests_settings) -> None: diff --git a/pixcdust/tests/test_converters.py b/pixcdust/tests/test_converters.py index 5633df7..a094e96 100644 --- a/pixcdust/tests/test_converters.py +++ b/pixcdust/tests/test_converters.py @@ -1,36 +1,43 @@ import random -from pathlib import PosixPath, Path +from pathlib import Path, PosixPath from typing import List, Union import fiona -import numpy as np import geopandas as gpd +import numpy as np import pytest -from shapely.geometry import Polygon import xarray as xr - -from pixcdust.converters.gpkg import Nc2GpkgConverter, GpkgDGGSProjecter +from pixcdust.converters.gpkg import GpkgDGGSProjecter, Nc2GpkgConverter from pixcdust.converters.shapefile import Nc2ShpConverter from pixcdust.converters.zarr import Nc2ZarrConverter from pixcdust.readers import GpkgReader -from pixcdust.readers.zarr import ZarrReader from pixcdust.readers.netcdf import NcSimpleReader +from pixcdust.readers.zarr import ZarrReader +from shapely.geometry import Polygon LIM_AREA_POL = Polygon( - [(-1.50580, 43.39543), (-1.36597, 43.39543), (-1.36597, 43.56471), (-1.50580, 43.56471), (-1.50580, 43.39543)]) -LIM_AREA_GEOM = gpd.GeoDataFrame(index=[0], crs='epsg:4326', geometry=[LIM_AREA_POL]) + [ + (-1.50580, 43.39543), + (-1.36597, 43.39543), + (-1.36597, 43.56471), + (-1.50580, 43.56471), + (-1.50580, 43.39543), + ] +) +LIM_AREA_GEOM = gpd.GeoDataFrame(index=[0], crs="epsg:4326", geometry=[LIM_AREA_POL]) """Geometry used as area of interest of limited area tests.""" def test_nc_simple_reader_conditions(input_files): """Test NcSimpleReader with conditions on variables.""" # Define conditions - conditions = {"classification": {'operator': "ge", 'threshold': 4}, # classification >= 4 - "classification": {'operator': "le", 'threshold': 3}, # classification <= 3 - "sig0": {'operator': "gt", 'threshold': 15} # sig0 > 15 - } + conditions = { + "classification": {"operator": "ge", "threshold": 4}, # classification >= 4 + "classification": {"operator": "le", "threshold": 3}, # classification <= 3 + "sig0": {"operator": "gt", "threshold": 15}, # sig0 > 15 + } - converted_vars = ['height', 'sig0', 'classification'] + converted_vars = ["height", "sig0", "classification"] # Instantiate the NcSimpleReader with conditions reader = NcSimpleReader( @@ -46,18 +53,19 @@ def test_nc_simple_reader_conditions(input_files): for var, condition in conditions.items(): op = condition.get("operator") val = condition.get("threshold") - if op == 'ge': + if op == "ge": assert (reader.data[var] >= val).all(), f"{var} not >= {val}" - elif op == 'le': + elif op == "le": assert (reader.data[var] <= val).all(), f"{var} not <= {val}" - elif op == 'gt': + elif op == "gt": assert (reader.data[var] > val).all(), f"{var} not > {val}" - elif op == 'lt': + elif op == "lt": assert (reader.data[var] < val).all(), f"{var} not < {val}" -def validate_conversion_to_nc(read_data: xr.Dataset, converted_vars:List[str], first_file: Union[str, Path])\ - -> None: +def validate_conversion_to_nc( + read_data: xr.Dataset, converted_vars: List[str], first_file: Union[str, Path] +) -> None: """Compare the start of a converted database to the first original netcdf file. Args: @@ -69,15 +77,16 @@ def validate_conversion_to_nc(read_data: xr.Dataset, converted_vars:List[str], f """ ncsimple = NcSimpleReader(str(first_file)) ncsimple.open_dataset() - validate_conversion(read_data, converted_vars, ncsimple.data,is_longer=True) + validate_conversion(read_data, converted_vars, ncsimple.data, is_longer=True) + def validate_conversion( - read_data: xr.Dataset, - converted_vars:List[str], - expected_data: xr.Dataset, - is_longer: bool, - len_tol: int = 0, - sort_var: bool = False + read_data: xr.Dataset, + converted_vars: List[str], + expected_data: xr.Dataset, + is_longer: bool, + len_tol: int = 0, + sort_var: bool = False, ) -> None: """Compare the read data to the expected data. @@ -108,27 +117,33 @@ def validate_conversion( else: last = len(read_var) - np.testing.assert_allclose(read_var[last-30:last-1], expected_var[expected_last-30:expected_last-1]) - r = random.randrange(30,last) + np.testing.assert_allclose( + read_var[last - 30 : last - 1], + expected_var[expected_last - 30 : expected_last - 1], + ) + r = random.randrange(30, last) if len_tol == 0: - np.testing.assert_allclose(read_var[r-30:r-1], expected_var[r-30:r-1]) + np.testing.assert_allclose( + read_var[r - 30 : r - 1], expected_var[r - 30 : r - 1] + ) if is_longer: assert len(read_var) > expected_last else: assert expected_last + len_tol >= len(read_var) >= expected_last - len_tol +@pytest.mark.realdata def test_convert_zarr_full_area(input_files, first_file, tmp_folder): """Test zarr conversion without area_of_interest. It is compared to the input data. """ # Conversion - output = str(tmp_folder / "zarr_conv_test_full") - converted_vars = ['height', 'sig0', 'classification'] + output = str(tmp_folder / "zarr_conv_test_full") + converted_vars = ["height", "sig0", "classification"] pixc = Nc2ZarrConverter( - input_files, - variables=converted_vars, + input_files, + variables=converted_vars, ) pixc.database_from_nc(output, mode="o") @@ -137,19 +152,23 @@ def test_convert_zarr_full_area(input_files, first_file, tmp_folder): pixc_read.read() validate_conversion_to_nc(pixc_read.data, converted_vars, first_file) + @pytest.fixture(scope="session") def converted_lim_gpkg(input_files, tmp_folder): output_gpkg = str(tmp_folder / "gpkg_conv_test_lim") - converted_vars = ['height', 'sig0', 'classification'] + converted_vars = ["height", "sig0", "classification"] Nc2GpkgConverter( - input_files, - variables=converted_vars, - area_of_interest=LIM_AREA_GEOM, + input_files, + variables=converted_vars, + area_of_interest=LIM_AREA_GEOM, ).database_from_nc(output_gpkg, mode="o") return output_gpkg -def test_convert_gpkg_and_zarr_limited_area(input_files, first_file, tmp_folder, converted_lim_gpkg): +@pytest.mark.realdata +def test_convert_gpkg_and_zarr_limited_area( + input_files, first_file, tmp_folder, converted_lim_gpkg +): """Test geopackage and zarr conversion with area_of_interest. They are compared to each other. @@ -157,13 +176,13 @@ def test_convert_gpkg_and_zarr_limited_area(input_files, first_file, tmp_folder, """ # Conversion output_zarr = str(tmp_folder / "zarr_conv_test_lim") - converted_vars = ['height', 'sig0', 'classification'] + converted_vars = ["height", "sig0", "classification"] output_gpkg = converted_lim_gpkg Nc2ZarrConverter( - input_files, - variables=converted_vars, - area_of_interest=LIM_AREA_GEOM, + input_files, + variables=converted_vars, + area_of_interest=LIM_AREA_GEOM, ).database_from_nc(output_zarr, mode="o") # Validation @@ -171,20 +190,27 @@ def test_convert_gpkg_and_zarr_limited_area(input_files, first_file, tmp_folder, gpkg_read.read() zarr_read = ZarrReader(output_zarr) zarr_read.read() - validate_conversion(gpkg_read.data, converted_vars, zarr_read.data, is_longer=False, len_tol=2, sort_var="longitude") + validate_conversion( + gpkg_read.data, + converted_vars, + zarr_read.data, + is_longer=False, + len_tol=2, + sort_var="longitude", + ) +@pytest.mark.realdata def test_convert_shape_limited_area(input_files, first_file, tmp_folder): - """Test shapefile conversion with area_of_interest. - """ + """Test shapefile conversion with area_of_interest.""" # Conversion - output = str(tmp_folder / "shp_conv_test_full") - converted_vars = ['height', 'sig0', 'classification'] + output = str(tmp_folder / "shp_conv_test_full") + converted_vars = ["height", "sig0", "classification"] pixc = Nc2ShpConverter( - input_files, - variables=converted_vars, - area_of_interest=LIM_AREA_GEOM, + input_files, + variables=converted_vars, + area_of_interest=LIM_AREA_GEOM, ) pixc.database_from_nc(output, mode="o") @@ -197,6 +223,7 @@ def test_convert_shape_limited_area(input_files, first_file, tmp_folder): # Test for GpkgDGGSProjecter +@pytest.mark.realdata def test_gpkg_dggs_projecter(converted_lim_gpkg, tmp_folder): """Test the GpkgDGGSProjecter class by converting a sample Gpkg to a DGGS projection.""" @@ -204,7 +231,7 @@ def test_gpkg_dggs_projecter(converted_lim_gpkg, tmp_folder): # Define parameters dggs_res = 10 healpix = False - dggs_layer_pattern = '_h3' + dggs_layer_pattern = "_h3" output_path = str(tmp_folder / "gpkg_dggs_output") # Create an instance of GpkgDGGSProjecter @@ -213,7 +240,7 @@ def test_gpkg_dggs_projecter(converted_lim_gpkg, tmp_folder): dggs_res=dggs_res, healpix=healpix, dggs_layer_pattern=dggs_layer_pattern, - path_out=output_path + path_out=output_path, ) # Validate initialization diff --git a/pixcdust/tests/test_dggs.py b/pixcdust/tests/test_dggs.py index d45141c..39b5d93 100644 --- a/pixcdust/tests/test_dggs.py +++ b/pixcdust/tests/test_dggs.py @@ -1,7 +1,9 @@ +import pytest import xarray as xr from pixcdust.readers import NcSimpleReader +@pytest.mark.realdata def test_h3_conversion(first_file): """ Test the conversion of the dataset to the H3 grid. @@ -12,18 +14,19 @@ def test_h3_conversion(first_file): reader = NcSimpleReader(first_file) reader.read() # Run the H3 conversion function - ds_h3 = reader.to_h3(variables='height', resolution=resolution) + ds_h3 = reader.to_h3(variables="height", resolution=resolution) # Assertions to verify correct output assert isinstance(ds_h3, xr.Dataset), "The result should be an xarray Dataset" # Verify that H3 cell IDs exist and have expected properties - assert 'cell_ids' in ds_h3.coords, "H3 cell IDs should be present in the output" + assert "cell_ids" in ds_h3.coords, "H3 cell IDs should be present in the output" # Ensure the data is not empty after conversion - assert len(ds_h3['cell_ids']) > 0, "The output dataset should have H3 cell IDs" + assert len(ds_h3["cell_ids"]) > 0, "The output dataset should have H3 cell IDs" +@pytest.mark.realdata def test_healpix_conversion(first_file): """ Test the conversion of the dataset to the HEALPix grid. @@ -33,13 +36,15 @@ def test_healpix_conversion(first_file): reader = NcSimpleReader(first_file) reader.read() # Run the HEALPix conversion function - ds_healpix = reader.to_healpix(variables='height', resolution=resolution) + ds_healpix = reader.to_healpix(variables="height", resolution=resolution) # Assertions to verify correct output assert isinstance(ds_healpix, xr.Dataset), "The result should be an xarray Dataset" # Verify that HEALPix cell IDs and coordinates exist - assert 'cell_ids' in ds_healpix.coords, "HEALPix cell IDs should be present" + assert "cell_ids" in ds_healpix.coords, "HEALPix cell IDs should be present" # Ensure the data is not empty after conversion - assert len(ds_healpix['cell_ids']) > 0, "The output dataset should have HEALPix cell IDs" + assert ( + len(ds_healpix["cell_ids"]) > 0 + ), "The output dataset should have HEALPix cell IDs" From 11f5c18beb016168f4d24a5ded068c43d7dcd40f Mon Sep 17 00:00:00 2001 From: pty Date: Mon, 31 Aug 2026 11:50:43 +0200 Subject: [PATCH 2/8] Migrate pyproject.toml to poetry 2.0.0 format (PEP621 Compliance) --- pyproject.toml | 110 +++++++++++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 54 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f1eeebf..251da84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,68 +1,70 @@ -[tool.poetry] +[project] name = "pixcdust" version = "0.2.0" description = "" -authors = ["zawadzl "] +authors = [ + {name = "zawadzl",email = "lionel.zawadzki@cnes.fr"} +] readme = "README.md" -packages = [{include = "pixcdust"}] - -license = "Apache 2.0" +license = "Apache-2.0" keywords = ["SWOT", "dggs", "hydroweb", "water", "hydrology", "inland water", "coastal", "scientific data", "search", "download", "cnes"] +requires-python = ">=3.12" +dependencies = [ + "numpy>=2.0,<3.0", + "fiona>=1.9.5,<2.0.0", + "pandas>=2.2.0,<3.0.0", + "geopandas>=1.1.3,<2.0.0", + "h3>=3.7.6,<4.0.0", + "astropy>=8.0.0,<9.0.0", + "astropy-healpix>=1.1.0,<2.0.0", + "xarray>=2024.2.0,<2025.0.0", + "shapely>=2.0.3,<3.0.0", + "click>=8.1.7,<9.0.0", + "zcollection>=2026.6.0,<2027.0.0", + "netcdf4>=1.6.5,<2.0.0", + "tqdm>=4.66.4,<5.0.0", + "pyogrio>=0.8.0,<0.9.0", + "pyarrow>=16.1.0,<17.0.0", + "py-hydroweb>=1.2.0,<2.0.0", + "folium>=0.16.0,<0.17.0", + "branca>=0.7.2,<0.8.0", + "xvec>=0.3.0,<0.4.0", + "eodag>=4.4.0,<5.0.0", + "xdggs>=0.6.0,<0.7.0", + "scipy>=1.18.0,<2.0.0", +] -[tool.poetry.urls] -Documentation = "https://pixcdust.readthedocs.io/en/latest/index.html" -#Changelog = -Source = "https://github.com/SWOT-community/PixCDust/tree/master" - +[project.urls] +documentation = "https://pixcdust.readthedocs.io/en/latest/index.html" +repository = "https://github.com/SWOT-community/PixCDust" -[tool.poetry.dependencies] -python = "~3.12" -numpy = ">=2.0, <3.0" -fiona = "^1.9.5" -pandas = "^2.2.0" -geopandas = "^1.1.3" -h3 = "^3.7.6" -astropy = "^8.0.0" -astropy-healpix = "^1.1.0" -xarray = "^2024.2.0" -shapely = "^2.0.3" -click = "^8.1.7" -zcollection = "^2026.6.0" -netcdf4 = "^1.6.5" -tqdm = "^4.66.4" -pyogrio = "^0.8.0" -pyarrow = "^16.1.0" -py-hydroweb = "^1.2.0" -folium = "^0.16.0" -branca = "^0.7.2" -xvec = "^0.3.0" -eodag = "^4.4.0" -xdggs = "^0.6.0" -scipy = "^1.18.0" - - -[tool.poetry.group.dev.dependencies] -decorator = "^5.1.1" -ipykernel = "^6.29.4" -matplotlib = "^3.9.0" -pytest = "^9.1.1" -black = "^26.5.1" -flake8 = "^7.3.0" +[dependency-groups] +dev = [ + "decorator>=5.1.1,<6.0.0", + "ipykernel>=6.29.4,<7.0.0", + "matplotlib>=3.9.0,<4.0.0", + "pytest>=9.1.1,<10.0.0", + "black>=26.5.1,<27.0.0", + "flake8>=7.3.0,<8.0.0", +] +doc = [ + "gitpython", + "sphinx", + "sphinx-rtd-theme", + "pydata-sphinx-theme", + "sphinx-autodoc-typehints", + "nbsphinx", + "sphinx-autoapi", + "ipykernel", + "sphinx-collections", +] [tool.poetry.group.docs] optional = true -[tool.poetry.group.docs.dependencies] -gitpython = "*" -sphinx = "*" -sphinx-rtd-theme = "*" -pydata-sphinx-theme = "*" -sphinx-autodoc-typehints = "*" -nbsphinx = "*" -sphinx-autoapi = "*" -ipykernel = "*" -sphinx-collections = "*" +[tool.poetry] +packages = [{include = "pixcdust"}] [build-system] -requires = ["poetry-core"] +requires = ["poetry-core>=2.0.0,<3.0.0"] build-backend = "poetry.core.masonry.api" From 2eb2557fbf309213b5e96bb8f226e6789030f238 Mon Sep 17 00:00:00 2001 From: pty Date: Mon, 31 Aug 2026 12:08:12 +0200 Subject: [PATCH 3/8] Fix optional dependenncies --- pyproject.toml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 251da84..25b7f66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ documentation = "https://pixcdust.readthedocs.io/en/latest/index.html" repository = "https://github.com/SWOT-community/PixCDust" -[dependency-groups] +[project.optional-dependencies] dev = [ "decorator>=5.1.1,<6.0.0", "ipykernel>=6.29.4,<7.0.0", @@ -59,9 +59,6 @@ doc = [ "sphinx-collections", ] -[tool.poetry.group.docs] -optional = true - [tool.poetry] packages = [{include = "pixcdust"}] From 66e7bdc65f248ee9428c192b3367d2f5d61eddfd Mon Sep 17 00:00:00 2001 From: pty Date: Mon, 31 Aug 2026 12:21:11 +0200 Subject: [PATCH 4/8] Added ruff to dev dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 25b7f66..364e36c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ dev = [ "pytest>=9.1.1,<10.0.0", "black>=26.5.1,<27.0.0", "flake8>=7.3.0,<8.0.0", + "ruff>=0.16.5,<0.17.5", ] doc = [ "gitpython", From 096b0752d2e353c85303ae0efa221840a233179c Mon Sep 17 00:00:00 2001 From: pty Date: Mon, 31 Aug 2026 12:22:13 +0200 Subject: [PATCH 5/8] ruff format fix --- doc/conf.py | 49 ++++--- pixcdust/converters/core.py | 37 +++-- pixcdust/converters/geo_utils.py | 4 +- pixcdust/converters/gpkg.py | 22 +-- pixcdust/converters/shapefile.py | 16 ++- pixcdust/converters/zarr.py | 43 +++--- pixcdust/dggs/dggs_converter.py | 73 ++++++---- pixcdust/dggs/h3_tools.py | 68 +++++---- pixcdust/downloaders/hydroweb_next.py | 131 +++++++++--------- .../convert/convert_to_zcollection.ipynb | 10 +- pixcdust/notebooks/convert/dggs_tuto.ipynb | 24 ++-- .../convert/download_pixc_to_gpkg.ipynb | 60 ++++---- .../convert/download_pixc_to_zarr.ipynb | 33 +++-- .../notebooks/read_and_use/read_netcdf.ipynb | 15 +- pixcdust/readers/__init__.py | 2 +- pixcdust/readers/base_reader.py | 31 +++-- pixcdust/readers/gpkg.py | 17 ++- pixcdust/readers/netcdf.py | 84 ++++++----- pixcdust/readers/zarr.py | 26 ++-- pixcdust/tests/init_tests.py | 80 ++++++----- pixcdust/tests/mock.py | 2 +- pixcdust/tests/test_converters_mock.py | 15 +- pixcdust/tests/test_dggs.py | 6 +- pixcdust/tests/test_downloaders.py | 5 +- pixcdust/tools/convert_pixc.py | 43 +++--- 25 files changed, 490 insertions(+), 406 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 6c96055..053812f 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -6,51 +6,56 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = 'pixcdust' -copyright = '2025, Zawadzki Lionel' -author = 'Zawadzki Lionel' -release = '0.1.0' +project = "pixcdust" +copyright = "2025, Zawadzki Lionel" +author = "Zawadzki Lionel" +release = "0.1.0" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = ['sphinx.ext.viewcode','autoapi.extension','nbsphinx',"sphinxcontrib.collections"] +extensions = [ + "sphinx.ext.viewcode", + "autoapi.extension", + "nbsphinx", + "sphinxcontrib.collections", +] html_show_sourcelink = False set_type_checking_flag = True nbsphinx_allow_errors = True -templates_path = ['_templates'] +templates_path = ["_templates"] exclude_patterns = [] -autoapi_dirs = ['../pixcdust'] -source_suffix = ['.rst'] +autoapi_dirs = ["../pixcdust"] +source_suffix = [".rst"] # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_theme = 'alabaster' -html_static_path = ['_static'] +html_theme = "alabaster" +html_static_path = ["_static"] # manual build path collections = { - 'notebooks': { - 'driver': 'copy_folder', - 'source': 'pixcdust/notebooks', - 'target': 'notebooks/', - 'ignore': ['*.py', '.sh'], - 'safe': False, + "notebooks": { + "driver": "copy_folder", + "source": "pixcdust/notebooks", + "target": "notebooks/", + "ignore": ["*.py", ".sh"], + "safe": False, } } # ReadTheDoc build path collections = { - 'notebooks': { - 'driver': 'copy_folder', - 'source': '../pixcdust/notebooks', - 'target': 'notebooks/', - 'ignore': ['*.py', '.sh'], - 'safe': False, + "notebooks": { + "driver": "copy_folder", + "source": "../pixcdust/notebooks", + "target": "notebooks/", + "ignore": ["*.py", ".sh"], + "safe": False, } } diff --git a/pixcdust/converters/core.py b/pixcdust/converters/core.py index 14107bd..503643b 100644 --- a/pixcdust/converters/core.py +++ b/pixcdust/converters/core.py @@ -91,8 +91,10 @@ class ConverterWSE(Converter): variables: Optionally only read these variables. area_of_interest: Optionally only read points in area_of_interest. """ - def database_from_nc(self, path_out: str | Path, mode: str = "w", compute_wse: bool = True) \ - -> None: + + def database_from_nc( + self, path_out: str | Path, mode: str = "w", compute_wse: bool = True + ) -> None: """Convert the path_in files to path_out. Args: path_out: Output path of the convertion. @@ -109,19 +111,21 @@ def _append_wse_vars(self): self.variables.append(var) def _compute_wse(self, gdf): - gdf[self._get_name_wse_var()] = \ - gdf[self._get_vars_wse_computation()[0]] - \ - gdf[self._get_vars_wse_computation()[1]] + gdf[self._get_name_wse_var()] = ( + gdf[self._get_vars_wse_computation()[0]] + - gdf[self._get_vars_wse_computation()[1]] + ) @staticmethod def _get_vars_wse_computation() -> list[str]: """Names of fields used to compute wse.""" - return ['height', 'geoid'] + return ["height", "geoid"] @staticmethod def _get_name_wse_var() -> str: """Output name for wse.""" - return 'wse' + return "wse" + @dataclass class GeoLayerH3Projecter: @@ -132,10 +136,13 @@ class GeoLayerH3Projecter: resolution: Resolution """ + data: gpd.GeoDataFrame resolution: int - def filter_variable(self, conditions: dict[str,dict[str, Union[str, float]]]) -> None: + def filter_variable( + self, conditions: dict[str, dict[str, Union[str, float]]] + ) -> None: """filters from xarray dataset based on operator and threshold on specific variables @@ -154,23 +161,23 @@ def filter_variable(self, conditions: dict[str,dict[str, Union[str, float]]]) -> AttributeError: if operator is not the function name of\ the operator module """ - _k_operator = 'operator' - _k_to = 'threshold' + _k_operator = "operator" + _k_to = "threshold" # Test if conditions dict meets specifications print(conditions) for k in conditions.keys(): if k not in self.data.columns: raise IOError( - f'dict conditions expected existing\ + f"dict conditions expected existing\ variables (in {self.data.columns}),\ - received {k}' + received {k}" ) for instructions in conditions[k].keys(): if instructions not in [_k_operator, _k_to]: raise ValueError( - f'dict conditions expected {_k_to} and {_k_operator}\ + f"dict conditions expected {_k_to} and {_k_operator}\ keys in dict {conditions},\ - received {instructions}' + received {instructions}" ) print(f"operator.{conditions[k][_k_operator]}") ope = getattr(operator, conditions[k][_k_operator]) @@ -184,6 +191,7 @@ def filter_variable(self, conditions: dict[str,dict[str, Union[str, float]]]) -> def compute_h3_layer(self) -> None: """Project data to h3.""" from pixcdust.dggs import h3_tools + self.data = h3_tools.gdf_to_h3_gdf( self.data, self.resolution, @@ -192,6 +200,7 @@ def compute_h3_layer(self) -> None: def compute_healpix_layer(self) -> None: """Project data to Healpix.""" from pixcdust.dggs import h3_tools + self.data = h3_tools.gdf_to_healpix_gdf( self.data, self.resolution, diff --git a/pixcdust/converters/geo_utils.py b/pixcdust/converters/geo_utils.py index 5a80ab2..f34ee35 100644 --- a/pixcdust/converters/geo_utils.py +++ b/pixcdust/converters/geo_utils.py @@ -19,9 +19,7 @@ import geopandas as gpd -def geoxarray_to_geodataframe( - ds: xr.Dataset, - *args, **kwargs) -> gpd.GeoDataFrame: +def geoxarray_to_geodataframe(ds: xr.Dataset, *args, **kwargs) -> gpd.GeoDataFrame: """Converts an xarray.Dataset with points coordinates into\ a geopandas.GeodataFrame with xvec diff --git a/pixcdust/converters/gpkg.py b/pixcdust/converters/gpkg.py index f323a47..0921f84 100644 --- a/pixcdust/converters/gpkg.py +++ b/pixcdust/converters/gpkg.py @@ -45,15 +45,16 @@ class Nc2GpkgConverter(ConverterWSE): """ - def database_from_nc(self, path_out: str | Path, mode: str = "w", compute_wse: bool = True) \ - -> None: + def database_from_nc( + self, path_out: str | Path, mode: str = "w", compute_wse: bool = True + ) -> None: path_out = str(path_out) if compute_wse: self._append_wse_vars() for path in tqdm(self.path_in): ncsimple = NcSimpleReader( path, - variables= self.variables, + variables=self.variables, area_of_interest=self.area_of_interest, conditions=self.conditions, ) @@ -62,7 +63,7 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w", compute_wse: b _, dt_time_start, cycle_number, pass_number, tile_number, swath_side = ( ncsimple.extract_info_from_nc_attrs(path) ) - time_start = dt_time_start.strftime('%Y%m%d') + time_start = dt_time_start.strftime("%Y%m%d") layer_name = f"{time_start}_{cycle_number}_\ {pass_number}_{tile_number}{swath_side}" @@ -77,8 +78,7 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w", compute_wse: b continue # converting data from xarray to geodataframe ncsimple.open_dataset() - gdf = ncsimple.to_geodataframe( - ) + gdf = ncsimple.to_geodataframe() if gdf.size == 0: tqdm.write( @@ -109,18 +109,19 @@ class GpkgDGGSProjecter: path: str dggs_res: int - conditions: Optional[dict[str,dict[str, Union[str, float]]]] = None + conditions: Optional[dict[str, dict[str, Union[str, float]]]] = None healpix: bool = False - dggs_layer_pattern: str = '_h3' + dggs_layer_pattern: str = "_h3" path_out: Optional[str] = None # database: GpkgReader def __post_init__(self) -> None: self.database = GpkgReader(self.path) self.database.layers = [ - layer for layer in fiona.listlayers(self.path) + layer + for layer in fiona.listlayers(self.path) if not layer.endswith(self.dggs_layer_pattern) - ] + ] if self.path_out is None: self.path_out = self.path @@ -165,6 +166,7 @@ class Zarr2GpkgConverter: Attributes: path: Gpkg pixelcloud to convert. """ + path: str data: gpd.GeoDataFrame = None diff --git a/pixcdust/converters/shapefile.py b/pixcdust/converters/shapefile.py index 0f5f9f4..fab4c42 100644 --- a/pixcdust/converters/shapefile.py +++ b/pixcdust/converters/shapefile.py @@ -45,14 +45,18 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: except FileExistsError: pass for path in tqdm(self.path_in): - ncsimple = NcSimpleReader(path, - variables=self.variables, - area_of_interest=self.area_of_interest, - conditions=self.conditions, - ) + ncsimple = NcSimpleReader( + path, + variables=self.variables, + area_of_interest=self.area_of_interest, + conditions=self.conditions, + ) filename_out = os.path.splitext(os.path.basename(path))[0] - path_shp = os.path.join(path_out, filename_out + '.shp', ) + path_shp = os.path.join( + path_out, + filename_out + ".shp", + ) # cheking if output file and layer already exist if os.path.exists(path_shp) and mode == "w": continue diff --git a/pixcdust/converters/zarr.py b/pixcdust/converters/zarr.py index b0fe07f..02702a2 100644 --- a/pixcdust/converters/zarr.py +++ b/pixcdust/converters/zarr.py @@ -33,7 +33,7 @@ from pixcdust.converters.core import Converter from pixcdust.readers.netcdf import NcSimpleReader, NcSimpleConstants -TIME_VARNAME = 'time' +TIME_VARNAME = "time" class Nc2ZarrConverter(Converter): @@ -71,29 +71,32 @@ def __init__( "classification":{'operator': "ge", 'threshold': 3},\ } """ - super().__init__(path_in=path_in, - variables=variables, - area_of_interest=area_of_interest, - conditions=conditions) + super().__init__( + path_in=path_in, + variables=variables, + area_of_interest=area_of_interest, + conditions=conditions, + ) self.collection: zcollection.collection.Collection = None self.__time_varname: str = TIME_VARNAME self.__fs = fsspec.filesystem("file") - self.__chunk_size = dask.utils.parse_bytes('2MiB') + self.__chunk_size = dask.utils.parse_bytes("2MiB") self.__cst = NcSimpleConstants() def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: - if mode in ['o', 'overwrite'] and os.path.exists(path_out): + if mode in ["o", "overwrite"] and os.path.exists(path_out): shutil.rmtree(path_out) - with dask.distributed.LocalCluster(processes=True) as cluster, \ - dask.distributed.Client(cluster) as client: - + with ( + dask.distributed.LocalCluster(processes=True) as cluster, + dask.distributed.Client(cluster) as client, + ): xr_ds = NcSimpleReader( path=self.path_in, variables=self.variables, area_of_interest=self.area_of_interest, - conditions=self.conditions + conditions=self.conditions, ) xr_ds.open_mfdataset( @@ -102,18 +105,15 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: zc_ds = zcollection.Dataset.from_xarray( xr_ds.to_xarray().drop_vars(self.__cst.default_added_points_name), - ) + ) zc_ds.block_size_limit = self.__chunk_size - zc_ds.chunks = { - list(zc_ds.dimensions.keys())[0]: self.__chunk_size - } + zc_ds.chunks = {list(zc_ds.dimensions.keys())[0]: self.__chunk_size} init = True if not os.path.exists(path_out) and init: - partition_handler = zcollection.partitioning.Date( - (xr_ds.cst.default_added_time_name, ), - 's', + (xr_ds.cst.default_added_time_name,), + "s", ) self.collection = zcollection.create_collection( @@ -129,9 +129,8 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: self.collection = zcollection.open_collection( path_out, filesystem=self.__fs, - mode='w', - ) + mode="w", + ) self.collection.insert( - zc_ds, - merge_callable=zcollection.collection.merging.merge_time_series + zc_ds, merge_callable=zcollection.collection.merging.merge_time_series ) diff --git a/pixcdust/dggs/dggs_converter.py b/pixcdust/dggs/dggs_converter.py index b3064d3..7df242b 100644 --- a/pixcdust/dggs/dggs_converter.py +++ b/pixcdust/dggs/dggs_converter.py @@ -25,7 +25,9 @@ from astropy import units as u -def prepare_dataset_h3(ds: Dataset, resolution: int, interp: bool=False, method: str = 'linear') -> Dataset: +def prepare_dataset_h3( + ds: Dataset, resolution: int, interp: bool = False, method: str = "linear" +) -> Dataset: """ Convert a Dataset with latitude and longitude coordinates into an H3-indexed grid. @@ -53,8 +55,14 @@ def prepare_dataset_h3(ds: Dataset, resolution: int, interp: bool=False, method: if interp: # Compute the bounding box for the dataset in lat/lon - lon_min, lon_max = ds.longitude.min().values.item(), ds.longitude.max().values.item() - lat_min, lat_max = ds.latitude.min().values.item(), ds.latitude.max().values.item() + lon_min, lon_max = ( + ds.longitude.min().values.item(), + ds.longitude.max().values.item(), + ) + lat_min, lat_max = ( + ds.latitude.min().values.item(), + ds.latitude.max().values.item(), + ) # Define the bounding box coordinates bbox_coords = [ @@ -85,17 +93,21 @@ def prepare_dataset_h3(ds: Dataset, resolution: int, interp: bool=False, method: # Interpolate the values onto the H3 grid interpolated_values = griddata( - points=(lat, lon), - values=values, - xi=ll_points, - method=method + points=(lat, lon), values=values, xi=ll_points, method=method ) data[var] = interpolated_values else: # Compute H3 index for each point in the dataset - h3_indices = np.array([h3.api.basic_int.geo_to_h3(lat_, lon_, resolution) for lat_, lon_ in zip(lat.values, lon.values)]) - ll_points = np.array([h3.api.basic_int.h3_to_geo(i) for i in np.unique(h3_indices)]) + h3_indices = np.array( + [ + h3.api.basic_int.geo_to_h3(lat_, lon_, resolution) + for lat_, lon_ in zip(lat.values, lon.values) + ] + ) + ll_points = np.array( + [h3.api.basic_int.h3_to_geo(i) for i in np.unique(h3_indices)] + ) # Create a dictionary to store values by H3 cell h3_data = {pix_id: [] for pix_id in np.unique(h3_indices)} @@ -107,18 +119,19 @@ def prepare_dataset_h3(ds: Dataset, resolution: int, interp: bool=False, method: h3_data[h3_id].append(values[idx]) # Compute the mean value for each variable in each H3 cell - data = {var: np.array([np.mean(np.array(h3_data[h3_id])) for h3_id in h3_data]) for var in ds.data_vars} + data = { + var: np.array([np.mean(np.array(h3_data[h3_id])) for h3_id in h3_data]) + for var in ds.data_vars + } coords = { "cell_ids": np.unique(h3_indices), - 'h3_lon': ('cell_ids', ll_points[:, 1]), - 'h3_lat': ('cell_ids', ll_points[:, 0]) + "h3_lon": ("cell_ids", ll_points[:, 1]), + "h3_lat": ("cell_ids", ll_points[:, 0]), } ds_h3 = xr.Dataset( - {var: (('cell_ids',), data[var]) for var in data}, - coords=coords, - attrs=ds.attrs + {var: (("cell_ids",), data[var]) for var in data}, coords=coords, attrs=ds.attrs ) ds_h3.cell_ids.attrs = {"grid_name": "h3", "resolution": resolution} @@ -126,7 +139,9 @@ def prepare_dataset_h3(ds: Dataset, resolution: int, interp: bool=False, method: return ds_h3 -def prepare_dataset_healpix(ds: Dataset, resolution: int = 8, interp: bool=False, method: str = 'linear') -> Dataset: +def prepare_dataset_healpix( + ds: Dataset, resolution: int = 8, interp: bool = False, method: str = "linear" +) -> Dataset: """ Convert a Dataset with latitude and longitude coordinates into an HEALPix-indexed grid. @@ -154,8 +169,8 @@ def prepare_dataset_healpix(ds: Dataset, resolution: int = 8, interp: bool=False healpix = HEALPix(nside=nside, order="nested") # Get HEALPix pixel centers - lats = ds['latitude'].values - lons = ds['longitude'].values + lats = ds["latitude"].values + lons = ds["longitude"].values # pix_indices = np.array(hp.ang2pix(nside, lons, lats, nest=nest, lonlat=True)) # healpix_lon, healpix_lat = hp.pix2ang(nside, np.unique(pix_indices), nest=nest, lonlat=True) pix_indices = healpix.lonlat_to_healpix(lons * u.deg, lats * u.deg) @@ -178,10 +193,7 @@ def prepare_dataset_healpix(ds: Dataset, resolution: int = 8, interp: bool=False # Interpolate the values onto the HEALPix grid interpolated_values = griddata( - points=(lons, lats), - values=values, - xi=interp_points, - method=method + points=(lons, lats), values=values, xi=interp_points, method=method ) data[var] = interpolated_values @@ -196,19 +208,22 @@ def prepare_dataset_healpix(ds: Dataset, resolution: int = 8, interp: bool=False healpix_data[h3_id].append(values[idx]) # Compute the mean value for each variable in each HEALPix cell - data = {var: np.array([np.mean(np.array(healpix_data[h3_id])) for h3_id in healpix_data]) for var in ds.data_vars} + data = { + var: np.array( + [np.mean(np.array(healpix_data[h3_id])) for h3_id in healpix_data] + ) + for var in ds.data_vars + } coords = { - 'cell_ids': np.unique(pix_indices), - 'healpix_lon': ('cell_ids', healpix_lon), - 'healpix_lat': ('cell_ids', healpix_lat) + "cell_ids": np.unique(pix_indices), + "healpix_lon": ("cell_ids", healpix_lon), + "healpix_lat": ("cell_ids", healpix_lat), } # Create the new dataset with the aggregated or interpolated data ds_healpix = xr.Dataset( - {var: (('cell_ids',), data[var]) for var in data}, - coords=coords, - attrs=ds.attrs + {var: (("cell_ids",), data[var]) for var in data}, coords=coords, attrs=ds.attrs ) ds_healpix.cell_ids.attrs = { diff --git a/pixcdust/dggs/h3_tools.py b/pixcdust/dggs/h3_tools.py index e2fad98..72ca124 100644 --- a/pixcdust/dggs/h3_tools.py +++ b/pixcdust/dggs/h3_tools.py @@ -21,6 +21,7 @@ from shapely.geometry import Polygon + def h3_to_polygon(h3_index): boundary = h3.api.basic_int.h3_to_geo_boundary(h3_index, geo_json=True) return Polygon(boundary) @@ -33,30 +34,34 @@ def get_h3_res_name(res: int) -> str: def gdf_to_h3_gdf( gdf: gpd.GeoDataFrame, resolution: int, - ) -> gpd.GeoDataFrame: +) -> gpd.GeoDataFrame: """ - Convert a GeoDataFrame with latitude and longitude columns to a GeoDataFrame - where rows are aggregated into H3 hexagons based on a given resolution. + Convert a GeoDataFrame with latitude and longitude columns to a GeoDataFrame + where rows are aggregated into H3 hexagons based on a given resolution. - Args: - gdf (gpd.GeoDataFrame): The input GeoDataFrame containing 'latitude' and 'longitude' columns. - resolution (int): The H3 resolution level for hexagon indexing. + Args: + gdf (gpd.GeoDataFrame): The input GeoDataFrame containing 'latitude' and 'longitude' columns. + resolution (int): The H3 resolution level for hexagon indexing. - Returns: - gpd.GeoDataFrame: A new GeoDataFrame where the rows are grouped by H3 hexagons, - containing the mean of the grouped variables, and a geometry column with polygons - representing the H3 hexagons. - """ + Returns: + gpd.GeoDataFrame: A new GeoDataFrame where the rows are grouped by H3 hexagons, + containing the mean of the grouped variables, and a geometry column with polygons + representing the H3 hexagons. + """ # Get the column name for H3 index based on the resolution h3_col = get_h3_res_name(resolution) # Apply the H3 function to each row to calculate the H3 index based on latitude, longitude, and resolution - gdf[h3_col] = gdf.apply(lambda row: h3.api.basic_int.geo_to_h3(row['latitude'], row['longitude'], resolution), - axis=1) + gdf[h3_col] = gdf.apply( + lambda row: h3.api.basic_int.geo_to_h3( + row["latitude"], row["longitude"], resolution + ), + axis=1, + ) # Drop the latitude, longitude, and geometry columns as they are no longer needed - h3_df = gdf.drop(columns=['latitude', 'longitude', 'geometry']) + h3_df = gdf.drop(columns=["latitude", "longitude", "geometry"]) # Group by the H3 index column, and compute the mean of all other columns in each group h3_df = h3_df.groupby(h3_col).mean().reset_index() @@ -82,40 +87,45 @@ def get_healpix_res_name(res: int) -> str: def gdf_to_healpix_gdf( gdf: gpd.GeoDataFrame, resolution: int, - ) -> gpd.GeoDataFrame: +) -> gpd.GeoDataFrame: """ - Convert a GeoDataFrame with latitude and longitude columns to a GeoDataFrame - where rows are aggregated into Healpix geometry based on a given resolution. + Convert a GeoDataFrame with latitude and longitude columns to a GeoDataFrame + where rows are aggregated into Healpix geometry based on a given resolution. - Args: - gdf (gpd.GeoDataFrame): The input GeoDataFrame containing 'latitude' and 'longitude' columns. - resolution (int): The Healpix resolution level for indexing. + Args: + gdf (gpd.GeoDataFrame): The input GeoDataFrame containing 'latitude' and 'longitude' columns. + resolution (int): The Healpix resolution level for indexing. - Returns: - gpd.GeoDataFrame: A new GeoDataFrame where the rows are grouped by H3 hexagons, - containing the mean of the grouped variables, and a geometry column with polygons - representing the Healpix geometry. - """ + Returns: + gpd.GeoDataFrame: A new GeoDataFrame where the rows are grouped by H3 hexagons, + containing the mean of the grouped variables, and a geometry column with polygons + representing the Healpix geometry. + """ nside = 2**resolution # Init heaplix grid - healpix = HEALPix(nside=nside, order='nested') + healpix = HEALPix(nside=nside, order="nested") # Get the column name for Healpix index based on the resolution healpix_col = get_healpix_res_name(resolution) # Apply the Healpix function to each row to calculate the Healpix index based on latitude, longitude, and resolution gdf[healpix_col] = gdf.apply( - lambda row: healpix.lonlat_to_healpix(row['longitude'] * u.deg, row['latitude'] * u.deg), axis=1 + lambda row: healpix.lonlat_to_healpix( + row["longitude"] * u.deg, row["latitude"] * u.deg + ), + axis=1, ) # Drop the latitude, longitude, and geometry columns as they are no longer needed - healpix_df = gdf.drop(columns=['latitude', 'longitude', 'geometry']) + healpix_df = gdf.drop(columns=["latitude", "longitude", "geometry"]) # Group by the Healpix index column, and compute the mean of all other columns in each group healpix_df = healpix_df.groupby(healpix_col).mean().reset_index() # Convert each Healpix index into a polygon geometry representing its boundaries - geometry = healpix_df[healpix_col].apply(lambda pix: healpix_to_polygon(healpix, pix)) + geometry = healpix_df[healpix_col].apply( + lambda pix: healpix_to_polygon(healpix, pix) + ) return gpd.GeoDataFrame(data=healpix_df, geometry=geometry, crs=4326) diff --git a/pixcdust/downloaders/hydroweb_next.py b/pixcdust/downloaders/hydroweb_next.py index 65d953b..1a13c44 100644 --- a/pixcdust/downloaders/hydroweb_next.py +++ b/pixcdust/downloaders/hydroweb_next.py @@ -60,6 +60,7 @@ class Downloader(ABC): """ + PROVIDER = "hydroweb_next" def __init__( @@ -100,8 +101,6 @@ def __init__( self.setup() self.query_args = self.define_query() - - @staticmethod def _explode_simplify_geometry( @@ -141,15 +140,16 @@ def _explode_simplify_geometry( axis=1, ) if (geom["nodes_count"] > 200).any(): - raise AttributeError(( - "One or several of your search polygons have too many nodes," - "consider using the tolerance parameter" - "in order to simplify the polygons." - )) + raise AttributeError( + ( + "One or several of your search polygons have too many nodes," + "consider using the tolerance parameter" + "in order to simplify the polygons." + ) + ) return geom - def search_download(self, tolerance: Optional[float] = None) -> None: """Search files according to the query and download them. @@ -171,10 +171,12 @@ def search_download(self, tolerance: Optional[float] = None) -> None: for geom in geometries.geometry.values: self._search(geom.__geo_interface__) else: - raise AttributeError(( - "geometry should string (WKT) or geopandas.GeoDataFrame, " - f"received {type(self.geometry)} instead" - )) + raise AttributeError( + ( + "geometry should string (WKT) or geopandas.GeoDataFrame, " + f"received {type(self.geometry)} instead" + ) + ) # This command actually downloads the matching products downloaded_paths = self._download() @@ -191,17 +193,19 @@ def setup(self) -> None: @abstractmethod def define_query(self) -> dict: pass - + @abstractmethod - def _search(self, geom:Optional[str] = None) -> None: + def _search(self, geom: Optional[str] = None) -> None: pass @abstractmethod def _download(self) -> List: pass + class EODownloader(Downloader): """Downloader for SWOT Pixel Cloud files from hydroweb.next.""" + def __init__(self, *args, **kwargs): """Downloader for SWOT Pixel Cloud files from hydroweb.next initialization. @@ -220,7 +224,6 @@ def __init__(self, *args, **kwargs): """ super().__init__(*args, **kwargs) - def setup(self) -> None: self.dag = EODataAccessGateway() @@ -233,19 +236,20 @@ def setup(self) -> None: self.__check_collection_name() - def __check_collection_name(self) -> None: list_collections = [ d.id for d in self.dag.list_collections(provider=self.PROVIDER) ] if self.collection_name not in list_collections: - raise ValueError(( - "Did not find collection_name in " - f"list of available collections in {self.PROVIDER}." - f"\nAvailable collections are: {list_collections}" - )) - + raise ValueError( + ( + "Did not find collection_name in " + f"list of available collections in {self.PROVIDER}." + f"\nAvailable collections are: {list_collections}" + ) + ) + def define_query(self) -> dict: # Default search criteria when iterating over collection pages default_search_criteria = { @@ -258,35 +262,33 @@ def define_query(self) -> dict: } if self.dates is not None: - self.query_args["start"] = \ - self.dates[0].strftime("%Y-%m-%dT%H:%M:%SZ") - self.query_args["end"] = \ - self.dates[1].strftime("%Y-%m-%dT%H:%M:%SZ") + self.query_args["start"] = self.dates[0].strftime("%Y-%m-%dT%H:%M:%SZ") + self.query_args["end"] = self.dates[1].strftime("%Y-%m-%dT%H:%M:%SZ") self.query_args.update(default_search_criteria) return self.query_args - def _search(self, geom:Optional[str] = None) -> None: + def _search(self, geom: Optional[str] = None) -> None: if geom is not None: self.query_args["geom"] = geom self.search_results = self.dag.search_all(**self.query_args) # Iterate over all pages to find all products - #for page_results in self.dag.search_iter_page(**self.query_args): + # for page_results in self.dag.search_iter_page(**self.query_args): # self.search_results.extend(page_results) def _download(self) -> List: - #donwload only .nc asset + # donwload only .nc asset downloaded_paths = self.dag.download_all( - self.search_results, asset=r".*\.nc$", - output_dir=self.path_download + self.search_results, asset=r".*\.nc$", output_dir=self.path_download ) return downloaded_paths class DefaultDownloader(Downloader): """Downloader for SWOT Pixel Cloud files from hydroweb.next.""" + def __init__(self, *args, **kwargs): """Downloader for SWOT Pixel Cloud files from hydroweb.next initialization. @@ -306,38 +308,45 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def setup(self) -> None: - apikey = os.environ['HYDROWEB_API_KEY'] + apikey = os.environ["HYDROWEB_API_KEY"] self.client = py_hydroweb.Client(api_key=apikey) self.downloaded_paths = [] def define_query(self) -> dict: if self.dates is not None: - self.query_args["start_datetime"] = {'gte': self.dates[0].isoformat(timespec='milliseconds')+"Z"} - self.query_args["end_datetime"] = {'lte':self.dates[1].isoformat(timespec='milliseconds')+"Z"} + self.query_args["start_datetime"] = { + "gte": self.dates[0].isoformat(timespec="milliseconds") + "Z" + } + self.query_args["end_datetime"] = { + "lte": self.dates[1].isoformat(timespec="milliseconds") + "Z" + } return self.query_args - def _search(self, geom:Optional[str] = None) -> None: + def _search(self, geom: Optional[str] = None) -> None: # This command actually downloads the matching products basket = py_hydroweb.DownloadBasket("pixcdust_basket") - - kwargs= {"collection_id": self.collection_name, "query": self.query_args,"folder": self.collection_name} + + kwargs = { + "collection_id": self.collection_name, + "query": self.query_args, + "folder": self.collection_name, + } if geom is not None: kwargs.update({"intersects": geom}) basket.add_collection(**kwargs) - self.download_id = self.client.submit_download(download_basket=basket) - - def _download(self) -> List: - downloaded_zip_path = self.client.download_zip(download_id=self.download_id, output_folder=self.path_download) + downloaded_zip_path = self.client.download_zip( + download_id=self.download_id, output_folder=self.path_download + ) with zipfile.ZipFile(downloaded_zip_path, "r") as zf: # Liste des chemins à extraire files = [name for name in zf.namelist() if name.lower().endswith(".nc")] - for member in tqdm.tqdm(files, desc = "Extracting zip"): + for member in tqdm.tqdm(files, desc="Extracting zip"): zf.extract(member, path=self.path_download) downloaded_path = os.path.join(self.path_download, member) self.downloaded_paths.append(downloaded_path) @@ -345,31 +354,29 @@ def _download(self) -> List: os.remove(downloaded_zip_path) self.client.delete_download(download_id=self.download_id) - - + return self.downloaded_paths -def PixCDownloader(*args, backend='default',**kwargs): +def PixCDownloader(*args, backend="default", **kwargs): """Downloader for SWOT Pixel Cloud files from hydroweb.next initialization. - Keyword Args: - geometry: A geometry used as search criteria. Defaults to None. - dates: Minimum and maximum dates to be used as search criteria. - Defaults to None. - path_download: - download path. Defaults to "/tmp/hydroweb_next". - verbose: Verbose level (0: nothing, 1: only progress bars, 2: INFO, 3: DEBUG). - Defaults to 0. - - Raises: - AttributeError: if the geometry is not one - of (str, tuple, list, geopandas.GeoDataFrame) + Keyword Args: + geometry: A geometry used as search criteria. Defaults to None. + dates: Minimum and maximum dates to be used as search criteria. + Defaults to None. + path_download: + download path. Defaults to "/tmp/hydroweb_next". + verbose: Verbose level (0: nothing, 1: only progress bars, 2: INFO, 3: DEBUG). + Defaults to 0. + + Raises: + AttributeError: if the geometry is not one + of (str, tuple, list, geopandas.GeoDataFrame) """ if backend == "eodag": - print(f'using backend: {backend}') - return EODownloader("SWOT_L2_HR_PIXC",*args, **kwargs) + print(f"using backend: {backend}") + return EODownloader("SWOT_L2_HR_PIXC", *args, **kwargs) else: - print('using default backend: py-hydroweb') - return DefaultDownloader("SWOT_L2_HR_PIXC",*args, **kwargs) - + print("using default backend: py-hydroweb") + return DefaultDownloader("SWOT_L2_HR_PIXC", *args, **kwargs) diff --git a/pixcdust/notebooks/convert/convert_to_zcollection.ipynb b/pixcdust/notebooks/convert/convert_to_zcollection.ipynb index c590c3d..6531fc7 100644 --- a/pixcdust/notebooks/convert/convert_to_zcollection.ipynb +++ b/pixcdust/notebooks/convert/convert_to_zcollection.ipynb @@ -53,10 +53,10 @@ ], "source": [ "pixc = Nc2ZarrConverter(\n", - " path_in = sorted(glob('/tmp/pixc/**/*.nc',recursive=True)),\n", - " variables=['height', 'sig0', 'classification'],\n", - " )\n", - "pixc.database_from_nc(path_out= \"/tmp/pixc_zarr\")" + " path_in=sorted(glob(\"/tmp/pixc/**/*.nc\", recursive=True)),\n", + " variables=[\"height\", \"sig0\", \"classification\"],\n", + ")\n", + "pixc.database_from_nc(path_out=\"/tmp/pixc_zarr\")" ] }, { @@ -104,7 +104,7 @@ ], "source": [ "pixcr = ZarrReader(\"/tmp/pixc_zarr\")\n", - "pixcr.read((datetime.datetime(2023,4,6), datetime.datetime(2023,4,8)))\n", + "pixcr.read((datetime.datetime(2023, 4, 6), datetime.datetime(2023, 4, 8)))\n", "pixc" ] }, diff --git a/pixcdust/notebooks/convert/dggs_tuto.ipynb b/pixcdust/notebooks/convert/dggs_tuto.ipynb index d8a38ba..859d598 100644 --- a/pixcdust/notebooks/convert/dggs_tuto.ipynb +++ b/pixcdust/notebooks/convert/dggs_tuto.ipynb @@ -47,8 +47,8 @@ "\n", "# Limiting time period\n", "dates = (\n", - " datetime(2023,4,6),\n", - " datetime(2023,4,8),\n", + " datetime(2023, 4, 6),\n", + " datetime(2023, 4, 8),\n", ")" ] }, @@ -106,8 +106,8 @@ " gdf_geom,\n", " dates,\n", " verbose=1,\n", - " path_download='/tmp/pixc',\n", - " )\n", + " path_download=\"/tmp/pixc\",\n", + ")\n", "pixcdownloader.search_download()" ] }, @@ -141,7 +141,7 @@ ], "source": [ "# Search swot nc files\n", - "swot_nc_files = glob.glob('/tmp/pixc/**/*.nc',recursive=True)\n", + "swot_nc_files = glob.glob(\"/tmp/pixc/**/*.nc\", recursive=True)\n", "swot_nc_files" ] }, @@ -847,7 +847,7 @@ ], "source": [ "# Chose one or more variables\n", - "reader.data['height']" + "reader.data[\"height\"]" ] }, { @@ -1451,7 +1451,9 @@ ], "source": [ "# Reproject variables into h3 grid\n", - "ds_h3 = reader.to_h3(variables = 'height', resolution = 8) # Modify the resolution if needeed\n", + "ds_h3 = reader.to_h3(\n", + " variables=\"height\", resolution=8\n", + ") # Modify the resolution if needeed\n", "ds_h3" ] }, @@ -1479,7 +1481,7 @@ ], "source": [ "# Show\n", - "ds_h3['height'].dggs.explore()" + "ds_h3[\"height\"].dggs.explore()" ] }, { @@ -2082,7 +2084,9 @@ ], "source": [ "# Reproject variables into healpix grid\n", - "ds_healpix = reader.to_healpix(variables = 'height', resolution = 13) # modify resolution if needed\n", + "ds_healpix = reader.to_healpix(\n", + " variables=\"height\", resolution=13\n", + ") # modify resolution if needed\n", "ds_healpix" ] }, @@ -2110,7 +2114,7 @@ ], "source": [ "# Show\n", - "ds_healpix['height'].dggs.explore()" + "ds_healpix[\"height\"].dggs.explore()" ] }, { diff --git a/pixcdust/notebooks/convert/download_pixc_to_gpkg.ipynb b/pixcdust/notebooks/convert/download_pixc_to_gpkg.ipynb index e4d9361..8e14544 100644 --- a/pixcdust/notebooks/convert/download_pixc_to_gpkg.ipynb +++ b/pixcdust/notebooks/convert/download_pixc_to_gpkg.ipynb @@ -41,8 +41,8 @@ "\n", "# Limiting time period\n", "dates = (\n", - " datetime(2023,4,6),\n", - " datetime(2023,4,8),\n", + " datetime(2023, 4, 6),\n", + " datetime(2023, 4, 8),\n", ")" ] }, @@ -80,8 +80,8 @@ " gdf_geom,\n", " dates,\n", " verbose=1,\n", - " path_download='/tmp/pixc1',\n", - " )\n", + " path_download=\"/tmp/pixc1\",\n", + ")\n", "pixcdownloader.search_download()" ] }, @@ -216,16 +216,17 @@ ], "source": [ "# You can specify conditions on variables to filter data\n", - "conditions= {\"sig0\":{'operator': \"gt\", 'threshold': 20}, # sig0 > 20\n", - " \"classification\":{'operator': \"ge\", 'threshold': 3}, # classification >= 3\n", - " }\n", + "conditions = {\n", + " \"sig0\": {\"operator\": \"gt\", \"threshold\": 20}, # sig0 > 20\n", + " \"classification\": {\"operator\": \"ge\", \"threshold\": 3}, # classification >= 3\n", + "}\n", "\n", "pixc = Nc2GpkgConverter(\n", - " path_in = glob(pixcdownloader.path_download+'/*/*.nc'),\n", - " variables=['height', 'sig0', 'classification'],\n", - " area_of_interest=gdf_geom,\n", - " conditions=conditions\n", - " )\n", + " path_in=glob(pixcdownloader.path_download + \"/*/*.nc\"),\n", + " variables=[\"height\", \"sig0\", \"classification\"],\n", + " area_of_interest=gdf_geom,\n", + " conditions=conditions,\n", + ")\n", "pixc.database_from_nc(path_out=\"/tmp/pixc_gpkg.gpkg\")" ] }, @@ -867,10 +868,8 @@ "source": [ "from pixcdust.readers.gpkg import GpkgReader\n", "\n", - "# nb: you may specify \n", - "pixc_read = GpkgReader(\n", - " \"/tmp/pixc_gpkg.gpkg\"\n", - ")\n", + "# nb: you may specify\n", + "pixc_read = GpkgReader(\"/tmp/pixc_gpkg.gpkg\")\n", "pixc_read.read()\n", "pixc_read.data" ] @@ -897,7 +896,10 @@ ], "source": [ "from pixcdust.converters.gpkg import GpkgDGGSProjecter\n", - "h3_grid = GpkgDGGSProjecter(\"/tmp/pixc_gpkg.gpkg\", 10, path_out = '../data/h3_gpkg.gpkg', healpix=False) # True for healpix projection\n", + "\n", + "h3_grid = GpkgDGGSProjecter(\n", + " \"/tmp/pixc_gpkg.gpkg\", 10, path_out=\"../data/h3_gpkg.gpkg\", healpix=False\n", + ") # True for healpix projection\n", "h3_grid.compute_layers()" ] }, @@ -1521,9 +1523,7 @@ } ], "source": [ - "pixc_read = GpkgReader(\n", - " '../data/h3_gpkg.gpkg'\n", - ")\n", + "pixc_read = GpkgReader(\"../data/h3_gpkg.gpkg\")\n", "pixc_read.read()\n", "pixc_read.data" ] @@ -2707,27 +2707,27 @@ "import folium\n", "import branca.colormap as cmp\n", "\n", - "layer_h3 = pixc_read.read_single_layer('20230407_483_16_78L_10__h3')\n", + "layer_h3 = pixc_read.read_single_layer(\"20230407_483_16_78L_10__h3\")\n", "\n", "# creating a colormap\n", "linear = cmp.LinearColormap(\n", - " ['blue', 'purple', 'orange', 'yellow'],\n", - " vmin=layer_h3['wse'].min(), # minimum wse value\n", - " vmax=layer_h3['wse'].max(), # maximum wse value\n", - " caption='Water Surface Elevation (m)' #Caption for Color scale or Legend\n", + " [\"blue\", \"purple\", \"orange\", \"yellow\"],\n", + " vmin=layer_h3[\"wse\"].min(), # minimum wse value\n", + " vmax=layer_h3[\"wse\"].max(), # maximum wse value\n", + " caption=\"Water Surface Elevation (m)\", # Caption for Color scale or Legend\n", ")\n", "# Initiating map\n", "m = folium.Map([43.6, 1.43], zoom_start=12, tiles=\"cartodbpositron\")\n", "\n", "folium.GeoJson(\n", " layer_h3,\n", - " style_function = lambda row: {\n", - " 'fillColor': linear(row['properties'][\"wse\"]),\n", - " 'weight': 0, #how thick the border has to be\n", - " 'fillOpacity': 1\n", + " style_function=lambda row: {\n", + " \"fillColor\": linear(row[\"properties\"][\"wse\"]),\n", + " \"weight\": 0, # how thick the border has to be\n", + " \"fillOpacity\": 1,\n", " },\n", ").add_to(m)\n", - "linear.add_to(m) #adds colorscale and legend\n", + "linear.add_to(m) # adds colorscale and legend\n", "m" ] }, diff --git a/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb b/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb index 74e8f96..e7d4ecc 100644 --- a/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb +++ b/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb @@ -40,8 +40,8 @@ "gdf_geom = gpd.read_file(\"../data/aoi.gpkg\")\n", "\n", "dates = (\n", - " datetime(2023,4,6),\n", - " datetime(2023,4,8),\n", + " datetime(2023, 4, 6),\n", + " datetime(2023, 4, 8),\n", ")" ] }, @@ -106,8 +106,8 @@ " gdf_geom,\n", " dates,\n", " verbose=1,\n", - " path_download='/tmp/pixc',\n", - " )\n", + " path_download=\"/tmp/pixc\",\n", + ")\n", "pixcdownloader.search_download()" ] }, @@ -138,17 +138,18 @@ "outputs": [], "source": [ "# You can specify conditions on variables to filter data\n", - "conditions= {\"sig0\":{'operator': \"gt\", 'threshold': 20}, # sig0 > 20\n", - " \"classification\":{'operator': \"ge\", 'threshold': 3}, # classification >= 3\n", - " }\n", + "conditions = {\n", + " \"sig0\": {\"operator\": \"gt\", \"threshold\": 20}, # sig0 > 20\n", + " \"classification\": {\"operator\": \"ge\", \"threshold\": 3}, # classification >= 3\n", + "}\n", "\n", "pixc = Nc2ZarrConverter(\n", - " path_in = glob(pixcdownloader.path_download+'/**/*.nc', recursive=True),\n", - " variables=['height', 'sig0', 'classification'],\n", - " area_of_interest=gdf_geom,\n", - " conditions=conditions,\n", - " )\n", - "pixc.database_from_nc(path_out='/tmp/pixc_zarr')" + " path_in=glob(pixcdownloader.path_download + \"/**/*.nc\", recursive=True),\n", + " variables=[\"height\", \"sig0\", \"classification\"],\n", + " area_of_interest=gdf_geom,\n", + " conditions=conditions,\n", + ")\n", + "pixc.database_from_nc(path_out=\"/tmp/pixc_zarr\")" ] }, { @@ -1295,10 +1296,8 @@ "from pixcdust.readers.zarr import ZarrReader\n", "import datetime\n", "\n", - "pixc_read = ZarrReader(\n", - " \"/tmp/pixc_zarr\"\n", - ")\n", - "pixc_read.read((datetime.datetime(2023,4,6), datetime.datetime(2023,4,8)))\n", + "pixc_read = ZarrReader(\"/tmp/pixc_zarr\")\n", + "pixc_read.read((datetime.datetime(2023, 4, 6), datetime.datetime(2023, 4, 8)))\n", "pixc_read.data" ] }, diff --git a/pixcdust/notebooks/read_and_use/read_netcdf.ipynb b/pixcdust/notebooks/read_and_use/read_netcdf.ipynb index f0379f6..1068aab 100644 --- a/pixcdust/notebooks/read_and_use/read_netcdf.ipynb +++ b/pixcdust/notebooks/read_and_use/read_netcdf.ipynb @@ -23,7 +23,7 @@ "metadata": {}, "outputs": [], "source": [ - "swot_nc_files = glob.glob('/tmp/pixc/**/*.nc',recursive=True)" + "swot_nc_files = glob.glob(\"/tmp/pixc/**/*.nc\", recursive=True)" ] }, { @@ -35,11 +35,14 @@ "path = swot_nc_files[0]\n", "\n", "# You can specify conditions on variables to filter data\n", - "conditions= {\"sig0\":{'operator': \"gt\", 'threshold': 20}, # sig0 > 20\n", - " \"classification\":{'operator': \"ge\", 'threshold': 3}, # classification >= 3\n", - " }\n", + "conditions = {\n", + " \"sig0\": {\"operator\": \"gt\", \"threshold\": 20}, # sig0 > 20\n", + " \"classification\": {\"operator\": \"ge\", \"threshold\": 3}, # classification >= 3\n", + "}\n", "\n", - "ncsimple = NcSimpleReader(path, variables=['height', 'sig0', 'classification'], conditions=conditions)" + "ncsimple = NcSimpleReader(\n", + " path, variables=[\"height\", \"sig0\", \"classification\"], conditions=conditions\n", + ")" ] }, { @@ -1318,7 +1321,7 @@ } ], "source": [ - "ncsimple.to_xarray() # same thing as ncsimple.data" + "ncsimple.to_xarray() # same thing as ncsimple.data" ] }, { diff --git a/pixcdust/readers/__init__.py b/pixcdust/readers/__init__.py index c869fa5..b9a46b9 100644 --- a/pixcdust/readers/__init__.py +++ b/pixcdust/readers/__init__.py @@ -20,4 +20,4 @@ from pixcdust.readers.netcdf import NcSimpleReader from pixcdust.readers.gpkg import GpkgReader -from pixcdust.readers.zarr import ZarrReader \ No newline at end of file +from pixcdust.readers.zarr import ZarrReader diff --git a/pixcdust/readers/base_reader.py b/pixcdust/readers/base_reader.py index 203b7e5..fd7e1d8 100644 --- a/pixcdust/readers/base_reader.py +++ b/pixcdust/readers/base_reader.py @@ -23,11 +23,12 @@ import geopandas as gpd -PIXC_DATE_RE=re.compile(r'_\d{8}T\d{6}_\d{8}T\d{6}_') +PIXC_DATE_RE = re.compile(r"_\d{8}T\d{6}_\d{8}T\d{6}_") """Regex patern used to extract the date (daystartThourstart_dayxendThoursend) from a pixc file name. """ + def sorted_by_date(file_list: Iterable[Union[str, Path]]) -> List[Union[str, Path]]: """Sort the filenames by date as some converters need monotonic dates. The date is parsed from the filename according to PIXC_DATE_RE. @@ -37,12 +38,14 @@ def sorted_by_date(file_list: Iterable[Union[str, Path]]) -> List[Union[str, Pat Returns: Sorted file_list. """ + def file_name_to_date(file_name: Union[str, Path]): date_founds = PIXC_DATE_RE.findall(str(file_name)) if date_founds: return date_founds[-1] return file_name - return sorted(file_list, key = file_name_to_date) # sort by date + + return sorted(file_list, key=file_name_to_date) # sort by date class BaseReader: @@ -63,13 +66,16 @@ class BaseReader: "classification":{'operator': "ge", 'threshold': 3},\ } """ - MULTI_FILE_SUPPORT=False - def __init__(self, - path: str | Iterable[str] | Path | Iterable[Path], - variables: Optional[list[str]] = None, - area_of_interest: Optional[gpd.GeoDataFrame] = None, - conditions: Optional[dict[str, dict[str, Union[str, float]]]] = None, - ): + + MULTI_FILE_SUPPORT = False + + def __init__( + self, + path: str | Iterable[str] | Path | Iterable[Path], + variables: Optional[list[str]] = None, + area_of_interest: Optional[gpd.GeoDataFrame] = None, + conditions: Optional[dict[str, dict[str, Union[str, float]]]] = None, + ): """Basic pixcdust database reader configuration. Args: @@ -83,21 +89,21 @@ def __init__(self, } """ if isinstance(path, str | Path): - self.path: str | Iterable[str] = str(path) + self.path: str | Iterable[str] = str(path) self.multi_file_db = False else: if not self.MULTI_FILE_SUPPORT: raise ValueError("This reader does not support opening multiple files.") self.multi_file_db = True # sort the filenames by date as some converters need monotonic dates. - self.path = [str(p) for p in sorted_by_date(path)] + self.path = [str(p) for p in sorted_by_date(path)] self.area_of_interest = area_of_interest self._data: Optional[xr.Dataset] = None self.variables = variables self.conditions = conditions @property - def data(self) -> xr.Dataset: + def data(self) -> xr.Dataset: """Return an xarray.Dataset view from the database loaded. Equivalent to to_xarray. @@ -128,7 +134,6 @@ def to_dataframe(self) -> pd.DataFrame: """ return self.data.to_dataframe() - def to_geodataframe( self, ) -> gpd.GeoDataFrame: diff --git a/pixcdust/readers/gpkg.py b/pixcdust/readers/gpkg.py index e6c6b29..93cb6c6 100644 --- a/pixcdust/readers/gpkg.py +++ b/pixcdust/readers/gpkg.py @@ -41,10 +41,9 @@ class GpkgReader(BaseReader): MULTI_FILE_SUPPORT: False, only support one file. """ - def __init__(self, - path: str | Path, - area_of_interest: Optional[gpd.GeoDataFrame] = None - ): + def __init__( + self, path: str | Path, area_of_interest: Optional[gpd.GeoDataFrame] = None + ): """Gpkg pixcdust database reader configuration. Read the list of layers from path. @@ -54,16 +53,17 @@ def __init__(self, """ super().__init__(path, area_of_interest=area_of_interest) self._gdf_data: Optional[gpd.GeoDataFrame] = None - self.layers: list[str] = fiona.listlayers(self.path) + self.layers: list[str] = fiona.listlayers(self.path) @property - def data(self) -> xr.Dataset: + def data(self) -> xr.Dataset: return self._gdf_data.to_xarray() @data.setter def data(self, obj: xr.Dataset) -> None: - raise NotImplementedError("PixCGpkgReader internal data representation is a GeoDataFrame.") - + raise NotImplementedError( + "PixCGpkgReader internal data representation is a GeoDataFrame." + ) def to_geodataframe( self, @@ -114,7 +114,6 @@ def read(self, layers: Optional[List[str]] = None) -> None: layers = self.layers for layer in tqdm(layers): - layer_data = self.read_single_layer( layer, ) diff --git a/pixcdust/readers/netcdf.py b/pixcdust/readers/netcdf.py index d5a58ae..91fd71d 100644 --- a/pixcdust/readers/netcdf.py +++ b/pixcdust/readers/netcdf.py @@ -56,11 +56,9 @@ class NcSimpleConstants: @dataclass class NcFormatCfg: - """Class configuring how a SWOT pixel cloud files is expected to be structured. - """ - constants: NcSimpleConstants = field( - default_factory=NcSimpleConstants - ) + """Class configuring how a SWOT pixel cloud files is expected to be structured.""" + + constants: NcSimpleConstants = field(default_factory=NcSimpleConstants) trusted_group: str = "pixel_cloud" forbidden_variables: list[str] = field( default_factory=lambda: [ @@ -90,15 +88,17 @@ class NcSimpleReader(BaseReader): "classification":{'operator': "ge", 'threshold': 3},\ } """ + MULTI_FILE_SUPPORT = True - def __init__(self, - path: str | Iterable[str] | Path | Iterable[Path], - variables: Optional[list[str]] = None, - area_of_interest: Optional[geopandas.GeoDataFrame] = None, - format_cfg : Optional[NcFormatCfg] = None, - conditions: Optional[dict[str, dict[str, Union[str, float]]]] = None, - ): + def __init__( + self, + path: str | Iterable[str] | Path | Iterable[Path], + variables: Optional[list[str]] = None, + area_of_interest: Optional[geopandas.GeoDataFrame] = None, + format_cfg: Optional[NcFormatCfg] = None, + conditions: Optional[dict[str, dict[str, Union[str, float]]]] = None, + ): """Netcdf pixcdust reader configuration. Args: @@ -122,7 +122,9 @@ def __init__(self, self.conditions = conditions @staticmethod - def extract_info_from_nc_attrs(filename: str) -> Tuple[str, datetime, int, int, int, str]: + def extract_info_from_nc_attrs( + filename: str, + ) -> Tuple[str, datetime, int, int, int, str]: """Extracts orbit information from global attributes\ in a SWOT pixel cloud netcdf. @@ -166,8 +168,8 @@ def filter_variable(self) -> None: ValueError: If 'operator' or 'threshold' keys are not in conditions. AttributeError: If operator is not the function name of the operator module. """ - _k_operator = 'operator' - _k_to = 'threshold' + _k_operator = "operator" + _k_to = "threshold" # Loop through each condition and apply the filter for var, condition in self.conditions.items(): @@ -178,14 +180,17 @@ def filter_variable(self) -> None: # Ensure the condition dictionary has the correct keys if _k_operator not in condition or _k_to not in condition: - raise ValueError(f"Condition for variable '{var}' must include '{_k_operator}' and '{_k_to}'") + raise ValueError( + f"Condition for variable '{var}' must include '{_k_operator}' and '{_k_to}'" + ) # Get the operator function dynamically from the operator module try: operator_func = getattr(operator, condition[_k_operator]) except AttributeError: raise AttributeError( - f"Operator '{condition[_k_operator]}' is not a valid operator in the operator module") + f"Operator '{condition[_k_operator]}' is not a valid operator in the operator module" + ) threshold = condition[_k_to] @@ -194,10 +199,12 @@ def filter_variable(self) -> None: self.data[var] = self.data[var].compute() # Apply the filter using .where() on the dataset - self.data = self.data.where(operator_func(self.data[var], threshold), drop=True) + self.data = self.data.where( + operator_func(self.data[var], threshold), drop=True + ) def read(self, orbit_info: bool = False) -> None: - """ Load self.path file(s). + """Load self.path file(s). You can then access from data or with methods like to_xarray, to_dataframe or to_geodataframe. @@ -212,7 +219,7 @@ def read(self, orbit_info: bool = False) -> None: return self.open_dataset() def open_dataset(self) -> None: - """ Load the self.path file (need only one file in self.path). + """Load the self.path file (need only one file in self.path). You can then access from data or with methods like to_xarray, to_dataframe or to_geodataframe. """ @@ -230,10 +237,10 @@ def open_dataset(self) -> None: self.__postprocess_points() def open_mfdataset( - self, - orbit_info: bool = False, + self, + orbit_info: bool = False, ) -> None: - """ Load self.path file(s) as a nested array. + """Load self.path file(s) as a nested array. You can then access from data or with methods like to_xarray, to_dataframe or to_geodataframe. @@ -288,11 +295,13 @@ def open_mfdataset( self.__postprocess_points() - def to_h3(self, - variables: str | list[str] | None=None, - resolution: int = 8, - interp: bool=False, - method: str = 'linear') -> xr.Dataset: + def to_h3( + self, + variables: str | list[str] | None = None, + resolution: int = 8, + interp: bool = False, + method: str = "linear", + ) -> xr.Dataset: """ Convert a Dataset with latitude and longitude coordinates into an H3-indexed grid. @@ -311,12 +320,17 @@ def to_h3(self, data = self.to_xarray()[variables] else: data = self.to_xarray() - return prepare_dataset_h3(data, resolution=resolution, interp=interp, method=method) + return prepare_dataset_h3( + data, resolution=resolution, interp=interp, method=method + ) - def to_healpix(self, variables: str | list[str] | None=None, - resolution: int = 8, - interp: bool= False, - method: str = 'linear') -> xr.Dataset: + def to_healpix( + self, + variables: str | list[str] | None = None, + resolution: int = 8, + interp: bool = False, + method: str = "linear", + ) -> xr.Dataset: """ Convert a Dataset with latitude and longitude coordinates into an HEALPix-indexed grid. @@ -335,7 +349,9 @@ def to_healpix(self, variables: str | list[str] | None=None, data = self.to_xarray()[variables] else: data = self.to_xarray() - return prepare_dataset_healpix(data, resolution=resolution, interp=interp, method=method) + return prepare_dataset_healpix( + data, resolution=resolution, interp=interp, method=method + ) def __postprocess_points(self) -> None: """Adds a points coordinates containing shapely.Points (longitude, latitude) diff --git a/pixcdust/readers/zarr.py b/pixcdust/readers/zarr.py index 3172d9e..22787ae 100644 --- a/pixcdust/readers/zarr.py +++ b/pixcdust/readers/zarr.py @@ -38,13 +38,11 @@ class ZarrReader(BaseReader): MULTI_FILE_SUPPORT: False, only support one file. """ - def read( self, - date_interval: Optional[ - Tuple[datetime.datetime, datetime.datetime] - ] | None = None, - ) -> None: + date_interval: Optional[Tuple[datetime.datetime, datetime.datetime]] + | None = None, + ) -> None: """Load a zarr database. You can then access from data or with methods like to_xarray, to_dataframe or to_geodataframe. @@ -56,17 +54,25 @@ def read( collection = zcollection.open_collection( self.path, - mode='r', + mode="r", ) if date_interval: date_min = date_interval[0] date_max = date_interval[1] data_z = collection.load( - filters=lambda keys: date_min <= datetime.datetime( - keys['year'], keys['month'], keys['day'], - keys['hour'], keys['minute'], keys['second'], - ) <= date_max + filters=lambda keys: ( + date_min + <= datetime.datetime( + keys["year"], + keys["month"], + keys["day"], + keys["hour"], + keys["minute"], + keys["second"], + ) + <= date_max + ) ) else: data_z = collection.load() diff --git a/pixcdust/tests/init_tests.py b/pixcdust/tests/init_tests.py index 2a34b9a..6f33a42 100644 --- a/pixcdust/tests/init_tests.py +++ b/pixcdust/tests/init_tests.py @@ -7,21 +7,22 @@ from pathlib import Path from pixcdust.downloaders.hydroweb_next import PixCDownloader + class JsonTestsSettings: - """ Reader-writer for the test configuration. - """ - CONFIG_FILE_NAME = 'conftest.json' + """Reader-writer for the test configuration.""" + + CONFIG_FILE_NAME = "conftest.json" + def __init__(self): try: with open(self._config_path) as f: self._settings = json.load(f) - except FileNotFoundError : + except FileNotFoundError: self._settings = {} @property def input_folder(self) -> Path: - """ Path to folder where test input data are downloaded and stored. - """ + """Path to folder where test input data are downloaded and stored.""" try: return Path(self._settings["input_folder"]) except KeyError: @@ -33,8 +34,7 @@ def input_folder(self, value: Union[Path, str]) -> None: @property def tmp_folder(self) -> Path: - """ Path to folder where test outputs data are written. - """ + """Path to folder where test outputs data are written.""" return Path(self._settings.get("tmp_folder", "/tmp/pixcdust-test")) @tmp_folder.setter @@ -43,8 +43,7 @@ def tmp_folder(self, value: Union[Path, str]) -> None: @property def hydroweb_auth(self) -> str: - """Hydroweb.next personal API key. - """ + """Hydroweb.next personal API key.""" return self._settings.get("hydroweb_auth", "") @hydroweb_auth.setter @@ -52,11 +51,10 @@ def hydroweb_auth(self, value: str) -> None: self._settings["hydroweb_auth"] = value def write(self) -> None: - """Write the config in JSON to self._config_path. - """ - with open(self._config_path, mode='w') as f: + """Write the config in JSON to self._config_path.""" + with open(self._config_path, mode="w") as f: json.dump(self._settings, f) - os.chmod(self._config_path,0o600) + os.chmod(self._config_path, 0o600) @property def _config_path(self) -> Path: @@ -64,7 +62,8 @@ def _config_path(self) -> Path: Should be the absolute path of tests/conftest.json """ - return Path(__file__).parent.absolute()/self.CONFIG_FILE_NAME + return Path(__file__).parent.absolute() / self.CONFIG_FILE_NAME + def init_hydroweb_env(test_settings: JsonTestsSettings) -> None: """Configure the Hydroweb.next API key of the current environment. @@ -85,21 +84,20 @@ def download_test_data(path_download: Path, backend: str) -> None: path_download: where to store the test data. """ dates = ( - datetime(2024,8,1), - datetime(2024,8,15), + datetime(2024, 8, 1), + datetime(2024, 8, 15), ) geometry = "POLYGON((-1.50580 43.39543,-1.36597 43.39543,-1.36597 43.56471,-1.50580 43.56471,-1.50580 43.39543))" pixcdownloader = PixCDownloader( - geometry, - dates, - backend=backend, - verbose=0, - path_download=str(path_download) - ) + geometry, dates, backend=backend, verbose=0, path_download=str(path_download) + ) pixcdownloader.search_download() + TEST_DATA_COUNT = 2 + + def check_test_data(path_download: Path) -> bool: data_list = list(path_download.glob("**/*.nc")) return len(data_list) == TEST_DATA_COUNT @@ -107,27 +105,38 @@ def check_test_data(path_download: Path) -> bool: if __name__ == "__main__": parser = argparse.ArgumentParser( - prog='init_tests', - description='Configure the tests and if missing download the test data' + prog="init_tests", + description="Configure the tests and if missing download the test data", + ) + parser.add_argument( + "-I", "--input_folder", help="path where is downloaded the test input data" + ) + parser.add_argument( + "-T", "--tmp_folder", help="path where is writen the temporary data" + ) + parser.add_argument( + "-H", "--hydroweb_auth", help="api key to download from hydroweb" + ) + parser.add_argument( + "-D", + "--download", + help="force download of the test data", + nargs="?", + const="true", ) - parser.add_argument("-I","--input_folder", help="path where is downloaded the test input data") - parser.add_argument("-T","--tmp_folder", help="path where is writen the temporary data") - parser.add_argument("-H","--hydroweb_auth", help="api key to download from hydroweb") - parser.add_argument("-D","--download", help="force download of the test data", nargs='?', const='true') args = parser.parse_args() settings = JsonTestsSettings() - dl_cfg_changed=False + dl_cfg_changed = False if args.input_folder: - dl_cfg_changed=True + dl_cfg_changed = True settings.input_folder = args.input_folder if args.tmp_folder: settings.tmp_folder = args.tmp_folder if args.hydroweb_auth: - dl_cfg_changed=True + dl_cfg_changed = True settings.hydroweb_auth = args.hydroweb_auth - settings.write() path_download = settings.input_folder @@ -135,9 +144,8 @@ def check_test_data(path_download: Path) -> bool: if args.download is None: if not check_test_data(path_download): # config changed and the data is missing. - download_test_data(path_download,'default') + download_test_data(path_download, "default") else: if args.download.lower() == "true": # download requested by user - download_test_data(path_download,'default') - + download_test_data(path_download, "default") diff --git a/pixcdust/tests/mock.py b/pixcdust/tests/mock.py index d191ab0..2af690f 100644 --- a/pixcdust/tests/mock.py +++ b/pixcdust/tests/mock.py @@ -79,4 +79,4 @@ def mock_xarray(length: int = 10000) -> xr.Dataset: def mock_area_of_interest() -> gpd.GeoDataFrame: - raise NotImplementedError \ No newline at end of file + raise NotImplementedError diff --git a/pixcdust/tests/test_converters_mock.py b/pixcdust/tests/test_converters_mock.py index 64bba3d..2be9c2f 100644 --- a/pixcdust/tests/test_converters_mock.py +++ b/pixcdust/tests/test_converters_mock.py @@ -6,20 +6,15 @@ class TestConverters(unittest.TestCase): - """Class for testing Converters, to be implemented - """ + """Class for testing Converters, to be implemented""" def setUp(self): - """function to set up the test environment - """ - self.list_vars = [ - "height", "sig0", "classification", "geoid", "cross_track" - ] + """function to set up the test environment""" + self.list_vars = ["height", "sig0", "classification", "geoid", "cross_track"] self.data = mock_xarray() def test_convert_ds_to_gpkg(self): - """function for testing the conversion to geopackage - """ + """function for testing the conversion to geopackage""" pixc = Nc2GpkgConverter( "/tmp", variables=self.list_vars, @@ -31,7 +26,7 @@ def test_convert_ds_to_gpkg(self): # TODO: add relevant tests def test_convert_ds_to_zarr(self): - """function for testing the conversion from + """function for testing the conversion from netcdf to zarr with zcollection """ diff --git a/pixcdust/tests/test_dggs.py b/pixcdust/tests/test_dggs.py index 39b5d93..86aa5a8 100644 --- a/pixcdust/tests/test_dggs.py +++ b/pixcdust/tests/test_dggs.py @@ -45,6 +45,6 @@ def test_healpix_conversion(first_file): assert "cell_ids" in ds_healpix.coords, "HEALPix cell IDs should be present" # Ensure the data is not empty after conversion - assert ( - len(ds_healpix["cell_ids"]) > 0 - ), "The output dataset should have HEALPix cell IDs" + assert len(ds_healpix["cell_ids"]) > 0, ( + "The output dataset should have HEALPix cell IDs" + ) diff --git a/pixcdust/tests/test_downloaders.py b/pixcdust/tests/test_downloaders.py index 22de58b..9a2a8d0 100644 --- a/pixcdust/tests/test_downloaders.py +++ b/pixcdust/tests/test_downloaders.py @@ -11,7 +11,7 @@ def test_hydroweb_next(hydroweb_env, input_folder, tmp_folder): Only run with the option --ddl. """ dl_dir = tmp_folder / "download_test" - download_test_data(dl_dir,'default') + download_test_data(dl_dir, "default") dl_files = sorted(dl_dir.glob("**/*.nc")) all_input_files = sorted(input_folder.glob("**/*.nc")) @@ -20,6 +20,7 @@ def test_hydroweb_next(hydroweb_env, input_folder, tmp_folder): for dl_f, input_f in zip(dl_files, all_input_files): assert dl_f.stat().st_size == input_f.stat().st_size + @pytest.mark.downloader def test_hydroweb_next_eodag(hydroweb_env, input_folder, tmp_folder): """Test hydroweb.next eodag downloader. @@ -28,7 +29,7 @@ def test_hydroweb_next_eodag(hydroweb_env, input_folder, tmp_folder): Only run with the option --ddl. """ dl_dir = tmp_folder / "download_test_eodag" - download_test_data(dl_dir,'eodag') + download_test_data(dl_dir, "eodag") # fix: avoid listing eodag's .download repo dl_files = sorted(dl_dir.glob("**/[!.]*.nc")) diff --git a/pixcdust/tools/convert_pixc.py b/pixcdust/tools/convert_pixc.py index 9ea7761..9f4d692 100644 --- a/pixcdust/tools/convert_pixc.py +++ b/pixcdust/tools/convert_pixc.py @@ -23,6 +23,7 @@ from pixcdust.converters.shapefile import Nc2ShpConverter from pixcdust.converters.core import Converter + def paths_glob(ctx, param, paths): return list(paths) @@ -36,26 +37,24 @@ def paths_glob(ctx, param, paths): help="list of variables of interest to extract from SWOT PIXC files,\ separated with commas ','", ) -@click.option("--aoi", type=click.File(mode='r'), default=None) +@click.option("--aoi", type=click.File(mode="r"), default=None) @click.option( - "-m", "--mode", - type=click.Choice(['w', 'o']), + "-m", + "--mode", + type=click.Choice(["w", "o"]), help="Mode for writing in database", - default=('w'), + default=("w"), ) @click.argument( - 'format_out', - type=click.Choice( - ['gpkg', 'zarr', 'shp'], - case_sensitive=False - ), + "format_out", + type=click.Choice(["gpkg", "zarr", "shp"], case_sensitive=False), ) @click.argument( - 'path_out', + "path_out", type=click.Path(), ) @click.argument( - 'paths_in', + "paths_in", nargs=-1, callback=paths_glob, ) @@ -66,7 +65,7 @@ def cli( variables: str, aoi: str, mode: str, - ): +): """_summary_ Args: @@ -84,13 +83,13 @@ def cli( NotImplementedError: _description_ """ if variables is not None: - variables.strip('()') - variables.strip('[]') - list_vars = variables.split(',') + variables.strip("()") + variables.strip("[]") + list_vars = variables.split(",") for var in list_vars: if any(not c.isalnum() for c in var): raise click.BadOptionUsage( - 'variables', + "variables", "apart from the commas, no special caracter may be used", ) @@ -102,19 +101,19 @@ def cli( else: gdf_aoi = None - if format_out.lower() == 'gpkg': - pixc : Converter = Nc2GpkgConverter( + if format_out.lower() == "gpkg": + pixc: Converter = Nc2GpkgConverter( paths_in, variables=list_vars, area_of_interest=gdf_aoi, ) - elif format_out.lower() == 'zarr': + elif format_out.lower() == "zarr": pixc = Nc2ZarrConverter( sorted(paths_in), variables=list_vars, area_of_interest=gdf_aoi, ) - elif format_out.lower() == 'shp': + elif format_out.lower() == "shp": pixc = Nc2ShpConverter( paths_in, variables=list_vars, @@ -122,8 +121,8 @@ def cli( ) else: raise NotImplementedError( - f'the conversion format {format_out} has not been implemented yet', - ) + f"the conversion format {format_out} has not been implemented yet", + ) pixc.database_from_nc(path_out, mode=mode) From 4275ab45dc479131d681605390f9a98d3fab97f0 Mon Sep 17 00:00:00 2001 From: pty Date: Mon, 31 Aug 2026 16:38:33 +0200 Subject: [PATCH 6/8] Fix ruff lint --- pixcdust/converters/core.py | 34 ++++---- pixcdust/converters/geo_utils.py | 2 +- pixcdust/converters/gpkg.py | 17 ++-- pixcdust/converters/zarr.py | 21 ++--- pixcdust/dggs/dggs_converter.py | 6 +- pixcdust/dggs/h3_tools.py | 3 +- pixcdust/downloaders/hydroweb_next.py | 46 +++++----- .../convert/convert_to_zcollection.ipynb | 14 +-- pixcdust/notebooks/convert/dggs_tuto.ipynb | 19 ++-- .../convert/download_pixc_to_gpkg.ipynb | 21 +++-- .../convert/download_pixc_to_zarr.ipynb | 27 +++--- .../notebooks/read_and_use/read_netcdf.ipynb | 5 +- pixcdust/readers/__init__.py | 8 +- pixcdust/readers/base_reader.py | 20 ++--- pixcdust/readers/gpkg.py | 13 ++- pixcdust/readers/netcdf.py | 86 +++++++++---------- pixcdust/readers/zarr.py | 8 +- pixcdust/tests/conftest.py | 5 +- pixcdust/tests/init_tests.py | 17 ++-- pixcdust/tests/mock.py | 9 +- pixcdust/tests/test_converters.py | 12 +-- pixcdust/tests/test_converters_mock.py | 2 +- pixcdust/tests/test_dggs.py | 1 + pixcdust/tools/convert_pixc.py | 5 +- 24 files changed, 205 insertions(+), 196 deletions(-) diff --git a/pixcdust/converters/core.py b/pixcdust/converters/core.py index 503643b..8b3c6ab 100644 --- a/pixcdust/converters/core.py +++ b/pixcdust/converters/core.py @@ -16,12 +16,11 @@ """Interface used by all Pixcdust Converters.""" import copy -from dataclasses import dataclass import operator +from collections.abc import Iterable +from dataclasses import dataclass from pathlib import Path -from typing import Optional, Union, Iterable - import geopandas as gpd @@ -44,9 +43,9 @@ class Converter: def __init__( self, path_in: str | Iterable[str] | Path | Iterable[Path], - variables: Optional[list[str]] = None, - area_of_interest: Optional[gpd.GeoDataFrame] = None, - conditions: Optional[dict[str, dict[str, Union[str, float]]]] = None, + variables: list[str] | None = None, + area_of_interest: gpd.GeoDataFrame | None = None, + conditions: dict[str, dict[str, str | float]] | None = None, ): """Basic initialisation of a pixcdust converter. @@ -141,7 +140,7 @@ class GeoLayerH3Projecter: resolution: int def filter_variable( - self, conditions: dict[str, dict[str, Union[str, float]]] + self, conditions: dict[str, dict[str, str | float]] ) -> None: """filters from xarray dataset based on operator and threshold on specific variables @@ -165,26 +164,29 @@ def filter_variable( _k_to = "threshold" # Test if conditions dict meets specifications print(conditions) - for k in conditions.keys(): - if k not in self.data.columns: - raise IOError( + for var, condition in conditions.items(): + if var not in self.data.columns: + raise OSError( f"dict conditions expected existing\ variables (in {self.data.columns}),\ - received {k}" + received {var}" ) - for instructions in conditions[k].keys(): + for instructions in condition: if instructions not in [_k_operator, _k_to]: raise ValueError( f"dict conditions expected {_k_to} and {_k_operator}\ keys in dict {conditions},\ received {instructions}" ) - print(f"operator.{conditions[k][_k_operator]}") - ope = getattr(operator, conditions[k][_k_operator]) + op_name = condition[_k_operator] + # Typing issue, improvement would be to use TypedDict for conditions + assert isinstance(op_name, str) + print(f"operator.{op_name}") + ope = getattr(operator, op_name) self.data = self.data[ ope( - self.data[k], - conditions[k][_k_to], + self.data[var], + condition[_k_to], ) ] diff --git a/pixcdust/converters/geo_utils.py b/pixcdust/converters/geo_utils.py index f34ee35..b959ee3 100644 --- a/pixcdust/converters/geo_utils.py +++ b/pixcdust/converters/geo_utils.py @@ -15,8 +15,8 @@ # """Converters utility""" -import xarray as xr import geopandas as gpd +import xarray as xr def geoxarray_to_geodataframe(ds: xr.Dataset, *args, **kwargs) -> gpd.GeoDataFrame: diff --git a/pixcdust/converters/gpkg.py b/pixcdust/converters/gpkg.py index 0921f84..7b406f5 100644 --- a/pixcdust/converters/gpkg.py +++ b/pixcdust/converters/gpkg.py @@ -16,18 +16,17 @@ """Geopackage converters.""" import os -from pathlib import Path -from typing import Optional, Union from dataclasses import dataclass +from pathlib import Path -from tqdm import tqdm import fiona import geopandas as gpd +from tqdm import tqdm from pixcdust.converters.core import ConverterWSE, GeoLayerH3Projecter +from pixcdust.readers.gpkg import GpkgReader from pixcdust.readers.netcdf import NcSimpleReader from pixcdust.readers.zarr import ZarrReader -from pixcdust.readers.gpkg import GpkgReader class Nc2GpkgConverter(ConverterWSE): @@ -65,12 +64,10 @@ def database_from_nc( ) time_start = dt_time_start.strftime("%Y%m%d") - layer_name = f"{time_start}_{cycle_number}_\ -{pass_number}_{tile_number}{swath_side}" + layer_name = f"{time_start}_{cycle_number}_{pass_number}_{tile_number}{swath_side}" # cheking if output file and layer already exist - if os.path.exists(path_out) and mode == "w": - if layer_name in fiona.listlayers(path_out): + if os.path.exists(path_out) and mode == "w" and layer_name in fiona.listlayers(path_out): tqdm.write( f"skipping layer {layer_name} \ (already in geopackage {path_out})" @@ -109,10 +106,10 @@ class GpkgDGGSProjecter: path: str dggs_res: int - conditions: Optional[dict[str, dict[str, Union[str, float]]]] = None + conditions: dict[str, dict[str, str | float]] | None = None healpix: bool = False dggs_layer_pattern: str = "_h3" - path_out: Optional[str] = None + path_out: str | None = None # database: GpkgReader def __post_init__(self) -> None: diff --git a/pixcdust/converters/zarr.py b/pixcdust/converters/zarr.py index 02702a2..d69a5d2 100644 --- a/pixcdust/converters/zarr.py +++ b/pixcdust/converters/zarr.py @@ -17,21 +17,18 @@ import os import shutil +from collections.abc import Iterable from pathlib import Path -from typing import Optional, Iterable +import dask +import dask.utils import fsspec -from typing import Tuple, List, Union - import geopandas as gpd import zcollection import zcollection.indexing -import dask -import dask.utils - from pixcdust.converters.core import Converter -from pixcdust.readers.netcdf import NcSimpleReader, NcSimpleConstants +from pixcdust.readers.netcdf import NcSimpleConstants, NcSimpleReader TIME_VARNAME = "time" @@ -53,9 +50,9 @@ class Nc2ZarrConverter(Converter): def __init__( self, path_in: str | Iterable[str] | Path | Iterable[Path], - variables: Optional[list[str]] = None, - area_of_interest: Optional[gpd.GeoDataFrame] = None, - conditions: Optional[dict[str, dict[str, Union[str, float]]]] = None, + variables: list[str] | None = None, + area_of_interest: gpd.GeoDataFrame | None = None, + conditions: dict[str, dict[str, str | float]] | None = None, ): """Basic initialisation of a pixcdust converter. @@ -90,7 +87,7 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: with ( dask.distributed.LocalCluster(processes=True) as cluster, - dask.distributed.Client(cluster) as client, + dask.distributed.Client(cluster) as _, ): xr_ds = NcSimpleReader( path=self.path_in, @@ -107,7 +104,7 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: xr_ds.to_xarray().drop_vars(self.__cst.default_added_points_name), ) zc_ds.block_size_limit = self.__chunk_size - zc_ds.chunks = {list(zc_ds.dimensions.keys())[0]: self.__chunk_size} + zc_ds.chunks = {next(iter(zc_ds.dimensions.keys())): self.__chunk_size} init = True if not os.path.exists(path_out) and init: diff --git a/pixcdust/dggs/dggs_converter.py b/pixcdust/dggs/dggs_converter.py index 7df242b..812d49f 100644 --- a/pixcdust/dggs/dggs_converter.py +++ b/pixcdust/dggs/dggs_converter.py @@ -18,11 +18,11 @@ import h3 import numpy as np import xarray as xr -from xarray import Dataset import xdggs -from scipy.interpolate import griddata -from astropy_healpix import HEALPix from astropy import units as u +from astropy_healpix import HEALPix +from scipy.interpolate import griddata +from xarray import Dataset def prepare_dataset_h3( diff --git a/pixcdust/dggs/h3_tools.py b/pixcdust/dggs/h3_tools.py index 72ca124..93d2e77 100644 --- a/pixcdust/dggs/h3_tools.py +++ b/pixcdust/dggs/h3_tools.py @@ -16,9 +16,8 @@ import geopandas as gpd import h3 -from astropy_healpix import HEALPix from astropy import units as u - +from astropy_healpix import HEALPix from shapely.geometry import Polygon diff --git a/pixcdust/downloaders/hydroweb_next.py b/pixcdust/downloaders/hydroweb_next.py index 1a13c44..2ad22ce 100644 --- a/pixcdust/downloaders/hydroweb_next.py +++ b/pixcdust/downloaders/hydroweb_next.py @@ -15,19 +15,17 @@ # """Downloaders for hydroweb.next. Require an API-Key see HELP_MESSAGE.""" -from abc import ABC, abstractmethod -import os -from pathlib import Path -from typing import Optional, Union, Tuple, List import datetime +import os import zipfile +from abc import ABC, abstractmethod +from pathlib import Path import geopandas +import py_hydroweb import shapely import tqdm -import py_hydroweb -from eodag import EODataAccessGateway, SearchResult -from eodag import setup_logging +from eodag import EODataAccessGateway, SearchResult, setup_logging HELP_MESSAGE = """ Download products from hydroweb.next (https://hydroweb.next.theia-land.fr) @@ -66,10 +64,10 @@ class Downloader(ABC): def __init__( self, collection_name: str, - geometry: Union[str, list[str], geopandas.GeoDataFrame, None] = (None,), - dates: Optional[Tuple[datetime.date, datetime.date]] = None, + geometry: str | list[str] | geopandas.GeoDataFrame | None = (None,), + dates: tuple[datetime.date, datetime.date] | None = None, path_download: str | Path = "/tmp/hydroweb_next", - verbose: Optional[int] = 0, + verbose: int | None = 0, ): """Downloader for hydroweb.next STAC API initialization. @@ -94,7 +92,7 @@ def __init__( self.verbose = verbose self.query_args = {} - self.search_results: List[SearchResult] = [] + self.search_results: list[SearchResult] = [] if not os.path.isdir(self.path_download): os.mkdir(self.path_download) @@ -141,16 +139,16 @@ def _explode_simplify_geometry( ) if (geom["nodes_count"] > 200).any(): raise AttributeError( - ( + "One or several of your search polygons have too many nodes," "consider using the tolerance parameter" "in order to simplify the polygons." - ) + ) return geom - def search_download(self, tolerance: Optional[float] = None) -> None: + def search_download(self, tolerance: float | None = None) -> None: """Search files according to the query and download them. Args: @@ -172,10 +170,10 @@ def search_download(self, tolerance: Optional[float] = None) -> None: self._search(geom.__geo_interface__) else: raise AttributeError( - ( + "geometry should string (WKT) or geopandas.GeoDataFrame, " f"received {type(self.geometry)} instead" - ) + ) # This command actually downloads the matching products @@ -195,11 +193,11 @@ def define_query(self) -> dict: pass @abstractmethod - def _search(self, geom: Optional[str] = None) -> None: + def _search(self, geom: str | None = None) -> None: pass @abstractmethod - def _download(self) -> List: + def _download(self) -> list: pass @@ -243,11 +241,11 @@ def __check_collection_name(self) -> None: if self.collection_name not in list_collections: raise ValueError( - ( + "Did not find collection_name in " f"list of available collections in {self.PROVIDER}." f"\nAvailable collections are: {list_collections}" - ) + ) def define_query(self) -> dict: @@ -269,7 +267,7 @@ def define_query(self) -> dict: return self.query_args - def _search(self, geom: Optional[str] = None) -> None: + def _search(self, geom: str | None = None) -> None: if geom is not None: self.query_args["geom"] = geom @@ -278,7 +276,7 @@ def _search(self, geom: Optional[str] = None) -> None: # for page_results in self.dag.search_iter_page(**self.query_args): # self.search_results.extend(page_results) - def _download(self) -> List: + def _download(self) -> list: # donwload only .nc asset downloaded_paths = self.dag.download_all( self.search_results, asset=r".*\.nc$", output_dir=self.path_download @@ -323,7 +321,7 @@ def define_query(self) -> dict: return self.query_args - def _search(self, geom: Optional[str] = None) -> None: + def _search(self, geom: str | None = None) -> None: # This command actually downloads the matching products basket = py_hydroweb.DownloadBasket("pixcdust_basket") @@ -338,7 +336,7 @@ def _search(self, geom: Optional[str] = None) -> None: self.download_id = self.client.submit_download(download_basket=basket) - def _download(self) -> List: + def _download(self) -> list: downloaded_zip_path = self.client.download_zip( download_id=self.download_id, output_folder=self.path_download ) diff --git a/pixcdust/notebooks/convert/convert_to_zcollection.ipynb b/pixcdust/notebooks/convert/convert_to_zcollection.ipynb index 6531fc7..2e95710 100644 --- a/pixcdust/notebooks/convert/convert_to_zcollection.ipynb +++ b/pixcdust/notebooks/convert/convert_to_zcollection.ipynb @@ -6,8 +6,9 @@ "metadata": {}, "outputs": [], "source": [ - "from pixcdust.converters.zarr import Nc2ZarrConverter\n", - "from glob import glob" + "from glob import glob\n", + "\n", + "from pixcdust.converters.zarr import Nc2ZarrConverter" ] }, { @@ -72,13 +73,14 @@ "metadata": {}, "outputs": [], "source": [ - "from pixcdust.readers.zarr import ZarrReader\n", - "import datetime" + "import datetime\n", + "\n", + "from pixcdust.readers.zarr import ZarrReader" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -104,7 +106,7 @@ ], "source": [ "pixcr = ZarrReader(\"/tmp/pixc_zarr\")\n", - "pixcr.read((datetime.datetime(2023, 4, 6), datetime.datetime(2023, 4, 8)))\n", + "pixcr.read((datetime.datetime(2023, 4, 6, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo), datetime.datetime(2023, 4, 8, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo)))\n", "pixc" ] }, diff --git a/pixcdust/notebooks/convert/dggs_tuto.ipynb b/pixcdust/notebooks/convert/dggs_tuto.ipynb index 859d598..f85730b 100644 --- a/pixcdust/notebooks/convert/dggs_tuto.ipynb +++ b/pixcdust/notebooks/convert/dggs_tuto.ipynb @@ -13,18 +13,19 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "5a3343de-dc8e-492b-b5e6-9f6cf87f55c8", "metadata": {}, "outputs": [], "source": [ "# imports\n", - "from pixcdust.downloaders.hydroweb_next import PixCDownloader\n", - "from pixcdust.readers import NcSimpleReader\n", + "import glob\n", + "from datetime import UTC, datetime\n", + "\n", "import geopandas as gpd\n", - "from datetime import datetime\n", - "import xarray as xr\n", - "import glob" + "\n", + "from pixcdust.downloaders.hydroweb_next import PixCDownloader\n", + "from pixcdust.readers import NcSimpleReader" ] }, { @@ -37,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "1bc7ccfd-8edb-4170-b933-c0fb33822521", "metadata": {}, "outputs": [], @@ -47,8 +48,8 @@ "\n", "# Limiting time period\n", "dates = (\n", - " datetime(2023, 4, 6),\n", - " datetime(2023, 4, 8),\n", + " datetime(2023, 4, 6, tzinfo=datetime.now(UTC).astimezone().tzinfo),\n", + " datetime(2023, 4, 8, tzinfo=datetime.now(UTC).astimezone().tzinfo),\n", ")" ] }, diff --git a/pixcdust/notebooks/convert/download_pixc_to_gpkg.ipynb b/pixcdust/notebooks/convert/download_pixc_to_gpkg.ipynb index 8e14544..e69255a 100644 --- a/pixcdust/notebooks/convert/download_pixc_to_gpkg.ipynb +++ b/pixcdust/notebooks/convert/download_pixc_to_gpkg.ipynb @@ -21,18 +21,20 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "from pixcdust.downloaders.hydroweb_next import PixCDownloader\n", + "from datetime import UTC, datetime\n", + "\n", "import geopandas as gpd\n", - "from datetime import datetime" + "\n", + "from pixcdust.downloaders.hydroweb_next import PixCDownloader" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -41,8 +43,8 @@ "\n", "# Limiting time period\n", "dates = (\n", - " datetime(2023, 4, 6),\n", - " datetime(2023, 4, 8),\n", + " datetime(2023, 4, 6, tzinfo=datetime.now(UTC).astimezone().tzinfo),\n", + " datetime(2023, 4, 8, tzinfo=datetime.now(UTC).astimezone().tzinfo),\n", ")" ] }, @@ -169,8 +171,9 @@ "metadata": {}, "outputs": [], "source": [ - "from pixcdust.converters.gpkg import Nc2GpkgConverter\n", - "from glob import glob" + "from glob import glob\n", + "\n", + "from pixcdust.converters.gpkg import Nc2GpkgConverter" ] }, { @@ -2704,8 +2707,8 @@ } ], "source": [ - "import folium\n", "import branca.colormap as cmp\n", + "import folium\n", "\n", "layer_h3 = pixc_read.read_single_layer(\"20230407_483_16_78L_10__h3\")\n", "\n", diff --git a/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb b/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb index e7d4ecc..70ec716 100644 --- a/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb +++ b/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb @@ -20,19 +20,20 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "import pixcdust\n", - "from pixcdust.downloaders.hydroweb_next import PixCDownloader\n", + "from datetime import UTC, datetime\n", + "\n", "import geopandas as gpd\n", - "from datetime import datetime" + "\n", + "from pixcdust.downloaders.hydroweb_next import PixCDownloader" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -40,8 +41,8 @@ "gdf_geom = gpd.read_file(\"../data/aoi.gpkg\")\n", "\n", "dates = (\n", - " datetime(2023, 4, 6),\n", - " datetime(2023, 4, 8),\n", + " datetime(2023, 4, 6, tzinfo=datetime.now(UTC).astimezone().tzinfo),\n", + " datetime(2023, 4, 8, tzinfo=datetime.now(UTC).astimezone().tzinfo),\n", ")" ] }, @@ -127,8 +128,9 @@ "metadata": {}, "outputs": [], "source": [ - "from pixcdust.converters.zarr import Nc2ZarrConverter\n", - "from glob import glob" + "from glob import glob\n", + "\n", + "from pixcdust.converters.zarr import Nc2ZarrConverter" ] }, { @@ -181,7 +183,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -1293,11 +1295,12 @@ } ], "source": [ - "from pixcdust.readers.zarr import ZarrReader\n", "import datetime\n", "\n", + "from pixcdust.readers.zarr import ZarrReader\n", + "\n", "pixc_read = ZarrReader(\"/tmp/pixc_zarr\")\n", - "pixc_read.read((datetime.datetime(2023, 4, 6), datetime.datetime(2023, 4, 8)))\n", + "pixc_read.read((datetime.datetime(2023, 4, 6, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo), datetime.datetime(2023, 4, 8, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo)))\n", "pixc_read.data" ] }, diff --git a/pixcdust/notebooks/read_and_use/read_netcdf.ipynb b/pixcdust/notebooks/read_and_use/read_netcdf.ipynb index 1068aab..3647e52 100644 --- a/pixcdust/notebooks/read_and_use/read_netcdf.ipynb +++ b/pixcdust/notebooks/read_and_use/read_netcdf.ipynb @@ -13,8 +13,9 @@ "metadata": {}, "outputs": [], "source": [ - "from pixcdust.readers.netcdf import NcSimpleReader\n", - "import glob" + "import glob\n", + "\n", + "from pixcdust.readers.netcdf import NcSimpleReader" ] }, { diff --git a/pixcdust/readers/__init__.py b/pixcdust/readers/__init__.py index b9a46b9..f16e27d 100644 --- a/pixcdust/readers/__init__.py +++ b/pixcdust/readers/__init__.py @@ -18,6 +18,12 @@ They support the Netcdf official format and converted Zarr or Geopackage database. """ -from pixcdust.readers.netcdf import NcSimpleReader from pixcdust.readers.gpkg import GpkgReader +from pixcdust.readers.netcdf import NcSimpleReader from pixcdust.readers.zarr import ZarrReader + +__all__ = [ + "GpkgReader", + "NcSimpleReader", + "ZarrReader", +] diff --git a/pixcdust/readers/base_reader.py b/pixcdust/readers/base_reader.py index fd7e1d8..6c1b87d 100644 --- a/pixcdust/readers/base_reader.py +++ b/pixcdust/readers/base_reader.py @@ -16,12 +16,12 @@ """Interface used by all Pixcdust Readers.""" import re -from typing import Optional, Iterable, Union, List +from collections.abc import Iterable from pathlib import Path -import xarray as xr -import pandas as pd -import geopandas as gpd +import geopandas as gpd +import pandas as pd +import xarray as xr PIXC_DATE_RE = re.compile(r"_\d{8}T\d{6}_\d{8}T\d{6}_") """Regex patern used to extract the date (daystartThourstart_dayxendThoursend) @@ -29,7 +29,7 @@ """ -def sorted_by_date(file_list: Iterable[Union[str, Path]]) -> List[Union[str, Path]]: +def sorted_by_date(file_list: Iterable[str | Path]) -> list[str | Path]: """Sort the filenames by date as some converters need monotonic dates. The date is parsed from the filename according to PIXC_DATE_RE. Args: @@ -39,7 +39,7 @@ def sorted_by_date(file_list: Iterable[Union[str, Path]]) -> List[Union[str, Pat Sorted file_list. """ - def file_name_to_date(file_name: Union[str, Path]): + def file_name_to_date(file_name: str | Path): date_founds = PIXC_DATE_RE.findall(str(file_name)) if date_founds: return date_founds[-1] @@ -72,9 +72,9 @@ class BaseReader: def __init__( self, path: str | Iterable[str] | Path | Iterable[Path], - variables: Optional[list[str]] = None, - area_of_interest: Optional[gpd.GeoDataFrame] = None, - conditions: Optional[dict[str, dict[str, Union[str, float]]]] = None, + variables: list[str] | None = None, + area_of_interest: gpd.GeoDataFrame | None = None, + conditions: dict[str, dict[str, str | float]] | None = None, ): """Basic pixcdust database reader configuration. @@ -98,7 +98,7 @@ def __init__( # sort the filenames by date as some converters need monotonic dates. self.path = [str(p) for p in sorted_by_date(path)] self.area_of_interest = area_of_interest - self._data: Optional[xr.Dataset] = None + self._data: xr.Dataset | None = None self.variables = variables self.conditions = conditions diff --git a/pixcdust/readers/gpkg.py b/pixcdust/readers/gpkg.py index 93cb6c6..af5228a 100644 --- a/pixcdust/readers/gpkg.py +++ b/pixcdust/readers/gpkg.py @@ -16,13 +16,12 @@ """Converted Pixcdust GeoPackage Reader.""" from pathlib import Path -from typing import Optional, List -from tqdm import tqdm import fiona -import xarray as xr -import pandas as pd import geopandas as gpd +import pandas as pd +import xarray as xr +from tqdm import tqdm from pixcdust.readers.base_reader import BaseReader @@ -42,7 +41,7 @@ class GpkgReader(BaseReader): """ def __init__( - self, path: str | Path, area_of_interest: Optional[gpd.GeoDataFrame] = None + self, path: str | Path, area_of_interest: gpd.GeoDataFrame | None = None ): """Gpkg pixcdust database reader configuration. Read the list of layers from path. @@ -52,7 +51,7 @@ def __init__( area_of_interest: Optionally only read points in area_of_interest. """ super().__init__(path, area_of_interest=area_of_interest) - self._gdf_data: Optional[gpd.GeoDataFrame] = None + self._gdf_data: gpd.GeoDataFrame | None = None self.layers: list[str] = fiona.listlayers(self.path) @property @@ -99,7 +98,7 @@ def read_single_layer(self, layer: str) -> gpd.GeoDataFrame: return layer_data - def read(self, layers: Optional[List[str]] = None) -> None: + def read(self, layers: list[str] | None = None) -> None: """Load all layers, or subset of layers, from geopackage database. You can then access from data or with methods like to_xarray, to_dataframe or to_geodataframe. diff --git a/pixcdust/readers/netcdf.py b/pixcdust/readers/netcdf.py index 91fd71d..6c00c7f 100644 --- a/pixcdust/readers/netcdf.py +++ b/pixcdust/readers/netcdf.py @@ -15,20 +15,19 @@ # """Pre-conversion SWOT Pixel Cloud Netcdf reader.""" +import operator +from collections.abc import Iterable from dataclasses import dataclass, field -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path -from typing import Tuple, Optional, Iterable, Union +import dask.array as da +import geopandas import numpy as np -import xvec # noqa # pylint: disable=unused-import # xvec provide xvec accessor to xarray. - import xarray as xr -import geopandas -import operator -import dask.array as da +import xvec # noqa # pylint: disable=unused-import from pixcdust.dggs.dggs_converter import prepare_dataset_h3, prepare_dataset_healpix from pixcdust.readers.base_reader import BaseReader @@ -94,10 +93,10 @@ class NcSimpleReader(BaseReader): def __init__( self, path: str | Iterable[str] | Path | Iterable[Path], - variables: Optional[list[str]] = None, - area_of_interest: Optional[geopandas.GeoDataFrame] = None, - format_cfg: Optional[NcFormatCfg] = None, - conditions: Optional[dict[str, dict[str, Union[str, float]]]] = None, + variables: list[str] | None = None, + area_of_interest: geopandas.GeoDataFrame | None = None, + format_cfg: NcFormatCfg | None = None, + conditions: dict[str, dict[str, str | float]] | None = None, ): """Netcdf pixcdust reader configuration. @@ -124,7 +123,7 @@ def __init__( @staticmethod def extract_info_from_nc_attrs( filename: str, - ) -> Tuple[str, datetime, int, int, int, str]: + ) -> tuple[str, datetime, int, int, int, str]: """Extracts orbit information from global attributes\ in a SWOT pixel cloud netcdf. @@ -149,7 +148,7 @@ def extract_info_from_nc_attrs( time_granule_start = ds_glob.attrs[cst.default_time_start_name] dt_time_start = datetime.strptime( time_granule_start, cst.default_time_format_attrs - ).replace(microsecond=0) + ).replace(microsecond=0).astimezone(UTC) return ( time_granule_start, @@ -172,37 +171,38 @@ def filter_variable(self) -> None: _k_to = "threshold" # Loop through each condition and apply the filter - for var, condition in self.conditions.items(): - if var not in self.data.variables: - raise IOError( - f"Variable '{var}' not found in dataset variables (available: {list(self.data.variables)})" - ) - - # Ensure the condition dictionary has the correct keys - if _k_operator not in condition or _k_to not in condition: - raise ValueError( - f"Condition for variable '{var}' must include '{_k_operator}' and '{_k_to}'" - ) - - # Get the operator function dynamically from the operator module - try: - operator_func = getattr(operator, condition[_k_operator]) - except AttributeError: - raise AttributeError( - f"Operator '{condition[_k_operator]}' is not a valid operator in the operator module" + if self.conditions: + for var, condition in self.conditions.items(): + if var not in self.data.variables: + raise OSError( + f"Variable '{var}' not found in dataset variables (available: {list(self.data.variables)})" + ) + + # Ensure the condition dictionary has the correct keys + if _k_operator not in condition or _k_to not in condition: + raise ValueError( + f"Condition for variable '{var}' must include '{_k_operator}' and '{_k_to}'" + ) + + # Get the operator function dynamically from the operator module + try: + operator_func = getattr(operator, condition[_k_operator]) + except AttributeError: + raise AttributeError( + f"Operator '{condition[_k_operator]}' is not a valid operator in the operator module" + ) + + threshold = condition[_k_to] + + # Compute the boolean condition if it's a Dask array + if isinstance(self.data[var].data, da.Array): + self.data[var] = self.data[var].compute() + + # Apply the filter using .where() on the dataset + self.data = self.data.where( + operator_func(self.data[var], threshold), drop=True ) - threshold = condition[_k_to] - - # Compute the boolean condition if it's a Dask array - if isinstance(self.data[var].data, da.Array): - self.data[var] = self.data[var].compute() - - # Apply the filter using .where() on the dataset - self.data = self.data.where( - operator_func(self.data[var], threshold), drop=True - ) - def read(self, orbit_info: bool = False) -> None: """Load self.path file(s). You can then access from data or with methods like @@ -274,7 +274,7 @@ def open_mfdataset( if self.variables: # check if variables in forbidden variables before loading if len(set(self.variables).intersection(set(self.forbidden_variables))) > 0: - raise IOError( + raise OSError( f"variables from {self.forbidden_variables} \ cannot be extracted" ) diff --git a/pixcdust/readers/zarr.py b/pixcdust/readers/zarr.py index 22787ae..ee8fd66 100644 --- a/pixcdust/readers/zarr.py +++ b/pixcdust/readers/zarr.py @@ -15,9 +15,8 @@ # """Converted Pixcdust zarr database Reader.""" -from typing import Optional, Tuple - import datetime + import xarray as xr import zcollection @@ -40,8 +39,7 @@ class ZarrReader(BaseReader): def read( self, - date_interval: Optional[Tuple[datetime.datetime, datetime.datetime]] - | None = None, + date_interval: tuple[datetime.datetime, datetime.datetime] | None = None, ) -> None: """Load a zarr database. You can then access from data or with methods like @@ -60,6 +58,7 @@ def read( if date_interval: date_min = date_interval[0] date_max = date_interval[1] + # LOCALE timezone data_z = collection.load( filters=lambda keys: ( date_min @@ -70,6 +69,7 @@ def read( keys["hour"], keys["minute"], keys["second"], + tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo, ) <= date_max ) diff --git a/pixcdust/tests/conftest.py b/pixcdust/tests/conftest.py index e6f2162..fe36819 100644 --- a/pixcdust/tests/conftest.py +++ b/pixcdust/tests/conftest.py @@ -1,8 +1,7 @@ -import os from pathlib import Path -from typing import List import pytest + from pixcdust.tests.init_tests import JsonTestsSettings, init_hydroweb_env MOCK_DATA_DIR = Path(__file__).parent / "mock_data" @@ -61,7 +60,7 @@ def input_folder(tests_settings) -> Path: @pytest.fixture(scope="session") -def input_files(input_folder) -> List[Path]: +def input_files(input_folder) -> list[Path]: return list(input_folder.glob("**/*nc")) diff --git a/pixcdust/tests/init_tests.py b/pixcdust/tests/init_tests.py index 6f33a42..3d0558c 100644 --- a/pixcdust/tests/init_tests.py +++ b/pixcdust/tests/init_tests.py @@ -1,14 +1,14 @@ import argparse -import os -from typing import Union - -from datetime import datetime import json +import os +from datetime import UTC, datetime from pathlib import Path + from pixcdust.downloaders.hydroweb_next import PixCDownloader class JsonTestsSettings: + """Reader-writer for the test configuration.""" CONFIG_FILE_NAME = "conftest.json" @@ -29,7 +29,7 @@ def input_folder(self) -> Path: raise KeyError("Test input folder not set. Configure it with init_tests.py") @input_folder.setter - def input_folder(self, value: Union[Path, str]) -> None: + def input_folder(self, value: Path | str) -> None: self._settings["input_folder"] = str(value) @property @@ -38,7 +38,7 @@ def tmp_folder(self) -> Path: return Path(self._settings.get("tmp_folder", "/tmp/pixcdust-test")) @tmp_folder.setter - def tmp_folder(self, value: Union[Path, str]) -> None: + def tmp_folder(self, value: Path | str) -> None: self._settings["tmp_folder"] = str(value) @property @@ -84,8 +84,9 @@ def download_test_data(path_download: Path, backend: str) -> None: path_download: where to store the test data. """ dates = ( - datetime(2024, 8, 1), - datetime(2024, 8, 15), + # LOCALE timezone + datetime(2024, 8, 1, tzinfo=datetime.now(UTC).astimezone().tzinfo), + datetime(2024, 8, 15, tzinfo=datetime.now(UTC).astimezone().tzinfo), ) geometry = "POLYGON((-1.50580 43.39543,-1.36597 43.39543,-1.36597 43.56471,-1.50580 43.56471,-1.50580 43.39543))" diff --git a/pixcdust/tests/mock.py b/pixcdust/tests/mock.py index 2af690f..5ad13e1 100644 --- a/pixcdust/tests/mock.py +++ b/pixcdust/tests/mock.py @@ -1,8 +1,8 @@ -from datetime import datetime -import numpy as np +from datetime import UTC, datetime -import xarray as xr import geopandas as gpd +import numpy as np +import xarray as xr from pixcdust.readers.netcdf import NcSimpleConstants @@ -42,7 +42,8 @@ def mock_xarray(length: int = 10000) -> xr.Dataset: # mocking data x = coords[cst.default_lat_name] cst_time_array = np.ones(len(x)).astype(datetime) - cst_time_array[:] = datetime(2024, 6, 5, 11, 40, 12) + # LOCALE timezone + cst_time_array[:] = datetime(2024, 6, 5, 11, 40, 12, tzinfo=datetime.now(UTC).astimezone().tzinfo) data_vars = { "height": (dims, np.sin(x) + np.random.normal(scale=10, size=len(x))), diff --git a/pixcdust/tests/test_converters.py b/pixcdust/tests/test_converters.py index a094e96..95211f1 100644 --- a/pixcdust/tests/test_converters.py +++ b/pixcdust/tests/test_converters.py @@ -1,19 +1,19 @@ import random -from pathlib import Path, PosixPath -from typing import List, Union +from pathlib import Path import fiona import geopandas as gpd import numpy as np import pytest import xarray as xr +from shapely.geometry import Polygon + from pixcdust.converters.gpkg import GpkgDGGSProjecter, Nc2GpkgConverter from pixcdust.converters.shapefile import Nc2ShpConverter from pixcdust.converters.zarr import Nc2ZarrConverter from pixcdust.readers import GpkgReader from pixcdust.readers.netcdf import NcSimpleReader from pixcdust.readers.zarr import ZarrReader -from shapely.geometry import Polygon LIM_AREA_POL = Polygon( [ @@ -32,7 +32,7 @@ def test_nc_simple_reader_conditions(input_files): """Test NcSimpleReader with conditions on variables.""" # Define conditions conditions = { - "classification": {"operator": "ge", "threshold": 4}, # classification >= 4 + "height": {"operator": "ge", "threshold": 4}, # height >= 4 "classification": {"operator": "le", "threshold": 3}, # classification <= 3 "sig0": {"operator": "gt", "threshold": 15}, # sig0 > 15 } @@ -64,7 +64,7 @@ def test_nc_simple_reader_conditions(input_files): def validate_conversion_to_nc( - read_data: xr.Dataset, converted_vars: List[str], first_file: Union[str, Path] + read_data: xr.Dataset, converted_vars: list[str], first_file: str | Path ) -> None: """Compare the start of a converted database to the first original netcdf file. @@ -82,7 +82,7 @@ def validate_conversion_to_nc( def validate_conversion( read_data: xr.Dataset, - converted_vars: List[str], + converted_vars: list[str], expected_data: xr.Dataset, is_longer: bool, len_tol: int = 0, diff --git a/pixcdust/tests/test_converters_mock.py b/pixcdust/tests/test_converters_mock.py index 2be9c2f..2c54310 100644 --- a/pixcdust/tests/test_converters_mock.py +++ b/pixcdust/tests/test_converters_mock.py @@ -1,8 +1,8 @@ import unittest -from pixcdust.tests.mock import mock_xarray from pixcdust.converters.gpkg import Nc2GpkgConverter from pixcdust.converters.zarr import Nc2ZarrConverter +from pixcdust.tests.mock import mock_xarray class TestConverters(unittest.TestCase): diff --git a/pixcdust/tests/test_dggs.py b/pixcdust/tests/test_dggs.py index 86aa5a8..56df9c0 100644 --- a/pixcdust/tests/test_dggs.py +++ b/pixcdust/tests/test_dggs.py @@ -1,5 +1,6 @@ import pytest import xarray as xr + from pixcdust.readers import NcSimpleReader diff --git a/pixcdust/tools/convert_pixc.py b/pixcdust/tools/convert_pixc.py index 9f4d692..59245fa 100644 --- a/pixcdust/tools/convert_pixc.py +++ b/pixcdust/tools/convert_pixc.py @@ -15,13 +15,12 @@ # import click - import geopandas as gpd +from pixcdust.converters.core import Converter from pixcdust.converters.gpkg import Nc2GpkgConverter -from pixcdust.converters.zarr import Nc2ZarrConverter from pixcdust.converters.shapefile import Nc2ShpConverter -from pixcdust.converters.core import Converter +from pixcdust.converters.zarr import Nc2ZarrConverter def paths_glob(ctx, param, paths): From 0a119edd4fd4192668bbd8f3f5e0159ec49233e8 Mon Sep 17 00:00:00 2001 From: pty Date: Mon, 31 Aug 2026 16:39:00 +0200 Subject: [PATCH 7/8] Ruff format files --- pixcdust/converters/core.py | 4 +--- pixcdust/converters/gpkg.py | 18 ++++++++++----- pixcdust/downloaders/hydroweb_next.py | 22 +++++++------------ .../convert/convert_to_zcollection.ipynb | 11 +++++++++- .../convert/download_pixc_to_zarr.ipynb | 11 +++++++++- pixcdust/readers/netcdf.py | 8 ++++--- pixcdust/tests/init_tests.py | 1 - pixcdust/tests/mock.py | 4 +++- 8 files changed, 49 insertions(+), 30 deletions(-) diff --git a/pixcdust/converters/core.py b/pixcdust/converters/core.py index 8b3c6ab..4a8113b 100644 --- a/pixcdust/converters/core.py +++ b/pixcdust/converters/core.py @@ -139,9 +139,7 @@ class GeoLayerH3Projecter: data: gpd.GeoDataFrame resolution: int - def filter_variable( - self, conditions: dict[str, dict[str, str | float]] - ) -> None: + def filter_variable(self, conditions: dict[str, dict[str, str | float]]) -> None: """filters from xarray dataset based on operator and threshold on specific variables diff --git a/pixcdust/converters/gpkg.py b/pixcdust/converters/gpkg.py index 7b406f5..7258d9e 100644 --- a/pixcdust/converters/gpkg.py +++ b/pixcdust/converters/gpkg.py @@ -64,15 +64,21 @@ def database_from_nc( ) time_start = dt_time_start.strftime("%Y%m%d") - layer_name = f"{time_start}_{cycle_number}_{pass_number}_{tile_number}{swath_side}" + layer_name = ( + f"{time_start}_{cycle_number}_{pass_number}_{tile_number}{swath_side}" + ) # cheking if output file and layer already exist - if os.path.exists(path_out) and mode == "w" and layer_name in fiona.listlayers(path_out): - tqdm.write( - f"skipping layer {layer_name} \ + if ( + os.path.exists(path_out) + and mode == "w" + and layer_name in fiona.listlayers(path_out) + ): + tqdm.write( + f"skipping layer {layer_name} \ (already in geopackage {path_out})" - ) - continue + ) + continue # converting data from xarray to geodataframe ncsimple.open_dataset() gdf = ncsimple.to_geodataframe() diff --git a/pixcdust/downloaders/hydroweb_next.py b/pixcdust/downloaders/hydroweb_next.py index 2ad22ce..7b65cbe 100644 --- a/pixcdust/downloaders/hydroweb_next.py +++ b/pixcdust/downloaders/hydroweb_next.py @@ -139,11 +139,9 @@ def _explode_simplify_geometry( ) if (geom["nodes_count"] > 200).any(): raise AttributeError( - - "One or several of your search polygons have too many nodes," - "consider using the tolerance parameter" - "in order to simplify the polygons." - + "One or several of your search polygons have too many nodes," + "consider using the tolerance parameter" + "in order to simplify the polygons." ) return geom @@ -170,10 +168,8 @@ def search_download(self, tolerance: float | None = None) -> None: self._search(geom.__geo_interface__) else: raise AttributeError( - - "geometry should string (WKT) or geopandas.GeoDataFrame, " - f"received {type(self.geometry)} instead" - + "geometry should string (WKT) or geopandas.GeoDataFrame, " + f"received {type(self.geometry)} instead" ) # This command actually downloads the matching products @@ -241,11 +237,9 @@ def __check_collection_name(self) -> None: if self.collection_name not in list_collections: raise ValueError( - - "Did not find collection_name in " - f"list of available collections in {self.PROVIDER}." - f"\nAvailable collections are: {list_collections}" - + "Did not find collection_name in " + f"list of available collections in {self.PROVIDER}." + f"\nAvailable collections are: {list_collections}" ) def define_query(self) -> dict: diff --git a/pixcdust/notebooks/convert/convert_to_zcollection.ipynb b/pixcdust/notebooks/convert/convert_to_zcollection.ipynb index 2e95710..9e35c80 100644 --- a/pixcdust/notebooks/convert/convert_to_zcollection.ipynb +++ b/pixcdust/notebooks/convert/convert_to_zcollection.ipynb @@ -106,7 +106,16 @@ ], "source": [ "pixcr = ZarrReader(\"/tmp/pixc_zarr\")\n", - "pixcr.read((datetime.datetime(2023, 4, 6, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo), datetime.datetime(2023, 4, 8, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo)))\n", + "pixcr.read(\n", + " (\n", + " datetime.datetime(\n", + " 2023, 4, 6, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo\n", + " ),\n", + " datetime.datetime(\n", + " 2023, 4, 8, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo\n", + " ),\n", + " )\n", + ")\n", "pixc" ] }, diff --git a/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb b/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb index 70ec716..f41cb0f 100644 --- a/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb +++ b/pixcdust/notebooks/convert/download_pixc_to_zarr.ipynb @@ -1300,7 +1300,16 @@ "from pixcdust.readers.zarr import ZarrReader\n", "\n", "pixc_read = ZarrReader(\"/tmp/pixc_zarr\")\n", - "pixc_read.read((datetime.datetime(2023, 4, 6, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo), datetime.datetime(2023, 4, 8, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo)))\n", + "pixc_read.read(\n", + " (\n", + " datetime.datetime(\n", + " 2023, 4, 6, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo\n", + " ),\n", + " datetime.datetime(\n", + " 2023, 4, 8, tzinfo=datetime.datetime.now(datetime.UTC).astimezone().tzinfo\n", + " ),\n", + " )\n", + ")\n", "pixc_read.data" ] }, diff --git a/pixcdust/readers/netcdf.py b/pixcdust/readers/netcdf.py index 6c00c7f..59bf527 100644 --- a/pixcdust/readers/netcdf.py +++ b/pixcdust/readers/netcdf.py @@ -146,9 +146,11 @@ def extract_info_from_nc_attrs( pass_number = np.uint16(ds_glob.attrs[cst.default_pass_num_name]) cycle_number = np.uint16(ds_glob.attrs[cst.default_cyc_num_name]) time_granule_start = ds_glob.attrs[cst.default_time_start_name] - dt_time_start = datetime.strptime( - time_granule_start, cst.default_time_format_attrs - ).replace(microsecond=0).astimezone(UTC) + dt_time_start = ( + datetime.strptime(time_granule_start, cst.default_time_format_attrs) + .replace(microsecond=0) + .astimezone(UTC) + ) return ( time_granule_start, diff --git a/pixcdust/tests/init_tests.py b/pixcdust/tests/init_tests.py index 3d0558c..dfd2e44 100644 --- a/pixcdust/tests/init_tests.py +++ b/pixcdust/tests/init_tests.py @@ -8,7 +8,6 @@ class JsonTestsSettings: - """Reader-writer for the test configuration.""" CONFIG_FILE_NAME = "conftest.json" diff --git a/pixcdust/tests/mock.py b/pixcdust/tests/mock.py index 5ad13e1..a52ea23 100644 --- a/pixcdust/tests/mock.py +++ b/pixcdust/tests/mock.py @@ -43,7 +43,9 @@ def mock_xarray(length: int = 10000) -> xr.Dataset: x = coords[cst.default_lat_name] cst_time_array = np.ones(len(x)).astype(datetime) # LOCALE timezone - cst_time_array[:] = datetime(2024, 6, 5, 11, 40, 12, tzinfo=datetime.now(UTC).astimezone().tzinfo) + cst_time_array[:] = datetime( + 2024, 6, 5, 11, 40, 12, tzinfo=datetime.now(UTC).astimezone().tzinfo + ) data_vars = { "height": (dims, np.sin(x) + np.random.normal(scale=10, size=len(x))), From 6a43f7e1b7729159cc0612df4ccf4424594fbd65 Mon Sep 17 00:00:00 2001 From: pty Date: Mon, 31 Aug 2026 16:40:06 +0200 Subject: [PATCH 8/8] Added CI worklow --- .github/workflows/ci.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..015823f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +name: CI + +on: + push: + branches: ["**"] + pull_request: + branches: ["**"] + +jobs: + lint-and-test: + name: Lint & Tests check + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: "pip" + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install --upgrade build + pip install ".[dev]" + + - name: Run Ruff linter + run: ruff check . + + - name: Run Ruff formatter check + run: ruff format --check . + + - name: Run tests + run: pytest