From 0ac436c5687dc2a992aee9d9840b8046bababbe6 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Wed, 10 Jun 2026 15:32:12 +0200 Subject: [PATCH 1/2] perf: optimized imports and initializations --- eodag_cube/api/product/_assets.py | 10 +++++-- eodag_cube/api/product/_product.py | 48 +++++++++++++++++++++--------- tests/units/test_eoproduct.py | 6 ++-- tests/units/test_utils.py | 10 +++---- 4 files changed, 50 insertions(+), 24 deletions(-) diff --git a/eodag_cube/api/product/_assets.py b/eodag_cube/api/product/_assets.py index e191152..cd1d376 100644 --- a/eodag_cube/api/product/_assets.py +++ b/eodag_cube/api/product/_assets.py @@ -20,7 +20,6 @@ import logging from typing import TYPE_CHECKING, Any, Dict, Union -import xarray as xr from eodag.api.product._assets import Asset as Asset_core from eodag.api.product._assets import AssetsDict as AssetsDict_core from eodag.utils import DEFAULT_DOWNLOAD_TIMEOUT, DEFAULT_DOWNLOAD_WAIT @@ -28,6 +27,7 @@ if TYPE_CHECKING: from contextlib import nullcontext + import xarray as xr from fsspec.core import OpenFile from rasterio.env import Env @@ -44,7 +44,13 @@ class AssetsDict(AssetsDict_core): """ def __setitem__(self, key: str, value: Dict[str, Any]) -> None: - super(AssetsDict_core, self).__setitem__(key, Asset(self.product, key, value)) + # Avoid re-wrapping a value that is already an Asset bound to this + # product and key (e.g. when updating from another AssetsDict). + if isinstance(value, Asset) and value.product is self.product and value.key == key: + asset = value + else: + asset = Asset(self.product, key, value) + super(AssetsDict_core, self).__setitem__(key, asset) class Asset(Asset_core): diff --git a/eodag_cube/api/product/_product.py b/eodag_cube/api/product/_product.py index 680c78f..89f8a2f 100644 --- a/eodag_cube/api/product/_product.py +++ b/eodag_cube/api/product/_product.py @@ -17,18 +17,13 @@ # limitations under the License. from __future__ import annotations -import concurrent.futures import logging import os from contextlib import nullcontext from pathlib import Path -from typing import Any, Iterable, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Iterable, Optional, Union, cast from urllib.parse import urlparse -import fsspec -import rasterio -from boto3 import Session -from boto3.resources.base import ServiceResource from eodag.api.product._product import EOProduct as EOProduct_core from eodag.api.product.metadata_mapping import OFFLINE_STATUS from eodag.plugins.authentication.aws_auth import AwsAuth @@ -38,16 +33,17 @@ USER_AGENT, ) from eodag.utils.exceptions import UnsupportedDatasetAddressScheme -from fsspec.core import OpenFile from requests import PreparedRequest from requests.auth import AuthBase from requests.structures import CaseInsensitiveDict -from eodag_cube.api.product._assets import AssetsDict -from eodag_cube.types import XarrayDict from eodag_cube.utils.exceptions import DatasetCreationError -from eodag_cube.utils.metadata import build_bands, build_stac_metadata, merge_bands -from eodag_cube.utils.xarray import try_open_dataset + +if TYPE_CHECKING: + import rasterio + from fsspec.core import OpenFile + + from eodag_cube.types import XarrayDict logger = logging.getLogger("eodag-cube.api.product") @@ -89,10 +85,10 @@ class EOProduct(EOProduct_core): """ def __init__(self, provider: str, properties: dict[str, Any], **kwargs: Any) -> None: + # ``EOProduct_core.__init__`` already builds ``self.assets`` using the + # eodag-cube ``AssetsDict`` (eodag resolves it to eodag-cube when installed), + # so no asset rebuild is needed here. super(EOProduct, self).__init__(provider=provider, properties=properties, **kwargs) - core_assets_data = self.assets.data - self.assets = AssetsDict(self) - self.assets.update(core_assets_data) def _get_rio_env(self, dataset_address: str) -> dict[str, Any]: """Get rasterio environment variables needed for data access. @@ -100,6 +96,8 @@ def _get_rio_env(self, dataset_address: str) -> dict[str, Any]: :param dataset_address: address of the data to read :return: The rasterio environment variables """ + import rasterio + product_location_scheme = dataset_address.split("://")[0] if "s3" in product_location_scheme and isinstance(self.downloader_auth, AwsAuth): rio_env_dict = {"session": rasterio.session.AWSSession(**self.downloader_auth.get_rio_env())} @@ -127,6 +125,9 @@ def _get_storage_options( """ Get fsspec storage_options keyword arguments """ + from boto3 import Session + from boto3.resources.base import ServiceResource + auth = self.downloader_auth.authenticate() if self.downloader_auth else None if self.downloader is None: return {} @@ -186,6 +187,9 @@ def get_file_obj( stop checking order status :returns: product data file object """ + import fsspec + from fsspec.core import OpenFile + storage_options = self._get_storage_options(asset_key, wait, timeout) path = storage_options.pop("path", None) @@ -207,6 +211,8 @@ def rio_env(self, dataset_address: Optional[str] = None) -> Union[rasterio.env.E :param dataset_address: address of the data to read :return: The rasterio environment """ + import rasterio + if dataset_address: if env_dict := self._get_rio_env(dataset_address): return rasterio.Env(**env_dict) @@ -225,6 +231,11 @@ def _build_local_xarray_dict(self, local_path: str, **xarray_kwargs: Any) -> Xar :param xarray_kwargs: (optional) keyword arguments passed to :func:`xarray.open_dataset` :returns: a dictionary of :class:`xarray.Dataset` """ + import fsspec + + from eodag_cube.types import XarrayDict + from eodag_cube.utils.xarray import try_open_dataset + xarray_dict = XarrayDict() fs = fsspec.filesystem("file") @@ -273,6 +284,13 @@ def to_xarray( :param xarray_kwargs: (optional) keyword arguments passed to :func:`xarray.open_dataset` :returns: a dictionary of :class:`xarray.Dataset` """ + import concurrent.futures + + import rasterio + + from eodag_cube.types import XarrayDict + from eodag_cube.utils.xarray import try_open_dataset + if asset_key is None and len(self.assets) > 0: # assets @@ -360,6 +378,8 @@ def augment_from_xarray( :param roles: (optional) roles of assets that must be fetched :returns: updated EOProduct """ + from eodag_cube.utils.metadata import build_bands, build_stac_metadata, merge_bands + if not self.assets: try: xd = self.to_xarray(roles=roles) diff --git a/tests/units/test_eoproduct.py b/tests/units/test_eoproduct.py index 6cea370..d7676ee 100644 --- a/tests/units/test_eoproduct.py +++ b/tests/units/test_eoproduct.py @@ -225,7 +225,7 @@ def test_get_storage_options_error(self): with self.assertRaises(DatasetCreationError, msg=f"foo not found in {product} assets"): product._get_storage_options(asset_key="foo") - @mock.patch("eodag_cube.api.product._product.fsspec.filesystem") + @mock.patch("fsspec.filesystem") @mock.patch("eodag_cube.api.product._product.EOProduct._get_storage_options", autospec=True) def test_get_file_obj(self, mock_storage_options, mock_fs): """get_file_obj should call fsspec open with appropriate args""" @@ -259,7 +259,7 @@ def test_get_file_obj(self, mock_storage_options, mock_fs): with self.assertRaises(UnsupportedDatasetAddressScheme, msg=f"Could not get {product} path"): product.get_file_obj() - @mock.patch("eodag_cube.api.product._product.try_open_dataset", autospec=True) + @mock.patch("eodag_cube.utils.xarray.try_open_dataset", autospec=True) @mock.patch("eodag_cube.api.product._product.EOProduct.get_file_obj", autospec=True) def test_to_xarray(self, mock_get_file, mock_open_ds): """to_xarrray should return well built XarrayDict""" @@ -273,7 +273,7 @@ def test_to_xarray(self, mock_get_file, mock_open_ds): self.assertTrue(xd["data"].equals(mock_open_ds.return_value)) self.assertDictEqual(product.properties, xd["data"].attrs) - @mock.patch("eodag_cube.api.product._product.try_open_dataset", autospec=True) + @mock.patch("eodag_cube.utils.xarray.try_open_dataset", autospec=True) @mock.patch("eodag_cube.api.product._product.EOProduct.get_file_obj", autospec=True) def test_to_xarray_assets(self, mock_get_file, mock_open_ds): """to_xarrray should return well built XarrayDict""" diff --git a/tests/units/test_utils.py b/tests/units/test_utils.py index 0756385..0b0de0a 100644 --- a/tests/units/test_utils.py +++ b/tests/units/test_utils.py @@ -161,7 +161,7 @@ def test_guess_engines(self, mock_head, mock_get): self.assertIn("cfgrib", guess_engines(file)) @mock.patch("eodag_cube.utils.xarray.guess_engines", return_value=["h5netcdf", "foo"]) - @mock.patch("eodag_cube.api.product._product.fsspec.open") + @mock.patch("fsspec.open") def test_try_open_dataset_local(self, mock_open, mock_guess_engines): """try_open_dataset must call xaray.open_dataset with appropriate args""" # local file : let xarray guess engine @@ -185,7 +185,7 @@ def test_try_open_dataset_local(self, mock_open, mock_guess_engines): ds = try_open_dataset(file, foo="bar", baz="qux") @mock.patch("eodag_cube.utils.xarray.guess_engines", return_value=["cfgrib"]) - @mock.patch("eodag_cube.api.product._product.fsspec.open") + @mock.patch("fsspec.open") def test_try_open_dataset_remote_grib(self, mock_open, mock_guess_engines): """try_open_dataset must call xaray.open_dataset with appropriate args""" # remote file + grib @@ -200,7 +200,7 @@ def test_try_open_dataset_remote_grib(self, mock_open, mock_guess_engines): try_open_dataset(file, foo="bar", baz="qux") @mock.patch("eodag_cube.utils.xarray.guess_engines", return_value=["cfgrib"]) - @mock.patch("eodag_cube.api.product._product.fsspec.open") + @mock.patch("fsspec.open") def test_try_open_dataset_local_grib(self, mock_open, mock_guess_engines): """try_open_dataset must call xaray.open_dataset with appropriate args""" # local file + grib @@ -217,7 +217,7 @@ def test_try_open_dataset_local_grib(self, mock_open, mock_guess_engines): mock_open_dataset.assert_called_once_with(file.path, engine="cfgrib", foo="bar", baz="qux") @mock.patch("eodag_cube.utils.xarray.guess_engines", return_value=["h5netcdf", "foo"]) - @mock.patch("eodag_cube.api.product._product.fsspec.open") + @mock.patch("fsspec.open") def test_try_open_dataset_remote_nc(self, mock_open, mock_guess_engines): """try_open_dataset must call xaray.open_dataset with appropriate args""" # remote file + nc @@ -234,7 +234,7 @@ def test_try_open_dataset_remote_nc(self, mock_open, mock_guess_engines): mock_open_dataset.assert_called_once_with(file, engine="h5netcdf", foo="bar", baz="qux") @mock.patch("eodag_cube.utils.xarray.guess_engines", return_value=["rasterio"]) - @mock.patch("eodag_cube.api.product._product.fsspec.open") + @mock.patch("fsspec.open") def test_try_open_dataset_remote_jp2(self, mock_open, mock_guess_engines): """try_open_dataset must call open_rasterio with appropriate args""" # remote file + nc From 1271cd0c2d1907c11cca54aebb09174e6eabf1c5 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato Date: Wed, 10 Jun 2026 15:43:33 +0200 Subject: [PATCH 2/2] refactor: restore previous assets population mechanism --- eodag_cube/api/product/_assets.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/eodag_cube/api/product/_assets.py b/eodag_cube/api/product/_assets.py index cd1d376..5665f75 100644 --- a/eodag_cube/api/product/_assets.py +++ b/eodag_cube/api/product/_assets.py @@ -44,13 +44,7 @@ class AssetsDict(AssetsDict_core): """ def __setitem__(self, key: str, value: Dict[str, Any]) -> None: - # Avoid re-wrapping a value that is already an Asset bound to this - # product and key (e.g. when updating from another AssetsDict). - if isinstance(value, Asset) and value.product is self.product and value.key == key: - asset = value - else: - asset = Asset(self.product, key, value) - super(AssetsDict_core, self).__setitem__(key, asset) + super(AssetsDict_core, self).__setitem__(key, Asset(self.product, key, value)) class Asset(Asset_core):