Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion eodag_cube/api/product/_assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,14 @@
import logging
from typing import TYPE_CHECKING, Any, Dict, Union

import xarray as xr
from eodag.api.product._assets import Asset as Asset_core
from eodag.api.product._assets import AssetsDict as AssetsDict_core
from eodag.utils import DEFAULT_DOWNLOAD_TIMEOUT, DEFAULT_DOWNLOAD_WAIT

if TYPE_CHECKING:
from contextlib import nullcontext

import xarray as xr
from fsspec.core import OpenFile
from rasterio.env import Env

Expand Down
48 changes: 34 additions & 14 deletions eodag_cube/api/product/_product.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,13 @@
# limitations under the License.
from __future__ import annotations

import concurrent.futures
import logging
import os
from contextlib import nullcontext
from pathlib import Path
from typing import Any, Iterable, Optional, Union, cast
from typing import TYPE_CHECKING, Any, Iterable, Optional, Union, cast
from urllib.parse import urlparse

import fsspec
import rasterio
from boto3 import Session
from boto3.resources.base import ServiceResource
from eodag.api.product._product import EOProduct as EOProduct_core
from eodag.api.product.metadata_mapping import OFFLINE_STATUS
from eodag.plugins.authentication.aws_auth import AwsAuth
Expand All @@ -38,16 +33,17 @@
USER_AGENT,
)
from eodag.utils.exceptions import UnsupportedDatasetAddressScheme
from fsspec.core import OpenFile
from requests import PreparedRequest
from requests.auth import AuthBase
from requests.structures import CaseInsensitiveDict

from eodag_cube.api.product._assets import AssetsDict
from eodag_cube.types import XarrayDict
from eodag_cube.utils.exceptions import DatasetCreationError
from eodag_cube.utils.metadata import build_bands, build_stac_metadata, merge_bands
from eodag_cube.utils.xarray import try_open_dataset

if TYPE_CHECKING:
import rasterio
from fsspec.core import OpenFile

from eodag_cube.types import XarrayDict

logger = logging.getLogger("eodag-cube.api.product")

Expand Down Expand Up @@ -89,17 +85,19 @@ class EOProduct(EOProduct_core):
"""

def __init__(self, provider: str, properties: dict[str, Any], **kwargs: Any) -> None:
# ``EOProduct_core.__init__`` already builds ``self.assets`` using the
# eodag-cube ``AssetsDict`` (eodag resolves it to eodag-cube when installed),
# so no asset rebuild is needed here.
super(EOProduct, self).__init__(provider=provider, properties=properties, **kwargs)
core_assets_data = self.assets.data
self.assets = AssetsDict(self)
self.assets.update(core_assets_data)

def _get_rio_env(self, dataset_address: str) -> dict[str, Any]:
"""Get rasterio environment variables needed for data access.

:param dataset_address: address of the data to read
:return: The rasterio environment variables
"""
import rasterio

product_location_scheme = dataset_address.split("://")[0]
if "s3" in product_location_scheme and isinstance(self.downloader_auth, AwsAuth):
rio_env_dict = {"session": rasterio.session.AWSSession(**self.downloader_auth.get_rio_env())}
Expand Down Expand Up @@ -127,6 +125,9 @@ def _get_storage_options(
"""
Get fsspec storage_options keyword arguments
"""
from boto3 import Session
from boto3.resources.base import ServiceResource

auth = self.downloader_auth.authenticate() if self.downloader_auth else None
if self.downloader is None:
return {}
Expand Down Expand Up @@ -186,6 +187,9 @@ def get_file_obj(
stop checking order status
:returns: product data file object
"""
import fsspec
from fsspec.core import OpenFile

storage_options = self._get_storage_options(asset_key, wait, timeout)

path = storage_options.pop("path", None)
Expand All @@ -207,6 +211,8 @@ def rio_env(self, dataset_address: Optional[str] = None) -> Union[rasterio.env.E
:param dataset_address: address of the data to read
:return: The rasterio environment
"""
import rasterio

if dataset_address:
if env_dict := self._get_rio_env(dataset_address):
return rasterio.Env(**env_dict)
Expand All @@ -225,6 +231,11 @@ def _build_local_xarray_dict(self, local_path: str, **xarray_kwargs: Any) -> Xar
:param xarray_kwargs: (optional) keyword arguments passed to :func:`xarray.open_dataset`
:returns: a dictionary of :class:`xarray.Dataset`
"""
import fsspec

from eodag_cube.types import XarrayDict
from eodag_cube.utils.xarray import try_open_dataset

xarray_dict = XarrayDict()
fs = fsspec.filesystem("file")

Expand Down Expand Up @@ -273,6 +284,13 @@ def to_xarray(
:param xarray_kwargs: (optional) keyword arguments passed to :func:`xarray.open_dataset`
:returns: a dictionary of :class:`xarray.Dataset`
"""
import concurrent.futures

import rasterio

from eodag_cube.types import XarrayDict
from eodag_cube.utils.xarray import try_open_dataset

if asset_key is None and len(self.assets) > 0:
# assets

Expand Down Expand Up @@ -360,6 +378,8 @@ def augment_from_xarray(
:param roles: (optional) roles of assets that must be fetched
:returns: updated EOProduct
"""
from eodag_cube.utils.metadata import build_bands, build_stac_metadata, merge_bands

if not self.assets:
try:
xd = self.to_xarray(roles=roles)
Expand Down
6 changes: 3 additions & 3 deletions tests/units/test_eoproduct.py
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ def test_get_storage_options_error(self):
with self.assertRaises(DatasetCreationError, msg=f"foo not found in {product} assets"):
product._get_storage_options(asset_key="foo")

@mock.patch("eodag_cube.api.product._product.fsspec.filesystem")
@mock.patch("fsspec.filesystem")
@mock.patch("eodag_cube.api.product._product.EOProduct._get_storage_options", autospec=True)
def test_get_file_obj(self, mock_storage_options, mock_fs):
"""get_file_obj should call fsspec open with appropriate args"""
Expand Down Expand Up @@ -259,7 +259,7 @@ def test_get_file_obj(self, mock_storage_options, mock_fs):
with self.assertRaises(UnsupportedDatasetAddressScheme, msg=f"Could not get {product} path"):
product.get_file_obj()

@mock.patch("eodag_cube.api.product._product.try_open_dataset", autospec=True)
@mock.patch("eodag_cube.utils.xarray.try_open_dataset", autospec=True)
@mock.patch("eodag_cube.api.product._product.EOProduct.get_file_obj", autospec=True)
def test_to_xarray(self, mock_get_file, mock_open_ds):
"""to_xarrray should return well built XarrayDict"""
Expand All @@ -273,7 +273,7 @@ def test_to_xarray(self, mock_get_file, mock_open_ds):
self.assertTrue(xd["data"].equals(mock_open_ds.return_value))
self.assertDictEqual(product.properties, xd["data"].attrs)

@mock.patch("eodag_cube.api.product._product.try_open_dataset", autospec=True)
@mock.patch("eodag_cube.utils.xarray.try_open_dataset", autospec=True)
@mock.patch("eodag_cube.api.product._product.EOProduct.get_file_obj", autospec=True)
def test_to_xarray_assets(self, mock_get_file, mock_open_ds):
"""to_xarrray should return well built XarrayDict"""
Expand Down
10 changes: 5 additions & 5 deletions tests/units/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ def test_guess_engines(self, mock_head, mock_get):
self.assertIn("cfgrib", guess_engines(file))

@mock.patch("eodag_cube.utils.xarray.guess_engines", return_value=["h5netcdf", "foo"])
@mock.patch("eodag_cube.api.product._product.fsspec.open")
@mock.patch("fsspec.open")
def test_try_open_dataset_local(self, mock_open, mock_guess_engines):
"""try_open_dataset must call xaray.open_dataset with appropriate args"""
# local file : let xarray guess engine
Expand All @@ -185,7 +185,7 @@ def test_try_open_dataset_local(self, mock_open, mock_guess_engines):
ds = try_open_dataset(file, foo="bar", baz="qux")

@mock.patch("eodag_cube.utils.xarray.guess_engines", return_value=["cfgrib"])
@mock.patch("eodag_cube.api.product._product.fsspec.open")
@mock.patch("fsspec.open")
def test_try_open_dataset_remote_grib(self, mock_open, mock_guess_engines):
"""try_open_dataset must call xaray.open_dataset with appropriate args"""
# remote file + grib
Expand All @@ -200,7 +200,7 @@ def test_try_open_dataset_remote_grib(self, mock_open, mock_guess_engines):
try_open_dataset(file, foo="bar", baz="qux")

@mock.patch("eodag_cube.utils.xarray.guess_engines", return_value=["cfgrib"])
@mock.patch("eodag_cube.api.product._product.fsspec.open")
@mock.patch("fsspec.open")
def test_try_open_dataset_local_grib(self, mock_open, mock_guess_engines):
"""try_open_dataset must call xaray.open_dataset with appropriate args"""
# local file + grib
Expand All @@ -217,7 +217,7 @@ def test_try_open_dataset_local_grib(self, mock_open, mock_guess_engines):
mock_open_dataset.assert_called_once_with(file.path, engine="cfgrib", foo="bar", baz="qux")

@mock.patch("eodag_cube.utils.xarray.guess_engines", return_value=["h5netcdf", "foo"])
@mock.patch("eodag_cube.api.product._product.fsspec.open")
@mock.patch("fsspec.open")
def test_try_open_dataset_remote_nc(self, mock_open, mock_guess_engines):
"""try_open_dataset must call xaray.open_dataset with appropriate args"""
# remote file + nc
Expand All @@ -234,7 +234,7 @@ def test_try_open_dataset_remote_nc(self, mock_open, mock_guess_engines):
mock_open_dataset.assert_called_once_with(file, engine="h5netcdf", foo="bar", baz="qux")

@mock.patch("eodag_cube.utils.xarray.guess_engines", return_value=["rasterio"])
@mock.patch("eodag_cube.api.product._product.fsspec.open")
@mock.patch("fsspec.open")
def test_try_open_dataset_remote_jp2(self, mock_open, mock_guess_engines):
"""try_open_dataset must call open_rasterio with appropriate args"""
# remote file + nc
Expand Down
Loading