adjusted code to work with new sample product locations

integration/test_sen1_native.py

@@ -6,6 +6,7 @@
 
 import xarray as xr
 
+# TODO: adjust path to new locations
 bucket = "e05ab01a9d56408d82ac32d69a5aae2a:sample-data"
 path_prefix = "tutorial_data/cpm_v253"
 url_prefix = f"s3://{bucket}/{path_prefix}"

integration/test_sen2_analysis.py

@@ -11,27 +11,36 @@
 from xarray_eopf.spatial import get_spatial_vars
 from xarray_eopf.utils import timeit
 
-bucket = "e05ab01a9d56408d82ac32d69a5aae2a:sample-data"
-path_prefix = "tutorial_data/cpm_v253"
-s3_prefix = f"s3://{bucket}/{path_prefix}"
-https_prefix = f"https://{DEFAULT_ENDPOINT_URL}/{bucket}/{path_prefix}"
+s02msil1c_bucket = "e05ab01a9d56408d82ac32d69a5aae2a:202504-s02msil1c"
+s02msil2a_bucket = "e05ab01a9d56408d82ac32d69a5aae2a:202504-s02msil2a"
+path_prefix = "15/products/cpm_v256"
+l1c_filename = "S2B_MSIL1C_20250415T142749_N0511_R139_T25WEV_20250415T180239.zarr"
+l2a_filename = "S2B_MSIL2A_20250415T142749_N0511_R139_T25WEV_20250415T181516.zarr"
 
 allowed_open_time = 5  # seconds
 show_chunking = False
 
 
 class Sentinel2AnalysisTest(TestCase):
     def test_open_dataset_sen2_l1c_s3(self):
-        self._test_open_dataset_sen2_l1c(s3_prefix)
+        self._test_open_dataset_sen2_l1c(f"s3://{s02msil1c_bucket}/{path_prefix}")
+
+    def test_open_dataset_sen2_l2a_s3(self):
+        self._test_open_dataset_sen2_l2a(f"s3://{s02msil2a_bucket}/{path_prefix}")
 
     def test_open_dataset_sen2_l1c_https(self):
-        self._test_open_dataset_sen2_l1c(https_prefix)
+        self._test_open_dataset_sen2_l1c(
+            f"{DEFAULT_ENDPOINT_URL}/{s02msil1c_bucket}/{path_prefix}"
+        )
 
-    def _test_open_dataset_sen2_l1c(self, url_prefix):
-        url = (
-            f"{url_prefix}/"
-            "S2B_MSIL1C_20250113T103309_N0511_R108_T32TLQ_20250113T122458.zarr"
+    def test_open_dataset_sen2_l2a_https(self):
+        self._test_open_dataset_sen2_l2a(
+            f"{DEFAULT_ENDPOINT_URL}/{s02msil2a_bucket}/{path_prefix}"
         )
+
+    def _test_open_dataset_sen2_l1c(self, url_prefix):
+        # See https://stac.browser.user.eopf.eodc.eu/collections/sentinel-2-l1c/items/S2B_MSIL1C_20250415T142749_N0511_R139_T25WEV_20250415T180239
+        url = f"{url_prefix}/{l1c_filename}"
         with timeit("open " + url) as result:
             # noinspection PyTypeChecker
             ds = xr.open_dataset(
@@ -51,17 +60,8 @@ def _test_open_dataset_sen2_l1c(self, url_prefix):
         for var_name in spatial_vars.keys():
             self.assertEqual((10980, 10980), ds[var_name].shape[-2:], msg=var_name)
 
-    def test_open_dataset_sen2_l2a_s3(self):
-        self._test_open_dataset_sen2_l2a(s3_prefix)
-
-    def test_open_dataset_sen2_l2a_https(self):
-        self._test_open_dataset_sen2_l2a(https_prefix)
-
     def _test_open_dataset_sen2_l2a(self, url_prefix):
-        url = (
-            f"{url_prefix}/"
-            "S2A_MSIL2A_20240101T102431_N0510_R065_T32TNT_20240101T144052.zarr"
-        )
+        url = f"{url_prefix}/{l2a_filename}"
         with timeit("open " + url) as result:
             # noinspection PyTypeChecker
             ds = xr.open_dataset(
@@ -83,3 +83,13 @@ def _test_open_dataset_sen2_l2a(self, url_prefix):
         assert_data_arrays_are_chunked(self, spatial_vars, verbose=show_chunking)
         for var_name in spatial_vars.keys():
             self.assertEqual((10980, 10980), ds[var_name].shape[-2:], msg=var_name)
+
+    def test_production(self):
+        url = (
+            "https://objectstore.eodc.eu:2222/"
+            "e05ab01a9d56408d82ac32d69a5aae2a:202504-s02msil2a/15/products/"
+            "cpm_v256/"
+            "S2B_MSIL2A_20250415T142749_N0511_R139_T25WEU_20250415T181516.zarr"
+        )
+        ds = xr.open_dataset(url, engine="eopf-zarr")
+        self.assertIsInstance(ds, xr.Dataset)

integration/test_sen2_native.py

@@ -11,27 +11,43 @@
 from xarray_eopf.spatial import get_spatial_vars
 from xarray_eopf.utils import timeit
 
-bucket = "e05ab01a9d56408d82ac32d69a5aae2a:sample-data"
-path_prefix = "tutorial_data/cpm_v253"
-s3_prefix = f"s3://{bucket}/{path_prefix}"
-https_prefix = f"https://{DEFAULT_ENDPOINT_URL}/{bucket}/{path_prefix}"
+s02msil1c_bucket = "e05ab01a9d56408d82ac32d69a5aae2a:202504-s02msil1c"
+s02msil2a_bucket = "e05ab01a9d56408d82ac32d69a5aae2a:202504-s02msil2a"
+path_prefix = "15/products/cpm_v256"
+l1c_filename = "S2B_MSIL1C_20250415T142749_N0511_R139_T25WEV_20250415T180239.zarr"
+l2a_filename = "S2B_MSIL2A_20250415T142749_N0511_R139_T25WEV_20250415T181516.zarr"
 
 allowed_open_time = 5  # seconds
 
 
 class Sentinel2NativeTest(TestCase):
     def test_open_datatree_sen2_l1c_s3(self):
-        self._test_open_datatree_sen2_l1c(s3_prefix)
+        self._test_open_datatree_sen2_l1c(f"s3://{s02msil1c_bucket}/{path_prefix}")
+
+    def test_open_dataset_sen2_l1c_s3(self):
+        self._test_open_dataset_sen2_l1c(f"s3://{s02msil1c_bucket}/{path_prefix}")
+
+    def test_open_dataset_sen2_l2a_s3(self):
+        self._test_open_dataset_sen2_l2a(f"s3://{s02msil2a_bucket}/{path_prefix}")
 
     def test_open_datatree_sen2_l1c_https(self):
-        self._test_open_datatree_sen2_l1c(https_prefix)
+        self._test_open_datatree_sen2_l1c(
+            f"{DEFAULT_ENDPOINT_URL}/{s02msil1c_bucket}/{path_prefix}"
+        )
+
+    def test_open_dataset_sen2_l1c_https(self):
+        self._test_open_dataset_sen2_l1c(
+            f"{DEFAULT_ENDPOINT_URL}/{s02msil1c_bucket}/{path_prefix}"
+        )
+
+    def test_open_dataset_sen2_l2a_https(self):
+        self._test_open_dataset_sen2_l2a(
+            f"{DEFAULT_ENDPOINT_URL}/{s02msil2a_bucket}/{path_prefix}"
+        )
 
     def _test_open_datatree_sen2_l1c(self, url_prefix: str):
         # noinspection PyTypeChecker
-        url = (
-            f"{url_prefix}/"
-            f"S2B_MSIL1C_20250113T103309_N0511_R108_T32TLQ_20250113T122458.zarr"
-        )
+        url = f"{url_prefix}/{l1c_filename}"
         with timeit("open " + url) as result:
             # noinspection PyTypeChecker
             dt = xr.open_datatree(url, engine="eopf-zarr", op_mode="native", chunks={})
@@ -49,17 +65,8 @@ def _test_open_datatree_sen2_l1c(self, url_prefix: str):
         )
         assert_data_arrays_are_chunked(self, spatial_vars, verbose=True)
 
-    def test_open_datatree_sen2_l2a_s3(self):
-        self._test_open_datatree_sen2_l2a(s3_prefix)
-
-    def test_open_datatree_sen2_l2a_https(self):
-        self._test_open_datatree_sen2_l2a(https_prefix)
-
     def _test_open_datatree_sen2_l2a(self, url_prefix: str):
-        url = (
-            f"{url_prefix}/"
-            "S2A_MSIL2A_20240101T102431_N0510_R065_T32TNT_20240101T144052.zarr"
-        )
+        url = f"{url_prefix}/{l2a_filename}.zarr"
         with timeit("open " + url) as result:
             # noinspection PyTypeChecker
             dt = xr.open_datatree(url, engine="eopf-zarr", op_mode="native", chunks={})
@@ -77,17 +84,8 @@ def _test_open_datatree_sen2_l2a(self, url_prefix: str):
         )
         assert_data_arrays_are_chunked(self, spatial_vars, verbose=True)
 
-    def test_open_dataset_sen2_l1c_s3(self):
-        self._test_open_dataset_sen2_l1c(s3_prefix)
-
-    def test_open_dataset_sen2_l1c_https(self):
-        self._test_open_dataset_sen2_l1c(https_prefix)
-
     def _test_open_dataset_sen2_l1c(self, url_prefix: str):
-        url = (
-            f"{url_prefix}/"
-            "S2B_MSIL1C_20250113T103309_N0511_R108_T32TLQ_20250113T122458.zarr"
-        )
+        url = f"{url_prefix}/{l1c_filename}"
         with timeit(url) as result:
             # noinspection PyTypeChecker
             ds = xr.open_dataset(url, engine="eopf-zarr", op_mode="native", chunks={})
@@ -105,17 +103,8 @@ def _test_open_dataset_sen2_l1c(self, url_prefix: str):
         self.assertEqual(43, len(spatial_vars))
         assert_data_arrays_are_chunked(self, spatial_vars, verbose=True)
 
-    def test_open_dataset_sen2_l2a_s3(self):
-        self._test_open_dataset_sen2_l2a(s3_prefix)
-
-    def test_open_dataset_sen2_l2a_https(self):
-        self._test_open_dataset_sen2_l2a(https_prefix)
-
     def _test_open_dataset_sen2_l2a(self, url_prefix: str):
-        url = (
-            f"{url_prefix}/"
-            "S2A_MSIL2A_20240101T102431_N0510_R065_T32TNT_20240101T144052.zarr"
-        )
+        url = f"{url_prefix}/{l2a_filename}"
         with timeit(url) as result:
             # noinspection PyTypeChecker
             ds = xr.open_dataset(url, engine="eopf-zarr", op_mode="native", chunks={})

tests/test_source.py

@@ -0,0 +1,46 @@
+#  Copyright (c) 2025 by EOPF Sample Service team and contributors
+#  Permissions are hereby granted under the terms of the Apache 2.0 License:
+#  https://opensource.org/license/apache-2-0.
+from pathlib import Path
+from unittest import TestCase
+
+import fsspec
+import pytest
+import s3fs
+
+from xarray_eopf.source import normalize_source
+
+
+class NormalizeSourceTest(TestCase):
+    def test_s3_url(self):
+        url = "s3://no-bucket/test.zarr"
+        store = normalize_source(url, None)
+        self.assertIsInstance(store, fsspec.FSMap)
+        self.assertIsInstance(store.fs, s3fs.S3FileSystem)
+        self.assertEqual("no-bucket/test.zarr", store.root)
+
+    def test_ceph_s3_url(self):
+        ceph_url = "s3://no-bucket:e6f4/test.zarr"
+        store = normalize_source(ceph_url, None)
+        self.assertIsInstance(store, fsspec.FSMap)
+        self.assertIsInstance(store.fs, s3fs.S3FileSystem)
+        self.assertEqual("no-bucket:e6f4/test.zarr", store.root)
+
+    def test_https_url(self):
+        path = "https://unknown.object.storage.com/no-bucket/test.zarr"
+        source = normalize_source(path, None)
+        self.assertEqual(path, source)
+
+    def test_other(self):
+        mapping = {}
+        self.assertIs(mapping, normalize_source(mapping, None))
+
+        path = Path("data/test.zarr")
+        self.assertIs(path, normalize_source(path, None))
+
+    # noinspection PyMethodMayBeStatic
+    def test_fail(self):
+        with pytest.raises(
+            ValueError, match="storage_options argument applies only to paths or URLs"
+        ):
+            normalize_source({}, {})

xarray_eopf/backend.py

@@ -21,7 +21,7 @@
 )
 from .filter import filter_dataset
 from .flatten import flatten_datatree, flatten_datatree_as_dict
-from .store import open_store
+from .source import normalize_source
 from .utils import assert_arg_is_one_of
 
 
@@ -40,7 +40,6 @@ def open_datatree(
         *,
         op_mode: OpMode = OP_MODE_ANALYSIS,
         product_type: str | None = None,
-        protocol: str | None = None,
         storage_options: Mapping[str, Any] | None = None,
         drop_variables: str | Iterable[str] | None = None,
         decode_timedelta: (
@@ -59,10 +58,6 @@ def open_datatree(
                 Only used if `op_mode="analysis"`; typically not required
                 if the filename inherent to `filename_or_obj`
                 adheres to EOPF naming conventions.
-            protocol: If `filename_or_obj` is a file path or URL,
-                it forces using the specified filesystem protocol.
-                Otherwise, the protocol will be derived from the file path or URL.
-                Will be passed to [`fsspec.filesystem()`](https://filesystem-spec.readthedocs.io/en/latest/usage.html).
             storage_options: If `filename_or_obj` is a file path or URL,
                 these options specify the source filesystem.
                 Will be passed to [`fsspec.filesystem()`](https://filesystem-spec.readthedocs.io/en/latest/usage.html).
@@ -78,7 +73,7 @@ def open_datatree(
 
         assert_arg_is_one_of(op_mode, "op_mode", OP_MODES)
 
-        fs_store = open_store(filename_or_obj, protocol, storage_options)
+        fs_store = normalize_source(filename_or_obj, storage_options)
 
         datatree = xr.open_datatree(
             fs_store,
@@ -107,7 +102,6 @@ def open_dataset(
         *,
         op_mode: OpMode = OP_MODE_ANALYSIS,
         # params for op_mode=native/analysis
-        protocol: str | None = None,
         storage_options: Mapping[str, Any] | None = None,
         group_sep: str = "_",
         variables: str | Iterable[str] | None = None,
@@ -133,10 +127,6 @@ def open_dataset(
                 Only used if `op_mode="analysis"`; typically not required
                 if the filename inherent to `filename_or_obj`
                 adheres to EOPF naming conventions.
-            protocol: If `filename_or_obj` is a file path or URL,
-                it forces using the specified filesystem protocol.
-                Otherwise, the protocol will be derived from the file path or URL.
-                Will be passed to [`fsspec.filesystem()`](https://filesystem-spec.readthedocs.io/en/latest/usage.html).
             storage_options: If `filename_or_obj` is a file path or URL,
                 these options specify the source filesystem.
                 Will be passed to [`fsspec.filesystem()`](https://filesystem-spec.readthedocs.io/en/latest/usage.html).
@@ -167,7 +157,6 @@ def open_dataset(
         datatree = self.open_datatree(
             filename_or_obj,
             op_mode="native",
-            protocol=protocol,
             storage_options=storage_options,
             # here as it is required for all backends
             drop_variables=drop_variables,