Error during DataArray.copy

[lukesdm]

When trying to copy a loaded dataarray, e.g.,

da = lazycogs.open(...)
da.copy()

you get the error:

TypeError: cannot pickle 'builtins.DuckdbClient' object

This can be reproduced from the quickstart notebook - try to run da.copy() after da is loaded.

Full traceback:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[10], line 1
----> 1 da.copy()

File ~/.venvs/py312/lib/python3.12/site-packages/xarray/core/dataarray.py:1342, in DataArray.copy(self, deep, data)
   1274 def copy(self, deep: bool = True, data: Any = None) -> Self:
   1275     """Returns a copy of this array.
   1276 
   1277     If `deep=True`, a deep copy is made of the data array.
   (...)   1340     pandas.DataFrame.copy
   1341     """
-> 1342     return self._copy(deep=deep, data=data)

File ~/.venvs/py312/lib/python3.12/site-packages/xarray/core/dataarray.py:1350, in DataArray._copy(self, deep, data, memo)
   1344 def _copy(
   1345     self,
   1346     deep: bool = True,
   1347     data: Any = None,
   1348     memo: dict[int, Any] | None = None,
   1349 ) -> Self:
-> 1350     variable = self.variable._copy(deep=deep, data=data, memo=memo)
   1351     indexes, index_vars = self.xindexes.copy_indexes(deep=deep)
   1353     coords = {}

File ~/.venvs/py312/lib/python3.12/site-packages/xarray/core/variable.py:959, in Variable._copy(self, deep, data, memo)
    956         ndata = indexing.MemoryCachedArray(data_old.array)  # type: ignore[assignment]
    958     if deep:
--> 959         ndata = copy.deepcopy(ndata, memo)
    961 else:
    962     ndata = as_compatible_data(data)

File ~/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/copy.py:162, in deepcopy(x, memo, _nil)
    160                 y = x
    161             else:
--> 162                 y = _reconstruct(x, memo, *rv)
    164 # If is its own copy, don't memoize.
    165 if y is not x:

File ~/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/copy.py:259, in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    257 if state is not None:
    258     if deep:
--> 259         state = deepcopy(state, memo)
    260     if hasattr(y, '__setstate__'):
    261         y.__setstate__(state)

File ~/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/copy.py:136, in deepcopy(x, memo, _nil)
    134 copier = _deepcopy_dispatch.get(cls)
    135 if copier is not None:
--> 136     y = copier(x, memo)
    137 else:
    138     if issubclass(cls, type):

File ~/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/copy.py:201, in _deepcopy_tuple(x, memo, deepcopy)
    200 def _deepcopy_tuple(x, memo, deepcopy=deepcopy):
--> 201     y = [deepcopy(a, memo) for a in x]
    202     # We're not going to put the tuple in the memo, but it's still important we
    203     # check for it, in case the tuple contains recursive mutable structures.
    204     try:

File ~/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/copy.py:136, in deepcopy(x, memo, _nil)
    134 copier = _deepcopy_dispatch.get(cls)
    135 if copier is not None:
--> 136     y = copier(x, memo)
    137 else:
    138     if issubclass(cls, type):

File ~/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/copy.py:221, in _deepcopy_dict(x, memo, deepcopy)
    219 memo[id(x)] = y
    220 for key, value in x.items():
--> 221     y[deepcopy(key, memo)] = deepcopy(value, memo)
    222 return y

File ~/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/copy.py:162, in deepcopy(x, memo, _nil)
    160                 y = x
    161             else:
--> 162                 y = _reconstruct(x, memo, *rv)
    164 # If is its own copy, don't memoize.
    165 if y is not x:

File ~/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/copy.py:259, in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    257 if state is not None:
    258     if deep:
--> 259         state = deepcopy(state, memo)
    260     if hasattr(y, '__setstate__'):
    261         y.__setstate__(state)

File ~/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/copy.py:136, in deepcopy(x, memo, _nil)
    134 copier = _deepcopy_dispatch.get(cls)
    135 if copier is not None:
--> 136     y = copier(x, memo)
    137 else:
    138     if issubclass(cls, type):

File ~/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/copy.py:221, in _deepcopy_dict(x, memo, deepcopy)
    219 memo[id(x)] = y
    220 for key, value in x.items():
--> 221     y[deepcopy(key, memo)] = deepcopy(value, memo)
    222 return y

File ~/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/copy.py:151, in deepcopy(x, memo, _nil)
    149 reductor = getattr(x, "__reduce_ex__", None)
    150 if reductor is not None:
--> 151     rv = reductor(4)
    152 else:
    153     reductor = getattr(x, "__reduce__", None)

TypeError: cannot pickle 'builtins.DuckdbClient' object

[lukesdm]

P.S., This arose when trying to use rioxarray's to_raster on a derived DataArray.

[lukesdm]

Here's a workaround (the _stac_backend attr was holding a reference to a duckdb client... I guess only use this if you're not doing any further file reading with da):

da.attrs.pop("_stac_backend", None)
da_copy = da.copy()

[hrodmn]

Thanks for the report, I hit the same thing yesterday while testing out some write operations with rioxarray. Fix incoming!

[lukesdm]

Nice, thanks!

[hrodmn]

@lukesdm I made the minimal fix in #66 to avoid storing too much in the attrs which got released with v0.4.1, then I made a few more changes that should make lazycogs more interoperable with rioxarray in #71. The latter changes involved changing the public metadata a bit so I published them in a v0.5.0.