Skip to content

Reduce duplication between _generate_cubes and load_meshes #7211

Description

@trexfeathers

An artifact from when load_meshes was still experimental. Code should be factored out so that both these routines can call the same convenience.

Code

iris/lib/iris/loading.py

Lines 74 to 105 in 3432ca1

def _generate_cubes(uris, callback, constraints):
import iris.io
"""Return a generator of cubes given the URIs and a callback."""
if (
isinstance(uris, str)
or hasattr(uris, "fromcdl")
or not isinstance(uris, Iterable)
):
# Make a string, Dataset, or other single item, into an iterable.
uris = [uris]
# Group collections of uris by their iris handler
# Create list of tuples relating schemes to part names
uri_tuples = sorted(iris.io.decode_uri(uri) for uri in uris)
for scheme, groups in itertools.groupby(uri_tuples, key=lambda x: x[0]):
# Call each scheme handler with the appropriate URIs
if scheme == "file":
part_names = [x[1] for x in groups]
for cube in iris.io.load_files(part_names, callback, constraints):
yield cube
elif scheme in ["http", "https"]:
urls = [":".join(x) for x in groups]
for cube in iris.io.load_http(urls, callback):
yield cube
elif scheme == "data":
data_objects = [x[1] for x in groups]
for cube in iris.io.load_data_objects(data_objects, callback):
yield cube
else:
raise ValueError("Iris cannot handle the URI scheme: %s" % scheme)

if isinstance(uris, str):
uris = [uris]
# Group collections of uris by their iris handler
# Create list of tuples relating schemes to part names.
uri_tuples = sorted(decode_uri(uri) for uri in uris)
valid_sources = []
for scheme, groups in groupby(uri_tuples, key=lambda x: x[0]):
# Call each scheme handler with the appropriate URIs
if scheme == "file":
filenames = [x[1] for x in groups]
sources = expand_filespecs(filenames)
elif scheme in ["http", "https"]:
sources = [":".join(x) for x in groups]
else:
message = f"Iris cannot handle the URI scheme: {scheme}"
raise ValueError(message)
for source in sources:
if scheme == "file":
with open(source, "rb") as fh:
handling_format_spec = FORMAT_AGENT.get_spec(Path(source).name, fh)
else:
handling_format_spec = FORMAT_AGENT.get_spec(source, None)
if handling_format_spec.handler == nc_loader.load_cubes:
valid_sources.append(source)
else:
message = f"Ignoring non-NetCDF file: {source}"
logger.info(msg=message, extra=dict(cls=None))
result = {}
for source in valid_sources:
with CFReader(source) as cf_reader:
meshes_dict = _meshes_from_cf(cf_reader)
meshes = list(meshes_dict.values())
if var_name is not None:
meshes = list(filter(lambda m: m.var_name == var_name, meshes))
if meshes:
result[source] = meshes
return result

Metadata

Metadata

Assignees

No one assigned

    Labels

    Good First IssueA good issue to take on if you're just getting started with Iris development

    Type

    No type

    Projects

    Status
    No status

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions