From 3429b2c20be0b0c1d76c5a88aa392e4a86d82ccb Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Wed, 23 Oct 2024 14:39:31 -0500 Subject: [PATCH 01/20] adding draft for fixing behaviour for group parameter --- xarray/backends/zarr.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 50755ebf1f7..b7126446790 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -580,7 +580,7 @@ def open_store( use_zarr_fill_value_as_mask=use_zarr_fill_value_as_mask, zarr_format=zarr_format, ) - group_paths = [node for node in _iter_zarr_groups(zarr_group, parent=group)] + group_paths = list(set([node for node in _iter_zarr_groups(zarr_group, parent=group)])) return { group: cls( zarr_group.get(group), @@ -1531,8 +1531,12 @@ def open_groups_as_dict( decode_timedelta=decode_timedelta, ) group_name = str(NodePath(path_group)) - groups_dict[group_name] = group_ds - + # groups_dict[group_name] = group_ds + if group: + group_name = str(NodePath(NodePath(group).stem) / NodePath(group_name).stem) + groups_dict[group_name] = group_ds + else: + groups_dict[group_name] = group_ds return groups_dict From 1507f4d07682d39cd6e22caffd79556643922445 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 19:56:38 +0000 Subject: [PATCH 02/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/backends/zarr.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index b7126446790..2ceb4eeae24 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -580,7 +580,9 @@ def open_store( use_zarr_fill_value_as_mask=use_zarr_fill_value_as_mask, zarr_format=zarr_format, ) - group_paths = list(set([node for node in _iter_zarr_groups(zarr_group, parent=group)])) + group_paths = list( + set([node for node in _iter_zarr_groups(zarr_group, parent=group)]) + ) return { group: cls( zarr_group.get(group), @@ -1533,7 +1535,9 @@ def open_groups_as_dict( group_name = str(NodePath(path_group)) # groups_dict[group_name] = group_ds if group: - group_name = str(NodePath(NodePath(group).stem) / NodePath(group_name).stem) + group_name = str( + NodePath(NodePath(group).stem) / NodePath(group_name).stem + ) groups_dict[group_name] = group_ds else: groups_dict[group_name] = group_ds From e24e88b7bb63bfdcd5a5d64fcc262f4c489bca93 Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Wed, 23 Oct 2024 15:42:11 -0500 Subject: [PATCH 03/20] new trial --- xarray/backends/zarr.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index b7126446790..bbb5a6aa34e 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -580,7 +580,7 @@ def open_store( use_zarr_fill_value_as_mask=use_zarr_fill_value_as_mask, zarr_format=zarr_format, ) - group_paths = list(set([node for node in _iter_zarr_groups(zarr_group, parent=group)])) + group_paths = sorted(list(set([node for node in _iter_zarr_groups(zarr_group, parent=group)]))) return { group: cls( zarr_group.get(group), @@ -1530,13 +1530,12 @@ def open_groups_as_dict( use_cftime=use_cftime, decode_timedelta=decode_timedelta, ) - group_name = str(NodePath(path_group)) - # groups_dict[group_name] = group_ds if group: - group_name = str(NodePath(NodePath(group).stem) / NodePath(group_name).stem) - groups_dict[group_name] = group_ds + relative = str(NodePath("/") / NodePath(group).parent) + group_name = str(NodePath(path_group).relative_to(relative)) else: - groups_dict[group_name] = group_ds + group_name = str(NodePath(path_group)) + groups_dict[group_name] = group_ds return groups_dict From b6fac5b1c393f3dee9bc7758bf74895cb2b14eb6 Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Wed, 23 Oct 2024 15:44:46 -0500 Subject: [PATCH 04/20] new trial --- xarray/backends/zarr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 47f8bc7b449..c765b18540a 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -1536,7 +1536,8 @@ def open_groups_as_dict( relative = str(NodePath("/") / NodePath(group).parent) group_name = str(NodePath(path_group).relative_to(relative)) else: - groups_dict[group_name] = group_ds + group_name = str(NodePath(path_group)) + groups_dict[group_name] = group_ds return groups_dict From ce83c89f429ee8ad54768d16bdccf8d993242849 Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Thu, 24 Oct 2024 10:31:39 -0500 Subject: [PATCH 05/20] fixing duplicate pahts and path in the root group --- xarray/backends/zarr.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index c765b18540a..4a05dcc2b74 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -580,9 +580,7 @@ def open_store( use_zarr_fill_value_as_mask=use_zarr_fill_value_as_mask, zarr_format=zarr_format, ) - group_paths = list( - set([node for node in _iter_zarr_groups(zarr_group, parent=group)]) - ) + group_paths = [node for node in _iter_zarr_groups(zarr_group, parent=group)] return { group: cls( zarr_group.get(group), @@ -1468,7 +1466,12 @@ def open_datatree( zarr_format=zarr_format, **kwargs, ) - return datatree_from_dict_with_io_cleanup(groups_dict) + if group: + dt = datatree_from_dict_with_io_cleanup(groups_dict) + dt.encoding["source_group"] = group + return dt + else: + return datatree_from_dict_with_io_cleanup(groups_dict) def open_groups_as_dict( self, @@ -1533,8 +1536,7 @@ def open_groups_as_dict( decode_timedelta=decode_timedelta, ) if group: - relative = str(NodePath("/") / NodePath(group).parent) - group_name = str(NodePath(path_group).relative_to(relative)) + group_name = str(NodePath(path_group).relative_to(parent)) else: group_name = str(NodePath(path_group)) groups_dict[group_name] = group_ds @@ -1546,7 +1548,6 @@ def _iter_zarr_groups(root: ZarrGroup, parent: str = "/") -> Iterable[str]: yield str(parent_nodepath) for path, group in root.groups(): gpath = parent_nodepath / path - yield str(gpath) yield from _iter_zarr_groups(group, parent=str(gpath)) From 72fcee6c03a74f07e10939fe0294a2f4c0068623 Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Thu, 24 Oct 2024 11:13:48 -0500 Subject: [PATCH 06/20] removing yield str(gpath) --- xarray/backends/common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xarray/backends/common.py b/xarray/backends/common.py index 12382c3f39b..8d1d089a913 100644 --- a/xarray/backends/common.py +++ b/xarray/backends/common.py @@ -135,7 +135,6 @@ def _iter_nc_groups(root, parent="/"): yield str(parent) for path, group in root.groups.items(): gpath = parent / path - yield str(gpath) yield from _iter_nc_groups(group, parent=gpath) From bd853c8b467c38c9b92d89fac7a3869603b347fd Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Thu, 24 Oct 2024 11:15:07 -0500 Subject: [PATCH 07/20] implementing the proposed solution to hdf5 and netcdf backends --- xarray/backends/h5netcdf_.py | 12 ++++++++++-- xarray/backends/netCDF4_.py | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 888489c0c04..3d31fca9899 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -494,7 +494,12 @@ def open_datatree( driver_kwds=driver_kwds, **kwargs, ) - return datatree_from_dict_with_io_cleanup(groups_dict) + if group: + dt = datatree_from_dict_with_io_cleanup(groups_dict) + dt.encoding["source_group"] = group + return dt + else: + return datatree_from_dict_with_io_cleanup(groups_dict) def open_groups_as_dict( self, @@ -556,7 +561,10 @@ def open_groups_as_dict( decode_timedelta=decode_timedelta, ) - group_name = str(NodePath(path_group)) + if group: + group_name = str(NodePath(path_group).relative_to(parent)) + else: + group_name = str(NodePath(path_group)) groups_dict[group_name] = group_ds return groups_dict diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index b4609e626b5..00caafad225 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -729,7 +729,12 @@ def open_datatree( autoclose=autoclose, **kwargs, ) - return datatree_from_dict_with_io_cleanup(groups_dict) + if group: + dt = datatree_from_dict_with_io_cleanup(groups_dict) + dt.encoding["source_group"] = group + return dt + else: + return datatree_from_dict_with_io_cleanup(groups_dict) def open_groups_as_dict( self, @@ -789,7 +794,10 @@ def open_groups_as_dict( use_cftime=use_cftime, decode_timedelta=decode_timedelta, ) - group_name = str(NodePath(path_group)) + if group: + group_name = str(NodePath(path_group).relative_to(parent)) + else: + group_name = str(NodePath(path_group)) groups_dict[group_name] = group_ds return groups_dict From d6e542253f2403aeb60a7fd171ec363bfd789d91 Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Thu, 24 Oct 2024 11:55:14 -0500 Subject: [PATCH 08/20] adding changes to whats-new.rst --- doc/whats-new.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4f08d17e825..90c5122ef8d 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -74,6 +74,8 @@ Bug fixes `_. - Fix binning by multiple variables where some bins have no observations. (:issue:`9630`). By `Deepak Cherian `_. +- Fix bug with `open_datatree(group='some_subgroup')` returning empty parent nodes. (:issue:`9665`, :pull:`9666`) + by `Alfonso Ladino `_. Documentation ~~~~~~~~~~~~~ From 12005e2917e2c1ecb87ce0132ead0e7b1da996a7 Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Thu, 24 Oct 2024 12:00:27 -0500 Subject: [PATCH 09/20] removing encoding['source_group'] line to avoid conflicts with PR #9660 --- xarray/backends/h5netcdf_.py | 8 ++------ xarray/backends/netCDF4_.py | 8 ++------ xarray/backends/zarr.py | 8 ++------ 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 3d31fca9899..f51f5873817 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -494,12 +494,8 @@ def open_datatree( driver_kwds=driver_kwds, **kwargs, ) - if group: - dt = datatree_from_dict_with_io_cleanup(groups_dict) - dt.encoding["source_group"] = group - return dt - else: - return datatree_from_dict_with_io_cleanup(groups_dict) + + return datatree_from_dict_with_io_cleanup(groups_dict) def open_groups_as_dict( self, diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 00caafad225..7a08a1da8d4 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -729,12 +729,8 @@ def open_datatree( autoclose=autoclose, **kwargs, ) - if group: - dt = datatree_from_dict_with_io_cleanup(groups_dict) - dt.encoding["source_group"] = group - return dt - else: - return datatree_from_dict_with_io_cleanup(groups_dict) + + return datatree_from_dict_with_io_cleanup(groups_dict) def open_groups_as_dict( self, diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 4a05dcc2b74..9fe9d4cdf9f 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -1466,12 +1466,8 @@ def open_datatree( zarr_format=zarr_format, **kwargs, ) - if group: - dt = datatree_from_dict_with_io_cleanup(groups_dict) - dt.encoding["source_group"] = group - return dt - else: - return datatree_from_dict_with_io_cleanup(groups_dict) + + return datatree_from_dict_with_io_cleanup(groups_dict) def open_groups_as_dict( self, From e4384d6bcbb96b3a765ec0f32af181667f481c1a Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Thu, 24 Oct 2024 13:29:14 -0500 Subject: [PATCH 10/20] adding test --- xarray/tests/test_backends_datatree.py | 52 ++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/xarray/tests/test_backends_datatree.py b/xarray/tests/test_backends_datatree.py index 16598194e1d..7a2ec1934c1 100644 --- a/xarray/tests/test_backends_datatree.py +++ b/xarray/tests/test_backends_datatree.py @@ -240,6 +240,38 @@ def test_open_groups_to_dict(self, tmpdir) -> None: for ds in aligned_dict_of_datasets.values(): ds.close() + def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None: + """Test opening a specific group within a NetCDF file using `open_datatree`.""" + # Open the specific group '/Group1/subgroup1' and check if it loads correctly + filepath = tmpdir / "test.nc" + group = "/set1" + original_dt = simple_datatree + original_dt.to_netcdf(filepath) + + with open_datatree(filepath, group=group, engine=self.engine) as subgroup_tree: + # Check that the subtree is not None + assert subgroup_tree is not None + # Check the expected number of children within node + assert list(subgroup_tree.children) == list(original_dt[group].children) + # Check the dimensions of the group '/set1' + assert len(subgroup_tree.dataset.dims) == len(original_dt[group].dataset.dims) + # Check if the variables in this "set1" group are correctly read + assert list(subgroup_tree.dataset.data_vars) == list(original_dt[group].dataset.data_vars) + # Check the values of the variables 'a' and 'b' + assert subgroup_tree.dataset["a"].values == original_dt[group].dataset["a"].values + assert subgroup_tree.dataset["b"].values == original_dt[group].dataset["b"].values + + def test_open_datatree_nonexistent_group(self, tmpdir, simple_datatree) -> None: + """Test `open_datatree` behavior when attempting to open a non-existent group.""" + filepath = tmpdir / "test.nc" + original_dt = simple_datatree + original_dt.to_netcdf(filepath, engine=self.engine) + + # Attempt to open a non-existent group, which should raise a ValueError + with pytest.raises(OSError, match="Group '/nonexistent_group' not found"): + open_datatree(filepath, group="/nonexistent_group", engine=self.engine) + + @requires_h5netcdf class TestH5NetCDFDatatreeIO(DatatreeIOBase): @@ -382,3 +414,23 @@ def test_open_groups(self, unaligned_datatree_zarr) -> None: for ds in unaligned_dict_of_datasets.values(): ds.close() + + def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None: + """Test opening a specific group within a Zarr store using `open_datatree`.""" + filepath = tmpdir / "test.zarr" + group = "/set2" + original_dt = simple_datatree + original_dt.to_zarr(filepath) + + with open_datatree(filepath, group=group, engine=self.engine) as subgroup_tree: + # Check that the subtree is not None + assert subgroup_tree is not None + # Check the expected number of children within node + assert list(subgroup_tree.children) == list(original_dt[group].children) + # Check the dimensions of the group '/set2' + assert len(subgroup_tree.dataset.dims) == len(original_dt[group].dataset.dims) + # Check if the variables in this "set2" group are correctly read + assert list(subgroup_tree.dataset.data_vars) == list(original_dt[group].dataset.data_vars) + # Check the values of the variables 'a' and 'b' + assert_equal(subgroup_tree.dataset["a"], original_dt[group].dataset["a"]) + assert_equal(subgroup_tree.dataset["b"], original_dt[group].dataset["b"]) \ No newline at end of file From e935e4e34ab8fadfc31695143396926a5f9cec03 Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Thu, 24 Oct 2024 13:37:56 -0500 Subject: [PATCH 11/20] adding test --- xarray/tests/test_backends_datatree.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/xarray/tests/test_backends_datatree.py b/xarray/tests/test_backends_datatree.py index 7a2ec1934c1..1a07d1995df 100644 --- a/xarray/tests/test_backends_datatree.py +++ b/xarray/tests/test_backends_datatree.py @@ -242,22 +242,16 @@ def test_open_groups_to_dict(self, tmpdir) -> None: def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None: """Test opening a specific group within a NetCDF file using `open_datatree`.""" - # Open the specific group '/Group1/subgroup1' and check if it loads correctly filepath = tmpdir / "test.nc" group = "/set1" original_dt = simple_datatree original_dt.to_netcdf(filepath) with open_datatree(filepath, group=group, engine=self.engine) as subgroup_tree: - # Check that the subtree is not None assert subgroup_tree is not None - # Check the expected number of children within node assert list(subgroup_tree.children) == list(original_dt[group].children) - # Check the dimensions of the group '/set1' assert len(subgroup_tree.dataset.dims) == len(original_dt[group].dataset.dims) - # Check if the variables in this "set1" group are correctly read assert list(subgroup_tree.dataset.data_vars) == list(original_dt[group].dataset.data_vars) - # Check the values of the variables 'a' and 'b' assert subgroup_tree.dataset["a"].values == original_dt[group].dataset["a"].values assert subgroup_tree.dataset["b"].values == original_dt[group].dataset["b"].values @@ -423,14 +417,9 @@ def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None: original_dt.to_zarr(filepath) with open_datatree(filepath, group=group, engine=self.engine) as subgroup_tree: - # Check that the subtree is not None assert subgroup_tree is not None - # Check the expected number of children within node assert list(subgroup_tree.children) == list(original_dt[group].children) - # Check the dimensions of the group '/set2' assert len(subgroup_tree.dataset.dims) == len(original_dt[group].dataset.dims) - # Check if the variables in this "set2" group are correctly read assert list(subgroup_tree.dataset.data_vars) == list(original_dt[group].dataset.data_vars) - # Check the values of the variables 'a' and 'b' assert_equal(subgroup_tree.dataset["a"], original_dt[group].dataset["a"]) assert_equal(subgroup_tree.dataset["b"], original_dt[group].dataset["b"]) \ No newline at end of file From 9a41b682fff401296e97105656774ad210ba354c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 24 Oct 2024 18:38:53 +0000 Subject: [PATCH 12/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/tests/test_backends_datatree.py | 27 +++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/xarray/tests/test_backends_datatree.py b/xarray/tests/test_backends_datatree.py index 59469d2e72e..4559f080c7f 100644 --- a/xarray/tests/test_backends_datatree.py +++ b/xarray/tests/test_backends_datatree.py @@ -347,10 +347,20 @@ def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None: with open_datatree(filepath, group=group, engine=self.engine) as subgroup_tree: assert subgroup_tree is not None assert list(subgroup_tree.children) == list(original_dt[group].children) - assert len(subgroup_tree.dataset.dims) == len(original_dt[group].dataset.dims) - assert list(subgroup_tree.dataset.data_vars) == list(original_dt[group].dataset.data_vars) - assert subgroup_tree.dataset["a"].values == original_dt[group].dataset["a"].values - assert subgroup_tree.dataset["b"].values == original_dt[group].dataset["b"].values + assert len(subgroup_tree.dataset.dims) == len( + original_dt[group].dataset.dims + ) + assert list(subgroup_tree.dataset.data_vars) == list( + original_dt[group].dataset.data_vars + ) + assert ( + subgroup_tree.dataset["a"].values + == original_dt[group].dataset["a"].values + ) + assert ( + subgroup_tree.dataset["b"].values + == original_dt[group].dataset["b"].values + ) def test_open_datatree_nonexistent_group(self, tmpdir, simple_datatree) -> None: """Test `open_datatree` behavior when attempting to open a non-existent group.""" @@ -363,7 +373,6 @@ def test_open_datatree_nonexistent_group(self, tmpdir, simple_datatree) -> None: open_datatree(filepath, group="/nonexistent_group", engine=self.engine) - @requires_h5netcdf class TestH5NetCDFDatatreeIO(DatatreeIOBase): engine: T_DataTreeNetcdfEngine | None = "h5netcdf" @@ -538,8 +547,12 @@ def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None: with open_datatree(filepath, group=group, engine=self.engine) as subgroup_tree: assert subgroup_tree is not None assert list(subgroup_tree.children) == list(original_dt[group].children) - assert len(subgroup_tree.dataset.dims) == len(original_dt[group].dataset.dims) - assert list(subgroup_tree.dataset.data_vars) == list(original_dt[group].dataset.data_vars) + assert len(subgroup_tree.dataset.dims) == len( + original_dt[group].dataset.dims + ) + assert list(subgroup_tree.dataset.data_vars) == list( + original_dt[group].dataset.data_vars + ) assert_equal(subgroup_tree.dataset["a"], original_dt[group].dataset["a"]) assert_equal(subgroup_tree.dataset["b"], original_dt[group].dataset["b"]) From f5d3073f61973297b8dafd1fd4972b7265821cb5 Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Thu, 24 Oct 2024 13:40:35 -0500 Subject: [PATCH 13/20] adding assert subgroup_tree.root.parent is None --- xarray/tests/test_backends_datatree.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xarray/tests/test_backends_datatree.py b/xarray/tests/test_backends_datatree.py index 59469d2e72e..c8be98712c2 100644 --- a/xarray/tests/test_backends_datatree.py +++ b/xarray/tests/test_backends_datatree.py @@ -346,6 +346,7 @@ def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None: with open_datatree(filepath, group=group, engine=self.engine) as subgroup_tree: assert subgroup_tree is not None + assert subgroup_tree.root.parent is None assert list(subgroup_tree.children) == list(original_dt[group].children) assert len(subgroup_tree.dataset.dims) == len(original_dt[group].dataset.dims) assert list(subgroup_tree.dataset.data_vars) == list(original_dt[group].dataset.data_vars) @@ -537,6 +538,7 @@ def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None: with open_datatree(filepath, group=group, engine=self.engine) as subgroup_tree: assert subgroup_tree is not None + assert subgroup_tree.root.parent is None assert list(subgroup_tree.children) == list(original_dt[group].children) assert len(subgroup_tree.dataset.dims) == len(original_dt[group].dataset.dims) assert list(subgroup_tree.dataset.data_vars) == list(original_dt[group].dataset.data_vars) From bb6d4139e9b584a4a97430c2740f6bd71503a765 Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Thu, 24 Oct 2024 13:46:14 -0500 Subject: [PATCH 14/20] modifying tests --- xarray/tests/test_backends_datatree.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/xarray/tests/test_backends_datatree.py b/xarray/tests/test_backends_datatree.py index 05a50d6158e..232fefa3bd4 100644 --- a/xarray/tests/test_backends_datatree.py +++ b/xarray/tests/test_backends_datatree.py @@ -363,17 +363,6 @@ def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None: == original_dt[group].dataset["b"].values ) - def test_open_datatree_nonexistent_group(self, tmpdir, simple_datatree) -> None: - """Test `open_datatree` behavior when attempting to open a non-existent group.""" - filepath = tmpdir / "test.nc" - original_dt = simple_datatree - original_dt.to_netcdf(filepath, engine=self.engine) - - # Attempt to open a non-existent group, which should raise a ValueError - with pytest.raises(OSError, match="Group '/nonexistent_group' not found"): - open_datatree(filepath, group="/nonexistent_group", engine=self.engine) - - @requires_h5netcdf class TestH5NetCDFDatatreeIO(DatatreeIOBase): engine: T_DataTreeNetcdfEngine | None = "h5netcdf" From fcf3dc6070fc55927aca86caa0f05386cdb693d1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 24 Oct 2024 18:47:25 +0000 Subject: [PATCH 15/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/tests/test_backends_datatree.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xarray/tests/test_backends_datatree.py b/xarray/tests/test_backends_datatree.py index 232fefa3bd4..a0a4ec9dc10 100644 --- a/xarray/tests/test_backends_datatree.py +++ b/xarray/tests/test_backends_datatree.py @@ -363,6 +363,7 @@ def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None: == original_dt[group].dataset["b"].values ) + @requires_h5netcdf class TestH5NetCDFDatatreeIO(DatatreeIOBase): engine: T_DataTreeNetcdfEngine | None = "h5netcdf" From 195e036c0263e8daf691eaa87f0763ff09b94624 Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Thu, 24 Oct 2024 14:18:18 -0500 Subject: [PATCH 16/20] Update xarray/tests/test_backends_datatree.py Co-authored-by: Justus Magin --- xarray/tests/test_backends_datatree.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xarray/tests/test_backends_datatree.py b/xarray/tests/test_backends_datatree.py index a0a4ec9dc10..bd690007a5e 100644 --- a/xarray/tests/test_backends_datatree.py +++ b/xarray/tests/test_backends_datatree.py @@ -545,8 +545,7 @@ def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None: assert list(subgroup_tree.dataset.data_vars) == list( original_dt[group].dataset.data_vars ) - assert_equal(subgroup_tree.dataset["a"], original_dt[group].dataset["a"]) - assert_equal(subgroup_tree.dataset["b"], original_dt[group].dataset["b"]) + assert_equal(subgroup_tree.dataset, original_dt[group].dataset) @requires_dask def test_open_groups_chunks(self, tmpdir) -> None: From 5ef3a56d8a6a823912bce5f77a6d47777d23a204 Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Thu, 24 Oct 2024 14:22:36 -0500 Subject: [PATCH 17/20] applying suggested changes --- doc/whats-new.rst | 2 -- xarray/tests/test_backends_datatree.py | 21 +-------------------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 90c5122ef8d..4f08d17e825 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -74,8 +74,6 @@ Bug fixes `_. - Fix binning by multiple variables where some bins have no observations. (:issue:`9630`). By `Deepak Cherian `_. -- Fix bug with `open_datatree(group='some_subgroup')` returning empty parent nodes. (:issue:`9665`, :pull:`9666`) - by `Alfonso Ladino `_. Documentation ~~~~~~~~~~~~~ diff --git a/xarray/tests/test_backends_datatree.py b/xarray/tests/test_backends_datatree.py index bd690007a5e..6f450a33d8c 100644 --- a/xarray/tests/test_backends_datatree.py +++ b/xarray/tests/test_backends_datatree.py @@ -348,20 +348,7 @@ def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None: assert subgroup_tree is not None assert subgroup_tree.root.parent is None assert list(subgroup_tree.children) == list(original_dt[group].children) - assert len(subgroup_tree.dataset.dims) == len( - original_dt[group].dataset.dims - ) - assert list(subgroup_tree.dataset.data_vars) == list( - original_dt[group].dataset.data_vars - ) - assert ( - subgroup_tree.dataset["a"].values - == original_dt[group].dataset["a"].values - ) - assert ( - subgroup_tree.dataset["b"].values - == original_dt[group].dataset["b"].values - ) + assert_equal(subgroup_tree.dataset, original_dt[group].dataset) @requires_h5netcdf @@ -539,12 +526,6 @@ def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None: assert subgroup_tree is not None assert subgroup_tree.root.parent is None assert list(subgroup_tree.children) == list(original_dt[group].children) - assert len(subgroup_tree.dataset.dims) == len( - original_dt[group].dataset.dims - ) - assert list(subgroup_tree.dataset.data_vars) == list( - original_dt[group].dataset.data_vars - ) assert_equal(subgroup_tree.dataset, original_dt[group].dataset) @requires_dask From 90c5b4d35c536e3c56ae1da8b7d379971c649376 Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Thu, 24 Oct 2024 14:49:40 -0500 Subject: [PATCH 18/20] updating test --- xarray/tests/test_backends_datatree.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/xarray/tests/test_backends_datatree.py b/xarray/tests/test_backends_datatree.py index 6f450a33d8c..6e2d25249fb 100644 --- a/xarray/tests/test_backends_datatree.py +++ b/xarray/tests/test_backends_datatree.py @@ -343,12 +343,11 @@ def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None: group = "/set1" original_dt = simple_datatree original_dt.to_netcdf(filepath) - + expected_subtree = original_dt[group].copy() + expected_subtree.orphan() with open_datatree(filepath, group=group, engine=self.engine) as subgroup_tree: - assert subgroup_tree is not None assert subgroup_tree.root.parent is None - assert list(subgroup_tree.children) == list(original_dt[group].children) - assert_equal(subgroup_tree.dataset, original_dt[group].dataset) + assert_equal(subgroup_tree, expected_subtree) @requires_h5netcdf @@ -521,12 +520,11 @@ def test_open_datatree_specific_group(self, tmpdir, simple_datatree) -> None: group = "/set2" original_dt = simple_datatree original_dt.to_zarr(filepath) - + expected_subtree = original_dt[group].copy() + expected_subtree.orphan() with open_datatree(filepath, group=group, engine=self.engine) as subgroup_tree: - assert subgroup_tree is not None assert subgroup_tree.root.parent is None - assert list(subgroup_tree.children) == list(original_dt[group].children) - assert_equal(subgroup_tree.dataset, original_dt[group].dataset) + assert_equal(subgroup_tree, expected_subtree) @requires_dask def test_open_groups_chunks(self, tmpdir) -> None: From e78d576596f451adc3ca3f31b04431e45cdec718 Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Thu, 24 Oct 2024 15:26:16 -0500 Subject: [PATCH 19/20] adding Justus and Alfonso to the list of contributors to the DataTree entry --- doc/whats-new.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4f08d17e825..17ee0b931d1 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -28,8 +28,11 @@ New Features By `Owen Littlejohns `_, `Eni Awowale `_, `Matt Savoie `_, - `Stephan Hoyer `_ and - `Tom Nicholas `_. + `Stephan Hoyer `_, + `Tom Nicholas `_, + `Justus Magin `_, and + `Alfonso Ladino `_. + - A migration guide for users of the prototype `xarray-contrib/datatree repository `_ has been added, and can be found in the `DATATREE_MIGRATION_GUIDE.md` file in the repository root. By `Tom Nicholas `_. - Added zarr backends for :py:func:`open_groups` (:issue:`9430`, :pull:`9469`). From 762587bf07790b78db823500260a79c3729b3133 Mon Sep 17 00:00:00 2001 From: Alfonso Ladino Date: Thu, 24 Oct 2024 15:27:27 -0500 Subject: [PATCH 20/20] adding Justus and Alfonso to the list of contributors to the DataTree entry --- doc/whats-new.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 17ee0b931d1..c3dd6776c27 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -32,7 +32,6 @@ New Features `Tom Nicholas `_, `Justus Magin `_, and `Alfonso Ladino `_. - - A migration guide for users of the prototype `xarray-contrib/datatree repository `_ has been added, and can be found in the `DATATREE_MIGRATION_GUIDE.md` file in the repository root. By `Tom Nicholas `_. - Added zarr backends for :py:func:`open_groups` (:issue:`9430`, :pull:`9469`).