Merge pull request #629 from shoyer/tolerance

ENH: add tolerance argument to .sel and .reindex

Author: shoyer

ci/requirements-py26.yml

@@ -13,5 +13,6 @@ dependencies:
   - unittest2
   - pip:
     - coveralls
+    - cyordereddict
     - dask
     - h5netcdf

doc/indexing.rst

@@ -214,7 +214,7 @@ Nearest neighbor lookups
 
 The label based selection methods :py:meth:`~xray.Dataset.sel`,
 :py:meth:`~xray.Dataset.reindex` and :py:meth:`~xray.Dataset.reindex_like` all
-support a ``method`` keyword argument. The method parameter allows for
+support ``method`` and ``tolerance`` keyword argument. The method parameter allows for
 enabling nearest neighbor (inexact) lookups by use of the methods ``'pad'``,
 ``'backfill'`` or ``'nearest'``:
 
@@ -225,8 +225,14 @@ enabling nearest neighbor (inexact) lookups by use of the methods ``'pad'``,
     data.sel(x=0.1, method='backfill')
     data.reindex(x=[0.5, 1, 1.5, 2, 2.5], method='pad')
 
+Tolerance limits the maximum distance for valid matches with an inexact lookup:
+
+.. ipython:: python
+
+    data.reindex(x=[1.1, 1.5], method='nearest', tolerance=0.2)
+
 Using ``method='nearest'`` or a scalar argument with ``.sel()`` requires pandas
-version 0.16 or newer.
+version 0.16 or newer. Using ``tolerance`` requries pandas version 0.17 or newer.
 
 The method parameter is not yet supported if any of the arguments
 to ``.sel()`` is a ``slice`` object:

doc/whats-new.rst

@@ -12,7 +12,11 @@ What's New
 v0.6.1
 ------
 
-The minimum required version of dask for use with xray is now version 0.6.
+This pull request contains a number of bug and compatibility fixes, as well
+as enhancements to indexing, plotting and writing files to disk.
+
+Note that the minimum required version of dask for use with xray is now
+version 0.6.
 
 API Changes
 ~~~~~~~~~~~
@@ -26,24 +30,41 @@ API Changes
 Enhancements
 ~~~~~~~~~~~~
 
+- :py:meth:`~xray.Dataset.sel` and :py:meth:`~xray.Dataset.reindex` now support
+  the ``tolerance`` argument for controlling nearest-neighbor selection
+  (:issue:`629`)::
+
+  .. ipython::
+    :verbatim:
+
+    In [5]: array = xray.DataArray([1, 2, 3], dims='x')
+
+    In [6]: array.reindex(x=[0.9, 1.5], method='nearest', tolerance=0.2)
+    Out[6]:
+    <xray.DataArray (x: 2)>
+    array([  2.,  nan])
+    Coordinates:
+      * x        (x) float64 0.9 1.5
+
+  This feature requires pandas v0.17 or newer.
+- Faceted plotting through :py:class:`~xray.plot.FacetGrid` and the
+  :py:meth:`~xray.plot.plot` method.
+- New ``encoding`` argument in :py:meth:`~xray.Dataset.to_netcdf` for writing
+  netCDF files with compression, as described in the new documentation
+  section on :ref:`io.netcdf.writing_encoded`.
 - Add :py:attr:`~xray.Dataset.real` and :py:attr:`~xray.Dataset.imag`
   attributes to Dataset and DataArray (:issue:`553`).
 - More informative error message with :py:meth:`~xray.Dataset.from_dataframe`
   if the frame has duplicate columns.
 - xray now uses deterministic names for dask arrays it creates or opens from
   disk. This allows xray users to take advantage of dask's nascent support for
   caching intermediate computation results. See :issue:`555` for an example.
-- Faceted plotting through :py:class:`~xray.plot.FacetGrid` and the
-  :py:meth:`~xray.plot.plot` method.
-- New ``encoding`` argument in :py:meth:`~xray.Dataset.to_netcdf` for writing
-  netCDF files with compression, as described in the new documentation
-  section on :ref:`io.netcdf.writing_encoded`.
 
 Bug fixes
 ~~~~~~~~~
 
-- Forwards compatibility with the next pandas release of changes (v0.17.0).
-  We were using some internal pandas routines for datetime conversion, which
+- Forwards compatibility with the latest pandas release (v0.17.0). We were
+  using some internal pandas routines for datetime conversion, which
   unfortunately have now changed upstream (:issue:`569`).
 - Aggregation functions now correctly skip ``NaN`` for data for ``complex128``
   dtype (:issue:`554`).

xray/core/alignment.py

@@ -1,5 +1,6 @@
 import functools
 import operator
+import pandas as pd
 from collections import defaultdict
 
 import numpy as np
@@ -99,7 +100,8 @@ def partial_align(*objects, **kwargs):
     return tuple(obj.reindex(copy=copy, **joined_indexes) for obj in objects)
 
 
-def reindex_variables(variables, indexes, indexers, method=None, copy=True):
+def reindex_variables(variables, indexes, indexers, method=None,
+                      tolerance=None, copy=True):
     """Conform a dictionary of aligned variables onto a new set of variables,
     filling in missing values with NaN.
 
@@ -123,6 +125,10 @@ def reindex_variables(variables, indexes, indexers, method=None, copy=True):
           * pad / ffill: propgate last valid index value forward
           * backfill / bfill: propagate next valid index value backward
           * nearest: use nearest valid index value
+    tolerance : optional
+        Maximum distance between original and new labels for inexact matches.
+        The values of the index at the matching locations most satisfy the
+        equation ``abs(index[indexer] - target) <= tolerance``.
     copy : bool, optional
         If `copy=True`, the returned dataset contains only copied
         variables. If `copy=False` and no reindexing is required then
@@ -137,11 +143,21 @@ def reindex_variables(variables, indexes, indexers, method=None, copy=True):
     to_indexers = {}
     to_shape = {}
     from_indexers = {}
+
+    # for compat with older versions of pandas that don't support tolerance
+    get_indexer_kwargs = {}
+    if tolerance is not None:
+        if pd.__version__ < '0.17':
+            raise NotImplementedError(
+                'the tolerance argument requires pandas v0.17 or newer')
+        get_indexer_kwargs['tolerance'] = tolerance
+
     for name, index in iteritems(indexes):
         to_shape[name] = index.size
         if name in indexers:
             target = utils.safe_cast_to_index(indexers[name])
-            indexer = index.get_indexer(target, method=method)
+            indexer = index.get_indexer(target, method=method,
+                                        **get_indexer_kwargs)
 
             to_shape[name] = len(target)
             # Note pandas uses negative values from get_indexer to signify

xray/core/dataarray.py

@@ -539,7 +539,7 @@ def isel(self, **indexers):
         ds = self._dataset.isel(**indexers)
         return self._with_replaced_dataset(ds)
 
-    def sel(self, method=None, **indexers):
+    def sel(self, method=None, tolerance=None, **indexers):
         """Return a new DataArray whose dataset is given by selecting
         index labels along the specified dimension(s).
 
@@ -548,8 +548,8 @@ def sel(self, method=None, **indexers):
         Dataset.sel
         DataArray.isel
         """
-        return self.isel(**indexing.remap_label_indexers(self, indexers,
-                                                         method=method))
+        return self.isel(**indexing.remap_label_indexers(
+            self, indexers, method=method, tolerance=tolerance))
 
     def isel_points(self, dim='points', **indexers):
         """Return a new DataArray whose dataset is given by pointwise integer
@@ -562,18 +562,20 @@ def isel_points(self, dim='points', **indexers):
         ds = self._dataset.isel_points(dim=dim, **indexers)
         return self._with_replaced_dataset(ds)
 
-    def sel_points(self, dim='points', method=None, **indexers):
+    def sel_points(self, dim='points', method=None, tolerance=None,
+                   **indexers):
         """Return a new DataArray whose dataset is given by pointwise selection
         of index labels along the specified dimension(s).
 
         See Also
         --------
         Dataset.sel_points
         """
-        ds = self._dataset.sel_points(dim=dim, method=method, **indexers)
+        ds = self._dataset.sel_points(dim=dim, method=method,
+                                      tolerance=tolerance, **indexers)
         return self._with_replaced_dataset(ds)
 
-    def reindex_like(self, other, method=None, copy=True):
+    def reindex_like(self, other, method=None, tolerance=None, copy=True):
         """Conform this object onto the indexes of another object, filling
         in missing values with NaN.
 
@@ -594,6 +596,11 @@ def reindex_like(self, other, method=None, copy=True):
             * pad / ffill: propgate last valid index value forward
             * backfill / bfill: propagate next valid index value backward
             * nearest: use nearest valid index value (requires pandas>=0.16)
+        tolerance : optional
+            Maximum distance between original and new labels for inexact
+            matches. The values of the index at the matching locations most
+            satisfy the equation ``abs(index[indexer] - target) <= tolerance``.
+            Requires pandas>=0.17.
         copy : bool, optional
             If `copy=True`, the returned array's dataset contains only copied
             variables. If `copy=False` and no reindexing is required then
@@ -610,9 +617,10 @@ def reindex_like(self, other, method=None, copy=True):
         DataArray.reindex
         align
         """
-        return self.reindex(method=method, copy=copy, **other.indexes)
+        return self.reindex(method=method, tolerance=tolerance, copy=copy,
+                            **other.indexes)
 
-    def reindex(self, method=None, copy=True, **indexers):
+    def reindex(self, method=None, tolerance=None, copy=True, **indexers):
         """Conform this object onto a new set of indexes, filling in
         missing values with NaN.
 
@@ -630,6 +638,11 @@ def reindex(self, method=None, copy=True, **indexers):
             * pad / ffill: propgate last valid index value forward
             * backfill / bfill: propagate next valid index value backward
             * nearest: use nearest valid index value (requires pandas>=0.16)
+        tolerance : optional
+            Maximum distance between original and new labels for inexact
+            matches. The values of the index at the matching locations most
+            satisfy the equation ``abs(index[indexer] - target) <= tolerance``.
+            Requires pandas>=0.17.
         **indexers : dict
             Dictionary with keys given by dimension names and values given by
             arrays of coordinates tick labels. Any mis-matched coordinate values
@@ -647,7 +660,8 @@ def reindex(self, method=None, copy=True, **indexers):
         DataArray.reindex_like
         align
         """
-        ds = self._dataset.reindex(method=method, copy=copy, **indexers)
+        ds = self._dataset.reindex(method=method, tolerance=tolerance,
+                                   copy=copy, **indexers)
         return self._with_replaced_dataset(ds)
 
     def rename(self, new_name_or_name_dict):

xray/core/dataset.py

@@ -1010,7 +1010,7 @@ def isel(self, **indexers):
             variables[name] = var.isel(**var_indexers)
         return self._replace_vars_and_dims(variables)
 
-    def sel(self, method=None, **indexers):
+    def sel(self, method=None, tolerance=None, **indexers):
         """Returns a new dataset with each array indexed by tick labels
         along the specified dimension(s).
 
@@ -1036,6 +1036,11 @@ def sel(self, method=None, **indexers):
             * pad / ffill: propgate last valid index value forward
             * backfill / bfill: propagate next valid index value backward
             * nearest: use nearest valid index value
+        tolerance : optional
+            Maximum distance between original and new labels for inexact
+            matches. The values of the index at the matching locations most
+            satisfy the equation ``abs(index[indexer] - target) <= tolerance``.
+            Requires pandas>=0.17.
         **indexers : {dim: indexer, ...}
             Keyword arguments with names matching dimensions and values given
             by scalars, slices or arrays of tick labels.
@@ -1056,8 +1061,8 @@ def sel(self, method=None, **indexers):
         Dataset.isel_points
         DataArray.sel
         """
-        return self.isel(**indexing.remap_label_indexers(self, indexers,
-                                                         method=method))
+        return self.isel(**indexing.remap_label_indexers(
+            self, indexers, method=method, tolerance=tolerance))
 
     def isel_points(self, dim='points', **indexers):
         """Returns a new dataset with each array indexed pointwise along the
@@ -1146,7 +1151,8 @@ def relevant_keys(mapping):
                         zip(*[v for k, v in indexers])]],
                       dim=dim, coords=coords, data_vars=data_vars)
 
-    def sel_points(self, dim='points', method=None, **indexers):
+    def sel_points(self, dim='points', method=None, tolerance=None,
+                   **indexers):
         """Returns a new dataset with each array indexed pointwise by tick
         labels along the specified dimension(s).
 
@@ -1172,6 +1178,11 @@ def sel_points(self, dim='points', method=None, **indexers):
             * pad / ffill: propagate last valid index value forward
             * backfill / bfill: propagate next valid index value backward
             * nearest: use nearest valid index value
+        tolerance : optional
+            Maximum distance between original and new labels for inexact
+            matches. The values of the index at the matching locations most
+            satisfy the equation ``abs(index[indexer] - target) <= tolerance``.
+            Requires pandas>=0.17.
         **indexers : {dim: indexer, ...}
             Keyword arguments with names matching dimensions and values given
             by array-like objects. All indexers must be the same length and
@@ -1192,11 +1203,11 @@ def sel_points(self, dim='points', method=None, **indexers):
         Dataset.isel_points
         DataArray.sel_points
         """
-        pos_indexers = indexing.remap_label_indexers(self, indexers,
-                                                     method=method)
+        pos_indexers = indexing.remap_label_indexers(
+            self, indexers, method=method, tolerance=tolerance)
         return self.isel_points(dim=dim, **pos_indexers)
 
-    def reindex_like(self, other, method=None, copy=True):
+    def reindex_like(self, other, method=None, tolerance=None, copy=True):
         """Conform this object onto the indexes of another object, filling
         in missing values with NaN.
 
@@ -1217,6 +1228,11 @@ def reindex_like(self, other, method=None, copy=True):
             * pad / ffill: propagate last valid index value forward
             * backfill / bfill: propagate next valid index value backward
             * nearest: use nearest valid index value (requires pandas>=0.16)
+        tolerance : optional
+            Maximum distance between original and new labels for inexact
+            matches. The values of the index at the matching locations most
+            satisfy the equation ``abs(index[indexer] - target) <= tolerance``.
+            Requires pandas>=0.17.
         copy : bool, optional
             If `copy=True`, the returned dataset contains only copied
             variables. If `copy=False` and no reindexing is required then
@@ -1233,9 +1249,10 @@ def reindex_like(self, other, method=None, copy=True):
         Dataset.reindex
         align
         """
-        return self.reindex(method=method, copy=copy, **other.indexes)
+        return self.reindex(method=method, copy=copy, tolerance=tolerance,
+                            **other.indexes)
 
-    def reindex(self, indexers=None, method=None, copy=True, **kw_indexers):
+    def reindex(self, indexers=None, method=None, tolerance=None, copy=True, **kw_indexers):
         """Conform this object onto a new set of indexes, filling in
         missing values with NaN.
 
@@ -1254,6 +1271,11 @@ def reindex(self, indexers=None, method=None, copy=True, **kw_indexers):
             * pad / ffill: propagate last valid index value forward
             * backfill / bfill: propagate next valid index value backward
             * nearest: use nearest valid index value (requires pandas>=0.16)
+        tolerance : optional
+            Maximum distance between original and new labels for inexact
+            matches. The values of the index at the matching locations most
+            satisfy the equation ``abs(index[indexer] - target) <= tolerance``.
+            Requires pandas>=0.17.
         copy : bool, optional
             If `copy=True`, the returned dataset contains only copied
             variables. If `copy=False` and no reindexing is required then
@@ -1279,7 +1301,7 @@ def reindex(self, indexers=None, method=None, copy=True, **kw_indexers):
             return self.copy(deep=True) if copy else self
 
         variables = alignment.reindex_variables(
-            self.variables, self.indexes, indexers, method, copy=copy)
+            self.variables, self.indexes, indexers, method, tolerance, copy=copy)
         return self._replace_vars_and_dims(variables)
 
     def rename(self, name_dict, inplace=False):