From 4bd0594096b699f6a2d36cf8340db5946783abdc Mon Sep 17 00:00:00 2001 From: David Jun Date: Tue, 18 Mar 2025 09:47:14 -0400 Subject: [PATCH 1/6] Resolving #61086 : BUG: Collision between equivalent frequencies 'QS-FEB' and 'QS-NOV' --- doc/source/whatsnew/v3.0.0.rst | 1 + pandas/core/arrays/datetimelike.py | 7 ++++- pandas/core/indexes/datetimes.py | 28 +++++++++++++++++++ .../indexes/datetimes/test_constructors.py | 9 ++++++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 2bb1e4af21860..2c23a15ffddc8 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -75,6 +75,7 @@ Other enhancements - :meth:`pandas.concat` will raise a ``ValueError`` when ``ignore_index=True`` and ``keys`` is not ``None`` (:issue:`59274`) - :py:class:`frozenset` elements in pandas objects are now natively printed (:issue:`60690`) - Add ``"delete_rows"`` option to ``if_exists`` argument in :meth:`DataFrame.to_sql` deleting all records of the table before inserting data (:issue:`37210`). +- Added :func:`DateTimeIndex.set_freq` and :class:`DateTimeIndex` now supports assigning equivalent quarerly start frequency (:issue:`61086`) - Added half-year offset classes :class:`HalfYearBegin`, :class:`HalfYearEnd`, :class:`BHalfYearBegin` and :class:`BHalfYearEnd` (:issue:`60928`) - Errors occurring during SQL I/O will now throw a generic :class:`.DatabaseError` instead of the raw Exception type from the underlying driver manager library (:issue:`60748`) - Implemented :meth:`Series.str.isascii` and :meth:`Series.str.isascii` (:issue:`59091`) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index eba738c926497..64e59635b7ec4 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -2519,8 +2519,13 @@ def _validate_inferred_freq( ------- freq : DateOffset or None """ + offset1 = to_offset(freq) + offset2 = to_offset(inferred_freq) + + freq_equal = type(offset1) == type(offset2) + if inferred_freq is not None: - if freq is not None and freq != inferred_freq: + if freq is not None and not freq_equal: raise ValueError( f"Inferred frequency {inferred_freq} from passed " "values does not conform to passed frequency " diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 9adbaadbdcdc8..cc3393297fe50 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -421,6 +421,34 @@ def _can_range_setop(self, other) -> bool: # -------------------------------------------------------------------- + def _set_freq(self, freq, *, inplace: bool = False): + """ + Set a new frequency for this DatetimeIndex. + + Parameters + ---------- + freq : str, Timedelta, datetime.timedelta, or DateOffset, default 'S' + Frequency strings can have multiples, e.g. '5h'. See + :ref:`here ` for a list of + frequency aliases. + inplace : bool, default False + If True, modifies object in place. Otherwise, returns a new DateTimeIndex + + Returns + ------- + DatetimeIndex + Fixed frequency DatetimeIndex + """ + + if inplace: + self._freq = to_offset(freq) + else: + new_index = self.copy() + new_index.freq = to_offset(freq) + return new_index + + # -------------------------------------------------------------------- + def _get_time_micros(self) -> npt.NDArray[np.int64]: """ Return the number of microseconds since midnight. diff --git a/pandas/tests/indexes/datetimes/test_constructors.py b/pandas/tests/indexes/datetimes/test_constructors.py index c418b2a18008b..cb39981c6be4b 100644 --- a/pandas/tests/indexes/datetimes/test_constructors.py +++ b/pandas/tests/indexes/datetimes/test_constructors.py @@ -1204,3 +1204,12 @@ def test_dti_constructor_object_dtype_dayfirst_yearfirst_with_tz(self): result2 = DatetimeIndex([val], tz="US/Pacific", yearfirst=True) expected2 = DatetimeIndex([yfirst]).as_unit("s") tm.assert_index_equal(result2, expected2) + + def test_datetimeindex_equivalent_freq(self): + idx2 = date_range("2020-02-01", freq="QS-FEB", periods=4) + new_idx2 = idx2._set_freq("QS-MAY") + tm.assert_index_equal(new_idx2, idx2) + assert new_idx2.freq == "QS-MAY" + + idx2._set_freq("QS-MAY", inplace=True) + assert idx2.freq == "QS-MAY" From 492816019f660ac12bb37abce55362457f33ce94 Mon Sep 17 00:00:00 2001 From: David Jun Date: Wed, 19 Mar 2025 10:52:53 -0400 Subject: [PATCH 2/6] Resolving #61086 : BUG: Collision between equivalent frequencies 'QS-FEB' and 'QS-NOV' --- doc/source/whatsnew/v3.0.0.rst | 2 +- pandas/core/indexes/datetimes.py | 28 ------------------- .../indexes/datetimes/test_constructors.py | 14 ++++------ 3 files changed, 7 insertions(+), 37 deletions(-) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 2c23a15ffddc8..224ff859cf5a4 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -75,7 +75,6 @@ Other enhancements - :meth:`pandas.concat` will raise a ``ValueError`` when ``ignore_index=True`` and ``keys`` is not ``None`` (:issue:`59274`) - :py:class:`frozenset` elements in pandas objects are now natively printed (:issue:`60690`) - Add ``"delete_rows"`` option to ``if_exists`` argument in :meth:`DataFrame.to_sql` deleting all records of the table before inserting data (:issue:`37210`). -- Added :func:`DateTimeIndex.set_freq` and :class:`DateTimeIndex` now supports assigning equivalent quarerly start frequency (:issue:`61086`) - Added half-year offset classes :class:`HalfYearBegin`, :class:`HalfYearEnd`, :class:`BHalfYearBegin` and :class:`BHalfYearEnd` (:issue:`60928`) - Errors occurring during SQL I/O will now throw a generic :class:`.DatabaseError` instead of the raw Exception type from the underlying driver manager library (:issue:`60748`) - Implemented :meth:`Series.str.isascii` and :meth:`Series.str.isascii` (:issue:`59091`) @@ -655,6 +654,7 @@ Datetimelike - Bug in :meth:`DatetimeIndex.is_year_start` and :meth:`DatetimeIndex.is_quarter_start` returning ``False`` on double-digit frequencies (:issue:`58523`) - Bug in :meth:`DatetimeIndex.union` and :meth:`DatetimeIndex.intersection` when ``unit`` was non-nanosecond (:issue:`59036`) - Bug in :meth:`Series.dt.microsecond` producing incorrect results for pyarrow backed :class:`Series`. (:issue:`59154`) +- Bug in :meth:`_validate_inferred_freq` where DateTimeIndex incorrectly raised a ``ValueError`` when assigning a logically equivalent frequency (:issue:`61086`) - Bug in :meth:`to_datetime` not respecting dayfirst if an uncommon date string was passed. (:issue:`58859`) - Bug in :meth:`to_datetime` on float array with missing values throwing ``FloatingPointError`` (:issue:`58419`) - Bug in :meth:`to_datetime` on float32 df with year, month, day etc. columns leads to precision issues and incorrect result. (:issue:`60506`) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index cc3393297fe50..9adbaadbdcdc8 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -421,34 +421,6 @@ def _can_range_setop(self, other) -> bool: # -------------------------------------------------------------------- - def _set_freq(self, freq, *, inplace: bool = False): - """ - Set a new frequency for this DatetimeIndex. - - Parameters - ---------- - freq : str, Timedelta, datetime.timedelta, or DateOffset, default 'S' - Frequency strings can have multiples, e.g. '5h'. See - :ref:`here ` for a list of - frequency aliases. - inplace : bool, default False - If True, modifies object in place. Otherwise, returns a new DateTimeIndex - - Returns - ------- - DatetimeIndex - Fixed frequency DatetimeIndex - """ - - if inplace: - self._freq = to_offset(freq) - else: - new_index = self.copy() - new_index.freq = to_offset(freq) - return new_index - - # -------------------------------------------------------------------- - def _get_time_micros(self) -> npt.NDArray[np.int64]: """ Return the number of microseconds since midnight. diff --git a/pandas/tests/indexes/datetimes/test_constructors.py b/pandas/tests/indexes/datetimes/test_constructors.py index cb39981c6be4b..7babf8332b411 100644 --- a/pandas/tests/indexes/datetimes/test_constructors.py +++ b/pandas/tests/indexes/datetimes/test_constructors.py @@ -1205,11 +1205,9 @@ def test_dti_constructor_object_dtype_dayfirst_yearfirst_with_tz(self): expected2 = DatetimeIndex([yfirst]).as_unit("s") tm.assert_index_equal(result2, expected2) - def test_datetimeindex_equivalent_freq(self): - idx2 = date_range("2020-02-01", freq="QS-FEB", periods=4) - new_idx2 = idx2._set_freq("QS-MAY") - tm.assert_index_equal(new_idx2, idx2) - assert new_idx2.freq == "QS-MAY" - - idx2._set_freq("QS-MAY", inplace=True) - assert idx2.freq == "QS-MAY" + def test_validate_inferred_freq_equivalence(self): + idx = date_range("2020-02-01", freq="QS-FEB", periods=4) + + new_idx = DatetimeIndex(idx, freq="QS-MAY") + + assert isinstance(new_idx, DatetimeIndex) From db653a9bc1f3e78c903da04cca815a9b792d8c77 Mon Sep 17 00:00:00 2001 From: David Jun Date: Wed, 19 Mar 2025 10:52:53 -0400 Subject: [PATCH 3/6] Resolving #61086 : BUG: Collision between equivalent frequencies 'QS-FEB' and 'QS-NOV' From 6bb7366de6c50d32e15ab1abd0e63b81997cda22 Mon Sep 17 00:00:00 2001 From: YongChamp Date: Wed, 19 Mar 2025 21:37:25 -0400 Subject: [PATCH 4/6] BUG: Collision between equivalent frequencies 'QS-FEB' and 'QS-NOV' --- pandas/core/arrays/datetimelike.py | 33 +++++++++++++++---- .../indexes/datetimes/test_constructors.py | 13 ++++++-- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 64e59635b7ec4..c96232e2eb1ef 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -2519,20 +2519,41 @@ def _validate_inferred_freq( ------- freq : DateOffset or None """ - offset1 = to_offset(freq) - offset2 = to_offset(inferred_freq) - freq_equal = type(offset1) == type(offset2) +def _validate_inferred_freq(freq, inferred_freq): + """ + Ensure that freq matches inferred_freq unless they are logically equivalent. + """ if inferred_freq is not None: - if freq is not None and not freq_equal: + if freq is not None: + offset1, offset2 = to_offset(freq), to_offset(inferred_freq) + if type(offset1) == type(offset2): + if hasattr(offset1, "startingMonth") and hasattr( + offset2, "startingMonth" + ): + if (offset1.startingMonth - offset2.startingMonth) % 3 != 0: + raise ValueError( + f"Inferred frequency {inferred_freq} from passed " + "values does not conform to passed frequency " + f"{freq.freqstr}" + ) + if ( + hasattr(offset1, "n") + and hasattr(offset2, "n") + and offset1.n != offset2.n + ): + raise ValueError( + f"Inferred frequency {inferred_freq} from passed " + "values does not conform to passed frequency " + f"{freq.freqstr}" + ) raise ValueError( f"Inferred frequency {inferred_freq} from passed " "values does not conform to passed frequency " f"{freq.freqstr}" ) - if freq is None: - freq = inferred_freq + freq = inferred_freq return freq diff --git a/pandas/tests/indexes/datetimes/test_constructors.py b/pandas/tests/indexes/datetimes/test_constructors.py index 7babf8332b411..c95e42ecd2258 100644 --- a/pandas/tests/indexes/datetimes/test_constructors.py +++ b/pandas/tests/indexes/datetimes/test_constructors.py @@ -1206,8 +1206,17 @@ def test_dti_constructor_object_dtype_dayfirst_yearfirst_with_tz(self): tm.assert_index_equal(result2, expected2) def test_validate_inferred_freq_equivalence(self): - idx = date_range("2020-02-01", freq="QS-FEB", periods=4) + idx = date_range("2022-02-01", freq="QS-FEB", periods=4) new_idx = DatetimeIndex(idx, freq="QS-MAY") - assert isinstance(new_idx, DatetimeIndex) + + msg = ( + "Inferred frequency .* from passed " + "values does not conform to passed frequency .*" + ) + with pytest.raises( + ValueError, + match=msg, + ): + DatetimeIndex(idx, freq="QS-MAR") From 251f184fd5ddf029ea2ff76fa59546c1cb2d9800 Mon Sep 17 00:00:00 2001 From: YongChamp Date: Wed, 19 Mar 2025 21:38:27 -0400 Subject: [PATCH 5/6] BUG: Collision between equivalent frequencies 'QS-FEB' and 'QS-NOV' --- pandas/core/arrays/datetimelike.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index c96232e2eb1ef..3570c564c2d6b 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -2538,6 +2538,7 @@ def _validate_inferred_freq(freq, inferred_freq): "values does not conform to passed frequency " f"{freq.freqstr}" ) + return freq if ( hasattr(offset1, "n") and hasattr(offset2, "n") @@ -2548,6 +2549,7 @@ def _validate_inferred_freq(freq, inferred_freq): "values does not conform to passed frequency " f"{freq.freqstr}" ) + return freq raise ValueError( f"Inferred frequency {inferred_freq} from passed " "values does not conform to passed frequency " From 12ef20af8a4fd717214f6aab8193a9609876511e Mon Sep 17 00:00:00 2001 From: YongChamp Date: Wed, 19 Mar 2025 22:11:42 -0400 Subject: [PATCH 6/6] BUG: Collision between equivalent frequencies 'QS-FEB' and 'QS-NOV' --- pandas/core/arrays/datetimelike.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 3570c564c2d6b..76dee09c6f813 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -2520,11 +2520,6 @@ def _validate_inferred_freq( freq : DateOffset or None """ - -def _validate_inferred_freq(freq, inferred_freq): - """ - Ensure that freq matches inferred_freq unless they are logically equivalent. - """ if inferred_freq is not None: if freq is not None: offset1, offset2 = to_offset(freq), to_offset(inferred_freq)