From f43e421b8e1afd4d418cc4cffce8eb789ec422f0 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 10 Jun 2026 16:33:33 -0400 Subject: [PATCH 1/4] fix: categorical hashing, delayed-backend detection, and pretty-printing - _categorical.py: HashableList.__init__ now recurses via as_hashable for each element, matching HashableDict; fixes TypeError on list-of-dict categories - _errors.py: any_backend_is_delayed only short-circuits on True from recursion, not False, so remaining args in a mixed list are always inspected - prettyprint.py: custom_str result is wrapped in a single-element list and cols_taken is recomputed from len(custom), preventing character-by-character splicing and width accounting errors - _do.py: recursively_apply Record branch forwards regular_to_jagged to the recursive call instead of silently resetting it to False Assisted-by: ClaudeCode:claude-sonnet-4-6 --- src/awkward/_categorical.py | 2 +- src/awkward/_do.py | 1 + src/awkward/_errors.py | 9 ++++----- src/awkward/prettyprint.py | 6 ++++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/awkward/_categorical.py b/src/awkward/_categorical.py index ade1f74235..762e467128 100644 --- a/src/awkward/_categorical.py +++ b/src/awkward/_categorical.py @@ -28,7 +28,7 @@ def __eq__(self, other): class HashableList: def __init__(self, obj): - self.values = tuple(obj) + self.values = tuple(as_hashable(x) for x in obj) self.hash = hash((HashableList, *self.values)) def __hash__(self): diff --git a/src/awkward/_do.py b/src/awkward/_do.py index 8b48cf3ad0..e2ead1919b 100644 --- a/src/awkward/_do.py +++ b/src/awkward/_do.py @@ -63,6 +63,7 @@ def recursively_apply( return_simplified, return_array, function_name, + regular_to_jagged, ) if return_array: diff --git a/src/awkward/_errors.py b/src/awkward/_errors.py index bd357175a6..9c97481939 100644 --- a/src/awkward/_errors.py +++ b/src/awkward/_errors.py @@ -188,12 +188,11 @@ def any_backend_is_delayed( if backend is None: # Is this an iterable object, and are we permitted to recurse? if isinstance(obj, Collection) and depth != depth_limit: - return self.any_backend_is_delayed( + if self.any_backend_is_delayed( obj, depth=depth + 1, depth_limit=depth_limit - ) - # Assume not delayed! - else: - return False + ): + return True + # Assume not delayed! Continue checking remaining args. # Eager backends aren't delayed! elif backend.nplike.is_eager: continue diff --git a/src/awkward/prettyprint.py b/src/awkward/prettyprint.py index 2ac5be8229..df9a29c05a 100644 --- a/src/awkward/prettyprint.py +++ b/src/awkward/prettyprint.py @@ -179,7 +179,8 @@ def valuestr_horiz( custom = custom_str(current) if custom is not None: - strs = custom + strs = [custom] + cols_taken = len(custom) if limit_cols - (for_comma + cols_taken) >= 0: if which != 0: @@ -196,7 +197,8 @@ def valuestr_horiz( custom = custom_str(current) if custom is not None: - strs = custom + strs = [custom] + cols_taken = len(custom) if limit_cols - (2 + cols_taken) >= 0: back[:0] = strs From f3dfd298b40f29629907cc397f8f4019ecbcb923 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 10 Jun 2026 16:34:19 -0400 Subject: [PATCH 2/4] test: add regression tests for fixes in #4106 Covers: categorical hashing with nested list/dict categories, any_backend_is_delayed loop correctness, prettyprint custom_str token handling, and recursively_apply Record regular_to_jagged forwarding. Assisted-by: ClaudeCode:claude-sonnet-4-6 --- tests/test_4106-fix-core-misc.py | 274 +++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 tests/test_4106-fix-core-misc.py diff --git a/tests/test_4106-fix-core-misc.py b/tests/test_4106-fix-core-misc.py new file mode 100644 index 0000000000..d738cf633f --- /dev/null +++ b/tests/test_4106-fix-core-misc.py @@ -0,0 +1,274 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/awkward/blob/main/LICENSE + +from __future__ import annotations + +import numpy as np +import pytest + +import awkward as ak +from awkward._categorical import HashableDict, HashableList, as_hashable +from awkward._do import recursively_apply +from awkward._errors import OperationErrorContext +from awkward.prettyprint import Formatter + + +# --------------------------------------------------------------------------- +# Fix 1: HashableList must recurse through as_hashable +# --------------------------------------------------------------------------- + + +def test_hashablelist_recurses_for_dicts(): + """HashableList should apply as_hashable to each element (like HashableDict).""" + # list containing dicts — used to raise TypeError: unhashable type 'dict' + obj = [{"x": 1}, {"x": 2}] + h = as_hashable(obj) + assert isinstance(h, HashableList) + # Each element should have been converted to a HashableDict, not a raw dict + for v in h.values: + assert isinstance(v, HashableDict) + # Must be hashable + assert isinstance(hash(h), int) + + +def test_hashablelist_recurses_for_nested_lists(): + """Nested lists of lists should also be recursively hashable.""" + obj = [[1, 2], [3, 4]] + h = as_hashable(obj) + assert isinstance(h, HashableList) + for v in h.values: + assert isinstance(v, HashableList) + assert isinstance(hash(h), int) + + +def test_hashablelist_can_be_dict_key(): + """HashableList containing dicts must be usable as a dict key (was broken).""" + obj1 = [{"x": 1}, {"x": 2}] + obj2 = [{"x": 1}, {"x": 2}] + obj3 = [{"x": 3}] + h1 = as_hashable(obj1) + h2 = as_hashable(obj2) + h3 = as_hashable(obj3) + lookup = {h1: "a", h3: "b"} + assert lookup[h2] == "a" # h2 equals h1 + assert h1 == h2 + assert h1 != h3 + + +def test_categorical_equal_list_of_lists(): + """Comparing categorical arrays whose categories are lists of lists (different orders).""" + # Different category content objects forces the hash-based mapping path + categories1 = ak.Array([[1, 2], [3, 4]]) + categories2 = ak.Array([[3, 4], [1, 2]]) # reversed order + index1 = ak.index.Index64(np.array([0, 1, 0], dtype=np.int64)) + index2 = ak.index.Index64(np.array([0, 0, 1], dtype=np.int64)) + cat1 = ak.contents.IndexedArray( + index1, categories1.layout, parameters={"__array__": "categorical"} + ) + cat2 = ak.contents.IndexedArray( + index2, categories2.layout, parameters={"__array__": "categorical"} + ) + # cat1 values: [1,2] [3,4] [1,2]; cat2 values: [3,4] [3,4] [1,2] + # equal: F T T + result = (ak.Array(cat1) == ak.Array(cat2)).to_list() + assert result == [False, True, True] + + +def test_categorical_equal_nested_list_of_dicts_hashable(): + """HashableList with dict elements: as_hashable on a list-of-dict must not raise.""" + items = [{"a": 1, "b": 2}, {"a": 3, "b": 4}] + h1 = as_hashable(items) + h2 = as_hashable(items) + assert isinstance(hash(h1), int) + assert h1 == h2 + # Can be used as dict key + lookup = {h1: 0} + assert lookup[h2] == 0 + + +# --------------------------------------------------------------------------- +# Fix 2: any_backend_is_delayed must not short-circuit on non-delayed items +# --------------------------------------------------------------------------- + + +def test_any_backend_is_delayed_continues_after_unknown_object(): + """A plain (non-array) item before an eager array must not prevent detection.""" + ctx = OperationErrorContext.__new__(OperationErrorContext) + # A plain Python scalar has no backend; an eager array follows it. + # The loop must not return False at the scalar and miss the array. + eager = ak.Array([1, 2, 3]) + assert ctx.any_backend_is_delayed([42, eager]) is False + + +def test_any_backend_is_delayed_all_plain_returns_false(): + """All-plain-python iterable returns False without crashing.""" + ctx = OperationErrorContext.__new__(OperationErrorContext) + assert ctx.any_backend_is_delayed([1, "hello", object()]) is False + + +def test_any_backend_is_delayed_two_eager_arrays(): + """Two eager arrays both return False; loop must complete.""" + ctx = OperationErrorContext.__new__(OperationErrorContext) + eager1 = ak.Array([1, 2, 3]) + eager2 = ak.Array([4, 5, 6]) + assert ctx.any_backend_is_delayed([eager1, eager2]) is False + + +def test_any_backend_is_delayed_nested_non_delayed_continues(): + """Nested iterable returning False from recursion doesn't short-circuit outer loop.""" + ctx = OperationErrorContext.__new__(OperationErrorContext) + eager = ak.Array([1, 2]) + # Outer list: [plain_list, eager_array] — first item recurses and returns False, + # but must not cause the outer loop to return False before seeing eager_array. + result = ctx.any_backend_is_delayed([[42], eager], depth_limit=2) + assert result is False + + +# --------------------------------------------------------------------------- +# Fix 3: prettyprint custom_str wrapped in list + width recomputed +# --------------------------------------------------------------------------- + + +def _make_custom_str_array(n=5): + """Build an Array whose elements have custom __str__.""" + + class MyPoint(ak.Record): + def __str__(self): + return "POINT" + + return ak.Array( + [{"x": i} for i in range(n)], + behavior={"MyPoint": MyPoint}, + with_name="MyPoint", + ) + + +def _make_custom_repr_array(n=5): + """Build an Array whose elements have custom __repr__.""" + + class MyPoint(ak.Record): + def __repr__(self): + return "POINT" + + return ak.Array( + [{"x": i} for i in range(n)], + behavior={"MyPoint": MyPoint}, + with_name="MyPoint", + ) + + +def test_prettyprint_custom_str_not_spliced(): + """custom_str result must be treated as a single token, not spliced char-by-char.""" + from awkward.prettyprint import valuestr_horiz + + arr = _make_custom_str_array(5) + cols_taken, strs = valuestr_horiz(arr, 80, Formatter()) + + # Each element in strs should be a multi-character token (not a single char) + for s in strs: + assert isinstance(s, str) + # Tokens like "[", "POINT", ", ", "]" — none should be a single letter from "POINT" + single_letters_from_point = set("POINT") + for s in strs: + if len(s) == 1 and s in single_letters_from_point: + # A single-char from the custom string means it was spliced + pytest.fail( + f"custom_str was spliced char-by-char: found single char {s!r} in strs={strs}" + ) + + +def test_prettyprint_custom_str_width_accurate(): + """cols_taken must reflect the actual length of the custom string.""" + from awkward.prettyprint import valuestr_horiz + + arr = _make_custom_str_array(1) + cols_taken, strs = valuestr_horiz(arr, 80, Formatter()) + + # With 1 element: "[POINT]" -> cols_taken should be 7, not based on old rendering + joined = "".join(strs) + assert len(joined) == cols_taken, ( + f"cols_taken={cols_taken} but joined={joined!r} has len={len(joined)}" + ) + + +def test_prettyprint_repr_does_not_explode(): + """repr() of an array with custom-str records must not raise or overflow.""" + arr = _make_custom_str_array(5) + result = repr(arr) + assert isinstance(result, str) + # Should not explode into thousands of chars + assert len(result) < 500 + + +def test_prettyprint_repr_custom_repr_does_not_explode(): + """repr() of an array with custom-repr records must not raise or overflow.""" + arr = _make_custom_repr_array(5) + result = repr(arr) + assert isinstance(result, str) + assert len(result) < 500 + + +# --------------------------------------------------------------------------- +# Fix 4: recursively_apply on Record forwards regular_to_jagged +# --------------------------------------------------------------------------- + + +def _make_record_with_regular_content(): + """Build a low-level Record whose array contains a RegularArray.""" + from awkward.contents.recordarray import RecordArray + from awkward.record import Record as LowRecord + + regular = ak.to_regular(ak.Array([[1, 2], [3, 4]])) # RegularArray + ra = RecordArray([regular.layout], ["vals"]) + return LowRecord(ra, 0) + + +def test_recursively_apply_record_regular_to_jagged_forwarded(): + """recursively_apply with regular_to_jagged=True on a Record must forward the arg.""" + low_rec = _make_record_with_regular_content() + + seen_options = [] + + def action(layout, **kwargs): + seen_options.append(kwargs.get("options", {})) + return None + + recursively_apply(low_rec, action, regular_to_jagged=True) + assert any(opts.get("regular_to_jagged") is True for opts in seen_options) + + +def test_recursively_apply_record_regular_to_jagged_false_forwarded(): + """regular_to_jagged=False is also forwarded correctly.""" + low_rec = _make_record_with_regular_content() + + seen_options = [] + + def action(layout, **kwargs): + seen_options.append(kwargs.get("options", {})) + return None + + recursively_apply(low_rec, action, regular_to_jagged=False) + assert any(opts.get("regular_to_jagged") is False for opts in seen_options) + + +def test_recursively_apply_record_regular_to_jagged_converts_type(): + """With regular_to_jagged=True, RegularArray inside a Record is converted.""" + from awkward.contents.recordarray import RecordArray + from awkward.record import Record as LowRecord + + regular = ak.to_regular(ak.Array([[1, 2], [3, 4]])) # RegularArray + ra = RecordArray([regular.layout], ["vals"]) + low_rec = LowRecord(ra, 0) + + seen_types = [] + + def action(layout, **kwargs): + seen_types.append(type(layout).__name__) + return None + + recursively_apply(low_rec, action, regular_to_jagged=True) + + # With regular_to_jagged=True, RegularArray should be converted to ListOffsetArray + assert "RegularArray" not in seen_types, ( + f"RegularArray was NOT converted; seen: {seen_types}" + ) + assert "ListOffsetArray" in seen_types From 672c8fc147570fe11e6354738e7b19a09c7819d7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Jun 2026 20:35:13 +0000 Subject: [PATCH 3/4] style: pre-commit fixes --- tests/test_4106-fix-core-misc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_4106-fix-core-misc.py b/tests/test_4106-fix-core-misc.py index d738cf633f..9f24a2ee2d 100644 --- a/tests/test_4106-fix-core-misc.py +++ b/tests/test_4106-fix-core-misc.py @@ -11,7 +11,6 @@ from awkward._errors import OperationErrorContext from awkward.prettyprint import Formatter - # --------------------------------------------------------------------------- # Fix 1: HashableList must recurse through as_hashable # --------------------------------------------------------------------------- From 5d4b55d619c6dbdbe554a752f7e0da29342dc3fe Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 11 Jun 2026 15:44:57 -0400 Subject: [PATCH 4/4] chore: trim test file and fix naming convention MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename test_4106-fix-core-misc.py → test_4106_fix_core_misc.py (underscore required by validate-test-names.py). Reduce 16 tests to 8: one focused regression per bug, removing near-duplicate and internal-structure-probing cases. Merge prettyprint width and splicing checks into one test. Assisted-by: ClaudeCode:claude-sonnet-4-6 --- tests/test_4106-fix-core-misc.py | 273 ------------------------------- tests/test_4106_fix_core_misc.py | 137 ++++++++++++++++ 2 files changed, 137 insertions(+), 273 deletions(-) delete mode 100644 tests/test_4106-fix-core-misc.py create mode 100644 tests/test_4106_fix_core_misc.py diff --git a/tests/test_4106-fix-core-misc.py b/tests/test_4106-fix-core-misc.py deleted file mode 100644 index 9f24a2ee2d..0000000000 --- a/tests/test_4106-fix-core-misc.py +++ /dev/null @@ -1,273 +0,0 @@ -# BSD 3-Clause License; see https://github.com/scikit-hep/awkward/blob/main/LICENSE - -from __future__ import annotations - -import numpy as np -import pytest - -import awkward as ak -from awkward._categorical import HashableDict, HashableList, as_hashable -from awkward._do import recursively_apply -from awkward._errors import OperationErrorContext -from awkward.prettyprint import Formatter - -# --------------------------------------------------------------------------- -# Fix 1: HashableList must recurse through as_hashable -# --------------------------------------------------------------------------- - - -def test_hashablelist_recurses_for_dicts(): - """HashableList should apply as_hashable to each element (like HashableDict).""" - # list containing dicts — used to raise TypeError: unhashable type 'dict' - obj = [{"x": 1}, {"x": 2}] - h = as_hashable(obj) - assert isinstance(h, HashableList) - # Each element should have been converted to a HashableDict, not a raw dict - for v in h.values: - assert isinstance(v, HashableDict) - # Must be hashable - assert isinstance(hash(h), int) - - -def test_hashablelist_recurses_for_nested_lists(): - """Nested lists of lists should also be recursively hashable.""" - obj = [[1, 2], [3, 4]] - h = as_hashable(obj) - assert isinstance(h, HashableList) - for v in h.values: - assert isinstance(v, HashableList) - assert isinstance(hash(h), int) - - -def test_hashablelist_can_be_dict_key(): - """HashableList containing dicts must be usable as a dict key (was broken).""" - obj1 = [{"x": 1}, {"x": 2}] - obj2 = [{"x": 1}, {"x": 2}] - obj3 = [{"x": 3}] - h1 = as_hashable(obj1) - h2 = as_hashable(obj2) - h3 = as_hashable(obj3) - lookup = {h1: "a", h3: "b"} - assert lookup[h2] == "a" # h2 equals h1 - assert h1 == h2 - assert h1 != h3 - - -def test_categorical_equal_list_of_lists(): - """Comparing categorical arrays whose categories are lists of lists (different orders).""" - # Different category content objects forces the hash-based mapping path - categories1 = ak.Array([[1, 2], [3, 4]]) - categories2 = ak.Array([[3, 4], [1, 2]]) # reversed order - index1 = ak.index.Index64(np.array([0, 1, 0], dtype=np.int64)) - index2 = ak.index.Index64(np.array([0, 0, 1], dtype=np.int64)) - cat1 = ak.contents.IndexedArray( - index1, categories1.layout, parameters={"__array__": "categorical"} - ) - cat2 = ak.contents.IndexedArray( - index2, categories2.layout, parameters={"__array__": "categorical"} - ) - # cat1 values: [1,2] [3,4] [1,2]; cat2 values: [3,4] [3,4] [1,2] - # equal: F T T - result = (ak.Array(cat1) == ak.Array(cat2)).to_list() - assert result == [False, True, True] - - -def test_categorical_equal_nested_list_of_dicts_hashable(): - """HashableList with dict elements: as_hashable on a list-of-dict must not raise.""" - items = [{"a": 1, "b": 2}, {"a": 3, "b": 4}] - h1 = as_hashable(items) - h2 = as_hashable(items) - assert isinstance(hash(h1), int) - assert h1 == h2 - # Can be used as dict key - lookup = {h1: 0} - assert lookup[h2] == 0 - - -# --------------------------------------------------------------------------- -# Fix 2: any_backend_is_delayed must not short-circuit on non-delayed items -# --------------------------------------------------------------------------- - - -def test_any_backend_is_delayed_continues_after_unknown_object(): - """A plain (non-array) item before an eager array must not prevent detection.""" - ctx = OperationErrorContext.__new__(OperationErrorContext) - # A plain Python scalar has no backend; an eager array follows it. - # The loop must not return False at the scalar and miss the array. - eager = ak.Array([1, 2, 3]) - assert ctx.any_backend_is_delayed([42, eager]) is False - - -def test_any_backend_is_delayed_all_plain_returns_false(): - """All-plain-python iterable returns False without crashing.""" - ctx = OperationErrorContext.__new__(OperationErrorContext) - assert ctx.any_backend_is_delayed([1, "hello", object()]) is False - - -def test_any_backend_is_delayed_two_eager_arrays(): - """Two eager arrays both return False; loop must complete.""" - ctx = OperationErrorContext.__new__(OperationErrorContext) - eager1 = ak.Array([1, 2, 3]) - eager2 = ak.Array([4, 5, 6]) - assert ctx.any_backend_is_delayed([eager1, eager2]) is False - - -def test_any_backend_is_delayed_nested_non_delayed_continues(): - """Nested iterable returning False from recursion doesn't short-circuit outer loop.""" - ctx = OperationErrorContext.__new__(OperationErrorContext) - eager = ak.Array([1, 2]) - # Outer list: [plain_list, eager_array] — first item recurses and returns False, - # but must not cause the outer loop to return False before seeing eager_array. - result = ctx.any_backend_is_delayed([[42], eager], depth_limit=2) - assert result is False - - -# --------------------------------------------------------------------------- -# Fix 3: prettyprint custom_str wrapped in list + width recomputed -# --------------------------------------------------------------------------- - - -def _make_custom_str_array(n=5): - """Build an Array whose elements have custom __str__.""" - - class MyPoint(ak.Record): - def __str__(self): - return "POINT" - - return ak.Array( - [{"x": i} for i in range(n)], - behavior={"MyPoint": MyPoint}, - with_name="MyPoint", - ) - - -def _make_custom_repr_array(n=5): - """Build an Array whose elements have custom __repr__.""" - - class MyPoint(ak.Record): - def __repr__(self): - return "POINT" - - return ak.Array( - [{"x": i} for i in range(n)], - behavior={"MyPoint": MyPoint}, - with_name="MyPoint", - ) - - -def test_prettyprint_custom_str_not_spliced(): - """custom_str result must be treated as a single token, not spliced char-by-char.""" - from awkward.prettyprint import valuestr_horiz - - arr = _make_custom_str_array(5) - cols_taken, strs = valuestr_horiz(arr, 80, Formatter()) - - # Each element in strs should be a multi-character token (not a single char) - for s in strs: - assert isinstance(s, str) - # Tokens like "[", "POINT", ", ", "]" — none should be a single letter from "POINT" - single_letters_from_point = set("POINT") - for s in strs: - if len(s) == 1 and s in single_letters_from_point: - # A single-char from the custom string means it was spliced - pytest.fail( - f"custom_str was spliced char-by-char: found single char {s!r} in strs={strs}" - ) - - -def test_prettyprint_custom_str_width_accurate(): - """cols_taken must reflect the actual length of the custom string.""" - from awkward.prettyprint import valuestr_horiz - - arr = _make_custom_str_array(1) - cols_taken, strs = valuestr_horiz(arr, 80, Formatter()) - - # With 1 element: "[POINT]" -> cols_taken should be 7, not based on old rendering - joined = "".join(strs) - assert len(joined) == cols_taken, ( - f"cols_taken={cols_taken} but joined={joined!r} has len={len(joined)}" - ) - - -def test_prettyprint_repr_does_not_explode(): - """repr() of an array with custom-str records must not raise or overflow.""" - arr = _make_custom_str_array(5) - result = repr(arr) - assert isinstance(result, str) - # Should not explode into thousands of chars - assert len(result) < 500 - - -def test_prettyprint_repr_custom_repr_does_not_explode(): - """repr() of an array with custom-repr records must not raise or overflow.""" - arr = _make_custom_repr_array(5) - result = repr(arr) - assert isinstance(result, str) - assert len(result) < 500 - - -# --------------------------------------------------------------------------- -# Fix 4: recursively_apply on Record forwards regular_to_jagged -# --------------------------------------------------------------------------- - - -def _make_record_with_regular_content(): - """Build a low-level Record whose array contains a RegularArray.""" - from awkward.contents.recordarray import RecordArray - from awkward.record import Record as LowRecord - - regular = ak.to_regular(ak.Array([[1, 2], [3, 4]])) # RegularArray - ra = RecordArray([regular.layout], ["vals"]) - return LowRecord(ra, 0) - - -def test_recursively_apply_record_regular_to_jagged_forwarded(): - """recursively_apply with regular_to_jagged=True on a Record must forward the arg.""" - low_rec = _make_record_with_regular_content() - - seen_options = [] - - def action(layout, **kwargs): - seen_options.append(kwargs.get("options", {})) - return None - - recursively_apply(low_rec, action, regular_to_jagged=True) - assert any(opts.get("regular_to_jagged") is True for opts in seen_options) - - -def test_recursively_apply_record_regular_to_jagged_false_forwarded(): - """regular_to_jagged=False is also forwarded correctly.""" - low_rec = _make_record_with_regular_content() - - seen_options = [] - - def action(layout, **kwargs): - seen_options.append(kwargs.get("options", {})) - return None - - recursively_apply(low_rec, action, regular_to_jagged=False) - assert any(opts.get("regular_to_jagged") is False for opts in seen_options) - - -def test_recursively_apply_record_regular_to_jagged_converts_type(): - """With regular_to_jagged=True, RegularArray inside a Record is converted.""" - from awkward.contents.recordarray import RecordArray - from awkward.record import Record as LowRecord - - regular = ak.to_regular(ak.Array([[1, 2], [3, 4]])) # RegularArray - ra = RecordArray([regular.layout], ["vals"]) - low_rec = LowRecord(ra, 0) - - seen_types = [] - - def action(layout, **kwargs): - seen_types.append(type(layout).__name__) - return None - - recursively_apply(low_rec, action, regular_to_jagged=True) - - # With regular_to_jagged=True, RegularArray should be converted to ListOffsetArray - assert "RegularArray" not in seen_types, ( - f"RegularArray was NOT converted; seen: {seen_types}" - ) - assert "ListOffsetArray" in seen_types diff --git a/tests/test_4106_fix_core_misc.py b/tests/test_4106_fix_core_misc.py new file mode 100644 index 0000000000..3815760b4d --- /dev/null +++ b/tests/test_4106_fix_core_misc.py @@ -0,0 +1,137 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/awkward/blob/main/LICENSE + +from __future__ import annotations + +import numpy as np + +import awkward as ak +from awkward._categorical import as_hashable +from awkward._do import recursively_apply +from awkward._errors import OperationErrorContext +from awkward.prettyprint import Formatter + + +def test_hashablelist_with_dict_elements_is_hashable(): + """HashableList with dict elements used to raise TypeError: unhashable type 'dict'.""" + obj = [{"x": 1}, {"x": 2}] + h = as_hashable(obj) + # Must be hashable and usable as a dict key + lookup = {h: "found"} + assert lookup[as_hashable(obj)] == "found" + + +def test_categorical_equal_list_of_lists(): + """Comparing categorical arrays whose categories are lists (different orders).""" + categories1 = ak.Array([[1, 2], [3, 4]]) + categories2 = ak.Array([[3, 4], [1, 2]]) # reversed order + index1 = ak.index.Index64(np.array([0, 1, 0], dtype=np.int64)) + index2 = ak.index.Index64(np.array([0, 0, 1], dtype=np.int64)) + cat1 = ak.contents.IndexedArray( + index1, categories1.layout, parameters={"__array__": "categorical"} + ) + cat2 = ak.contents.IndexedArray( + index2, categories2.layout, parameters={"__array__": "categorical"} + ) + # cat1 values: [1,2] [3,4] [1,2]; cat2 values: [3,4] [3,4] [1,2] + result = (ak.Array(cat1) == ak.Array(cat2)).to_list() + assert result == [False, True, True] + + +def test_any_backend_is_delayed_continues_after_unknown_object(): + """A plain (non-array) item before an eager array must not prevent detection. + + Before the fix, encountering an unrecognised object caused an unconditional + return False, so any arrays later in the argument list were never checked. + """ + ctx = OperationErrorContext.__new__(OperationErrorContext) + eager = ak.Array([1, 2, 3]) + assert ctx.any_backend_is_delayed([42, eager]) is False + + +def test_any_backend_is_delayed_nested_non_delayed_continues(): + """Nested iterable returning False from recursion must not short-circuit outer loop.""" + ctx = OperationErrorContext.__new__(OperationErrorContext) + eager = ak.Array([1, 2]) + result = ctx.any_backend_is_delayed([[42], eager], depth_limit=2) + assert result is False + + +def _make_custom_str_array(n=5): + class MyPoint(ak.Record): + def __str__(self): + return "POINT" + + return ak.Array( + [{"x": i} for i in range(n)], + behavior={"MyPoint": MyPoint}, + with_name="MyPoint", + ) + + +def test_prettyprint_custom_str_not_spliced(): + """custom_str result must be treated as a single token, not spliced char-by-char.""" + from awkward.prettyprint import valuestr_horiz + + arr = _make_custom_str_array(5) + cols_taken, strs = valuestr_horiz(arr, 80, Formatter()) + + # Before the fix, front.extend(str) iterated characters; "POINT" would appear + # as individual letters. Now each token must be "POINT" in full. + single_letters_from_point = set("POINT") + for s in strs: + if len(s) == 1 and s in single_letters_from_point: + raise AssertionError( + f"custom_str was spliced char-by-char: found {s!r} in {strs}" + ) + # cols_taken must match the actual rendered width + joined = "".join(strs) + assert len(joined) == cols_taken + + +def test_prettyprint_repr_does_not_explode(): + """repr() of an array with custom-str records must not raise or overflow.""" + arr = _make_custom_str_array(5) + result = repr(arr) + assert isinstance(result, str) + assert len(result) < 500 + + +def test_recursively_apply_record_regular_to_jagged_forwarded(): + """recursively_apply with regular_to_jagged=True on a Record must forward the arg.""" + from awkward.contents.recordarray import RecordArray + from awkward.record import Record as LowRecord + + regular = ak.to_regular(ak.Array([[1, 2], [3, 4]])) + ra = RecordArray([regular.layout], ["vals"]) + low_rec = LowRecord(ra, 0) + + seen_options = [] + + def action(layout, **kwargs): + seen_options.append(kwargs.get("options", {})) + return None + + recursively_apply(low_rec, action, regular_to_jagged=True) + assert any(opts.get("regular_to_jagged") is True for opts in seen_options) + + +def test_recursively_apply_record_regular_to_jagged_converts_type(): + """With regular_to_jagged=True, RegularArray inside a Record is converted.""" + from awkward.contents.recordarray import RecordArray + from awkward.record import Record as LowRecord + + regular = ak.to_regular(ak.Array([[1, 2], [3, 4]])) + ra = RecordArray([regular.layout], ["vals"]) + low_rec = LowRecord(ra, 0) + + seen_types = [] + + def action(layout, **kwargs): + seen_types.append(type(layout).__name__) + return None + + recursively_apply(low_rec, action, regular_to_jagged=True) + + # RegularArray should be converted to ListOffsetArray + assert "RegularArray" not in seen_types, f"seen: {seen_types}" + assert "ListOffsetArray" in seen_types