From 7118058f41600267f7b8292425ac717bee0dd40d Mon Sep 17 00:00:00 2001 From: Avasam Date: Wed, 19 Feb 2025 21:41:57 -0500 Subject: [PATCH 01/10] Merge type annotations from typeshed --- boltons/cacheutils.py | 141 +++++++++++++++++++++--------------- boltons/debugutils.py | 11 +-- boltons/deprutils.py | 7 +- boltons/dictutils.py | 160 ++++++++++++++++++++++++----------------- boltons/easterutils.py | 5 +- boltons/ecoutils.py | 12 ++-- boltons/excutils.py | 9 ++- boltons/fileutils.py | 89 +++++++++++++---------- boltons/formatutils.py | 50 +++++++------ boltons/funcutils.py | 2 +- boltons/gcutils.py | 12 ++-- boltons/ioutils.py | 18 ++--- boltons/iterutils.py | 33 ++++----- boltons/jsonutils.py | 25 ++++--- boltons/listutils.py | 45 +++++++----- boltons/mathutils.py | 49 ++++++++----- boltons/mboxutils.py | 10 ++- boltons/namedutils.py | 6 +- boltons/pathutils.py | 13 ++-- boltons/py.typed | 0 boltons/queueutils.py | 8 +-- boltons/setutils.py | 155 ++++++++++++++++++++++----------------- boltons/statsutils.py | 51 ++++++++----- boltons/strutils.py | 98 +++++++++++++------------ boltons/tableutils.py | 6 +- boltons/tbutils.py | 93 ++++++++++++++---------- boltons/timeutils.py | 43 +++++------ boltons/typeutils.py | 9 ++- boltons/urlutils.py | 6 +- 29 files changed, 681 insertions(+), 485 deletions(-) create mode 100644 boltons/py.typed diff --git a/boltons/cacheutils.py b/boltons/cacheutils.py index 856bf195..637c7ba9 100644 --- a/boltons/cacheutils.py +++ b/boltons/cacheutils.py @@ -62,11 +62,18 @@ # TODO: TimedLRI # TODO: support 0 max_size? +from __future__ import annotations import heapq -import weakref import itertools +import weakref +from collections.abc import Callable, Generator, Hashable, Iterable, Iterator, Mapping from operator import attrgetter +from typing import TYPE_CHECKING, Dict, Generic, TypeVar, overload + +if TYPE_CHECKING: + from _typeshed import SupportsItems, SupportsKeysAndGetItem + from typing_extensions import Self try: from threading import RLock @@ -87,11 +94,15 @@ def __exit__(self, exctype, excinst, exctb): _MISSING = object() _KWARG_MARK = object() +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") +_T = TypeVar("_T") + PREV, NEXT, KEY, VALUE = range(4) # names for the link fields DEFAULT_MAX_SIZE = 128 -class LRI(dict): +class LRI(Dict[_KT, _VT]): """The ``LRI`` implements the basic *Least Recently Inserted* strategy to caching. One could also think of this as a ``SizeLimitedDefaultDict``. @@ -113,8 +124,8 @@ class LRI(dict): >>> cap_cache.hit_count, cap_cache.miss_count, cap_cache.soft_miss_count (3, 1, 1) """ - def __init__(self, max_size=DEFAULT_MAX_SIZE, values=None, - on_miss=None): + def __init__(self, max_size: int = DEFAULT_MAX_SIZE, values=None, + on_miss: Callable[[_KT], _VT] | None=None): if max_size <= 0: raise ValueError('expected max_size > 0, not %r' % max_size) self.hit_count = self.miss_count = self.soft_miss_count = 0 @@ -125,7 +136,7 @@ def __init__(self, max_size=DEFAULT_MAX_SIZE, values=None, if on_miss is not None and not callable(on_miss): raise TypeError('expected on_miss to be a callable' ' (or None), not %r' % on_miss) - self.on_miss = on_miss + self.on_miss: Callable[[_KT], _VT] | None = on_miss if values: self.update(values) @@ -216,7 +227,7 @@ def _remove_from_ll(self, key): link[PREV][NEXT] = link[NEXT] link[NEXT][PREV] = link[PREV] - def __setitem__(self, key, value): + def __setitem__(self, key: _KT, value: _VT) -> None: with self._lock: try: link = self._get_link_and_move_to_front_of_ll(key) @@ -231,7 +242,7 @@ def __setitem__(self, key, value): super().__setitem__(key, value) return - def __getitem__(self, key): + def __getitem__(self, key: _KT) -> _VT: with self._lock: try: link = self._link_lookup[key] @@ -245,19 +256,27 @@ def __getitem__(self, key): self.hit_count += 1 return link[VALUE] - def get(self, key, default=None): + @overload + def get(self, key: _KT, default: None = None) -> _VT | None: ... + @overload + def get(self, key: _KT, default: _T) -> _T | _VT: ... + def get(self, key: _KT, default: _T | None = None) -> _T | _VT | None: try: return self[key] except KeyError: self.soft_miss_count += 1 return default - def __delitem__(self, key): + def __delitem__(self, key: _KT) -> None: with self._lock: super().__delitem__(key) self._remove_from_ll(key) - def pop(self, key, default=_MISSING): + @overload + def pop(self, key: _KT) -> _VT: ... + @overload + def pop(self, key: _KT, default: _T) -> _T | _VT: ... + def pop(self, key: _KT, default: _T = _MISSING) -> _T | _VT: # NB: hit/miss counts are bypassed for pop() with self._lock: try: @@ -270,21 +289,25 @@ def pop(self, key, default=_MISSING): self._remove_from_ll(key) return ret - def popitem(self): + def popitem(self) -> tuple[_KT, _VT]: with self._lock: item = super().popitem() self._remove_from_ll(item[0]) return item - def clear(self): + def clear(self) -> None: with self._lock: super().clear() self._init_ll() - def copy(self): + def copy(self) -> Self: return self.__class__(max_size=self.max_size, values=self) - def setdefault(self, key, default=None): + @overload + def setdefault(self, key: _KT, default: None = None) -> _VT | None: ... + @overload + def setdefault(self, key: _KT, default: _VT) -> _VT: ... + def setdefault(self, key:_KT, default: _VT | None = None) -> _VT | None: with self._lock: try: return self[key] @@ -293,7 +316,7 @@ def setdefault(self, key, default=None): self[key] = default return default - def update(self, E, **F): + def update(self, E: SupportsKeysAndGetItem[_KT, _VT] | Iterable[tuple[_KT, _VT]], **F: _VT) -> None: # E and F are throwback names to the dict() __doc__ with self._lock: if E is self: @@ -329,7 +352,7 @@ def __repr__(self): % (cn, self.max_size, self.on_miss, val_map)) -class LRU(LRI): +class LRU(LRI[_KT, _VT]): """The ``LRU`` is :class:`dict` subtype implementation of the *Least-Recently Used* caching strategy. @@ -363,7 +386,7 @@ class LRU(LRI): Other than the size-limiting caching behavior and statistics, ``LRU`` acts like its parent class, the built-in Python :class:`dict`. """ - def __getitem__(self, key): + def __getitem__(self, key: _KT) -> _VT: with self._lock: try: link = self._get_link_and_move_to_front_of_ll(key) @@ -398,9 +421,9 @@ def __repr__(self): return f'{self.__class__.__name__}({list.__repr__(self)})' -def make_cache_key(args, kwargs, typed=False, - kwarg_mark=_KWARG_MARK, - fasttypes=frozenset([int, str, frozenset, type(None)])): +def make_cache_key(args: Iterable[Hashable], kwargs: SupportsItems[Hashable, Hashable], typed: bool = False, + kwarg_mark: object = _KWARG_MARK, + fasttypes: frozenset[type] = frozenset([int, str, frozenset, type(None)])): """Make a generic key from a function's positional and keyword arguments, suitable for use in caches. Arguments within *args* and *kwargs* must be `hashable`_. If *typed* is ``True``, ``3`` and @@ -442,7 +465,7 @@ class CachedFunction: """This type is used by :func:`cached`, below. Instances of this class are used to wrap functions in caching logic. """ - def __init__(self, func, cache, scoped=True, typed=False, key=None): + def __init__(self, func, cache: Mapping | Callable, scoped=True, typed=False, key: Callable | None =None): self.func = func if callable(cache): self.get_cache = cache @@ -480,7 +503,7 @@ class CachedMethod: """Similar to :class:`CachedFunction`, this type is used by :func:`cachedmethod` to wrap methods in caching logic. """ - def __init__(self, func, cache, scoped=True, typed=False, key=None): + def __init__(self, func, cache: Mapping | Callable, scoped=True, typed=False, key: Callable | None = None): self.func = func self.__isabstractmethod__ = getattr(func, '__isabstractmethod__', False) if isinstance(cache, str): @@ -532,7 +555,7 @@ def __repr__(self): return ("%s(func=%r, scoped=%r, typed=%r)" % args) -def cached(cache, scoped=True, typed=False, key=None): +def cached(cache: Mapping | Callable, scoped: bool = True, typed: bool = False, key: Callable | None = None): """Cache any function with the cache object of your choosing. Note that the function wrapped should take only `hashable`_ arguments. @@ -570,7 +593,7 @@ def cached_func_decorator(func): return cached_func_decorator -def cachedmethod(cache, scoped=True, typed=False, key=None): +def cachedmethod(cache: Mapping | Callable, scoped: bool = True, typed: bool = False, key: Callable | None = None): """Similar to :func:`cached`, ``cachedmethod`` is used to cache methods based on their arguments, using any :class:`dict`-like *cache* object. @@ -612,7 +635,7 @@ def cached_method_decorator(func): return cached_method_decorator -class cachedproperty: +class cachedproperty(Generic[_KT, _VT]): """The ``cachedproperty`` is used similar to :class:`property`, except that the wrapped method is only called once. This is commonly used to implement lazy attributes. @@ -622,12 +645,16 @@ class cachedproperty: allows the cache to be cleared with :func:`delattr`, or through manipulating the object's ``__dict__``. """ - def __init__(self, func): + def __init__(self, func: Callable[[_KT], _VT]): self.__doc__ = getattr(func, '__doc__') self.__isabstractmethod__ = getattr(func, '__isabstractmethod__', False) self.func = func - def __get__(self, obj, objtype=None): + @overload + def __get__(self, obj: None, objtype: type | None = None) -> Self: ... + @overload + def __get__(self, obj: _KT, objtype: type | None = None) -> _VT: ... + def __get__(self, obj: _KT | None, objtype: type | None = None) -> Self | _VT: if obj is None: return self value = obj.__dict__[self.func.__name__] = self.func(obj) @@ -638,7 +665,7 @@ def __repr__(self): return f'<{cn} func={self.func}>' -class ThresholdCounter: +class ThresholdCounter(Generic[_KT]): """A **bounded** dict-like Mapping from keys to counts. The ThresholdCounter automatically compacts after every (1 / *threshold*) additions, maintaining exact counts for any keys @@ -682,7 +709,7 @@ class ThresholdCounter: """ # TODO: hit_count/miss_count? - def __init__(self, threshold=0.001): + def __init__(self, threshold: float = 0.001): if not 0 < threshold < 1: raise ValueError('expected threshold between 0 and 1, not: %r' % threshold) @@ -694,10 +721,10 @@ def __init__(self, threshold=0.001): self._cur_bucket = 1 @property - def threshold(self): + def threshold(self) -> float: return self._threshold - def add(self, key): + def add(self, key: _KT) -> None: """Increment the count of *key* by 1, automatically adding it if it does not exist. @@ -715,14 +742,14 @@ def add(self, key): self._cur_bucket += 1 return - def elements(self): + def elements(self) -> itertools.chain[_T]: """Return an iterator of all the common elements tracked by the counter. Yields each key as many times as it has been seen. """ repeaters = itertools.starmap(itertools.repeat, self.iteritems()) return itertools.chain.from_iterable(repeaters) - def most_common(self, n=None): + def most_common(self, n: int | None = None) -> list[tuple[_KT, int]]: """Get the top *n* keys and counts as tuples. If *n* is omitted, returns all the pairs. """ @@ -733,20 +760,20 @@ def most_common(self, n=None): return ret return ret[:n] - def get_common_count(self): + def get_common_count(self) -> int: """Get the sum of counts for keys exceeding the configured data threshold. """ return sum([count for count, _ in self._count_map.values()]) - def get_uncommon_count(self): + def get_uncommon_count(self) -> int: """Get the sum of counts for keys that were culled because the associated counts represented less than the configured threshold. The long-tail counts. """ return self.total - self.get_common_count() - def get_commonality(self): + def get_commonality(self) -> float: """Get a float representation of the effective count accuracy. The higher the number, the less uniform the keys being added, and the higher accuracy and efficiency of the ThresholdCounter. @@ -756,45 +783,45 @@ def get_commonality(self): """ return float(self.get_common_count()) / self.total - def __getitem__(self, key): + def __getitem__(self, key: _KT) -> int: return self._count_map[key][0] - def __len__(self): + def __len__(self) -> int: return len(self._count_map) - def __contains__(self, key): + def __contains__(self, key: _KT) -> bool: return key in self._count_map - def iterkeys(self): + def iterkeys(self) -> Iterator[_KT]: return iter(self._count_map) - def keys(self): + def keys(self) -> list[_KT]: return list(self.iterkeys()) - def itervalues(self): + def itervalues(self) -> Generator[int]: count_map = self._count_map for k in count_map: yield count_map[k][0] - def values(self): + def values(self) -> list[int]: return list(self.itervalues()) - def iteritems(self): + def iteritems(self) -> Generator[tuple[_KT, int]]: count_map = self._count_map for k in count_map: yield (k, count_map[k][0]) - def items(self): + def items(self) -> list[tuple[_KT, int]]: return list(self.iteritems()) - def get(self, key, default=0): + def get(self, key: _KT, default: int = 0) -> int: "Get count for *key*, defaulting to 0." try: return self[key] except KeyError: return default - def update(self, iterable, **kwargs): + def update(self, iterable: Iterable[_KT] | Mapping[_KT, int], **kwargs: Iterable[_KT] | Mapping[_KT, int]) -> None: """Like dict.update() but add counts instead of replacing them, used to add multiple items in one call. @@ -813,7 +840,7 @@ def update(self, iterable, **kwargs): self.update(kwargs) -class MinIDMap: +class MinIDMap(Generic[_KT]): """ Assigns arbitrary weakref-able objects the smallest possible unique integer IDs, such that no two objects have the same ID at the same @@ -824,11 +851,11 @@ class MinIDMap: Based on https://gist.github.com/kurtbrose/25b48114de216a5e55df """ def __init__(self): - self.mapping = weakref.WeakKeyDictionary() - self.ref_map = {} - self.free = [] + self.mapping: weakref.WeakKeyDictionary[_KT, int] = weakref.WeakKeyDictionary() + self.ref_map: dict[_T, int] = {} + self.free: list[int] = [] - def get(self, a): + def get(self, a: _KT) -> int: try: return self.mapping[a][0] # if object is mapped, return ID except KeyError: @@ -843,7 +870,7 @@ def get(self, a): self.ref_map[ref] = nxt return nxt - def drop(self, a): + def drop(self, a: _KT) -> None: freed, ref = self.mapping[a] del self.mapping[a] del self.ref_map[ref] @@ -854,16 +881,16 @@ def _clean(self, ref): heapq.heappush(self.free, self.ref_map[ref]) del self.ref_map[ref] - def __contains__(self, a): + def __contains__(self, a: _KT) -> bool: return a in self.mapping - def __iter__(self): + def __iter__(self) -> Iterator[_KT]: return iter(self.mapping) - def __len__(self): + def __len__(self) -> int: return self.mapping.__len__() - def iteritems(self): + def iteritems(self) -> Iterator[tuple[_KT, int]]: return iter((k, self.mapping[k][0]) for k in iter(self.mapping)) diff --git a/boltons/debugutils.py b/boltons/debugutils.py index 16d18d4b..a8c6e3c5 100644 --- a/boltons/debugutils.py +++ b/boltons/debugutils.py @@ -34,6 +34,9 @@ built-in Python debugger. """ +from __future__ import annotations + +from collections.abc import Callable import sys import time from reprlib import Repr @@ -47,7 +50,7 @@ __all__ = ['pdb_on_signal', 'pdb_on_exception', 'wrap_trace'] -def pdb_on_signal(signalnum=None): +def pdb_on_signal(signalnum: int | None = None) -> None: """Installs a signal handler for *signalnum*, which defaults to ``SIGINT``, or keyboard interrupt/ctrl-c. This signal handler launches a :mod:`pdb` breakpoint. Results vary in concurrent @@ -75,7 +78,7 @@ def pdb_int_handler(sig, frame): return -def pdb_on_exception(limit=100): +def pdb_on_exception(limit: int = 100) -> None: """Installs a handler which, instead of exiting, attaches a post-mortem pdb console whenever an unhandled exception is encountered. @@ -138,8 +141,8 @@ def trace_print_hook(event, label, obj, attr_name, return -def wrap_trace(obj, hook=trace_print_hook, - which=None, events=None, label=None): +def wrap_trace(obj, hook: Callable = trace_print_hook, + which: str | None = None, events: str | None = None, label: str | None = None): """Monitor an object for interactions. Whenever code calls a method, gets an attribute, or sets an attribute, an event is called. By default the trace output is printed, but a custom tracing *hook* diff --git a/boltons/deprutils.py b/boltons/deprutils.py index 6c56736f..a7dd35c8 100644 --- a/boltons/deprutils.py +++ b/boltons/deprutils.py @@ -31,18 +31,19 @@ import sys from types import ModuleType +from typing import Any from warnings import warn # todo: only warn once class DeprecatableModule(ModuleType): - def __init__(self, module): + def __init__(self, module: ModuleType): name = module.__name__ super().__init__(name=name) self.__dict__.update(module.__dict__) - def __getattribute__(self, name): + def __getattribute__(self, name: str) -> Any: get_attribute = super().__getattribute__ try: depros = get_attribute('_deprecated_members') @@ -55,7 +56,7 @@ def __getattribute__(self, name): return ret -def deprecate_module_member(mod_name, name, message): +def deprecate_module_member(mod_name: str, name: str, message: str) -> None: module = sys.modules[mod_name] if not isinstance(module, DeprecatableModule): sys.modules[mod_name] = module = DeprecatableModule(module) diff --git a/boltons/dictutils.py b/boltons/dictutils.py index f913f29f..fed69289 100644 --- a/boltons/dictutils.py +++ b/boltons/dictutils.py @@ -67,8 +67,22 @@ """ -from collections.abc import KeysView, ValuesView, ItemsView +from __future__ import annotations + +from collections.abc import ( + Callable, + Generator, + ItemsView, + Iterable, + KeysView, + ValuesView, +) from itertools import zip_longest +from typing import TYPE_CHECKING, Dict, FrozenSet, NoReturn, TypeVar, overload + +if TYPE_CHECKING: + from _typeshed import SupportsKeysAndGetItem + from typing_extensions import Self try: from .typeutils import make_sentinel @@ -76,14 +90,19 @@ except ImportError: _MISSING = object() +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") +_T = TypeVar("_T") PREV, NEXT, KEY, VALUE, SPREV, SNEXT = range(6) __all__ = ['MultiDict', 'OMD', 'OrderedMultiDict', 'OneToOne', 'ManyToMany', 'subdict', 'FrozenDict'] - -class OrderedMultiDict(dict): +# Whilst internally the data is stored using `Dict[_KT, list[_VT]]`, +# we want the publicly exposed type to match a `Mapping[_KT, _VT]` +# This does cause many inner type-checking issues to match the proper outer Protocol +class OrderedMultiDict(Dict[_KT, _VT]): """A MultiDict is a dictionary that can have multiple values per key and the OrderedMultiDict (OMD) is a MultiDict that retains original insertion order. Common use cases include: @@ -197,7 +216,7 @@ def _insert(self, k, v): last[NEXT] = root[PREV] = cell cells.append(cell) - def add(self, k, v): + def add(self, k: _KT, v: _VT) -> None: """Add a single value *v* under a key *k*. Existing values under *k* are preserved. """ @@ -205,7 +224,7 @@ def add(self, k, v): self._insert(k, v) values.append(v) - def addlist(self, k, v): + def addlist(self, k: _KT, v: Iterable[_VT]) -> None: """Add an iterable of values underneath a specific key, preserving any values already under that key. @@ -225,7 +244,11 @@ def addlist(self, k, v): self_insert(k, subv) values.extend(v) - def get(self, k, default=None): + @overload + def get(self, k: _KT, default: None = None) -> _VT | None:... + @overload + def get(self, k: _KT, default: _VT) -> _VT:... + def get(self, k: _KT, default: _VT | None = None) -> _VT | None: """Return the value for key *k* if present in the dictionary, else *default*. If *default* is not given, ``None`` is returned. This method never raises a :exc:`KeyError`. @@ -234,7 +257,7 @@ def get(self, k, default=None): """ return super().get(k, [default])[-1] - def getlist(self, k, default=_MISSING): + def getlist(self,k: _KT, default: list[_VT] = _MISSING) -> list[_VT]: """Get all values for key *k* as a list, if *k* is in the dictionary, else *default*. The list returned is a copy and can be safely mutated. If *default* is not given, an empty @@ -247,12 +270,16 @@ def getlist(self, k, default=_MISSING): return [] return default - def clear(self): + def clear(self) -> None: "Empty the dictionary." super().clear() self._clear_ll() - def setdefault(self, k, default=_MISSING): + @overload + def setdefault(self, k: _KT, default: None = _MISSING) -> _VT | None: ... + @overload + def setdefault(self, k: _KT, default: _VT ) -> _VT: ... + def setdefault(self, k: _KT, default: _VT | None = _MISSING) -> _VT | None: """If key *k* is in the dictionary, return its value. If not, insert *k* with a value of *default* and return *default*. *default* defaults to ``None``. See :meth:`dict.setdefault` for more @@ -262,18 +289,18 @@ def setdefault(self, k, default=_MISSING): self[k] = None if default is _MISSING else default return self[k] - def copy(self): + def copy(self) -> Self: "Return a shallow copy of the dictionary." return self.__class__(self.iteritems(multi=True)) @classmethod - def fromkeys(cls, keys, default=None): + def fromkeys(cls, keys: _KT, default: _VT | None = None) -> Self: """Create a dictionary from a list of keys, with all the values set to *default*, or ``None`` if *default* is not set. """ return cls([(k, default) for k in keys]) - def update(self, E, **F): + def update(self, E: SupportsKeysAndGetItem[_KT, _VT] | Iterable[tuple[_KT, _VT]], **F) -> None: """Add items from a dictionary or iterable (and/or keyword arguments), overwriting values under an existing key. See :meth:`dict.update` for more details. @@ -303,7 +330,7 @@ def update(self, E, **F): self[k] = F[k] return - def update_extend(self, E, **F): + def update_extend(self, E: SupportsKeysAndGetItem[_KT, _VT] | Iterable[tuple[_KT, _VT]], **F) -> None: """Add items from a dictionary, iterable, and/or keyword arguments without overwriting existing items present in the dictionary. Like :meth:`update`, but adds to existing keys @@ -328,7 +355,7 @@ def __setitem__(self, k, v): self._insert(k, v) super().__setitem__(k, [v]) - def __getitem__(self, k): + def __getitem__(self, k: _KT) -> _VT: return super().__getitem__(k)[-1] def __delitem__(self, k): @@ -371,7 +398,7 @@ def __ior__(self, other): self.update(other) return self - def pop(self, k, default=_MISSING): + def pop(self, k: _KT, default: _VT = _MISSING) -> _VT: """Remove all values under key *k*, returning the most-recently inserted value. Raises :exc:`KeyError` if the key is not present and no *default* is provided. @@ -383,7 +410,7 @@ def pop(self, k, default=_MISSING): raise KeyError(k) return default - def popall(self, k, default=_MISSING): + def popall(self, k: _KT, default: _VT = _MISSING) -> list[_VT]: """Remove all values under key *k*, returning them in the form of a list. Raises :exc:`KeyError` if the key is not present and no *default* is provided. @@ -395,7 +422,7 @@ def popall(self, k, default=_MISSING): return super_self.pop(k) return super_self.pop(k, default) - def poplast(self, k=_MISSING, default=_MISSING): + def poplast(self, k: _KT = _MISSING, default: _VT = _MISSING) -> _VT: """Remove and return the most-recently inserted value under the key *k*, or the most-recently inserted key if *k* is not provided. If no values remain under *k*, it will be removed @@ -435,7 +462,7 @@ def _remove_all(self, k): cell[PREV][NEXT], cell[NEXT][PREV] = cell[NEXT], cell[PREV] del self._map[k] - def iteritems(self, multi=False): + def iteritems(self, multi: bool = False) -> Generator[tuple[_KT, _VT]]: """Iterate over the OMD's items in insertion order. By default, yields only the most-recently inserted value for each key. Set *multi* to ``True`` to get all inserted items. @@ -450,7 +477,7 @@ def iteritems(self, multi=False): for key in self.iterkeys(): yield key, self[key] - def iterkeys(self, multi=False): + def iterkeys(self, multi: bool = False) -> Generator[_KT]: """Iterate over the OMD's keys in insertion order. By default, yields each key once, according to the most recent insertion. Set *multi* to ``True`` to get all keys, including duplicates, in @@ -472,7 +499,7 @@ def iterkeys(self, multi=False): yield k curr = curr[NEXT] - def itervalues(self, multi=False): + def itervalues(self, multi: bool = False) -> Generator[_VT]: """Iterate over the OMD's values in insertion order. By default, yields the most-recently inserted value per unique key. Set *multi* to ``True`` to get all values according to insertion @@ -481,7 +508,7 @@ def itervalues(self, multi=False): for k, v in self.iteritems(multi=multi): yield v - def todict(self, multi=False): + def todict(self, multi: bool = False) -> dict[_KT, _VT]: """Gets a basic :class:`dict` of the items in this dictionary. Keys are the same as the OMD, values are the most recently inserted values for each key. @@ -494,7 +521,7 @@ def todict(self, multi=False): return {k: self.getlist(k) for k in self} return {k: self[k] for k in self} - def sorted(self, key=None, reverse=False): + def sorted(self, key: _KT | None = None, reverse: bool = False) -> Self: """Similar to the built-in :func:`sorted`, except this method returns a new :class:`OrderedMultiDict` sorted by the provided key function, optionally reversed. @@ -519,7 +546,7 @@ def sorted(self, key=None, reverse=False): cls = self.__class__ return cls(sorted(self.iteritems(multi=True), key=key, reverse=reverse)) - def sortedvalues(self, key=None, reverse=False): + def sortedvalues(self, key: _KT | None = None, reverse: bool = False) -> Self: """Returns a copy of the :class:`OrderedMultiDict` with the same keys in the same order as the original OMD, but the values within each keyspace have been sorted according to *key* and @@ -561,7 +588,7 @@ def sortedvalues(self, key=None, reverse=False): ret.add(k, sorted_val_map[k].pop()) return ret - def inverted(self): + def inverted(self) -> Self: """Returns a new :class:`OrderedMultiDict` with values and keys swapped, like creating dictionary transposition or reverse index. Insertion order is retained and all keys and values @@ -578,7 +605,7 @@ def inverted(self): """ return self.__class__((v, k) for k, v in self.iteritems(multi=True)) - def counts(self): + def counts(self) -> Self: """Returns a mapping from key to number of values inserted under that key. Like :py:class:`collections.Counter`, but returns a new :class:`OrderedMultiDict`. @@ -588,19 +615,19 @@ def counts(self): super_getitem = super().__getitem__ return self.__class__((k, len(super_getitem(k))) for k in self) - def keys(self, multi=False): + def keys(self, multi: bool = False) -> list[_KT]: """Returns a list containing the output of :meth:`iterkeys`. See that method's docs for more details. """ return list(self.iterkeys(multi=multi)) - def values(self, multi=False): + def values(self, multi: bool = False) -> list[_VT]: """Returns a list containing the output of :meth:`itervalues`. See that method's docs for more details. """ return list(self.itervalues(multi=multi)) - def items(self, multi=False): + def items(self, multi: bool = False) -> list[tuple[_KT, _VT]]: """Returns a list containing the output of :meth:`iteritems`. See that method's docs for more details. """ @@ -628,15 +655,15 @@ def __repr__(self): kvs = ', '.join([repr((k, v)) for k, v in self.iteritems(multi=True)]) return f'{cn}([{kvs}])' - def viewkeys(self): + def viewkeys(self) -> KeysView[_KT]: "OMD.viewkeys() -> a set-like object providing a view on OMD's keys" return KeysView(self) - def viewvalues(self): + def viewvalues(self) -> ValuesView[_VT]: "OMD.viewvalues() -> an object providing a view on OMD's values" return ValuesView(self) - def viewitems(self): + def viewitems(self) -> ItemsView[_KT, _VT]: "OMD.viewitems() -> a set-like object providing a view on OMD's items" return ItemsView(self) @@ -645,8 +672,7 @@ def viewitems(self): OMD = OrderedMultiDict MultiDict = OrderedMultiDict - -class FastIterOrderedMultiDict(OrderedMultiDict): +class FastIterOrderedMultiDict(OrderedMultiDict[_KT, _VT]): """An OrderedMultiDict backed by a skip list. Iteration over keys is faster and uses constant memory but adding duplicate key-value pairs is slower. Brainchild of Mark Williams. @@ -663,7 +689,7 @@ def _clear_ll(self): None, None, self.root, self.root] - def _insert(self, k, v): + def _insert(self, k: _KT, v: _VT): root = self.root empty = [] cells = self._map.setdefault(k, empty) @@ -716,7 +742,7 @@ def _remove_all(self, k): cell[PREV][NEXT], cell[NEXT][PREV] = cell[NEXT], cell[PREV] cell[PREV][SNEXT] = cell[SNEXT] - def iteritems(self, multi=False): + def iteritems(self, multi: bool = False) -> Generator[tuple[_KT, _VT]]: next_link = NEXT if multi else SNEXT root = self.root curr = root[next_link] @@ -724,7 +750,7 @@ def iteritems(self, multi=False): yield curr[KEY], curr[VALUE] curr = curr[next_link] - def iterkeys(self, multi=False): + def iterkeys(self, multi: bool = False) -> Generator[_KT]: next_link = NEXT if multi else SNEXT root = self.root curr = root[next_link] @@ -748,7 +774,7 @@ def __reversed__(self): _OTO_UNIQUE_MARKER = object() -class OneToOne(dict): +class OneToOne(Dict[_KT, _VT]): """Implements a one-to-one mapping dictionary. In addition to inheriting from and behaving exactly like the builtin :class:`dict`, all values are automatically added as keys on a @@ -813,7 +839,7 @@ def __init__(self, *a, **kw): ' the following values: %r' % dupes) @classmethod - def unique(cls, *a, **kw): + def unique(cls, *a, **kw) -> Self: """This alternate constructor for OneToOne will raise an exception when input values overlap. For instance: @@ -845,14 +871,14 @@ def __delitem__(self, key): dict.__delitem__(self.inv, self[key]) dict.__delitem__(self, key) - def clear(self): + def clear(self) -> None: dict.clear(self) dict.clear(self.inv) - def copy(self): + def copy(self) -> Self: return self.__class__(self) - def pop(self, key, default=_MISSING): + def pop(self, key: _KT, default: _VT = _MISSING) -> _VT: if key in self: dict.__delitem__(self.inv, self[key]) return dict.pop(self, key) @@ -860,17 +886,17 @@ def pop(self, key, default=_MISSING): return default raise KeyError() - def popitem(self): + def popitem(self) -> tuple[_KT, _VT]: key, val = dict.popitem(self) dict.__delitem__(self.inv, val) return key, val - def setdefault(self, key, default=None): + def setdefault(self, key: _KT, default: _VT | None = None) -> _VT: if key not in self: self[key] = default return self[key] - def update(self, dict_or_iterable, **kw): + def update(self, dict_or_iterable, **kw) -> None: keys_vals = [] if isinstance(dict_or_iterable, dict): for val in dict_or_iterable.values(): @@ -896,8 +922,7 @@ def __repr__(self): # marker for the secret handshake used internally to set up the invert ManyToMany _PAIRING = object() - -class ManyToMany: +class ManyToMany(Dict[_KT, FrozenSet[_VT]]): """ a dict-like entity that represents a many-to-many relationship between two groups of objects @@ -907,7 +932,7 @@ class ManyToMany: also, can be used as a directed graph among hashable python objects """ - def __init__(self, items=None): + def __init__(self, items: ManyToMany[_KT, _VT] | SupportsKeysAndGetItem[_KT, _VT] | tuple[_KT, _VT] | None = None): self.data = {} if type(items) is tuple and items and items[0] is _PAIRING: self.inv = items[1] @@ -917,16 +942,16 @@ def __init__(self, items=None): self.update(items) return - def get(self, key, default=frozenset()): + def get(self, key: _KT, default: frozenset[_VT] = frozenset()) -> frozenset[_VT]: try: return self[key] except KeyError: return default - def __getitem__(self, key): + def __getitem__(self, key: _KT): return frozenset(self.data[key]) - def __setitem__(self, key, vals): + def __setitem__(self, key: _KT, vals: Iterable[_VT]) -> None: vals = set(vals) if key in self: to_remove = self.data[key] - vals @@ -936,13 +961,13 @@ def __setitem__(self, key, vals): for val in vals: self.add(key, val) - def __delitem__(self, key): + def __delitem__(self, key: _KT) -> None: for val in self.data.pop(key): self.inv.data[val].remove(key) if not self.inv.data[val]: del self.inv.data[val] - def update(self, iterable): + def update(self, iterable: ManyToMany[_KT, _VT] | SupportsKeysAndGetItem[_KT, _VT] | tuple[_KT, _VT]) -> None: """given an iterable of (key, val), add them all""" if type(iterable) is type(self): other = iterable @@ -964,7 +989,7 @@ def update(self, iterable): self.add(key, val) return - def add(self, key, val): + def add(self, key: _KT, val: _VT) -> None: if key not in self.data: self.data[key] = set() self.data[key].add(val) @@ -972,7 +997,7 @@ def add(self, key, val): self.inv.data[val] = set() self.inv.data[val].add(key) - def remove(self, key, val): + def remove(self, key: _KT, val: _VT) -> None: self.data[key].remove(val) if not self.data[key]: del self.data[key] @@ -980,7 +1005,7 @@ def remove(self, key, val): if not self.inv.data[val]: del self.inv.data[val] - def replace(self, key, newkey): + def replace(self, key: _KT, newkey: _KT) -> None: """ replace instances of key by newkey """ @@ -992,7 +1017,7 @@ def replace(self, key, newkey): revset.remove(key) revset.add(newkey) - def iteritems(self): + def iteritems(self) -> Generator[tuple[_KT, _VT]]: for key in self.data: for val in self.data[key]: yield key, val @@ -1017,7 +1042,7 @@ def __repr__(self): return f'{cn}({list(self.iteritems())!r})' -def subdict(d, keep=None, drop=None): +def subdict(d: dict[_KT, _VT], keep: Iterable[_KT] | None = None, drop: Iterable[_KT] | None = None) -> dict[_KT, _VT]: """Compute the "subdictionary" of a dict, *d*. A subdict is to a dict what a subset is a to set. If *A* is a @@ -1053,7 +1078,7 @@ class FrozenHashError(TypeError): pass -class FrozenDict(dict): +class FrozenDict(Dict[_KT, _VT]): """An immutable dict subtype that is hashable and can itself be used as a :class:`dict` key or :class:`set` entry. What :class:`frozenset` is to :class:`set`, FrozenDict is to @@ -1068,7 +1093,7 @@ class FrozenDict(dict): """ __slots__ = ('_hash',) - def updated(self, *a, **kw): + def updated(self, *a, **kw) -> Self: """Make a copy and add items from a dictionary or iterable (and/or keyword arguments), overwriting values under an existing key. See :meth:`dict.update` for more details. @@ -1078,7 +1103,7 @@ def updated(self, *a, **kw): return type(self)(data) @classmethod - def fromkeys(cls, keys, value=None): + def fromkeys(cls, keys: Iterable[_KT], value: _VT | None = None) -> Self: # one of the lesser known and used/useful dict methods return cls(dict.fromkeys(keys, value)) @@ -1103,18 +1128,23 @@ def __hash__(self): return ret - def __copy__(self): + def __copy__(self) -> Self: return self # immutable types don't copy, see tuple's behavior # block everything else - def _raise_frozen_typeerror(self, *a, **kw): + def _raise_frozen_typeerror(self, *a, **kw) -> NoReturn: "raises a TypeError, because FrozenDicts are immutable" raise TypeError('%s object is immutable' % self.__class__.__name__) - __ior__ = __setitem__ = __delitem__ = update = _raise_frozen_typeerror - setdefault = pop = popitem = clear = _raise_frozen_typeerror + __ior__: Callable[..., NoReturn] = _raise_frozen_typeerror + __setitem__: Callable[..., NoReturn] = _raise_frozen_typeerror + __delitem__: Callable[..., NoReturn] = _raise_frozen_typeerror + update: Callable[..., NoReturn] = _raise_frozen_typeerror + setdefault: Callable[..., NoReturn] = _raise_frozen_typeerror + pop: Callable[..., NoReturn] = _raise_frozen_typeerror + popitem: Callable[..., NoReturn] = _raise_frozen_typeerror + clear: Callable[..., NoReturn] = _raise_frozen_typeerror del _raise_frozen_typeerror - # end dictutils.py diff --git a/boltons/easterutils.py b/boltons/easterutils.py index 02a930a6..7194a069 100644 --- a/boltons/easterutils.py +++ b/boltons/easterutils.py @@ -30,7 +30,10 @@ -def gobs_program(): +from typing import NoReturn + + +def gobs_program() -> NoReturn: """ A pure-Python implementation of Gob's Algorithm (2006). A brief explanation can be found here: diff --git a/boltons/ecoutils.py b/boltons/ecoutils.py index 23fb25fb..336838b4 100644 --- a/boltons/ecoutils.py +++ b/boltons/ecoutils.py @@ -150,6 +150,7 @@ ``pip install boltons`` and try it yourself! """ +from __future__ import annotations import re import os @@ -162,6 +163,7 @@ import getpass import datetime import platform +from typing import Any ECO_VERSION = '1.1.0' # see version history below @@ -258,7 +260,7 @@ 'time_utc_offset': -time.timezone / 3600.0} -def get_python_info(): +def get_python_info() -> dict[str, Any]: ret = {} ret['argv'] = _escape_shell_args(sys.argv) ret['bin'] = sys.executable @@ -287,7 +289,7 @@ def get_python_info(): return ret -def get_profile(**kwargs): +def get_profile(**kwargs) -> dict[str, Any]: """The main entrypoint to ecoutils. Calling this will return a JSON-serializable dictionary of information about the current process. @@ -354,13 +356,13 @@ def get_profile(**kwargs): return ret -def dumps(val, indent): +def dumps(val: object, indent: int) -> str: if indent: return json.dumps(val, sort_keys=True, indent=indent) return json.dumps(val, sort_keys=True) -def get_profile_json(indent=False): +def get_profile_json(indent: bool = False) -> str: if indent: indent = 2 else: @@ -370,7 +372,7 @@ def get_profile_json(indent=False): return dumps(data_dict, indent) -def main(): +def main() -> None: print(get_profile_json(indent=True)) ############################################# diff --git a/boltons/excutils.py b/boltons/excutils.py index fa3262d4..9381ee13 100644 --- a/boltons/excutils.py +++ b/boltons/excutils.py @@ -28,11 +28,16 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import annotations import sys import traceback import linecache from collections import namedtuple +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing_extensions import Self # TODO: last arg or first arg? (last arg makes it harder to *args # into, but makes it more readable in the default exception @@ -63,7 +68,7 @@ class ExceptionCauseMixin(Exception): cause = None - def __new__(cls, *args, **kw): + def __new__(cls, *args, **kw) -> Self: cause = None if args and isinstance(args[0], Exception): cause, args = args[0], args[1:] @@ -99,7 +104,7 @@ def __new__(cls, *args, **kw): del exc_tb return ret - def get_str(self): + def get_str(self) -> str: """ Get formatted the formatted traceback and exception message. This function exists separately from __str__() diff --git a/boltons/fileutils.py b/boltons/fileutils.py index 2a057704..61b86041 100644 --- a/boltons/fileutils.py +++ b/boltons/fileutils.py @@ -33,7 +33,9 @@ most commonly-found gaps in the standard library. """ +from __future__ import annotations +from collections.abc import Callable, Iterable import os import re import sys @@ -41,7 +43,14 @@ import errno import fnmatch from shutil import copy2, copystat, Error +from types import TracebackType +from typing import IO, TYPE_CHECKING, Generator, NoReturn, TypeVar, overload +if TYPE_CHECKING: + from _typeshed import BytesPath, FileDescriptorOrPath, StrPath, StrOrBytesPath + from typing_extensions import Self + _StrPathT = TypeVar("_StrPathT", bound=StrPath) + _BytesPathT = TypeVar("_BytesPathT", bound=BytesPath) __all__ = ['mkdir_p', 'atomic_save', 'AtomicSaver', 'FilePerms', 'iter_find_files', 'copytree'] @@ -52,7 +61,7 @@ _SINGLE_FULL_PERM = 7 -def mkdir_p(path): +def mkdir_p(path: StrOrBytesPath) -> None: """Creates a directory and any parent directories that may need to be created along the way, without raising errors for any existing directories. This function mimics the behavior of the ``mkdir -p`` @@ -152,7 +161,7 @@ def _update_integer(self, fp_obj, value): mode |= (bit << (self.offset * 3)) fp_obj._integer |= mode - def __init__(self, user='', group='', other=''): + def __init__(self, user: str = '', group: str = '', other: str = ''): self._user, self._group, self._other = '', '', '' self._integer = 0 self.user = user @@ -160,7 +169,7 @@ def __init__(self, user='', group='', other=''): self.other = other @classmethod - def from_int(cls, i): + def from_int(cls, i: int) -> Self: """Create a :class:`FilePerms` object from an integer. >>> FilePerms.from_int(0o644) # note the leading zero-oh for octal @@ -176,7 +185,7 @@ def from_int(cls, i): return cls(*parts) @classmethod - def from_path(cls, path): + def from_path(cls, path: FileDescriptorOrPath) -> Self: """Make a new :class:`FilePerms` object based on the permissions assigned to the file or directory at *path*. @@ -192,7 +201,7 @@ def from_path(cls, path): stat_res = os.stat(path) return cls.from_int(stat.S_IMODE(stat_res.st_mode)) - def __int__(self): + def __int__(self) -> int: return self._integer # Sphinx tip: attribute docstrings come after the attribute @@ -216,7 +225,7 @@ def __repr__(self): _TEXT_OPENFLAGS |= os.O_NOINHERIT if hasattr(os, 'O_NOFOLLOW'): _TEXT_OPENFLAGS |= os.O_NOFOLLOW -_BIN_OPENFLAGS = _TEXT_OPENFLAGS +_BIN_OPENFLAGS: int = _TEXT_OPENFLAGS if hasattr(os, 'O_BINARY'): _BIN_OPENFLAGS |= os.O_BINARY @@ -245,7 +254,7 @@ def set_cloexec(fd): return -def atomic_save(dest_path, **kwargs): +def atomic_save(dest_path: str, **kwargs) -> AtomicSaver: """A convenient interface to the :class:`AtomicSaver` type. Example: >>> try: @@ -385,15 +394,15 @@ class AtomicSaver: _default_file_perms = RW_PERMS # TODO: option to abort if target file modify date has changed since start? - def __init__(self, dest_path, **kwargs): + def __init__(self, dest_path: str, **kwargs): self.dest_path = dest_path - self.overwrite = kwargs.pop('overwrite', True) - self.file_perms = kwargs.pop('file_perms', None) - self.overwrite_part = kwargs.pop('overwrite_part', False) - self.part_filename = kwargs.pop('part_file', None) - self.rm_part_on_exc = kwargs.pop('rm_part_on_exc', True) - self.text_mode = kwargs.pop('text_mode', False) - self.buffering = kwargs.pop('buffering', -1) + self.overwrite: bool = kwargs.pop('overwrite', True) + self.file_perms: int | None = kwargs.pop('file_perms', None) + self.overwrite_part: bool = kwargs.pop('overwrite_part', False) + self.part_filename: str | None = kwargs.pop('part_file', None) + self.rm_part_on_exc: bool = kwargs.pop('rm_part_on_exc', True) + self.text_mode: bool = kwargs.pop('text_mode', False) + self.buffering: int = kwargs.pop('buffering', -1) if kwargs: raise TypeError(f'unexpected kwargs: {kwargs.keys()!r}') @@ -435,7 +444,7 @@ def _open_part_file(self): raise return - def setup(self): + def setup(self) -> None: """Called on context manager entry (the :keyword:`with` statement), the ``setup()`` method creates the temporary file in the same directory as the destination file. @@ -459,11 +468,11 @@ def setup(self): self._open_part_file() return - def __enter__(self): + def __enter__(self) -> IO | None: self.setup() return self.part_file - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None) -> None: if self.part_file: # Ensure data is flushed and synced to disk before closing self.part_file.flush() @@ -489,7 +498,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): return -def iter_find_files(directory, patterns, ignored=None, include_dirs=False, max_depth=None): +def iter_find_files(directory: str, patterns: str | Iterable[str], ignored: str | Iterable[str] | None = None, include_dirs: bool = False, max_depth: int | None = None) -> Generator[str]: """Returns a generator that yields file paths under a *directory*, matching *patterns* using `glob`_ syntax (e.g., ``*.txt``). Also supports *ignored* patterns. @@ -550,8 +559,11 @@ def iter_find_files(directory, patterns, ignored=None, include_dirs=False, max_d yield filename return - -def copy_tree(src, dst, symlinks=False, ignore=None): +@overload +def copy_tree(src: _StrPathT, dst: _StrPathT, symlinks: bool = False, ignore: Callable[[_StrPathT, list[str]], Iterable[str]] | None = None) -> None: ... +@overload +def copy_tree(src: _BytesPathT, dst: _BytesPathT, symlinks: bool = False, ignore: Callable[[_BytesPathT, list[bytes]], Iterable[bytes]] | None = None) -> None: ... +def copy_tree(src: _StrPathT | _BytesPathT, dst: _StrPathT | _BytesPathT, symlinks: bool = False, ignore: Callable[[_StrPathT, list[str]], Iterable[str]] | Callable[[_BytesPathT, list[bytes]], Iterable[bytes]] | None = None) -> None: """The ``copy_tree`` function is an exact copy of the built-in :func:`shutil.copytree`, with one key difference: it will not raise an exception if part of the tree already exists. It achieves @@ -607,7 +619,6 @@ def copy_tree(src, dst, symlinks=False, ignore=None): if errors: raise Error(errors) - copytree = copy_tree # alias for drop-in replacement of shutil @@ -615,7 +626,7 @@ def copy_tree(src, dst, symlinks=False, ignore=None): class DummyFile: # TODO: raise ValueErrors on closed for all methods? # TODO: enforce read/write - def __init__(self, path, mode='r', buffering=None): + def __init__(self, path: StrOrBytesPath, mode: str = "r", buffering: int | None = None): self.name = path self.mode = mode self.closed = False @@ -625,73 +636,73 @@ def __init__(self, path, mode='r', buffering=None): self.newlines = None self.softspace = 0 - def close(self): + def close(self) -> None: self.closed = True - def fileno(self): + def fileno(self) -> int: return -1 - def flush(self): + def flush(self) -> None: if self.closed: raise ValueError('I/O operation on a closed file') return - def next(self): + def next(self) -> NoReturn: raise StopIteration() - def read(self, size=0): + def read(self, size: int = 0) -> str: if self.closed: raise ValueError('I/O operation on a closed file') return '' - def readline(self, size=0): + def readline(self, size: int = 0) -> str: if self.closed: raise ValueError('I/O operation on a closed file') return '' - def readlines(self, size=0): + def readlines(self, size: int = 0) -> list[str]: if self.closed: raise ValueError('I/O operation on a closed file') return [] - def seek(self): + def seek(self) -> None: if self.closed: raise ValueError('I/O operation on a closed file') return - def tell(self): + def tell(self) -> int: if self.closed: raise ValueError('I/O operation on a closed file') return 0 - def truncate(self): + def truncate(self) -> None: if self.closed: raise ValueError('I/O operation on a closed file') return - def write(self, string): + def write(self, string: str) -> None: if self.closed: raise ValueError('I/O operation on a closed file') return - def writelines(self, list_of_strings): + def writelines(self, list_of_strings: list[str]) -> None: if self.closed: raise ValueError('I/O operation on a closed file') return - def __next__(self): + def __next__(self) -> NoReturn: raise StopIteration() - def __enter__(self): + def __enter__(self) -> None: if self.closed: raise ValueError('I/O operation on a closed file') return - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type, exc_val, exc_tb) -> None: return -def rotate_file(filename, *, keep: int = 5): +def rotate_file(filename: StrPath, *, keep: int = 5) -> None: """ If *filename.ext* exists, it will be moved to *filename.1.ext*, with all conflicting filenames being moved up by one, dropping any files beyond *keep*. diff --git a/boltons/formatutils.py b/boltons/formatutils.py index 30340349..eea945da 100644 --- a/boltons/formatutils.py +++ b/boltons/formatutils.py @@ -65,9 +65,14 @@ # TODO: also include percent-formatting utils? # TODO: include lithoxyl.formatters.Formatter (or some adaptation)? +from __future__ import annotations +from collections.abc import Callable import re from string import Formatter +from typing import Generic, TypeVar + +_T = TypeVar("_T") __all__ = ['DeferredValue', 'get_format_args', 'tokenize_format_str', 'construct_format_field_str', 'infer_positional_format_args', @@ -79,7 +84,7 @@ r'({[:!.\[}])') # anon positional format arg -def construct_format_field_str(fname, fspec, conv): +def construct_format_field_str(fname: str | None, fspec: str | None, conv: str | None) -> str: """ Constructs a format field string from the field name, spec, and conversion character (``fname``, ``fspec``, ``conv``). See Python @@ -112,7 +117,7 @@ def split_format_str(fstr): return ret -def infer_positional_format_args(fstr): +def infer_positional_format_args(fstr: str) -> str: """Takes format strings with anonymous positional arguments, (e.g., "{}" and {:d}), and converts them into numbered ones for explicitness and compatibility with 2.6. @@ -140,12 +145,12 @@ def infer_positional_format_args(fstr): # This approach is hardly exhaustive but it works for most builtins _INTCHARS = 'bcdoxXn' _FLOATCHARS = 'eEfFgGn%' -_TYPE_MAP = dict([(x, int) for x in _INTCHARS] + +_TYPE_MAP: dict[str, type] = dict([(x, int) for x in _INTCHARS] + [(x, float) for x in _FLOATCHARS]) _TYPE_MAP['s'] = str -def get_format_args(fstr): +def get_format_args(fstr: str) -> tuple[list[tuple[int, type]], list[tuple[str, type]]]: """ Turn a format string into two lists of arguments referenced by the format string. One is positional arguments, and the other is named @@ -192,7 +197,7 @@ def _add_arg(argname, type_char='s'): return fargs, fkwargs -def tokenize_format_str(fstr, resolve_pos=True): +def tokenize_format_str(fstr: str, resolve_pos: bool = True) -> list[str | BaseFormatField]: """Takes a format string, turns it into a list of alternating string literals and :class:`BaseFormatField` tokens. By default, also infers anonymous positional references into explicit, numbered @@ -222,25 +227,25 @@ class BaseFormatField: .. _Format String Syntax: https://docs.python.org/2/library/string.html#string-formatting """ - def __init__(self, fname, fspec='', conv=None): + def __init__(self, fname: str, fspec: str = '', conv: str | None = None): self.set_fname(fname) self.set_fspec(fspec) self.set_conv(conv) - def set_fname(self, fname): + def set_fname(self, fname: str) -> None: "Set the field name." - path_list = re.split('[.[]', fname) # TODO + path_list: list[str] = re.split('[.[]', fname) # TODO self.base_name = path_list[0] self.fname = fname self.subpath = path_list[1:] self.is_positional = not self.base_name or self.base_name.isdigit() - def set_fspec(self, fspec): + def set_fspec(self, fspec: str) -> None: "Set the field spec." fspec = fspec or '' - subfields = [] + subfields: list[str] = [] for sublit, subfname, _, _ in Formatter().parse(fspec): if subfname is not None: subfields.append(subfname) @@ -249,7 +254,7 @@ def set_fspec(self, fspec): self.type_char = fspec[-1:] self.type_func = _TYPE_MAP.get(self.type_char, str) - def set_conv(self, conv): + def set_conv(self, conv: str | None) -> None: """There are only two built-in converters: ``s`` and ``r``. They are somewhat rare and appearlike ``"{ref!r}"``.""" # TODO @@ -257,7 +262,7 @@ def set_conv(self, conv): self.conv_func = None # TODO @property - def fstr(self): + def fstr(self) -> str: "The current state of the field in string format." return construct_format_field_str(self.fname, self.fspec, self.conv) @@ -274,11 +279,10 @@ def __repr__(self): def __str__(self): return self.fstr - _UNSET = object() -class DeferredValue: +class DeferredValue(Generic[_T]): """:class:`DeferredValue` is a wrapper type, used to defer computing values which would otherwise be expensive to stringify and format. This is most valuable in areas like logging, where one @@ -307,12 +311,12 @@ class DeferredValue: PROTIP: To keep lines shorter, use: ``from formatutils import DeferredValue as DV`` """ - def __init__(self, func, cache_value=True): + def __init__(self, func: Callable[[], _T], cache_value: bool = True): self.func = func self.cache_value = cache_value - self._value = _UNSET + self._value: _T = _UNSET - def get_value(self): + def get_value(self) -> _T: """Computes, optionally caches, and returns the value of the *func*. If ``get_value()`` has been called before, a cached value may be returned depending on the *cache_value* option @@ -326,22 +330,22 @@ def get_value(self): self._value = value return value - def __int__(self): + def __int__(self) -> int: return int(self.get_value()) - def __float__(self): + def __float__(self) -> float: return float(self.get_value()) - def __str__(self): + def __str__(self) -> str: return str(self.get_value()) - def __unicode__(self): + def __unicode__(self) -> str: return str(self.get_value()) - def __repr__(self): + def __repr__(self) -> str: return repr(self.get_value()) - def __format__(self, fmt): + def __format__(self, fmt: str) -> str: value = self.get_value() pt = fmt[-1:] # presentation type diff --git a/boltons/funcutils.py b/boltons/funcutils.py index 0bdc28e9..ea3c5d3f 100644 --- a/boltons/funcutils.py +++ b/boltons/funcutils.py @@ -281,7 +281,7 @@ def _partialmethod(self): def __partialmethod__(self): return functools.partialmethod(self.func, *self.args, **self.keywords) - def __set_name__(self, obj_type, name): + def __set_name__(self, obj_type, name) -> None: self.__name__ = name def __get__(self, obj, obj_type): diff --git a/boltons/gcutils.py b/boltons/gcutils.py index 1db18b46..de1b6b2b 100644 --- a/boltons/gcutils.py +++ b/boltons/gcutils.py @@ -58,14 +58,18 @@ """ # TODO: type survey +from __future__ import annotations import gc import sys +from typing import TypeVar + +_T = TypeVar("_T") __all__ = ['get_all', 'GCToggler', 'toggle_gc', 'toggle_gc_postcollect'] -def get_all(type_obj, include_subtypes=True): +def get_all(type_obj: type[_T], include_subtypes: bool = True) -> list[_T]: """Get a list containing all instances of a given type. This will work for the vast majority of types out there. @@ -142,13 +146,13 @@ class GCToggler: Between those two instances, the ``GCToggler`` type probably won't be used much directly, but is documented for inheritance purposes. """ - def __init__(self, postcollect=False): + def __init__(self, postcollect: bool = False): self.postcollect = postcollect - def __enter__(self): + def __enter__(self) -> None: gc.disable() - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type, exc_val, exc_tb) -> None: gc.enable() if self.postcollect: gc.collect() diff --git a/boltons/ioutils.py b/boltons/ioutils.py index bfcaf3fc..d076a238 100644 --- a/boltons/ioutils.py +++ b/boltons/ioutils.py @@ -98,7 +98,7 @@ def readline(self, length=None): def readlines(self, sizehint=0): """Returns a list of all lines from the current position forward""" - def writelines(self, lines): + def writelines(self, lines) -> None: """ Write lines to the file from an interable. @@ -232,7 +232,7 @@ def __enter__(self): self._checkClosed() return self - def __exit__(self, *args): + def __exit__(self, *args) -> None: self._file.close() def __eq__(self, other): @@ -270,7 +270,7 @@ def __ne__(self, other): def __bool__(self): return True - def __del__(self): + def __del__(self) -> None: """Can fail when called at program exit so suppress traceback.""" try: self.close() @@ -299,7 +299,7 @@ def read(self, n=-1): self._checkClosed() return self.buffer.read(n) - def write(self, s): + def write(self, s) -> None: self._checkClosed() if not isinstance(s, bytes): raise TypeError("bytes expected, got {}".format( @@ -324,7 +324,7 @@ def readline(self, length=None): def readlines(self, sizehint=0): return self.buffer.readlines(sizehint) - def rollover(self): + def rollover(self) -> None: """Roll the StringIO over to a TempFile""" if not self._rolled: tmp = TemporaryFile(dir=self._dir) @@ -392,7 +392,7 @@ def read(self, n=-1): self._tell = self.tell() + len(ret) return ret - def write(self, s): + def write(self, s) -> None: self._checkClosed() if not isinstance(s, str): raise TypeError("str expected, got {}".format( @@ -477,7 +477,7 @@ def buffer(self): def _rolled(self): return not isinstance(self.buffer.stream, BytesIO) - def rollover(self): + def rollover(self) -> None: """Roll the buffer over to a TempFile""" if not self._rolled: tmp = EncodedFile(TemporaryFile(dir=self._dir), @@ -508,7 +508,7 @@ def len(self): return total -def is_text_fileobj(fileobj): +def is_text_fileobj(fileobj) -> bool: if getattr(fileobj, 'encoding', False): # codecs.open and io.TextIOBase return True @@ -573,7 +573,7 @@ def read(self, amt=None): amt -= got return self._joiner.join(parts) - def seek(self, offset, whence=os.SEEK_SET): + def seek(self, offset, whence=os.SEEK_SET) -> None: """Enables setting position of the file cursor to a given *offset*. Currently only supports ``offset=0``. """ diff --git a/boltons/iterutils.py b/boltons/iterutils.py index e0a5b900..bcfded7a 100644 --- a/boltons/iterutils.py +++ b/boltons/iterutils.py @@ -39,14 +39,15 @@ following are based on examples in itertools docs. """ +from __future__ import annotations + import os import math import time -import codecs import random import itertools from itertools import zip_longest -from collections.abc import Mapping, Sequence, Set, ItemsView, Iterable +from collections.abc import Generator, Mapping, Sequence, Set, ItemsView, Iterable try: @@ -58,7 +59,7 @@ _UNSET = object() -def is_iterable(obj): +def is_iterable(obj) -> bool: """Similar in nature to :func:`callable`, ``is_iterable`` returns ``True`` if an object is `iterable`_, ``False`` if not. @@ -76,7 +77,7 @@ def is_iterable(obj): return True -def is_scalar(obj): +def is_scalar(obj) -> bool: """A near-mirror of :func:`is_iterable`. Returns ``False`` if an object is an iterable container type. Strings are considered scalar as well, because strings are more often treated as whole @@ -92,7 +93,7 @@ def is_scalar(obj): return not is_iterable(obj) or isinstance(obj, (str, bytes)) -def is_collection(obj): +def is_collection(obj) -> bool: """The opposite of :func:`is_scalar`. Returns ``True`` if an object is an iterable other than a string. @@ -118,7 +119,7 @@ def split(src, sep=None, maxsplit=None): return list(split_iter(src, sep, maxsplit)) -def split_iter(src, sep=None, maxsplit=None): +def split_iter(src, sep=None, maxsplit=None) -> Generator: """Splits an iterable based on a separator, *sep*, a max of *maxsplit* times (no max by default). *sep* can be: @@ -202,7 +203,7 @@ def lstrip(iterable, strip_value=None): return list(lstrip_iter(iterable, strip_value)) -def lstrip_iter(iterable, strip_value=None): +def lstrip_iter(iterable, strip_value=None) -> Generator: """Strips values from the beginning of an iterable. Stripped items will match the value of the argument strip_value. Functionality is analogous to that of the method str.lstrip. Returns a generator. @@ -232,7 +233,7 @@ def rstrip(iterable, strip_value=None): return list(rstrip_iter(iterable, strip_value)) -def rstrip_iter(iterable, strip_value=None): +def rstrip_iter(iterable, strip_value=None) -> Generator: """Strips values from the end of an iterable. Stripped items will match the value of the argument strip_value. Functionality is analogous to that of the method str.rstrip. Returns a generator. @@ -313,7 +314,7 @@ def _validate_positive_int(value, name, strictly_positive=True): return value -def chunked_iter(src, size, **kw): +def chunked_iter(src, size, **kw) -> Generator: """Generates *size*-sized chunks from *src* iterable. Unless the optional *fill* keyword argument is provided, iterables not evenly divisible by *size* will have a final chunk that is smaller than @@ -358,7 +359,7 @@ def postprocess(chk): return bytes(chk) return -def chunk_ranges(input_size, chunk_size, input_offset=0, overlap_size=0, align=False): +def chunk_ranges(input_size: int, chunk_size: int, input_offset: int = 0, overlap_size: int = 0, align: bool = False) -> Generator[tuple[int, int]]: """Generates *chunk_size*-sized chunk ranges for an input with length *input_size*. Optionally, a start of the input can be set via *input_offset*, and and overlap between the chunks may be specified via *overlap_size*. @@ -510,7 +511,7 @@ def windowed_iter(src, size, fill=_UNSET): return zip_longest(*tees, fillvalue=fill) -def xfrange(stop, start=None, step=1.0): +def xfrange(stop, start=None, step: float = 1.0) -> Generator: """Same as :func:`frange`, but generator-based instead of returning a list. @@ -580,7 +581,7 @@ def backoff(start, stop, count=None, factor=2.0, jitter=False): factor=factor, jitter=jitter)) -def backoff_iter(start, stop, count=None, factor=2.0, jitter=False): +def backoff_iter(start, stop, count=None, factor: float = 2.0, jitter: bool = False) -> Generator: """Generates a sequence of geometrically-increasing floats, suitable for usage with `exponential backoff`_. Starts with *start*, increasing by *factor* until *stop* is reached, optionally @@ -791,7 +792,7 @@ def unique(src, key=None): return list(unique_iter(src, key)) -def unique_iter(src, key=None): +def unique_iter(src, key=None) -> Generator: """Yield unique elements from the iterable, *src*, based on *key*, in the order in which they first appeared in *src*. @@ -954,7 +955,7 @@ def first(iterable, default=None, key=None): return next(filter(key, iterable), default) -def flatten_iter(iterable): +def flatten_iter(iterable) -> Generator: """``flatten_iter()`` yields all the elements from *iterable* while collapsing any nested iterables. @@ -1429,7 +1430,7 @@ def __init__(self, size=24): self.count = itertools.count() self.reseed() - def reseed(self): + def reseed(self) -> None: import socket self.pid = os.getpid() self.salt = '-'.join([str(self.pid), @@ -1478,7 +1479,7 @@ class SequentialGUIDerator(GUIDerator): """ - def reseed(self): + def reseed(self) -> None: super().reseed() start_str = self._sha1(self.salt.encode('utf8')).hexdigest() self.start = int(start_str[:self.size], 16) diff --git a/boltons/jsonutils.py b/boltons/jsonutils.py index bf61b03a..3d9254ff 100644 --- a/boltons/jsonutils.py +++ b/boltons/jsonutils.py @@ -35,20 +35,29 @@ .. _JSON Lines: http://jsonlines.org/ """ +from __future__ import annotations - +from collections.abc import Generator import io import os import json +from typing import IO, TYPE_CHECKING, Any, overload +if TYPE_CHECKING: + from typing_extensions import Self DEFAULT_BLOCKSIZE = 4096 __all__ = ['JSONLIterator', 'reverse_iter_lines'] - -def reverse_iter_lines(file_obj, blocksize=DEFAULT_BLOCKSIZE, preseek=True, encoding=None): +@overload +def reverse_iter_lines(file_obj: IO[bytes], blocksize: int = DEFAULT_BLOCKSIZE, preseek: bool = True, encoding: None = None) -> Generator[bytes]: ... +@overload +def reverse_iter_lines(file_obj: IO[str], blocksize: int = DEFAULT_BLOCKSIZE, preseek: bool = True, *, encoding: str) -> Generator[str]: ... +@overload +def reverse_iter_lines(file_obj: IO[str], blocksize: int, preseek: bool, encoding: str) -> Generator[str]: ... +def reverse_iter_lines(file_obj: IO[bytes] | IO[str], blocksize: int = DEFAULT_BLOCKSIZE, preseek: bool = True, encoding: str | None = None) -> Generator[bytes] | Generator[str]: """Returns an iterator over the lines from a file object, in reverse order, i.e., last line first, first line last. Uses the :meth:`file.seek` method of file objects, and is tested compatible with @@ -143,8 +152,8 @@ class JSONLIterator: .. _JSON Lines format: http://jsonlines.org/ """ - def __init__(self, file_obj, - ignore_errors=False, reverse=False, rel_seek=None): + def __init__(self, file_obj: IO[str], + ignore_errors: bool = False, reverse: bool = False, rel_seek: float | None = None): self._reverse = bool(reverse) self._file_obj = file_obj self.ignore_errors = ignore_errors @@ -169,7 +178,7 @@ def __init__(self, file_obj, self._line_iter = iter(self._file_obj) @property - def cur_byte_pos(self): + def cur_byte_pos(self) -> int: "A property representing where in the file the iterator is reading." return self._file_obj.tell() @@ -203,10 +212,10 @@ def _init_rel_seek(self): self._align_to_newline() self._cur_pos = fo.tell() - def __iter__(self): + def __iter__(self) -> Self: return self - def next(self): + def next(self) -> Any: """Yields one :class:`dict` loaded with :func:`json.loads`, advancing the file object by one line. Raises :exc:`StopIteration` upon reaching the end of the file (or beginning, if ``reverse`` was set to ``True``. diff --git a/boltons/listutils.py b/boltons/listutils.py index 06b9da28..3b085c83 100644 --- a/boltons/listutils.py +++ b/boltons/listutils.py @@ -38,10 +38,16 @@ :class:`collections.namedtuple`, check out :mod:`namedutils`. """ +from __future__ import annotations +from collections.abc import Iterable import operator from math import log as math_log from itertools import chain, islice +from typing import TYPE_CHECKING, List, TypeVar + +if TYPE_CHECKING: + from typing_extensions import Self, SupportsIndex try: from .typeutils import make_sentinel @@ -49,6 +55,8 @@ except ImportError: _MISSING = object() +_T = TypeVar("_T") + # TODO: expose splaylist? __all__ = ['BList', 'BarrelList'] @@ -57,7 +65,7 @@ # TODO: keep track of list lengths and bisect to the right list for # faster getitem (and slightly slower setitem and delitem ops) -class BarrelList(list): +class BarrelList(List[_T]): """The ``BarrelList`` is a :class:`list` subtype backed by many dynamically-scaled sublists, to provide better scaling and random insertion/deletion characteristics. It is a subtype of the builtin @@ -95,8 +103,8 @@ class BarrelList(list): _size_factor = 1520 "This size factor is the result of tuning using the tune() function below." - def __init__(self, iterable=None): - self.lists = [[]] + def __init__(self, iterable: Iterable[_T] | None = None): + self.lists: list[list[_T]] = [[]] if iterable: self.extend(iterable) @@ -132,7 +140,7 @@ def _balance_list(self, list_idx): return True return False - def insert(self, index, item): + def insert(self, index: SupportsIndex, item: _T) -> None: if len(self.lists) == 1: self.lists[0].insert(index, item) self._balance_list(0) @@ -144,13 +152,13 @@ def insert(self, index, item): self._balance_list(list_idx) return - def append(self, item): + def append(self, item: _T) -> None: self.lists[-1].append(item) - def extend(self, iterable): + def extend(self, iterable: Iterable[_T]) -> None: self.lists[-1].extend(iterable) - def pop(self, *a): + def pop(self, *a) -> _T: lists = self.lists if len(lists) == 1 and not a: return self.lists[0].pop() @@ -167,7 +175,7 @@ def pop(self, *a): self._balance_list(list_idx) return ret - def iter_slice(self, start, stop, step=None): + def iter_slice(self, start: int | None, stop: int | None, step: int | None = None) -> islice[_T]: iterable = self # TODO: optimization opportunities abound # start_list_idx, stop_list_idx = 0, len(self.lists) if start is None: @@ -186,7 +194,7 @@ def iter_slice(self, start, stop, step=None): # stop_list_idx, stop_rel_idx = self._translate_index(stop) return islice(iterable, start, stop, step) - def del_slice(self, start, stop, step=None): + def del_slice(self, start: int | None, stop: int | None, step: int | None = None) -> None: if step is not None and abs(step) > 1: # punt new_list = chain(self.iter_slice(0, start, step), self.iter_slice(stop, None, step)) @@ -217,7 +225,7 @@ def del_slice(self, start, stop, step=None): __delslice__ = del_slice @classmethod - def from_iterable(cls, it): + def from_iterable(cls, it: Iterable[_T]) -> Self: return cls(it) def __iter__(self): @@ -281,11 +289,11 @@ def __setitem__(self, index, item): raise IndexError() self.lists[list_idx][rel_idx] = item - def __getslice__(self, start, stop): + def __getslice__(self, start: int, stop: int): iter_slice = self.iter_slice(start, stop, 1) return self.from_iterable(iter_slice) - def __setslice__(self, start, stop, sequence): + def __setslice__(self, start: SupportsIndex, stop: SupportsIndex, sequence: Iterable[_T]) -> None: if len(self.lists) == 1: self.lists[0][start:stop] = sequence else: @@ -298,7 +306,7 @@ def __setslice__(self, start, stop, sequence): def __repr__(self): return f'{self.__class__.__name__}({list(self)!r})' - def sort(self): + def sort(self) -> None: # poor pythonist's mergesort, it's faster than sorted(self) # when the lists' average length is greater than 512. if len(self.lists) == 1: @@ -311,12 +319,12 @@ def sort(self): self.lists[0] = tmp_sorted self._balance_list(0) - def reverse(self): + def reverse(self) -> None: for cur in self.lists: cur.reverse() self.lists.reverse() - def count(self, item): + def count(self, item: _T) -> int: return sum([cur.count(item) for cur in self.lists]) def index(self, item): @@ -333,18 +341,19 @@ def index(self, item): BList = BarrelList -class SplayList(list): +class SplayList(List[_T]): """Like a `splay tree`_, the SplayList facilitates moving higher utility items closer to the front of the list for faster access. .. _splay tree: https://en.wikipedia.org/wiki/Splay_tree """ - def shift(self, item_index, dest_index=0): + def shift(self, item_index: SupportsIndex, dest_index: SupportsIndex = 0) -> None: if item_index == dest_index: return item = self.pop(item_index) self.insert(dest_index, item) - def swap(self, item_index, dest_index): + def swap(self, item_index: SupportsIndex, dest_index: SupportsIndex) -> None: + self.__getitem__ self[dest_index], self[item_index] = self[item_index], self[dest_index] diff --git a/boltons/mathutils.py b/boltons/mathutils.py index 1367a541..5771aac3 100644 --- a/boltons/mathutils.py +++ b/boltons/mathutils.py @@ -32,12 +32,16 @@ built-in :mod:`math` module. """ +from __future__ import annotations + +from collections.abc import Iterable from math import ceil as _ceil, floor as _floor import bisect import binascii +from typing import overload -def clamp(x, lower=float('-inf'), upper=float('inf')): +def clamp(x: float, lower: float = float('-inf'), upper: float = float('inf')) -> float: """Limit a value to a given range. Args: @@ -68,8 +72,11 @@ def clamp(x, lower=float('-inf'), upper=float('inf')): % (upper, lower)) return min(max(x, lower), upper) - -def ceil(x, options=None): +@overload +def ceil(x: float, options: None = None) -> int: ... +@overload +def ceil(x: float, options: Iterable[float]) -> float: ... +def ceil(x: float, options: Iterable[float] | None = None) -> float: """Return the ceiling of *x*. If *options* is set, return the smallest integer or float from *options* that is greater than or equal to *x*. @@ -94,7 +101,11 @@ def ceil(x, options=None): return options[i] -def floor(x, options=None): +@overload +def floor(x: float, options: None = None) -> int: ... +@overload +def floor(x: float, options: Iterable[float]) -> float: ... +def floor(x: float, options: Iterable[float] | None = None) -> float: """Return the floor of *x*. If *options* is set, return the largest integer or float from *options* that is less than or equal to *x*. @@ -137,7 +148,7 @@ class Bits: ''' __slots__ = ('val', 'len') - def __init__(self, val=0, len_=None): + def __init__(self, val: int | list[bool] | bytes | str = 0, len_: int | None = None): if type(val) is not int: if type(val) is list: val = ''.join(['1' if e else '0' for e in val]) @@ -166,7 +177,7 @@ def __init__(self, val=0, len_=None): self.val = val # data is stored internally as integer self.len = len_ - def __getitem__(self, k): + def __getitem__(self, k) -> Bits | bool: if type(k) is slice: return Bits(self.as_bin()[k]) if type(k) is int: @@ -175,49 +186,49 @@ def __getitem__(self, k): return bool((1 << (self.len - k - 1)) & self.val) raise TypeError(type(k)) - def __len__(self): + def __len__(self) -> int: return self.len - def __eq__(self, other): + def __eq__(self, other) -> bool: if type(self) is not type(other): return NotImplemented return self.val == other.val and self.len == other.len - def __or__(self, other): + def __or__(self, other: Bits) -> Bits: if type(self) is not type(other): return NotImplemented return Bits(self.val | other.val, max(self.len, other.len)) - def __and__(self, other): + def __and__(self, other: Bits) -> Bits: if type(self) is not type(other): return NotImplemented return Bits(self.val & other.val, max(self.len, other.len)) - def __lshift__(self, other): + def __lshift__(self, other: int) -> Bits: return Bits(self.val << other, self.len + other) - def __rshift__(self, other): + def __rshift__(self, other: int) -> Bits: return Bits(self.val >> other, self.len - other) - def __hash__(self): + def __hash__(self) -> int: return hash(self.val) - def as_list(self): + def as_list(self) -> list[bool]: return [c == '1' for c in self.as_bin()] - def as_bin(self): + def as_bin(self) -> str: return f'{{0:0{self.len}b}}'.format(self.val) - def as_hex(self): + def as_hex(self) -> str: # make template to pad out to number of bytes necessary to represent bits tmpl = f'%0{2 * (self.len // 8 + ((self.len % 8) != 0))}X' ret = tmpl % self.val return ret - def as_int(self): + def as_int(self) -> int: return self.val - def as_bytes(self): + def as_bytes(self) -> bytes: return binascii.unhexlify(self.as_hex()) @classmethod @@ -237,7 +248,7 @@ def from_hex(cls, hex): return cls(hex) @classmethod - def from_int(cls, int_, len_=None): + def from_int(cls, int_, len_: int | None = None): return cls(int_, len_) @classmethod diff --git a/boltons/mboxutils.py b/boltons/mboxutils.py index 299d1059..ac8a33fc 100644 --- a/boltons/mboxutils.py +++ b/boltons/mboxutils.py @@ -34,9 +34,15 @@ .. _mbox: https://en.wikipedia.org/wiki/Mbox """ +from __future__ import annotations + +from collections.abc import Callable import mailbox import tempfile +from typing import IO, TYPE_CHECKING +if TYPE_CHECKING: + from _typeshed import StrPath DEFAULT_MAXMEM = 4 * 1024 * 1024 # 4MB @@ -67,11 +73,11 @@ class mbox_readonlydir(mailbox.mbox): .. _Heirloom mailx: http://heirloom.sourceforge.net/mailx.html """ - def __init__(self, path, factory=None, create=True, maxmem=1024 * 1024): + def __init__(self, path: StrPath, factory: Callable[[IO], mailbox.mboxMessage] | None = None, create: bool = True, maxmem: int = 1024 * 1024): mailbox.mbox.__init__(self, path, factory, create) self.maxmem = maxmem - def flush(self): + def flush(self) -> None: """Write any pending changes to disk. This is called on mailbox close and is usually not called explicitly. diff --git a/boltons/namedutils.py b/boltons/namedutils.py index 0311f814..dae36c4d 100644 --- a/boltons/namedutils.py +++ b/boltons/namedutils.py @@ -45,7 +45,9 @@ skinnier approach, you'll probably have to look to C. """ +from __future__ import annotations +from collections.abc import Iterable import sys as _sys from collections import OrderedDict from keyword import iskeyword as _iskeyword @@ -120,7 +122,7 @@ def __getstate__(self): {field_defs} ''' -def namedtuple(typename, field_names, verbose=False, rename=False): +def namedtuple(typename: str, field_names: str | Iterable[str], verbose: bool = False, rename: bool = False): """Returns a new subclass of tuple with named fields. >>> Point = namedtuple('Point', ['x', 'y']) @@ -279,7 +281,7 @@ def __getstate__(self): ''' -def namedlist(typename, field_names, verbose=False, rename=False): +def namedlist(typename: str, field_names: str | Iterable[str], verbose: bool = False, rename: bool = False): """Returns a new subclass of list with named fields. >>> Point = namedlist('Point', ['x', 'y']) diff --git a/boltons/pathutils.py b/boltons/pathutils.py index a8cb2602..be2cb2f9 100644 --- a/boltons/pathutils.py +++ b/boltons/pathutils.py @@ -41,17 +41,22 @@ The :func:`shrinkuser` function replaces your home directory with a tilde. """ +from __future__ import annotations + from os.path import (expanduser, expandvars, join, normpath, split, splitext) import os +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from _typeshed import StrPath __all__ = [ 'augpath', 'shrinkuser', 'expandpath', ] -def augpath(path, suffix='', prefix='', ext=None, base=None, dpath=None, - multidot=False): +def augpath(path: StrPath, suffix: str = '', prefix: str = '', ext: str | None = None, base: str | None = None, dpath: str | None = None, + multidot: bool = False) -> str: """ Augment a path by modifying its components. @@ -131,7 +136,7 @@ def augpath(path, suffix='', prefix='', ext=None, base=None, dpath=None, return newpath -def shrinkuser(path, home='~'): +def shrinkuser(path: StrPath, home: str = '~') -> str: """ Inverse of :func:`os.path.expanduser`. @@ -162,7 +167,7 @@ def shrinkuser(path, home='~'): return path -def expandpath(path): +def expandpath(path: StrPath) -> str: """ Shell-like expansion of environment variables and tilde home directory. diff --git a/boltons/py.typed b/boltons/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/boltons/queueutils.py b/boltons/queueutils.py index f6e4e4c2..49e408c5 100644 --- a/boltons/queueutils.py +++ b/boltons/queueutils.py @@ -61,7 +61,7 @@ 3 """ - +from __future__ import annotations from heapq import heappush, heappop from bisect import insort @@ -121,7 +121,7 @@ def _push_entry(backend, entry): def _pop_entry(backend): pass # abstract - def add(self, task, priority=None): + def add(self, task, priority: int | None = None) -> None: """ Add a task to the queue, or change the *task*'s priority if *task* is already in the queue. *task* can be any hashable object, @@ -137,7 +137,7 @@ def add(self, task, priority=None): self._entry_map[task] = entry self._push_entry(self._pq, entry) - def remove(self, task): + def remove(self, task) -> None: """Remove a task from the priority queue. Raises :exc:`KeyError` if the *task* is absent. """ @@ -184,7 +184,7 @@ def pop(self, default=_REMOVED): raise IndexError('pop on empty queue') return task - def __len__(self): + def __len__(self) -> int: "Return the number of tasks in the queue." return len(self._entry_map) diff --git a/boltons/setutils.py b/boltons/setutils.py index 3ed5ab32..12292a3d 100644 --- a/boltons/setutils.py +++ b/boltons/setutils.py @@ -39,11 +39,23 @@ characteristics of Python's built-in set implementation. """ +from __future__ import annotations +import operator from bisect import bisect_left -from collections.abc import MutableSet +from collections.abc import ( + Collection, + Container, + Generator, + Iterator, + MutableSet, +) from itertools import chain, islice -import operator +import sys +from typing import TYPE_CHECKING, TypeVar, overload, Iterable + +if TYPE_CHECKING: + from typing_extensions import Self, Literal, SupportsIndex try: from .typeutils import make_sentinel @@ -64,6 +76,11 @@ # order of the 'other' inputs and put self last (to try and maintain # insertion order) +_T_co = TypeVar("_T_co", covariant=True) +if sys.version_info >= (3, 8): + from typing import Protocol + class _RSub(Iterable[_T_co], Protocol): + def __new__(cls: type[_RSub[_T_co]], __param: list[_T_co]) -> _RSub[_T_co]: ... class IndexedSet(MutableSet): """``IndexedSet`` is a :class:`collections.MutableSet` that maintains @@ -108,7 +125,7 @@ class IndexedSet(MutableSet): Otherwise, the API strives to be as complete a union of the :class:`list` and :class:`set` APIs as possible. """ - def __init__(self, other=None): + def __init__(self, other: Iterable | None = None): self.item_index_map = dict() self.item_list = [] self.dead_indices = [] @@ -201,16 +218,16 @@ def _add_dead(self, start, stop=None): return # common operations (shared by set and list) - def __len__(self): + def __len__(self) -> int: return len(self.item_index_map) - def __contains__(self, item): + def __contains__(self, item: object) -> bool: return item in self.item_index_map - def __iter__(self): + def __iter__(self) -> Iterator: return (item for item in self.item_list if item is not _MISSING) - def __reversed__(self): + def __reversed__(self) -> Generator: item_list = self.item_list return (item for item in reversed(item_list) if item is not _MISSING) @@ -226,18 +243,18 @@ def __eq__(self, other): return False @classmethod - def from_iterable(cls, it): + def from_iterable(cls, it: Iterable) -> Self: "from_iterable(it) -> create a set from an iterable" return cls(it) # set operations - def add(self, item): + def add(self, item) -> None: "add(item) -> add item to the set" if item not in self.item_index_map: self.item_index_map[item] = len(self.item_list) self.item_list.append(item) - def remove(self, item): + def remove(self, item) -> None: "remove(item) -> remove item from the set, raises if not present" try: didx = self.item_index_map.pop(item) @@ -247,20 +264,20 @@ def remove(self, item): self._add_dead(didx) self._cull() - def discard(self, item): + def discard(self, item) -> None: "discard(item) -> discard item from the set (does not raise)" try: self.remove(item) except KeyError: pass - def clear(self): + def clear(self) -> None: "clear() -> empty the set" del self.item_list[:] del self.dead_indices[:] self.item_index_map.clear() - def isdisjoint(self, other): + def isdisjoint(self, other: Iterable) -> bool: "isdisjoint(other) -> return True if no overlap with other" iim = self.item_index_map for k in other: @@ -268,7 +285,7 @@ def isdisjoint(self, other): return False return True - def issubset(self, other): + def issubset(self, other: Collection) -> bool: "issubset(other) -> return True if other contains this set" if len(other) < len(self): return False @@ -277,7 +294,7 @@ def issubset(self, other): return False return True - def issuperset(self, other): + def issuperset(self, other: Collection) -> bool: "issuperset(other) -> return True if set contains other" if len(other) > len(self): return False @@ -287,11 +304,11 @@ def issuperset(self, other): return False return True - def union(self, *others): + def union(self, *others) -> Self: "union(*others) -> return a new set containing this set and others" return self.from_iterable(chain(self, *others)) - def iter_intersection(self, *others): + def iter_intersection(self, *others: Container) -> Generator: "iter_intersection(*others) -> iterate over elements also in others" for k in self: for other in others: @@ -301,14 +318,14 @@ def iter_intersection(self, *others): yield k return - def intersection(self, *others): + def intersection(self, *others: Container) -> Self: "intersection(*others) -> get a set with overlap of this and others" if len(others) == 1: other = others[0] return self.from_iterable(k for k in self if k in other) return self.from_iterable(self.iter_intersection(*others)) - def iter_difference(self, *others): + def iter_difference(self, *others: Iterable) -> Generator: "iter_difference(*others) -> iterate over elements not in others" for k in self: for other in others: @@ -318,14 +335,14 @@ def iter_difference(self, *others): yield k return - def difference(self, *others): + def difference(self, *others: Iterable) -> Self: "difference(*others) -> get a new set with elements not in others" if len(others) == 1: other = others[0] return self.from_iterable(k for k in self if k not in other) return self.from_iterable(self.iter_difference(*others)) - def symmetric_difference(self, *others): + def symmetric_difference(self, *others: Container) -> Self: "symmetric_difference(*others) -> XOR set of this and others" ret = self.union(*others) return ret.difference(self.intersection(*others)) @@ -335,12 +352,12 @@ def symmetric_difference(self, *others): __sub__ = difference __xor__ = __rxor__ = symmetric_difference - def __rsub__(self, other): + def __rsub__(self, other: _RSub[_T_co]) -> _RSub[_T_co]: vals = [x for x in other if x not in self] return type(other)(vals) # in-place set operations - def update(self, *others): + def update(self, *others: Iterable) -> None: "update(*others) -> add values from one or more iterables" if not others: return # raise? @@ -351,19 +368,19 @@ def update(self, *others): for o in other: self.add(o) - def intersection_update(self, *others): + def intersection_update(self, *others: Iterable) -> None: "intersection_update(*others) -> discard self.difference(*others)" for val in self.difference(*others): self.discard(val) - def difference_update(self, *others): + def difference_update(self, *others: Container) -> None: "difference_update(*others) -> discard self.intersection(*others)" if self in others: self.clear() for val in self.intersection(*others): self.discard(val) - def symmetric_difference_update(self, other): # note singular 'other' + def symmetric_difference_update(self, other: Iterable) -> None: # note singular 'other' "symmetric_difference_update(other) -> in-place XOR with other" if self is other: self.clear() @@ -389,7 +406,7 @@ def __ixor__(self, *others): self.symmetric_difference_update(*others) return self - def iter_slice(self, start, stop, step=None): + def iter_slice(self, start: int | None, stop: int | None, step: int | None = None) -> islice: "iterate over a slice of the set" iterable = self if start is not None: @@ -402,7 +419,11 @@ def iter_slice(self, start, stop, step=None): return islice(iterable, start, stop, step) # list operations - def __getitem__(self, index): + @overload + def __getitem__(self, index: slice) -> Self: ... + @overload + def __getitem__(self, index: SupportsIndex): ... + def __getitem__(self, index: slice | SupportsIndex): try: start, stop, step = index.start, index.stop, index.step except AttributeError: @@ -419,7 +440,7 @@ def __getitem__(self, index): raise IndexError('IndexedSet index out of range') return ret - def pop(self, index=None): + def pop(self, index: int | None = None): "pop(index) -> remove the item at a given index (-1 by default)" item_index_map = self.item_index_map len_self = len(item_index_map) @@ -435,13 +456,13 @@ def pop(self, index=None): self._cull() return ret - def count(self, val): + def count(self, val) -> Literal[0, 1]: "count(val) -> count number of instances of value (0 or 1)" if val in self.item_index_map: return 1 return 0 - def reverse(self): + def reverse(self) -> None: "reverse() -> reverse the contents of the set in-place" reversed_list = list(reversed(self)) self.item_list[:] = reversed_list @@ -449,7 +470,7 @@ def reverse(self): self.item_index_map[item] = i del self.dead_indices[:] - def sort(self, **kwargs): + def sort(self, **kwargs) -> None: "sort() -> sort the contents of the set in-place" sorted_list = sorted(self, **kwargs) if sorted_list == self.item_list: @@ -459,7 +480,7 @@ def sort(self, **kwargs): self.item_index_map[item] = i del self.dead_indices[:] - def index(self, val): + def index(self, val) -> int: "index(val) -> get the index of a value, raises if not present" try: return self._get_apparent_index(self.item_index_map[val]) @@ -468,7 +489,7 @@ def index(self, val): raise ValueError(f'{val!r} is not in {cn}') -def complement(wrapped): +def complement(wrapped: Iterable) -> _ComplementSet: """Given a :class:`set`, convert it to a **complement set**. Whereas a :class:`set` keeps track of what it contains, a @@ -583,7 +604,7 @@ class _ComplementSet: """ __slots__ = ('_included', '_excluded') - def __init__(self, included=None, excluded=None): + def __init__(self, included: set | frozenset | None = None, excluded: set | frozenset | None = None): if included is None: assert type(excluded) in (set, frozenset) elif excluded is None: @@ -597,7 +618,7 @@ def __repr__(self): return f'complement({repr(self._excluded)})' return f'complement(complement({repr(self._included)}))' - def complemented(self): + def complemented(self) -> _ComplementSet: '''return a complement of the current set''' if type(self._included) is frozenset or type(self._excluded) is frozenset: return _ComplementSet(included=self._excluded, excluded=self._included) @@ -607,23 +628,23 @@ def complemented(self): __invert__ = complemented - def complement(self): + def complement(self) -> None: '''convert the current set to its complement in-place''' self._included, self._excluded = self._excluded, self._included - def __contains__(self, item): + def __contains__(self, item) -> bool: if self._included is None: - return not item in self._excluded + return item not in self._excluded return item in self._included - def add(self, item): + def add(self, item) -> None: if self._included is None: if item in self._excluded: self._excluded.remove(item) else: self._included.add(item) - def remove(self, item): + def remove(self, item) -> None: if self._included is None: self._excluded.add(item) else: @@ -634,13 +655,13 @@ def pop(self): raise NotImplementedError # self.missing.add(random.choice(gc.objects())) return self._included.pop() - def intersection(self, other): + def intersection(self, other: set | frozenset | _ComplementSet) -> _ComplementSet: try: return self & other except NotImplementedError: raise TypeError('argument must be another set or complement(set)') - def __and__(self, other): + def __and__(self, other: set | frozenset | _ComplementSet) -> _ComplementSet: inc, exc = _norm_args_notimplemented(other) if inc is NotImplemented: return NotImplemented @@ -657,7 +678,7 @@ def __and__(self, other): __rand__ = __and__ - def __iand__(self, other): + def __iand__(self, other: set | frozenset | _ComplementSet) -> Self: inc, exc = _norm_args_notimplemented(other) if inc is NotImplemented: return NotImplemented @@ -674,13 +695,13 @@ def __iand__(self, other): self._included &= inc return self - def union(self, other): + def union(self, other: set | frozenset | _ComplementSet) -> _ComplementSet: try: return self | other except NotImplementedError: raise TypeError('argument must be another set or complement(set)') - def __or__(self, other): + def __or__(self, other: set | frozenset | _ComplementSet) -> _ComplementSet: inc, exc = _norm_args_notimplemented(other) if inc is NotImplemented: return NotImplemented @@ -697,7 +718,7 @@ def __or__(self, other): __ror__ = __or__ - def __ior__(self, other): + def __ior__(self, other: set | frozenset | _ComplementSet) -> Self: inc, exc = _norm_args_notimplemented(other) if inc is NotImplemented: return NotImplemented @@ -713,7 +734,7 @@ def __ior__(self, other): self._included |= inc return self - def update(self, items): + def update(self, items: Iterable) -> None: if type(items) in (set, frozenset): inc, exc = items, None elif type(items) is _ComplementSet: @@ -732,7 +753,7 @@ def update(self, items): else: # + + self._included.update(inc) - def discard(self, items): + def discard(self, items: Iterable) -> None: if type(items) in (set, frozenset): inc, exc = items, None elif type(items) is _ComplementSet: @@ -750,13 +771,13 @@ def discard(self, items): else: # + + self._included.discard(inc) - def symmetric_difference(self, other): + def symmetric_difference(self, other: set | frozenset | _ComplementSet) -> _ComplementSet: try: return self ^ other except NotImplementedError: raise TypeError('argument must be another set or complement(set)') - def __xor__(self, other): + def __xor__(self, other: set | frozenset | _ComplementSet) -> _ComplementSet: inc, exc = _norm_args_notimplemented(other) if inc is NotImplemented: return NotImplemented @@ -775,7 +796,7 @@ def __xor__(self, other): __rxor__ = __xor__ - def symmetric_difference_update(self, other): + def symmetric_difference_update(self, other: set | frozenset | _ComplementSet) -> None: inc, exc = _norm_args_typeerror(other) if self._included is None: if exc is None: # - + @@ -790,7 +811,7 @@ def symmetric_difference_update(self, other): else: # + + self._included.symmetric_difference_update(inc) - def isdisjoint(self, other): + def isdisjoint(self, other: set | frozenset | _ComplementSet) -> bool: inc, exc = _norm_args_typeerror(other) if inc is NotImplemented: return NotImplemented @@ -805,14 +826,14 @@ def isdisjoint(self, other): else: # + + return self._included.isdisjoint(inc) - def issubset(self, other): + def issubset(self, other: set | frozenset | _ComplementSet) -> bool: '''everything missing from other is also missing from self''' try: return self <= other except NotImplementedError: raise TypeError('argument must be another set or complement(set)') - def __le__(self, other): + def __le__(self, other: set | frozenset | _ComplementSet) -> bool: inc, exc = _norm_args_notimplemented(other) if inc is NotImplemented: return NotImplemented @@ -829,7 +850,7 @@ def __le__(self, other): else: # + + return self._included.issubset(inc) - def __lt__(self, other): + def __lt__(self, other: set | frozenset | _ComplementSet) -> bool: inc, exc = _norm_args_notimplemented(other) if inc is NotImplemented: return NotImplemented @@ -846,14 +867,14 @@ def __lt__(self, other): else: # + + return self._included < inc - def issuperset(self, other): + def issuperset(self, other: set | frozenset | _ComplementSet) -> bool: '''everything missing from self is also missing from super''' try: return self >= other except NotImplementedError: raise TypeError('argument must be another set or complement(set)') - def __ge__(self, other): + def __ge__(self, other: set | frozenset | _ComplementSet) -> bool: inc, exc = _norm_args_notimplemented(other) if inc is NotImplemented: return NotImplemented @@ -868,7 +889,7 @@ def __ge__(self, other): else: # + + return self._included.issupserset(inc) - def __gt__(self, other): + def __gt__(self, other: set | frozenset | _ComplementSet) -> bool: inc, exc = _norm_args_notimplemented(other) if inc is NotImplemented: return NotImplemented @@ -883,13 +904,13 @@ def __gt__(self, other): else: # + + return self._included > inc - def difference(self, other): + def difference(self, other: set | frozenset | _ComplementSet) -> _ComplementSet: try: return self - other except NotImplementedError: raise TypeError('argument must be another set or complement(set)') - def __sub__(self, other): + def __sub__(self, other: set | frozenset | _ComplementSet) -> _ComplementSet: inc, exc = _norm_args_notimplemented(other) if inc is NotImplemented: return NotImplemented @@ -904,7 +925,7 @@ def __sub__(self, other): else: # + + return _ComplementSet(included=self._included.difference(inc)) - def __rsub__(self, other): + def __rsub__(self, other: set | frozenset | _ComplementSet) -> _ComplementSet: inc, exc = _norm_args_notimplemented(other) if inc is NotImplemented: return NotImplemented @@ -920,13 +941,13 @@ def __rsub__(self, other): else: # + + return _ComplementSet(included=inc.difference(self._included)) - def difference_update(self, other): + def difference_update(self, other: set | frozenset | _ComplementSet) -> None: try: self -= other except NotImplementedError: raise TypeError('argument must be another set or complement(set)') - def __isub__(self, other): + def __isub__(self, other: set | frozenset | _ComplementSet) -> Self: inc, exc = _norm_args_notimplemented(other) if inc is NotImplemented: return NotImplemented @@ -952,17 +973,17 @@ def __eq__(self, other): def __hash__(self): return hash(self._included) ^ hash(self._excluded) - def __len__(self): + def __len__(self) -> int: if self._included is not None: return len(self._included) raise NotImplementedError('complemented sets have undefined length') - def __iter__(self): + def __iter__(self) -> Iterator: if self._included is not None: return iter(self._included) raise NotImplementedError('complemented sets have undefined contents') - def __bool__(self): + def __bool__(self) -> bool: if self._included is not None: return bool(self._included) return True diff --git a/boltons/statsutils.py b/boltons/statsutils.py index 36a82230..4c52de27 100644 --- a/boltons/statsutils.py +++ b/boltons/statsutils.py @@ -125,14 +125,20 @@ """ +from __future__ import annotations import bisect -from math import floor, ceil from collections import Counter +from collections.abc import Callable, Iterable +from math import ceil, floor +from typing import TYPE_CHECKING, Iterator, overload +if TYPE_CHECKING: + from _typeshed import ConvertibleToFloat + from typing_extensions import Self, Literal class _StatsProperty: - def __init__(self, name, func): + def __init__(self, name: str, func: Callable[[Stats], float]): self.name = name self.func = func self.internal_name = '_' + name @@ -141,7 +147,11 @@ def __init__(self, name, func): pre_doctest_doc, _, _ = doc.partition('>>>') self.__doc__ = pre_doctest_doc - def __get__(self, obj, objtype=None): + @overload + def __get__(self, obj: None, objtype: object = None) -> Self: ... + @overload + def __get__(self, obj: Stats, objtype: object = None) -> float: ... + def __get__(self, obj: Stats | None, objtype: object = None) -> float | Self: if obj is None: return self if not obj.data: @@ -152,7 +162,6 @@ def __get__(self, obj, objtype=None): setattr(obj, self.internal_name, self.func(obj)) return getattr(obj, self.internal_name) - class Stats: """The ``Stats`` type is used to represent a group of unordered statistical datapoints for calculations such as mean, median, and @@ -171,11 +180,17 @@ class Stats: step for a little speed boost. Defaults to False. """ - def __init__(self, data, default=0.0, use_copy=True, is_sorted=False): + @overload + def __init__(self, data: list[float], default: float = 0.0, *, use_copy: Literal[False], is_sorted: bool = False) -> None: ... + @overload + def __init__(self, data: list[float], default: float, use_copy: Literal[False], is_sorted: bool = False) -> None: ... + @overload + def __init__(self, data: Iterable[float], default: float = 0.0, use_copy: Literal[True] = True, is_sorted: bool = False) -> None: ... + def __init__(self, data: Iterable[float], default: float = 0.0, use_copy: bool = True, is_sorted: bool = False) -> None: self._use_copy = use_copy self._is_sorted = is_sorted if use_copy: - self.data = list(data) + self.data: list[float] = list(data) else: self.data = data @@ -186,10 +201,10 @@ def __init__(self, data, default=0.0, use_copy=True, is_sorted=False): _StatsProperty)] self._pearson_precision = 0 - def __len__(self): + def __len__(self) -> int: return len(self.data) - def __iter__(self): + def __iter__(self) -> Iterator[float]: return iter(self.data) def _get_sorted_data(self): @@ -207,7 +222,7 @@ def _get_sorted_data(self): self.data.sort() return self.data - def clear_cache(self): + def clear_cache(self) -> None: """``Stats`` objects automatically cache intermediary calculations that can be reused. For instance, accessing the ``std_dev`` attribute after the ``variance`` attribute will be @@ -460,7 +475,7 @@ def _get_quantile(sorted_data, q): return data[idx_f] return (data[idx_f] * (idx_c - idx)) + (data[idx_c] * (idx - idx_f)) - def get_quantile(self, q): + def get_quantile(self, q: ConvertibleToFloat) -> float: """Get a quantile from the dataset. Quantiles are floating point values between ``0.0`` and ``1.0``, with ``0.0`` representing the minimum value in the dataset and ``1.0`` representing the @@ -476,7 +491,7 @@ def get_quantile(self, q): return self.default return self._get_quantile(self._get_sorted_data(), q) - def get_zscore(self, value): + def get_zscore(self, value: float) -> float: """Get the z-score for *value* in the group. If the standard deviation is 0, 0 inf or -inf will be returned to indicate whether the value is equal to, greater than or below the group's mean. @@ -491,7 +506,7 @@ def get_zscore(self, value): return float('-inf') return (float(value) - mean) / self.std_dev - def trim_relative(self, amount=0.15): + def trim_relative(self, amount: ConvertibleToFloat = 0.15) -> None: """A utility function used to cut a proportion of values off each end of a list of values. This has the effect of limiting the effect of outliers. @@ -552,7 +567,7 @@ def _get_bin_bounds(self, count=None, with_max=False): return bins - def get_histogram_counts(self, bins=None, **kw): + def get_histogram_counts(self, bins: int | list[float] | None = None, **kw) -> list[tuple[float, int]]: """Produces a list of ``(bin, count)`` pairs comprising a histogram of the Stats object's data, using fixed-width bins. See :meth:`Stats.format_histogram` for more details. @@ -602,7 +617,7 @@ def get_histogram_counts(self, bins=None, **kw): return bin_counts - def format_histogram(self, bins=None, **kw): + def format_histogram(self, bins: int | list[float] | None = None, **kw) -> str: """Produces a textual histogram of the data, using fixed-width bins, allowing for simple visualization, even in console environments. @@ -655,7 +670,7 @@ def format_histogram(self, bins=None, **kw): width=width, format_bin=format_bin) - def describe(self, quantiles=None, format=None): + def describe(self, quantiles: Iterable[float] | None = None, format: str | None = None) -> dict[str, float] | list[tuple[str, float]] | str: """Provides standard summary statistics for the data in the Stats object, in one of several convenient formats. @@ -722,8 +737,7 @@ def describe(self, quantiles=None, format=None): for label, val in items]) return ret - -def describe(data, quantiles=None, format=None): +def describe(data: Iterable[float], quantiles: Iterable[float] | None = None, format: str | None = None) -> dict[str, float] | list[tuple[str, float]] | str: """A convenience function to get standard summary statistics useful for describing most data. See :meth:`Stats.describe` for more details. @@ -767,8 +781,7 @@ def stats_helper(data, default=0.0): del attr_name del func - -def format_histogram_counts(bin_counts, width=None, format_bin=None): +def format_histogram_counts(bin_counts: Iterable[tuple[float, int]], width: int | None = None, format_bin: Callable | None = None) -> str: """The formatting logic behind :meth:`Stats.format_histogram`, which takes the output of :meth:`Stats.get_histogram_counts`, and passes them to this function. diff --git a/boltons/strutils.py b/boltons/strutils.py index 1d43f2b0..cc33262a 100644 --- a/boltons/strutils.py +++ b/boltons/strutils.py @@ -33,22 +33,25 @@ common capabilities missing from the standard library, several of them provided by ``strutils``. """ +from __future__ import annotations - -import builtins import re import sys +from typing import TYPE_CHECKING, Dict, overload import uuid import zlib import string import unicodedata import collections -from collections.abc import Mapping +from collections.abc import Callable, Generator, Iterable, Mapping, Sized from gzip import GzipFile from html.parser import HTMLParser from html import entities as htmlentitydefs from io import BytesIO as StringIO +if TYPE_CHECKING: + from _typeshed import ReadableBuffer + from typing_extensions import Literal __all__ = ['camel2under', 'under2camel', 'slugify', 'split_punct_ws', 'unit_len', 'ordinalize', 'cardinalize', 'pluralize', 'singularize', @@ -65,7 +68,7 @@ _camel2under_re = re.compile('((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))') -def camel2under(camel_string): +def camel2under(camel_string: str) -> str: """Converts a camelcased string to underscores. Useful for turning a class name into a function name. @@ -75,7 +78,7 @@ class name into a function name. return _camel2under_re.sub(r'_\1', camel_string).lower() -def under2camel(under_string): +def under2camel(under_string: str) -> str: """Converts an underscored string to camelcased. Useful for turning a function name into a class name. @@ -84,8 +87,13 @@ def under2camel(under_string): """ return ''.join(w.capitalize() or '_' for w in under_string.split('_')) - -def slugify(text, delim='_', lower=True, ascii=False): +@overload +def slugify(text: str, delim: str = "_", lower: bool = True, *, ascii: Literal[True]) -> bytes: ... +@overload +def slugify(text: str, delim: str, lower: bool, ascii: Literal[True]) -> bytes: ... +@overload +def slugify(text: str, delim: str = "_", lower: bool = True, ascii: Literal[False] = False) -> str: ... +def slugify(text: str, delim: str = "_", lower: bool = True, ascii: bool = False) -> bytes | str: """ A basic function that turns text full of scary characters (i.e., punctuation and whitespace), into a relatively safe @@ -111,7 +119,7 @@ def slugify(text, delim='_', lower=True, ascii=False): return ret -def split_punct_ws(text): +def split_punct_ws(text: str) -> list[str]: """While :meth:`str.split` will split on whitespace, :func:`split_punct_ws` will split on punctuation and whitespace. This used internally by :func:`slugify`, above. @@ -122,7 +130,7 @@ def split_punct_ws(text): return [w for w in _punct_re.split(text) if w] -def unit_len(sized_iterable, unit_noun='item'): # TODO: len_units()/unitize()? +def unit_len(sized_iterable: Sized, unit_noun: str = 'item') -> str: # TODO: len_units()/unitize()? """Returns a plain-English description of an iterable's :func:`len()`, conditionally pluralized with :func:`cardinalize`, detailed below. @@ -146,7 +154,7 @@ def unit_len(sized_iterable, unit_noun='item'): # TODO: len_units()/unitize()? '3': 'rd'} # 'th' is the default -def ordinalize(number, ext_only=False): +def ordinalize(number: int | str, ext_only: bool = False) -> str: """Turns *number* into its cardinal form, i.e., 1st, 2nd, 3rd, 4th, etc. If the last character isn't a digit, it returns the string value unchanged. @@ -182,7 +190,7 @@ def ordinalize(number, ext_only=False): return numstr + ext -def cardinalize(unit_noun, count): +def cardinalize(unit_noun: str, count: int) -> str: """Conditionally pluralizes a singular word *unit_noun* if *count* is not one, preserving case when possible. @@ -197,7 +205,7 @@ def cardinalize(unit_noun, count): return pluralize(unit_noun) -def singularize(word): +def singularize(word: str) -> str: """Semi-intelligently converts an English plural *word* to its singular form, preserving case pattern. @@ -231,7 +239,7 @@ def singularize(word): return _match_case(orig_word, singular) -def pluralize(word): +def pluralize(word: str) -> str: """Semi-intelligently converts an English *word* from singular form to plural, preserving case pattern. @@ -313,7 +321,7 @@ def _match_case(master, disciple): HASHTAG_RE = re.compile(r"(?:^|\s)[＃#]{1}(\w+)", re.UNICODE) -def find_hashtags(string): +def find_hashtags(string: str) -> list[str]: """Finds and returns all hashtags in a string, with the hashmark removed. Supports full-width hashmarks for Asian languages and does not false-positive on URL anchors. @@ -330,7 +338,7 @@ def find_hashtags(string): return HASHTAG_RE.findall(string) -def a10n(string): +def a10n(string: str) -> str: """That thing where "internationalization" becomes "i18n", what's it called? Abbreviation? Oh wait, no: ``a10n``. (It's actually a form of `numeronym`_.) @@ -367,7 +375,7 @@ def a10n(string): ''', re.VERBOSE) -def strip_ansi(text): +def strip_ansi(text: str | bytes | bytearray) -> str: """Strips ANSI escape codes from *text*. Useful for the occasional time when a log or redirected output accidentally captures console color codes and the like. @@ -405,7 +413,7 @@ def strip_ansi(text): return cleaned -def asciify(text, ignore=False): +def asciify(text: str | bytes | bytearray, ignore: bool = False) -> bytes: """Converts a unicode or bytestring, *text*, into a bytestring with just ascii characters. Performs basic deaccenting for all you Europhiles out there. @@ -438,7 +446,7 @@ def asciify(text, ignore=False): return ret -def is_ascii(text): +def is_ascii(text: str) -> bool: """Check if a string or bytestring, *text*, is composed of ascii characters only. Raises :exc:`ValueError` if argument is not text. @@ -465,9 +473,9 @@ def is_ascii(text): return True -class DeaccenterDict(dict): +class DeaccenterDict(Dict[int, int]): "A small caching dictionary for deaccenting." - def __missing__(self, key): + def __missing__(self, key: int) -> int: ch = self.get(key) if ch is not None: return ch @@ -547,7 +555,7 @@ def __missing__(self, key): _SIZE_RANGES = list(zip(_SIZE_BOUNDS, _SIZE_BOUNDS[1:])) -def bytes2human(nbytes, ndigits=0): +def bytes2human(nbytes: int, ndigits: int = 0) -> str: """Turns an integer value of *nbytes* into a human readable format. Set *ndigits* to control how many digits after the decimal point should be shown (default ``0``). @@ -574,19 +582,19 @@ def __init__(self): self.reset() self.strict = False self.convert_charrefs = True - self.result = [] + self.result: list[str] = [] - def handle_data(self, d): + def handle_data(self, d: str) -> None: self.result.append(d) - def handle_charref(self, number): + def handle_charref(self, number: str) -> None: if number[0] == 'x' or number[0] == 'X': codepoint = int(number[1:], 16) else: codepoint = int(number) self.result.append(chr(codepoint)) - def handle_entityref(self, name): + def handle_entityref(self, name: str) -> None: try: codepoint = htmlentitydefs.name2codepoint[name] except KeyError: @@ -594,11 +602,11 @@ def handle_entityref(self, name): else: self.result.append(chr(codepoint)) - def get_text(self): + def get_text(self) -> str: return ''.join(self.result) -def html2text(html): +def html2text(html: str) -> str: """Strips tags from HTML text, returning markup-free text. Also, does a best effort replacement of entities like " " @@ -616,7 +624,7 @@ def html2text(html): _NON_EMPTY_GZIP_BYTES = b'\x1f\x8b\x08\x08\xbc\xf7\xb9U\x00\x03not_empty\x00K\xaa,I-N\xcc\xc8\xafT\xe4\x02\x00\xf3nb\xbf\x0b\x00\x00\x00' -def gunzip_bytes(bytestring): +def gunzip_bytes(bytestring: ReadableBuffer) -> bytes: """The :mod:`gzip` module is great if you have a file or file-like object, but what if you just have bytes. StringIO is one possibility, but it's often faster, easier, and simpler to just @@ -631,7 +639,7 @@ def gunzip_bytes(bytestring): return zlib.decompress(bytestring, 16 + zlib.MAX_WBITS) -def gzip_bytes(bytestring, level=6): +def gzip_bytes(bytestring: ReadableBuffer, level: int = 6) -> bytes: """Turn some bytes into some compressed bytes. >>> len(gzip_bytes(b'a' * 10000)) @@ -658,7 +666,7 @@ def gzip_bytes(bytestring, level=6): re.UNICODE) -def iter_splitlines(text): +def iter_splitlines(text: str) -> Generator[str]: r"""Like :meth:`str.splitlines`, but returns an iterator of lines instead of a list. Also similar to :meth:`file.next`, as that also lazily reads and yields lines from a file. @@ -690,7 +698,7 @@ def iter_splitlines(text): return -def indent(text, margin, newline='\n', key=bool): +def indent(text: str, margin: str, newline: str = "\n", key: Callable[[str], bool] = bool) -> str: """The missing counterpart to the built-in :func:`textwrap.dedent`. Args: @@ -706,7 +714,7 @@ def indent(text, margin, newline='\n', key=bool): return newline.join(indented_lines) -def is_uuid(obj, version=4): +def is_uuid(obj, version: int = 4) -> bool: """Check the argument is either a valid UUID object or string. Args: @@ -730,7 +738,7 @@ def is_uuid(obj, version=4): return True -def escape_shell_args(args, sep=' ', style=None): +def escape_shell_args(args: Iterable[str], sep: str = ' ', style: Literal['cmd', 'sh'] | None = None) -> str: """Returns an escaped version of each string in *args*, according to *style*. @@ -759,7 +767,7 @@ def escape_shell_args(args, sep=' ', style=None): _find_sh_unsafe = re.compile(r'[^a-zA-Z0-9_@%+=:,./-]').search -def args2sh(args, sep=' '): +def args2sh(args: Iterable[str], sep: str = ' '): """Return a shell-escaped string version of *args*, separated by *sep*, based on the rules of sh, bash, and other shells in the Linux/BSD/MacOS ecosystem. @@ -793,7 +801,7 @@ def args2sh(args, sep=' '): return ' '.join(ret_list) -def args2cmd(args, sep=' '): +def args2cmd(args: Iterable[str], sep: str = ' '): r"""Return a shell-escaped string version of *args*, separated by *sep*, using the same rules as the Microsoft C runtime. @@ -872,7 +880,7 @@ def args2cmd(args, sep=' '): return ''.join(result) -def parse_int_list(range_string, delim=',', range_delim='-'): +def parse_int_list(range_string: str, delim: str = ",", range_delim: str = "-") -> list[int]: """Returns a sorted list of positive integers based on *range_string*. Reverse of :func:`format_int_list`. @@ -909,7 +917,7 @@ def parse_int_list(range_string, delim=',', range_delim='-'): return sorted(output) -def format_int_list(int_list, delim=',', range_delim='-', delim_space=False): +def format_int_list(int_list: list[int], delim: str = ",", range_delim: str = "-", delim_space: bool = False) -> str: """Returns a sorted range string from a list of positive integers (*int_list*). Contiguous ranges of integers are collapsed to min and max values. Reverse of :func:`parse_int_list`. @@ -1000,8 +1008,8 @@ def format_int_list(int_list, delim=',', range_delim='-', delim_space=False): def complement_int_list( - range_string, range_start=0, range_end=None, - delim=',', range_delim='-'): + range_string: str, range_start: int = 0, range_end: int | None = None, + delim: str = ',', range_delim: str = '-') -> str: """ Returns range string that is the complement of the one provided as *range_string* parameter. @@ -1087,7 +1095,7 @@ def complement_int_list( return format_int_list(complement_values, delim, range_delim) -def int_ranges_from_int_list(range_string, delim=',', range_delim='-'): +def int_ranges_from_int_list(range_string: str, delim: str = ',', range_delim: str = '-') -> tuple[int, int]: """ Transform a string of ranges (*range_string*) into a tuple of tuples. Args: @@ -1179,14 +1187,14 @@ class MultiReplace: of a dictionary. """ - def __init__(self, sub_map, **kwargs): + def __init__(self, sub_map: Mapping[str, str] | Iterable[tuple[str, str]], **kwargs): """Compile any regular expressions that have been passed.""" options = { 'regex': False, 'flags': 0, } options.update(kwargs) - self.group_map = {} + self.group_map: dict[str, str] = {} regex_values = [] if isinstance(sub_map, Mapping): @@ -1217,7 +1225,7 @@ def _get_value(self, match): key = [x for x in group_dict if group_dict[x]][0] return self.group_map[key] - def sub(self, text): + def sub(self, text: str) -> str: """ Run substitutions on the input text. @@ -1227,7 +1235,7 @@ def sub(self, text): return self.combined_pattern.sub(self._get_value, text) -def multi_replace(text, sub_map, **kwargs): +def multi_replace(text: str, sub_map: Mapping[str, str] | Iterable[tuple[str, str]], **kwargs) -> str: """ Shortcut function to invoke MultiReplace in a single call. @@ -1244,7 +1252,7 @@ def multi_replace(text, sub_map, **kwargs): return m.sub(text) -def unwrap_text(text, ending='\n\n'): +def unwrap_text(text: str, ending: str | None = "\n\n") -> str: r""" Unwrap text, the natural complement to :func:`textwrap.wrap`. diff --git a/boltons/tableutils.py b/boltons/tableutils.py index 899ee982..baea2e9c 100644 --- a/boltons/tableutils.py +++ b/boltons/tableutils.py @@ -168,7 +168,7 @@ class ListInputType(InputType): def check_type(self, obj): return isinstance(obj, MutableSequence) - def guess_headers(self, obj): + def guess_headers(self, obj) -> None: return None def get_entry(self, obj, headers): @@ -182,7 +182,7 @@ class TupleInputType(InputType): def check_type(self, obj): return isinstance(obj, tuple) - def guess_headers(self, obj): + def guess_headers(self, obj) -> None: return None def get_entry(self, obj, headers): @@ -272,7 +272,7 @@ def __init__(self, data=None, headers=_MISSING, metadata=None): self.extend(data) - def extend(self, data): + def extend(self, data) -> None: """ Append the given data to the end of the Table. """ diff --git a/boltons/tbutils.py b/boltons/tbutils.py index bfc49840..3a64f6d1 100644 --- a/boltons/tbutils.py +++ b/boltons/tbutils.py @@ -49,10 +49,17 @@ lines of code. """ +from __future__ import annotations +from collections.abc import Iterable, Iterator, Mapping import re import sys import linecache +from types import FrameType, TracebackType +from typing import TYPE_CHECKING, Any, Generic, TypeVar + +if TYPE_CHECKING: + from typing_extensions import Self, Literal # TODO: chaining primitives? what are real use cases where these help? @@ -87,8 +94,8 @@ class Callpoint: __slots__ = ('func_name', 'lineno', 'module_name', 'module_path', 'lasti', 'line') - def __init__(self, module_name, module_path, func_name, - lineno, lasti, line=None): + def __init__(self, module_name: str, module_path: str, func_name: str, + lineno: int, lasti: int, line: str | None = None): self.func_name = func_name self.lineno = lineno self.module_name = module_name @@ -96,7 +103,7 @@ def __init__(self, module_name, module_path, func_name, self.lasti = lasti self.line = line - def to_dict(self): + def to_dict(self) -> dict[str, Any]: "Get a :class:`dict` copy of the Callpoint. Useful for serialization." ret = {} for slot in self.__slots__: @@ -109,13 +116,13 @@ def to_dict(self): return ret @classmethod - def from_current(cls, level=1): + def from_current(cls, level: int = 1) -> Self: "Creates a Callpoint from the location of the calling function." frame = sys._getframe(level) return cls.from_frame(frame) @classmethod - def from_frame(cls, frame): + def from_frame(cls, frame: FrameType) -> Self: "Create a Callpoint object from data extracted from the given frame." func_name = frame.f_code.co_name lineno = frame.f_lineno @@ -127,7 +134,7 @@ def from_frame(cls, frame): lineno, lasti, line=line) @classmethod - def from_tb(cls, tb): + def from_tb(cls, tb: TracebackType) -> Self: """Create a Callpoint from the traceback of the current exception. Main difference with :meth:`from_frame` is that ``lineno`` and ``lasti`` come from the traceback, which is to @@ -151,7 +158,7 @@ def __repr__(self): else: return '{}({})'.format(cn, ', '.join([repr(a) for a in args])) - def tb_frame_str(self): + def tb_frame_str(self) -> str: """Render the Callpoint as it would appear in a standard printed Python traceback. Returns a string with filename, line number, function name, and the actual code line of the error on up to @@ -220,9 +227,13 @@ def __repr__(self): def __len__(self): return len(str(self)) +if sys.version_info >= (3 ,13): + _CallpointT = TypeVar("_CallpointT", bound=Callpoint, covariant=True, default=Callpoint) +else: + _CallpointT = TypeVar("_CallpointT", bound=Callpoint, covariant=True) # TODO: dedup frames, look at __eq__ on _DeferredLine -class TracebackInfo: +class TracebackInfo(Generic[_CallpointT]): """The TracebackInfo class provides a basic representation of a stack trace, be it from an exception being handled or just part of normal execution. It is basically a wrapper around a list of @@ -241,13 +252,13 @@ class TracebackInfo: :meth:`TracebackInfo.from_traceback` without the *tb* argument for an exception traceback. """ - callpoint_type = Callpoint + callpoint_type: type[_CallpointT] = Callpoint - def __init__(self, frames): + def __init__(self, frames: list[_CallpointT]): self.frames = frames @classmethod - def from_frame(cls, frame=None, level=1, limit=None): + def from_frame(cls, frame: FrameType | None = None, level: int = 1, limit: int | None = None) -> Self: """Create a new TracebackInfo *frame* by recurring up in the stack a max of *limit* times. If *frame* is unset, get the frame from :func:`sys._getframe` using *level*. @@ -278,7 +289,7 @@ def from_frame(cls, frame=None, level=1, limit=None): return cls(ret) @classmethod - def from_traceback(cls, tb=None, limit=None): + def from_traceback(cls, tb: TracebackType | None = None, limit: int | None = None) -> Self: """Create a new TracebackInfo from the traceback *tb* by recurring up in the stack a max of *limit* times. If *tb* is unset, get the traceback from the currently handled exception. If no @@ -311,21 +322,21 @@ def from_traceback(cls, tb=None, limit=None): return cls(ret) @classmethod - def from_dict(cls, d): + def from_dict(cls, d: Mapping[Literal['frames'], list[_CallpointT]]) -> Self: "Complements :meth:`TracebackInfo.to_dict`." # TODO: check this. return cls(d['frames']) - def to_dict(self): + def to_dict(self) -> dict[str, list[dict[str, _CallpointT]]]: """Returns a dict with a list of :class:`Callpoint` frames converted to dicts. """ return {'frames': [f.to_dict() for f in self.frames]} - def __len__(self): + def __len__(self) -> int: return len(self.frames) - def __iter__(self): + def __iter__(self) -> Iterator[_CallpointT]: return iter(self.frames) def __repr__(self): @@ -341,7 +352,7 @@ def __repr__(self): def __str__(self): return self.get_formatted() - def get_formatted(self): + def get_formatted(self) -> str: """Returns a string as formatted in the traditional Python built-in style observable when an exception is not caught. In other words, mimics :func:`traceback.format_tb` and @@ -351,8 +362,12 @@ def get_formatted(self): ret += ''.join([f.tb_frame_str() for f in self.frames]) return ret +if sys.version_info >= (3 ,13): + _TracebackInfoT = TypeVar("_TracebackInfoT", bound=TracebackInfo, covariant=True, default=TracebackInfo) +else: + _TracebackInfoT = TypeVar("_TracebackInfoT", bound=TracebackInfo, covariant=True) -class ExceptionInfo: +class ExceptionInfo(Generic[_TracebackInfoT]): """An ExceptionInfo object ties together three main fields suitable for representing an instance of an exception: The exception type name, a string representation of the exception itself (the @@ -378,16 +393,16 @@ class ExceptionInfo: """ #: Override this in inherited types to control the TracebackInfo type used - tb_info_type = TracebackInfo + tb_info_type: type[_TracebackInfoT] = TracebackInfo - def __init__(self, exc_type, exc_msg, tb_info): + def __init__(self, exc_type: str, exc_msg: str, tb_info: _TracebackInfoT) -> None: # TODO: additional fields for SyntaxErrors self.exc_type = exc_type self.exc_msg = exc_msg self.tb_info = tb_info @classmethod - def from_exc_info(cls, exc_type, exc_value, traceback): + def from_exc_info(cls, exc_type: type[BaseException], exc_value: BaseException, traceback: TracebackType) -> Self: """Create an :class:`ExceptionInfo` object from the exception's type, value, and traceback, as returned by :func:`sys.exc_info`. See also :meth:`from_current`. @@ -401,14 +416,14 @@ def from_exc_info(cls, exc_type, exc_value, traceback): return cls(type_str, val_str, tb_info) @classmethod - def from_current(cls): + def from_current(cls) -> Self: """Create an :class:`ExceptionInfo` object from the current exception being handled, by way of :func:`sys.exc_info`. Will raise an exception if no exception is currently being handled. """ return cls.from_exc_info(*sys.exc_info()) - def to_dict(self): + def to_dict(self) -> dict[str, Any]: """Get a :class:`dict` representation of the ExceptionInfo, suitable for JSON serialization. """ @@ -416,7 +431,7 @@ def to_dict(self): 'exc_msg': self.exc_msg, 'exc_tb': self.tb_info.to_dict()} - def __repr__(self): + def __repr__(self) -> str: cn = self.__class__.__name__ try: len_frames = len(self.tb_info.frames) @@ -427,7 +442,7 @@ def __repr__(self): args = (cn, self.exc_type, self.exc_msg, len_frames, last_frame) return '<%s [%s: %s] (%s frames%s)>' % args - def get_formatted(self): + def get_formatted(self) -> str: """Returns a string formatted in the traditional Python built-in style observable when an exception is not caught. In other words, mimics :func:`traceback.format_exception`. @@ -450,13 +465,13 @@ class ContextualCallpoint(Callpoint): The ContextualCallpoint is used by the :class:`ContextualTracebackInfo`. """ def __init__(self, *a, **kw): - self.local_reprs = kw.pop('local_reprs', {}) - self.pre_lines = kw.pop('pre_lines', []) - self.post_lines = kw.pop('post_lines', []) + self.local_reprs: dict = kw.pop('local_reprs', {}) + self.pre_lines: list[str] = kw.pop('pre_lines', []) + self.post_lines: list[str] = kw.pop('post_lines', []) super().__init__(*a, **kw) @classmethod - def from_frame(cls, frame): + def from_frame(cls, frame: FrameType) -> Self: "Identical to :meth:`Callpoint.from_frame`" ret = super().from_frame(frame) ret._populate_local_reprs(frame.f_locals) @@ -464,7 +479,7 @@ def from_frame(cls, frame): return ret @classmethod - def from_tb(cls, tb): + def from_tb(cls, tb: TracebackType) -> Self: "Identical to :meth:`Callpoint.from_tb`" ret = super().from_tb(tb) ret._populate_local_reprs(tb.tb_frame.f_locals) @@ -496,7 +511,7 @@ def _populate_local_reprs(self, f_locals): local_reprs[k] = surrogate return - def to_dict(self): + def to_dict(self) -> dict[str, Any]: """ Same principle as :meth:`Callpoint.to_dict`, but with the added contextual values. With ``ContextualCallpoint.to_dict()``, @@ -543,7 +558,7 @@ def to_dict(self): return ret -class ContextualTracebackInfo(TracebackInfo): +class ContextualTracebackInfo(TracebackInfo[ContextualCallpoint]): """The ContextualTracebackInfo type is a :class:`TracebackInfo` subtype that is used by :class:`ContextualExceptionInfo` and uses the :class:`ContextualCallpoint` as its frame-representing @@ -552,7 +567,7 @@ class ContextualTracebackInfo(TracebackInfo): callpoint_type = ContextualCallpoint -class ContextualExceptionInfo(ExceptionInfo): +class ContextualExceptionInfo(ExceptionInfo[ContextualTracebackInfo]): """The ContextualTracebackInfo type is a :class:`TracebackInfo` subtype that uses the :class:`ContextualCallpoint` as its frame-representing primitive. @@ -632,7 +647,7 @@ def _format_final_exc_line(etype, value): return line -def print_exception(etype, value, tb, limit=None, file=None): +def print_exception(etype: type[BaseException] | None, value: BaseException | None, tb: TracebackType | None, limit: int | None = None, file: str | None = None) -> None: """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if @@ -680,13 +695,13 @@ class ParsedException: Does not currently store SyntaxError details such as column. """ - def __init__(self, exc_type_name, exc_msg, frames=None): + def __init__(self, exc_type_name: str, exc_msg: str, frames: Iterable[Mapping[str, Any]] | None = None) -> None: self.exc_type = exc_type_name self.exc_msg = exc_msg self.frames = list(frames or []) @property - def source_file(self): + def source_file(self) -> str | None: """ The file path of module containing the function that raised the exception, or None if not available. @@ -696,7 +711,7 @@ def source_file(self): except IndexError: return None - def to_dict(self): + def to_dict(self) -> dict[str, Any]: "Get a copy as a JSON-serializable :class:`dict`." return {'exc_type': self.exc_type, 'exc_msg': self.exc_msg, @@ -707,7 +722,7 @@ def __repr__(self): return ('%s(%r, %r, frames=%r)' % (cn, self.exc_type, self.exc_msg, self.frames)) - def to_string(self): + def to_string(self) -> str: """Formats the exception and its traceback into the standard format, as returned by the traceback module. @@ -736,7 +751,7 @@ def to_string(self): return '\n'.join(lines) @classmethod - def from_string(cls, tb_str): + def from_string(cls, tb_str: str | bytes | bytearray) -> Self: """Parse a traceback and exception from the text *tb_str*. This text is expected to have been decoded, otherwise it will be interpreted as UTF-8. diff --git a/boltons/timeutils.py b/boltons/timeutils.py index 82f201ee..67efb82c 100644 --- a/boltons/timeutils.py +++ b/boltons/timeutils.py @@ -50,6 +50,9 @@ .. _dateutil: https://dateutil.readthedocs.io/en/stable/index.html """ +from __future__ import annotations + +from collections.abc import Generator import re import time import bisect @@ -62,7 +65,7 @@ total_seconds = timedelta.total_seconds -def dt_to_timestamp(dt): +def dt_to_timestamp(dt: datetime) -> float: """Converts from a :class:`~datetime.datetime` object to an integer timestamp, suitable interoperation with :func:`time.time` and other `Epoch-based timestamps`. @@ -94,7 +97,7 @@ def dt_to_timestamp(dt): _NONDIGIT_RE = re.compile(r'\D') -def isoparse(iso_str): +def isoparse(iso_str: str) -> datetime: """Parses the limited subset of `ISO8601-formatted time`_ strings as returned by :meth:`datetime.datetime.isoformat`. @@ -137,7 +140,7 @@ def isoparse(iso_str): for _, _, unit in reversed(_BOUNDS[:-2])} -def parse_timedelta(text): +def parse_timedelta(text: str) -> timedelta: """Robustly parses a short text description of a time period into a :class:`datetime.timedelta`. Supports weeks, days, hours, minutes, and seconds, with or without decimal points: @@ -190,7 +193,7 @@ def _cardinalize_time_unit(unit, value): return unit + 's' -def decimal_relative_time(d, other=None, ndigits=0, cardinalize=True): +def decimal_relative_time(d: datetime, other: datetime | None = None, ndigits: int = 0, cardinalize: bool = True) -> tuple[float, str]: """Get a tuple representing the relative time difference between two :class:`~datetime.datetime` objects or one :class:`~datetime.datetime` and now. @@ -236,7 +239,7 @@ def decimal_relative_time(d, other=None, ndigits=0, cardinalize=True): return rounded_diff, bname -def relative_time(d, other=None, ndigits=0): +def relative_time(d: datetime, other: datetime | None = None, ndigits: int = 0) -> str: """Get a string representation of the difference between two :class:`~datetime.datetime` objects or one :class:`~datetime.datetime` and the current time. Handles past and @@ -268,7 +271,7 @@ def relative_time(d, other=None, ndigits=0): return f'{abs(drt):g} {unit} {phrase}' -def strpdate(string, format): +def strpdate(string: str, format: str) -> date: """Parse the date string according to the format in `format`. Returns a :class:`date` object. Internally, :meth:`datetime.strptime` is used to parse the string and thus conversion specifiers for time fields (e.g. `%H`) @@ -295,7 +298,7 @@ def strpdate(string, format): return whence.date() -def daterange(start, stop, step=1, inclusive=False): +def daterange(start: date, stop: date, step: int = 1, inclusive: bool = False) -> Generator[date]: """In the spirit of :func:`range` and :func:`xrange`, the `daterange` generator that yields a sequence of :class:`~datetime.date` objects, starting at *start*, incrementing by *step*, until *stop* @@ -400,21 +403,21 @@ class ConstantTZInfo(tzinfo): name (str): Name of the timezone. offset (datetime.timedelta): Offset of the timezone. """ - def __init__(self, name="ConstantTZ", offset=ZERO): + def __init__(self, name: str = "ConstantTZ", offset: timedelta = ZERO): self.name = name self.offset = offset @property - def utcoffset_hours(self): + def utcoffset_hours(self) -> float: return timedelta.total_seconds(self.offset) / (60 * 60) - def utcoffset(self, dt): + def utcoffset(self, dt: datetime | None) -> timedelta: return self.offset - def tzname(self, dt): + def tzname(self, dt: datetime | None) -> str: return self.name - def dst(self, dt): + def dst(self, dt: datetime | None) -> timedelta: return ZERO def __repr__(self): @@ -446,23 +449,23 @@ class LocalTZInfo(tzinfo): if time.daylight: _dst_offset = timedelta(seconds=-time.altzone) - def is_dst(self, dt): + def is_dst(self, dt: datetime) -> bool: dt_t = (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.weekday(), 0, -1) local_t = time.localtime(time.mktime(dt_t)) return local_t.tm_isdst > 0 - def utcoffset(self, dt): + def utcoffset(self, dt: datetime) -> timedelta: if self.is_dst(dt): return self._dst_offset return self._std_offset - def dst(self, dt): + def dst(self, dt: datetime) -> timedelta: if self.is_dst(dt): return self._dst_offset - self._std_offset return ZERO - def tzname(self, dt): + def tzname(self, dt: datetime) -> str: return time.tzname[self.is_dst(dt)] def __repr__(self): @@ -511,7 +514,7 @@ class USTimeZone(tzinfo): :data:`Eastern`, :data:`Central`, :data:`Mountain`, and :data:`Pacific` tzinfo types. """ - def __init__(self, hours, reprname, stdname, dstname): + def __init__(self, hours: int, reprname: str, stdname: str, dstname: str): self.stdoffset = timedelta(hours=hours) self.reprname = reprname self.stdname = stdname @@ -520,16 +523,16 @@ def __init__(self, hours, reprname, stdname, dstname): def __repr__(self): return self.reprname - def tzname(self, dt): + def tzname(self, dt: datetime | None) -> str: if self.dst(dt): return self.dstname else: return self.stdname - def utcoffset(self, dt): + def utcoffset(self, dt: datetime | None) -> timedelta: return self.stdoffset + self.dst(dt) - def dst(self, dt): + def dst(self, dt: datetime | None) -> timedelta: if dt is None or dt.tzinfo is None: # An exception may be sensible here, in one or both cases. # It depends on how you want to treat them. The default diff --git a/boltons/typeutils.py b/boltons/typeutils.py index 875e2e2b..3a39f94d 100644 --- a/boltons/typeutils.py +++ b/boltons/typeutils.py @@ -33,13 +33,16 @@ ``typeutils`` attempts to do the same for metaprogramming with types and instances. """ +from __future__ import annotations + import sys from collections import deque + _issubclass = issubclass -def make_sentinel(name='_MISSING', var_name=None): +def make_sentinel(name: str = "_MISSING", var_name: str | None = None) -> object: """Creates and returns a new **instance** of a new class, suitable for usage as a "sentinel", a kind of singleton often used to indicate a value is missing when ``None`` is a valid input. @@ -105,7 +108,7 @@ def __deepcopy__(self, _memo): return Sentinel() -def issubclass(subclass, baseclass): +def issubclass(subclass: type, baseclass: type) -> bool: """Just like the built-in :func:`issubclass`, this function checks whether *subclass* is inherited from *baseclass*. Unlike the built-in function, this ``issubclass`` will simply return @@ -130,7 +133,7 @@ def issubclass(subclass, baseclass): return False -def get_all_subclasses(cls): +def get_all_subclasses(cls: type) -> list[type]: """Recursively finds and returns a :class:`list` of all types inherited from *cls*. diff --git a/boltons/urlutils.py b/boltons/urlutils.py index 45d3e73e..f4965001 100644 --- a/boltons/urlutils.py +++ b/boltons/urlutils.py @@ -120,7 +120,7 @@ class URLParseError(ValueError): DEFAULT_ENCODING = 'utf8' -def to_unicode(obj): +def to_unicode(obj: object) -> str: try: return str(obj) except UnicodeDecodeError: @@ -575,7 +575,7 @@ def path(self): for p in self.path_parts]) @path.setter - def path(self, path_text): + def path(self, path_text) -> None: self.path_parts = tuple([unquote(p) if '%' in p else p for p in to_unicode(path_text).split('/')]) return @@ -630,7 +630,7 @@ def default_port(self): except KeyError: return SCHEME_PORT_MAP.get(self.scheme.split('+')[-1]) - def normalize(self, with_case=True): + def normalize(self, with_case: bool = True) -> None: """Resolve any "." and ".." references in the path, as well as normalize scheme and host casing. To turn off case normalization, pass ``with_case=False``. From afaf397f0a02b43caf65a55e60b914c4e5745ce5 Mon Sep 17 00:00:00 2001 From: Avasam Date: Thu, 20 Feb 2025 20:31:53 -0500 Subject: [PATCH 02/10] Setup mypy and fix more types --- .github/workflows/tests.yaml | 2 +- boltons/cacheutils.py | 10 +++++----- boltons/strutils.py | 2 +- boltons/typeutils.py | 2 +- docs/conf.py | 4 ++-- mypy.ini | 30 ++++++++++++++++++++++++++++++ requirements-test.txt | 1 + tests/test_formatutils.py | 9 ++++++--- tests/test_funcutils_fb_py3.py | 7 ++++--- tests/test_statsutils.py | 5 ++--- tests/test_statsutils_histogram.py | 5 ++++- tox.ini | 13 +++++++++++-- 12 files changed, 68 insertions(+), 22 deletions(-) create mode 100644 mypy.ini diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 72f8e804..f87360d7 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -49,4 +49,4 @@ jobs: path: ${{ steps.pip-cache.outputs.dir }} key: pip|${{ runner.os }}|${{ matrix.python }}|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('requirements/*.txt') }} - run: pip install tox - - run: tox -e ${{ matrix.tox }} + - run: tox -e ${{ matrix.tox }},mypy diff --git a/boltons/cacheutils.py b/boltons/cacheutils.py index 637c7ba9..183e2792 100644 --- a/boltons/cacheutils.py +++ b/boltons/cacheutils.py @@ -616,11 +616,11 @@ def cachedmethod(cache: Mapping | Callable, scoped: bool = True, typed: bool = F values to be used as the key in the cache. >>> class Lowerer(object): - ... def __init__(self): + ... def __init__(self) -> None: ... self.cache = LRI() ... ... @cachedmethod('cache') - ... def lower(self, text): + ... def lower(self, text) -> str: ... return text.lower() ... >>> lowerer = Lowerer() @@ -850,9 +850,9 @@ class MinIDMap(Generic[_KT]): Based on https://gist.github.com/kurtbrose/25b48114de216a5e55df """ - def __init__(self): - self.mapping: weakref.WeakKeyDictionary[_KT, int] = weakref.WeakKeyDictionary() - self.ref_map: dict[_T, int] = {} + def __init__(self) -> None: + self.mapping: weakref.WeakKeyDictionary[_KT, tuple[int, weakref.ReferenceType[_KT]]] = weakref.WeakKeyDictionary() + self.ref_map: dict[weakref.ReferenceType[_KT], int] = {} self.free: list[int] = [] def get(self, a: _KT) -> int: diff --git a/boltons/strutils.py b/boltons/strutils.py index cc33262a..671ca44b 100644 --- a/boltons/strutils.py +++ b/boltons/strutils.py @@ -578,7 +578,7 @@ def bytes2human(nbytes: int, ndigits: int = 0) -> str: class HTMLTextExtractor(HTMLParser): - def __init__(self): + def __init__(self) -> None: self.reset() self.strict = False self.convert_charrefs = True diff --git a/boltons/typeutils.py b/boltons/typeutils.py index 3a39f94d..4ce5b50a 100644 --- a/boltons/typeutils.py +++ b/boltons/typeutils.py @@ -75,7 +75,7 @@ def make_sentinel(name: str = "_MISSING", var_name: str | None = None) -> object """ class Sentinel: - def __init__(self): + def __init__(self) -> None: self.name = name self.var_name = var_name diff --git a/docs/conf.py b/docs/conf.py index d10273e9..6dbf8314 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -228,7 +228,7 @@ def get_mod_stats(): # -- Options for LaTeX output --------------------------------------------- -latex_elements = { +# latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', @@ -240,7 +240,7 @@ def get_mod_stats(): # Latex figure (float) alignment #'figure_align': 'htbp', -} +# } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..4081f741 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,30 @@ +[mypy] +show_column_numbers = true +# Still contains Python 2 code +exclude = misc + +[mypy-boltons.*] +# TODO: Gradually fix these type issues in code and re-enable the codes (in no particular order) +disable_error_code = + assignment, + index, + arg-type, + attr-defined, + call-overload, + truthy-function, + operator, + var-annotated, + misc, + union-attr, + has-type, + override, + no-redef, + return-value, + type-var, + +[mypy-tests.*] +# TODO: Don't bother fully typing tests, but still validate usage +# check_untyped_defs = true + +[mypy-sphinx_rtd_theme.*] +follow_untyped_imports = true diff --git a/requirements-test.txt b/requirements-test.txt index 9ceb60ad..6dec3091 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,5 @@ coverage==7.2.7 +mypy[faster-cache]==1.15.0 pytest==7.4.4; python_version < "3.8" pytest==8.3.4; python_version >= "3.8" pytest-cov==4.1.0 diff --git a/tests/test_formatutils.py b/tests/test_formatutils.py index 8eba3c1f..a5d46a55 100644 --- a/tests/test_formatutils.py +++ b/tests/test_formatutils.py @@ -1,5 +1,5 @@ import re -from collections import namedtuple +from typing import NamedTuple, Tuple from boltons.formatutils import (get_format_args, split_format_str, @@ -7,8 +7,11 @@ infer_positional_format_args, DeferredValue as DV) - -PFAT = namedtuple("PositionalFormatArgTest", "fstr arg_vals res") +class PFAT(NamedTuple): + fstr: str + arg_vals: Tuple[object, ...] + res: str +PFAT.__name__ = "PositionalFormatArgTest" _PFATS = [PFAT('{} {} {}', ('hi', 'hello', 'bye'), "hi hello bye"), PFAT('{:d} {}', (1, 2), "1 2"), diff --git a/tests/test_funcutils_fb_py3.py b/tests/test_funcutils_fb_py3.py index 85cdf12c..2a986cca 100644 --- a/tests/test_funcutils_fb_py3.py +++ b/tests/test_funcutils_fb_py3.py @@ -2,6 +2,7 @@ import inspect import functools from collections import defaultdict +from typing import Tuple, Union import pytest @@ -31,13 +32,13 @@ def cedar_wrapper(*a, **kw): def test_wraps_py3(): @pita_wrap(flag=True) - def annotations(a: int, b: float=1, c: defaultdict=()) -> defaultdict: + def annotations(a: int, b: float=1, c: Union[defaultdict, Tuple[()]]=()) -> Tuple[int, float, Union[defaultdict, Tuple[()]]]: return a, b, c assert annotations(0) == (True, "annotations", (0, 1, ())) assert annotations.__annotations__ == {'a': int, 'b': float, - 'c': defaultdict, - 'return': defaultdict} + 'c': Union[defaultdict, Tuple[()]], + 'return': Tuple[int, float, Union[defaultdict, Tuple[()]]]} @pita_wrap(flag=False) def kwonly_arg(a, *, b, c=2): diff --git a/tests/test_statsutils.py b/tests/test_statsutils.py index c04a045e..30f9cd40 100644 --- a/tests/test_statsutils.py +++ b/tests/test_statsutils.py @@ -11,13 +11,12 @@ def test_stats_basic(): assert da.median == 9.5 -def _test_pearson(): +def test_pearson(): import random - from statsutils import pearson_type def get_pt(dist): vals = [dist() for x in range(10000)] - pt = pearson_type(vals) + pt = Stats(vals).pearson_type return pt for x in range(3): diff --git a/tests/test_statsutils_histogram.py b/tests/test_statsutils_histogram.py index b0b310f1..938afe71 100644 --- a/tests/test_statsutils_histogram.py +++ b/tests/test_statsutils_histogram.py @@ -1,5 +1,8 @@ +from typing import TYPE_CHECKING from boltons.statsutils import Stats +if TYPE_CHECKING: + from typing_extensions import Never # [round(random.normalvariate(10, 3), 3) for i in range(100)] NORM_DATA = [12.975, 8.341, 12.27, 12.799, 15.443, 6.739, 10.572, @@ -24,7 +27,7 @@ + list(range(20, 80)) + list(range(40, 60))) -EMPTY_DATA = [] +EMPTY_DATA: "list[Never]" = [] ALL_DATASETS = [EMPTY_DATA, LAYER_RANGE_DATA, SIMPLE_RANGE_DATA, NORM_DATA] diff --git a/tox.ini b/tox.ini index 12fb9288..d1cfb8f7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,16 @@ [tox] envlist = py37,py39,py310,py311,py312,py313,pypy3 + [testenv] -changedir = .tox -deps = -rrequirements-test.txt +deps = -r requirements-test.txt +# We want to test against the installed package instead of the source tree commands = pytest --doctest-modules {envsitepackagesdir}/boltons {toxinidir}/tests {posargs} +# Can't use pytest-mypy because we can't run mypy against a folder in the env where mypy is installed. +# So we just run mypy directly instead. +[testenv:mypy] +deps = + -r requirements-test.txt + -r docs/requirements-rtd.txt +setenv = MYPY_FORCE_COLOR=1 +commands = mypy {toxinidir} From 75876c76ca69d1c0dd3edc8f8f2e936006ce2d94 Mon Sep 17 00:00:00 2001 From: Avasam Date: Thu, 20 Feb 2025 22:12:03 -0500 Subject: [PATCH 03/10] Downgrade mypy for Python 3.7 --- boltons/cacheutils.py | 4 ++-- boltons/dictutils.py | 12 ++++++------ boltons/fileutils.py | 2 +- boltons/iterutils.py | 4 ++-- boltons/jsonutils.py | 8 ++++---- boltons/strutils.py | 2 +- boltons/timeutils.py | 2 +- mypy.ini | 4 +++- requirements-test.txt | 2 +- 9 files changed, 21 insertions(+), 19 deletions(-) diff --git a/boltons/cacheutils.py b/boltons/cacheutils.py index 183e2792..39c82021 100644 --- a/boltons/cacheutils.py +++ b/boltons/cacheutils.py @@ -798,7 +798,7 @@ def iterkeys(self) -> Iterator[_KT]: def keys(self) -> list[_KT]: return list(self.iterkeys()) - def itervalues(self) -> Generator[int]: + def itervalues(self) -> Generator[int, None, None]: count_map = self._count_map for k in count_map: yield count_map[k][0] @@ -806,7 +806,7 @@ def itervalues(self) -> Generator[int]: def values(self) -> list[int]: return list(self.itervalues()) - def iteritems(self) -> Generator[tuple[_KT, int]]: + def iteritems(self) -> Generator[tuple[_KT, int], None, None]: count_map = self._count_map for k in count_map: yield (k, count_map[k][0]) diff --git a/boltons/dictutils.py b/boltons/dictutils.py index fed69289..bbf8a179 100644 --- a/boltons/dictutils.py +++ b/boltons/dictutils.py @@ -462,7 +462,7 @@ def _remove_all(self, k): cell[PREV][NEXT], cell[NEXT][PREV] = cell[NEXT], cell[PREV] del self._map[k] - def iteritems(self, multi: bool = False) -> Generator[tuple[_KT, _VT]]: + def iteritems(self, multi: bool = False) -> Generator[tuple[_KT, _VT], None, None]: """Iterate over the OMD's items in insertion order. By default, yields only the most-recently inserted value for each key. Set *multi* to ``True`` to get all inserted items. @@ -477,7 +477,7 @@ def iteritems(self, multi: bool = False) -> Generator[tuple[_KT, _VT]]: for key in self.iterkeys(): yield key, self[key] - def iterkeys(self, multi: bool = False) -> Generator[_KT]: + def iterkeys(self, multi: bool = False) -> Generator[_KT, None, None]: """Iterate over the OMD's keys in insertion order. By default, yields each key once, according to the most recent insertion. Set *multi* to ``True`` to get all keys, including duplicates, in @@ -499,7 +499,7 @@ def iterkeys(self, multi: bool = False) -> Generator[_KT]: yield k curr = curr[NEXT] - def itervalues(self, multi: bool = False) -> Generator[_VT]: + def itervalues(self, multi: bool = False) -> Generator[_VT, None, None]: """Iterate over the OMD's values in insertion order. By default, yields the most-recently inserted value per unique key. Set *multi* to ``True`` to get all values according to insertion @@ -742,7 +742,7 @@ def _remove_all(self, k): cell[PREV][NEXT], cell[NEXT][PREV] = cell[NEXT], cell[PREV] cell[PREV][SNEXT] = cell[SNEXT] - def iteritems(self, multi: bool = False) -> Generator[tuple[_KT, _VT]]: + def iteritems(self, multi: bool = False) -> Generator[tuple[_KT, _VT], None, None]: next_link = NEXT if multi else SNEXT root = self.root curr = root[next_link] @@ -750,7 +750,7 @@ def iteritems(self, multi: bool = False) -> Generator[tuple[_KT, _VT]]: yield curr[KEY], curr[VALUE] curr = curr[next_link] - def iterkeys(self, multi: bool = False) -> Generator[_KT]: + def iterkeys(self, multi: bool = False) -> Generator[_KT, None, None]: next_link = NEXT if multi else SNEXT root = self.root curr = root[next_link] @@ -1017,7 +1017,7 @@ def replace(self, key: _KT, newkey: _KT) -> None: revset.remove(key) revset.add(newkey) - def iteritems(self) -> Generator[tuple[_KT, _VT]]: + def iteritems(self) -> Generator[tuple[_KT, _VT], None, None]: for key in self.data: for val in self.data[key]: yield key, val diff --git a/boltons/fileutils.py b/boltons/fileutils.py index 61b86041..21e8814e 100644 --- a/boltons/fileutils.py +++ b/boltons/fileutils.py @@ -498,7 +498,7 @@ def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException return -def iter_find_files(directory: str, patterns: str | Iterable[str], ignored: str | Iterable[str] | None = None, include_dirs: bool = False, max_depth: int | None = None) -> Generator[str]: +def iter_find_files(directory: str, patterns: str | Iterable[str], ignored: str | Iterable[str] | None = None, include_dirs: bool = False, max_depth: int | None = None) -> Generator[str, None, None]: """Returns a generator that yields file paths under a *directory*, matching *patterns* using `glob`_ syntax (e.g., ``*.txt``). Also supports *ignored* patterns. diff --git a/boltons/iterutils.py b/boltons/iterutils.py index bcfded7a..86c2c926 100644 --- a/boltons/iterutils.py +++ b/boltons/iterutils.py @@ -359,7 +359,7 @@ def postprocess(chk): return bytes(chk) return -def chunk_ranges(input_size: int, chunk_size: int, input_offset: int = 0, overlap_size: int = 0, align: bool = False) -> Generator[tuple[int, int]]: +def chunk_ranges(input_size: int, chunk_size: int, input_offset: int = 0, overlap_size: int = 0, align: bool = False) -> Generator[tuple[int, int], None, None]: """Generates *chunk_size*-sized chunk ranges for an input with length *input_size*. Optionally, a start of the input can be set via *input_offset*, and and overlap between the chunks may be specified via *overlap_size*. @@ -1315,7 +1315,7 @@ def get_path(root, path, default=_UNSET): cur = cur[seg] except (KeyError, IndexError) as exc: raise PathAccessError(exc, seg, path) - except TypeError as exc: + except TypeError: # either string index in a list, or a parent that # doesn't support indexing try: diff --git a/boltons/jsonutils.py b/boltons/jsonutils.py index 3d9254ff..f5cace97 100644 --- a/boltons/jsonutils.py +++ b/boltons/jsonutils.py @@ -52,12 +52,12 @@ __all__ = ['JSONLIterator', 'reverse_iter_lines'] @overload -def reverse_iter_lines(file_obj: IO[bytes], blocksize: int = DEFAULT_BLOCKSIZE, preseek: bool = True, encoding: None = None) -> Generator[bytes]: ... +def reverse_iter_lines(file_obj: IO[bytes], blocksize: int = DEFAULT_BLOCKSIZE, preseek: bool = True, encoding: None = None) -> Generator[bytes, None, None]: ... @overload -def reverse_iter_lines(file_obj: IO[str], blocksize: int = DEFAULT_BLOCKSIZE, preseek: bool = True, *, encoding: str) -> Generator[str]: ... +def reverse_iter_lines(file_obj: IO[str], blocksize: int = DEFAULT_BLOCKSIZE, preseek: bool = True, *, encoding: str) -> Generator[str, None, None]: ... @overload -def reverse_iter_lines(file_obj: IO[str], blocksize: int, preseek: bool, encoding: str) -> Generator[str]: ... -def reverse_iter_lines(file_obj: IO[bytes] | IO[str], blocksize: int = DEFAULT_BLOCKSIZE, preseek: bool = True, encoding: str | None = None) -> Generator[bytes] | Generator[str]: +def reverse_iter_lines(file_obj: IO[str], blocksize: int, preseek: bool, encoding: str) -> Generator[str, None, None]: ... +def reverse_iter_lines(file_obj: IO[bytes] | IO[str], blocksize: int = DEFAULT_BLOCKSIZE, preseek: bool = True, encoding: str | None = None) -> Generator[bytes, None, None] | Generator[str, None, None]: """Returns an iterator over the lines from a file object, in reverse order, i.e., last line first, first line last. Uses the :meth:`file.seek` method of file objects, and is tested compatible with diff --git a/boltons/strutils.py b/boltons/strutils.py index 671ca44b..f580d2da 100644 --- a/boltons/strutils.py +++ b/boltons/strutils.py @@ -666,7 +666,7 @@ def gzip_bytes(bytestring: ReadableBuffer, level: int = 6) -> bytes: re.UNICODE) -def iter_splitlines(text: str) -> Generator[str]: +def iter_splitlines(text: str) -> Generator[str, None, None]: r"""Like :meth:`str.splitlines`, but returns an iterator of lines instead of a list. Also similar to :meth:`file.next`, as that also lazily reads and yields lines from a file. diff --git a/boltons/timeutils.py b/boltons/timeutils.py index 67efb82c..c5743535 100644 --- a/boltons/timeutils.py +++ b/boltons/timeutils.py @@ -298,7 +298,7 @@ def strpdate(string: str, format: str) -> date: return whence.date() -def daterange(start: date, stop: date, step: int = 1, inclusive: bool = False) -> Generator[date]: +def daterange(start: date, stop: date, step: int = 1, inclusive: bool = False) -> Generator[date, None, None]: """In the spirit of :func:`range` and :func:`xrange`, the `daterange` generator that yields a sequence of :class:`~datetime.date` objects, starting at *start*, incrementing by *step*, until *stop* diff --git a/mypy.ini b/mypy.ini index 4081f741..c8878a3f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -27,4 +27,6 @@ disable_error_code = # check_untyped_defs = true [mypy-sphinx_rtd_theme.*] -follow_untyped_imports = true +ignore_missing_imports = True +# use the following with mypy 1.15+ +# follow_untyped_imports = true diff --git a/requirements-test.txt b/requirements-test.txt index 6dec3091..4d40a3c1 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,5 @@ coverage==7.2.7 -mypy[faster-cache]==1.15.0 +mypy==1.8.0 pytest==7.4.4; python_version < "3.8" pytest==8.3.4; python_version >= "3.8" pytest-cov==4.1.0 From 14d23bee25f7bd4458d0b1656a7b04b7ae9a2499 Mon Sep 17 00:00:00 2001 From: Avasam Date: Thu, 20 Feb 2025 22:21:50 -0500 Subject: [PATCH 04/10] Try splitting mypy versions across Python 3.7 --- requirements-test.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index 4d40a3c1..d8e0bc36 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,6 @@ coverage==7.2.7 -mypy==1.8.0 +mypy==1.4.1; python_version < "3.8" +mypy[faster-cache]==1.14.1; python_version >= "3.8" # Last mypy version to run on Python 3.8 pytest==7.4.4; python_version < "3.8" pytest==8.3.4; python_version >= "3.8" pytest-cov==4.1.0 From ac04b3614be412bdd76b3f949ab5aed394f9e0a3 Mon Sep 17 00:00:00 2001 From: Avasam Date: Thu, 20 Feb 2025 22:26:54 -0500 Subject: [PATCH 05/10] Add from __future__ import annotations to test --- tests/test_statsutils_histogram.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_statsutils_histogram.py b/tests/test_statsutils_histogram.py index 938afe71..60299412 100644 --- a/tests/test_statsutils_histogram.py +++ b/tests/test_statsutils_histogram.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import TYPE_CHECKING from boltons.statsutils import Stats @@ -27,7 +29,7 @@ + list(range(20, 80)) + list(range(40, 60))) -EMPTY_DATA: "list[Never]" = [] +EMPTY_DATA: list[Never] = [] ALL_DATASETS = [EMPTY_DATA, LAYER_RANGE_DATA, SIMPLE_RANGE_DATA, NORM_DATA] From c11798577f079b96b399134473d2c0e4c8fc08e9 Mon Sep 17 00:00:00 2001 From: Avasam Date: Thu, 20 Feb 2025 22:54:29 -0500 Subject: [PATCH 06/10] Don't run mypy tests on pypy and 3.7 --- .github/workflows/tests.yaml | 14 +++++++------- requirements-test.txt | 3 +-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index f87360d7..af7faf9f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -18,14 +18,14 @@ jobs: fail-fast: false matrix: include: - - { name: Linux, python: "3.13", os: ubuntu-latest, tox: py313 } - { name: Windows, python: "3.13", os: windows-latest, tox: py313 } - { name: Mac, python: "3.13", os: macos-latest, tox: py313 } - - { name: "3.12", python: "3.12", os: ubuntu-latest, tox: py312 } - - { name: "3.11", python: "3.11", os: ubuntu-latest, tox: py311 } - - { name: "3.10", python: "3.10", os: ubuntu-latest, tox: py310 } - - { name: "3.9", python: "3.9", os: ubuntu-latest, tox: py39 } - - { name: "3.8", python: "3.8", os: ubuntu-latest, tox: py38 } + - { name: Linux, python: "3.13", os: ubuntu-latest, tox: "py313,mypy" } + - { name: "3.12", python: "3.12", os: ubuntu-latest, tox: "py312,mypy" } + - { name: "3.11", python: "3.11", os: ubuntu-latest, tox: "py311,mypy" } + - { name: "3.10", python: "3.10", os: ubuntu-latest, tox: "py310,mypy" } + - { name: "3.9", python: "3.9", os: ubuntu-latest, tox: "py39,mypy" } + - { name: "3.8", python: "3.8", os: ubuntu-latest, tox: "py38,mypy" } - { name: "3.7", python: "3.7", os: ubuntu-22.04, tox: py37 } - { name: "PyPy3", python: "pypy-3.9", os: ubuntu-latest, tox: pypy3 } steps: @@ -49,4 +49,4 @@ jobs: path: ${{ steps.pip-cache.outputs.dir }} key: pip|${{ runner.os }}|${{ matrix.python }}|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('requirements/*.txt') }} - run: pip install tox - - run: tox -e ${{ matrix.tox }},mypy + - run: tox -e ${{ matrix.tox }} diff --git a/requirements-test.txt b/requirements-test.txt index d8e0bc36..99ebfd0f 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,5 @@ coverage==7.2.7 -mypy==1.4.1; python_version < "3.8" -mypy[faster-cache]==1.14.1; python_version >= "3.8" # Last mypy version to run on Python 3.8 +mypy[faster-cache]==1.14.1; python_version >= "3.8" # mypy isn't tested on 3.7 pytest==7.4.4; python_version < "3.8" pytest==8.3.4; python_version >= "3.8" pytest-cov==4.1.0 From 9f72b0415e8e036701d2d8610db5225c53fc638b Mon Sep 17 00:00:00 2001 From: Avasam Date: Thu, 20 Feb 2025 23:05:38 -0500 Subject: [PATCH 07/10] Don't even install mypy on pypy --- requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index 99ebfd0f..4905f193 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,5 @@ coverage==7.2.7 -mypy[faster-cache]==1.14.1; python_version >= "3.8" # mypy isn't tested on 3.7 +mypy[faster-cache]==1.14.1; implementation_name != "pypy"; python_version >= "3.8" # mypy isn't tested on 3.7 pytest==7.4.4; python_version < "3.8" pytest==8.3.4; python_version >= "3.8" pytest-cov==4.1.0 From dbdb92712dde2f1219dc605c739bc397fc4b806c Mon Sep 17 00:00:00 2001 From: Avasam Date: Thu, 20 Feb 2025 23:08:08 -0500 Subject: [PATCH 08/10] typo in marker --- requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index 4905f193..5dac9c66 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,5 @@ coverage==7.2.7 -mypy[faster-cache]==1.14.1; implementation_name != "pypy"; python_version >= "3.8" # mypy isn't tested on 3.7 +mypy[faster-cache]==1.14.1; implementation_name != "pypy" and python_version >= "3.8" # mypy isn't tested on 3.7 pytest==7.4.4; python_version < "3.8" pytest==8.3.4; python_version >= "3.8" pytest-cov==4.1.0 From dc5b51026475a34d827a6fbf2df5f30e92668bf9 Mon Sep 17 00:00:00 2001 From: Avasam Date: Thu, 20 Feb 2025 23:27:08 -0500 Subject: [PATCH 09/10] restore changedir --- tox.ini | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index d1cfb8f7..94cd2b3e 100644 --- a/tox.ini +++ b/tox.ini @@ -2,15 +2,17 @@ envlist = py37,py39,py310,py311,py312,py313,pypy3 [testenv] -deps = -r requirements-test.txt # We want to test against the installed package instead of the source tree +changedir = .tox +deps = -r requirements-test.txt commands = pytest --doctest-modules {envsitepackagesdir}/boltons {toxinidir}/tests {posargs} # Can't use pytest-mypy because we can't run mypy against a folder in the env where mypy is installed. # So we just run mypy directly instead. [testenv:mypy] +changedir = {toxinidir} deps = -r requirements-test.txt -r docs/requirements-rtd.txt setenv = MYPY_FORCE_COLOR=1 -commands = mypy {toxinidir} +commands = mypy . From 0aee8098baa0eb3ddf4b2b91e76f1bacbd594c73 Mon Sep 17 00:00:00 2001 From: Avasam Date: Thu, 20 Feb 2025 23:28:44 -0500 Subject: [PATCH 10/10] Add Typing :: Typed classifier --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 4dc6193d..4a8ea143 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ classifiers = [ "Topic :: Software Development :: Libraries", "Development Status :: 5 - Production/Stable", "Operating System :: OS Independent", + "Typing :: Typed", # List of python versions and their support status: # https://en.wikipedia.org/wiki/CPython#Version_history "Programming Language :: Python :: 3",