Manage recursion limit preventing RuntimeError

pyflakes has traditionally recursed with a handler for every
level of the ast.  The ast depth can become very large, especially
for an expression containing many binary operators.

Python has a maximum recursion limit, defaulting to a low number
like 1000, which resulted in a RuntimeError for the ast of:

    x = 1 + 2 + 3 + ... + 1001

To workaround this problem, pyflakes now increases the recursion
limit at runtime when it knows it will be exceeded.

Fixes lp:1507827

Author: jayvdb

pyflakes/checker.py

@@ -494,6 +494,7 @@ def __init__(self, tree, filename='(none)', builtins=None,
         self.scopeStack = [ModuleScope()]
         self.exceptHandlers = [()]
         self.root = tree
+        self._recursion_limit = sys.getrecursionlimit()
         self.handleChildren(tree)
         self.runDeferred(self._deferredFunctions)
         # Set _deferredFunctions to None so that deferFunction will fail
@@ -820,6 +821,15 @@ def on_conditional_branch():
                 self.report(messages.UndefinedName, node, name)
 
     def handleChildren(self, tree, omit=None):
+        # The recursion limit needs to be at least double nodeDepth
+        # as the recursion cycles between handleChildren and handleNode.
+        # Set it to triple nodeDepth to account for other items on the stack,
+        # and to reduce the frequency of changes to the limit.
+        acceptable_recursion_limit = self.nodeDepth * 3
+        if self._recursion_limit <= acceptable_recursion_limit:
+            sys.setrecursionlimit(acceptable_recursion_limit)
+            self._recursion_limit = acceptable_recursion_limit
+
         for node in iter_child_nodes(tree, omit=omit):
             self.handleNode(node, tree)
 

pyflakes/test/test_other.py

@@ -1,12 +1,19 @@
 """
 Tests for various Pyflakes behavior.
 """
+import sys
 
 from sys import version_info
 
 from pyflakes import messages as m
 from pyflakes.test.harness import TestCase, skip, skipIf
 
+try:
+    sys.pypy_version_info
+    PYPY = True
+except AttributeError:
+    PYPY = False
+
 
 class Test(TestCase):
 
@@ -1993,3 +2000,24 @@ def test_raise_notimplemented(self):
         self.flakes('''
         raise NotImplemented
         ''', m.RaiseNotImplemented)
+
+
+class TestMaximumRecursion(TestCase):
+
+    def setUp(self):
+        self._recursionlimit = sys.getrecursionlimit()
+
+    def test_recursion_limit(self):
+        # Using self._recursionlimit * 10 tends to cause CPython to core dump.
+        # Older PyPy tend to break with lower recusion limits.
+        if PYPY and version_info < (3, 5):
+            new_recursion_limit = self._recursionlimit * 3
+        else:
+            new_recursion_limit = self._recursionlimit * 6
+
+        r = range(new_recursion_limit)
+        s = 'x = ' + ' + '.join(str(n) for n in r)
+        self.flakes(s)
+
+    def tearDown(self):
+        sys.setrecursionlimit(self._recursionlimit)