PR nedbat#20 from Darius

Author: vrthra

AUTHORS

@@ -2,3 +2,4 @@ Paul Swartz
 Ned Batchelder
 Allison Kaptur
 Laura Lindzey
+Darius Bacon

byterun/pyobj.py

@@ -2,6 +2,7 @@
 
 import collections
 import inspect
+import re
 import types
 import dis
 
@@ -63,17 +64,18 @@ def __get__(self, instance, owner):
             return self
 
     def __call__(self, *args, **kwargs):
-        if PY2 and self.func_name in ["<setcomp>", "<dictcomp>", "<genexpr>"]:
+        if re.search(r'<(?:listcomp|setcomp|dictcomp|genexpr)>$', self.func_name):
             # D'oh! http://bugs.python.org/issue19611 Py2 doesn't know how to
             # inspect set comprehensions, dict comprehensions, or generator
             # expressions properly.  They are always functions of one argument,
-            # so just do the right thing.
+            # so just do the right thing.  Py3.4 also would fail without this
+            # hack, for list comprehensions too. (Haven't checked for other 3.x.)
             assert len(args) == 1 and not kwargs, "Surprising comprehension!"
             callargs = {".0": args[0]}
         else:
             callargs = inspect.getcallargs(self._func, *args, **kwargs)
         frame = self._vm.make_frame(
-            self.func_code, callargs, self.func_globals, {}
+            self.func_code, callargs, self.func_globals, {}, self.func_closure
         )
         CO_GENERATOR = 32           # flag for "this code uses yield"
         if self.func_code.co_flags & CO_GENERATOR:
@@ -137,7 +139,7 @@ def set(self, value):
 
 
 class Frame(object):
-    def __init__(self, f_code, f_globals, f_locals, f_back):
+    def __init__(self, f_code, f_globals, f_locals, f_closure, f_back):
         self.f_code = f_code
         self.py36_opcodes = list(dis.get_instructions(self.f_code)) \
             if six.PY3 and sys.version_info.minor >= 6 else None
@@ -155,24 +157,13 @@ def __init__(self, f_code, f_globals, f_locals, f_back):
         self.f_lineno = f_code.co_firstlineno
         self.f_lasti = 0
 
-        if f_code.co_cellvars:
-            self.cells = {}
-            if not f_back.cells:
-                f_back.cells = {}
-            for var in f_code.co_cellvars:
-                # Make a cell for the variable in our locals, or None.
-                cell = Cell(self.f_locals.get(var))
-                f_back.cells[var] = self.cells[var] = cell
-        else:
-            self.cells = None
-
+        self.cells = {} if f_code.co_cellvars or f_code.co_freevars else None
+        for var in f_code.co_cellvars:
+            # Make a cell for the variable in our locals, or None.
+            self.cells[var] = Cell(self.f_locals.get(var))
         if f_code.co_freevars:
-            if not self.cells:
-                self.cells = {}
-            for var in f_code.co_freevars:
-                assert self.cells is not None
-                assert f_back.cells, "f_back.cells: %r" % (f_back.cells,)
-                self.cells[var] = f_back.cells[var]
+            assert len(f_code.co_freevars) == len(f_closure)
+            self.cells.update(zip(f_code.co_freevars, f_closure))
 
         self.block_stack = []
         self.generator = None

byterun/pyvm2.py

@@ -16,7 +16,7 @@
 
 PY3, PY2 = six.PY3, not six.PY3
 
-from .pyobj import Frame, Block, Method, Function, Generator
+from .pyobj import Frame, Block, Method, Function, Generator, Cell
 
 log = logging.getLogger(__name__)
 
@@ -91,7 +91,7 @@ def push_block(self, type, handler=None, level=None):
     def pop_block(self):
         return self.frame.block_stack.pop()
 
-    def make_frame(self, code, callargs={}, f_globals=None, f_locals=None):
+    def make_frame(self, code, callargs={}, f_globals=None, f_locals=None, f_closure=None):
         log.info("make_frame: code=%r, callargs=%s" % (code, repper(callargs)))
         if f_globals is not None:
             f_globals = f_globals
@@ -108,7 +108,7 @@ def make_frame(self, code, callargs={}, f_globals=None, f_locals=None):
                 '__package__': None,
             }
         f_locals.update(callargs)
-        frame = Frame(code, f_globals, f_locals, self.frame)
+        frame = Frame(code, f_globals, f_locals, f_closure, self.frame)
         return frame
 
     def push_frame(self, frame):
@@ -446,7 +446,10 @@ def byte_LOAD_GLOBAL(self, name):
         elif name in f.f_builtins:
             val = f.f_builtins[name]
         else:
-            raise NameError("global name '%s' is not defined" % name)
+            if PY2:
+                raise NameError("global name '%s' is not defined" % name)
+            elif PY3:
+                raise NameError("name '%s' is not defined" % name)
         self.push(val)
 
     def byte_STORE_GLOBAL(self, name):
@@ -1173,11 +1176,59 @@ def byte_BUILD_CLASS(self):
     elif PY3:
         def byte_LOAD_BUILD_CLASS(self):
             # New in py3
-            self.push(__build_class__)
+            self.push(build_class)
 
         def byte_STORE_LOCALS(self):
             self.frame.f_locals = self.pop()
 
     if 0:   # Not in py2.7
         def byte_SET_LINENO(self, lineno):
             self.frame.f_lineno = lineno
+
+if PY3:
+    def build_class(func, name, *bases, **kwds):
+        "Like __build_class__ in bltinmodule.c, but running in the byterun VM."
+        if not isinstance(func, Function):
+            raise TypeError("func must be a function")
+        if not isinstance(name, str):
+            raise TypeError("name is not a string")
+        metaclass = kwds.pop('metaclass', None)
+        # (We don't just write 'metaclass=None' in the signature above
+        # because that's a syntax error in Py2.)
+        if metaclass is None:
+            metaclass = type(bases[0]) if bases else type
+        if isinstance(metaclass, type):
+            metaclass = calculate_metaclass(metaclass, bases)
+
+        try:
+            prepare = metaclass.__prepare__
+        except AttributeError:
+            namespace = {}
+        else:
+            namespace = prepare(name, bases, **kwds)
+
+        # Execute the body of func. This is the step that would go wrong if
+        # we tried to use the built-in __build_class__, because __build_class__
+        # does not call func, it magically executes its body directly, as we
+        # do here (except we invoke our VirtualMachine instead of CPython's).
+        frame = func._vm.make_frame(func.func_code,
+                                    f_globals=func.func_globals,
+                                    f_locals=namespace,
+                                    f_closure=func.func_closure)
+        cell = func._vm.run_frame(frame)
+
+        cls = metaclass(name, bases, namespace)
+        if isinstance(cell, Cell):
+            cell.set(cls)
+        return cls
+
+    def calculate_metaclass(metaclass, bases):
+        "Determine the most derived metatype."
+        winner = metaclass
+        for base in bases:
+            t = type(base)
+            if issubclass(t, winner):
+                winner = t
+            elif not issubclass(winner, t):
+                raise TypeError("metaclass conflict", winner, t)
+        return winner

tests/test_functions.py

@@ -207,6 +207,18 @@ def f4(g):
             assert answer == 54
             """)
 
+    def test_closure_vars_from_static_parent(self):
+        self.assert_ok("""\
+            def f(xs):
+                return lambda: xs[0]
+
+            def g(h):
+                xs = 5
+                lambda: xs
+                return h()
+
+            assert g(f([42])) == 42
+            """)
 
 class TestGenerators(vmtest.VmTestCase):
     def test_first(self):

tests/vmtest.py

@@ -40,6 +40,19 @@ def assert_ok(self, code, raises=None):
         # Print the disassembly so we'll see it if the test fails.
         dis_code(code)
 
+        # Run the code through our VM and the real Python interpreter, for comparison.
+        vm_value, vm_exc, vm_stdout = self.run_in_byterun(code)
+        py_value, py_exc, py_stdout = self.run_in_real_python(code)
+
+        self.assert_same_exception(vm_exc, py_exc)
+        self.assertEqual(vm_stdout.getvalue(), py_stdout.getvalue())
+        self.assertEqual(vm_value, py_value)
+        if raises:
+            self.assertIsInstance(vm_exc, raises)
+        else:
+            self.assertIsNone(vm_exc)
+
+    def run_in_byterun(self, code):
         real_stdout = sys.stdout
 
         # Run the code through our VM.
@@ -64,32 +77,36 @@ def assert_ok(self, code, raises=None):
                 raise
             vm_exc = e
         finally:
+            sys.stdout = real_stdout
             real_stdout.write("-- stdout ----------\n")
             real_stdout.write(vm_stdout.getvalue())
 
-        # Run the code through the real Python interpreter, for comparison.
+        return vm_value, vm_exc, vm_stdout
+
+    def run_in_real_python(self, code):
+        real_stdout = sys.stdout
 
         py_stdout = six.StringIO()
         sys.stdout = py_stdout
 
         py_value = py_exc = None
-        globs = {}
+        globs = {
+            '__builtins__': __builtins__,
+            '__name__': '__main__',
+            '__doc__': None,
+            '__package__': None,
+        }
+
         try:
             py_value = eval(code, globs, globs)
         except AssertionError:              # pragma: no cover
             raise
         except Exception as e:
             py_exc = e
+        finally:
+            sys.stdout = real_stdout
 
-        sys.stdout = real_stdout
-
-        self.assert_same_exception(vm_exc, py_exc)
-        self.assertEqual(vm_stdout.getvalue(), py_stdout.getvalue())
-        self.assertEqual(vm_value, py_value)
-        if raises:
-            self.assertIsInstance(vm_exc, raises)
-        else:
-            self.assertIsNone(vm_exc)
+        return py_value, py_exc, py_stdout
 
     def assert_same_exception(self, e1, e2):
         """Exceptions don't implement __eq__, check it ourselves."""