Skip to content

Commit e9cde5f

Browse files
wetorsbinet
authored andcommittedOct 17, 2023
compile,py: fix closure and decorator
1 parent 7102b79 commit e9cde5f

File tree

6 files changed

+466
-55
lines changed

6 files changed

+466
-55
lines changed
 

‎compile/compile.go

+13-2
Original file line numberDiff line numberDiff line change
@@ -213,13 +213,17 @@ func (c *compiler) compileAst(Ast ast.Ast, filename string, futureFlags int, don
213213
case *ast.Suite:
214214
panic("suite should not be possible")
215215
case *ast.Lambda:
216+
code.Argcount = int32(len(node.Args.Args))
217+
code.Kwonlyargcount = int32(len(node.Args.Kwonlyargs))
216218
// Make None the first constant as lambda can't have a docstring
217219
c.Const(py.None)
218220
code.Name = "<lambda>"
219221
c.setQualname()
220222
c.Expr(node.Body)
221223
valueOnStack = true
222224
case *ast.FunctionDef:
225+
code.Argcount = int32(len(node.Args.Args))
226+
code.Kwonlyargcount = int32(len(node.Args.Kwonlyargs))
223227
code.Name = string(node.Name)
224228
c.setQualname()
225229
c.Stmts(c.docString(node.Body, true))
@@ -299,6 +303,7 @@ func (c *compiler) compileAst(Ast ast.Ast, filename string, futureFlags int, don
299303
code.Stacksize = int32(c.OpCodes.StackDepth())
300304
code.Nlocals = int32(len(code.Varnames))
301305
code.Lnotab = string(c.OpCodes.Lnotab())
306+
code.InitCell2arg()
302307
return nil
303308
}
304309

@@ -479,7 +484,8 @@ func (c *compiler) makeClosure(code *py.Code, args uint32, child *compiler, qual
479484
if reftype == symtable.ScopeCell {
480485
arg = c.FindId(name, c.Code.Cellvars)
481486
} else { /* (reftype == FREE) */
482-
arg = c.FindId(name, c.Code.Freevars)
487+
// using CellAndFreeVars in closures requires skipping Cellvars
488+
arg = len(c.Code.Cellvars) + c.FindId(name, c.Code.Freevars)
483489
}
484490
if arg < 0 {
485491
panic(fmt.Sprintf("compile: makeClosure: lookup %q in %q %v %v\nfreevars of %q: %v\n", name, c.SymTable.Name, reftype, arg, code.Name, code.Freevars))
@@ -1363,7 +1369,12 @@ func (c *compiler) NameOp(name string, ctx ast.ExprContext) {
13631369
if op == 0 {
13641370
panic("NameOp: Op not set")
13651371
}
1366-
c.OpArg(op, c.Index(mangled, dict))
1372+
i := c.Index(mangled, dict)
1373+
// using CellAndFreeVars in closures requires skipping Cellvars
1374+
if scope == symtable.ScopeFree {
1375+
i += uint32(len(c.Code.Cellvars))
1376+
}
1377+
c.OpArg(op, i)
13671378
}
13681379

13691380
// Call a function which is already on the stack with n arguments already on the stack

‎py/code.go

+33-28
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,6 @@ func NewCode(argcount int32, kwonlyargcount int32,
112112
filename_ Object, name_ Object, firstlineno int32,
113113
lnotab_ Object) *Code {
114114

115-
var cell2arg []byte
116-
117115
// Type assert the objects
118116
consts := consts_.(Tuple)
119117
namesTuple := names_.(Tuple)
@@ -154,7 +152,6 @@ func NewCode(argcount int32, kwonlyargcount int32,
154152
// return nil;
155153
// }
156154

157-
n_cellvars := len(cellvars)
158155
intern_strings(namesTuple)
159156
intern_strings(varnamesTuple)
160157
intern_strings(freevarsTuple)
@@ -167,13 +164,40 @@ func NewCode(argcount int32, kwonlyargcount int32,
167164
}
168165
}
169166
}
167+
168+
co := &Code{
169+
Argcount: argcount,
170+
Kwonlyargcount: kwonlyargcount,
171+
Nlocals: nlocals,
172+
Stacksize: stacksize,
173+
Flags: flags,
174+
Code: code,
175+
Consts: consts,
176+
Names: names,
177+
Varnames: varnames,
178+
Freevars: freevars,
179+
Cellvars: cellvars,
180+
Filename: filename,
181+
Name: name,
182+
Firstlineno: firstlineno,
183+
Lnotab: lnotab,
184+
Weakreflist: nil,
185+
}
186+
co.InitCell2arg()
187+
return co
188+
}
189+
190+
// Create mapping between cells and arguments if needed.
191+
func (co *Code) InitCell2arg() {
192+
var cell2arg []byte
193+
n_cellvars := len(co.Cellvars)
170194
/* Create mapping between cells and arguments if needed. */
171195
if n_cellvars != 0 {
172-
total_args := argcount + kwonlyargcount
173-
if flags&CO_VARARGS != 0 {
196+
total_args := co.Argcount + co.Kwonlyargcount
197+
if co.Flags&CO_VARARGS != 0 {
174198
total_args++
175199
}
176-
if flags&CO_VARKEYWORDS != 0 {
200+
if co.Flags&CO_VARKEYWORDS != 0 {
177201
total_args++
178202
}
179203
used_cell2arg := false
@@ -182,9 +206,9 @@ func NewCode(argcount int32, kwonlyargcount int32,
182206
cell2arg[i] = CO_CELL_NOT_AN_ARG
183207
}
184208
// Find cells which are also arguments.
185-
for i, cell := range cellvars {
209+
for i, cell := range co.Cellvars {
186210
for j := int32(0); j < total_args; j++ {
187-
arg := varnames[j]
211+
arg := co.Varnames[j]
188212
if cell == arg {
189213
cell2arg[i] = byte(j)
190214
used_cell2arg = true
@@ -196,26 +220,7 @@ func NewCode(argcount int32, kwonlyargcount int32,
196220
cell2arg = nil
197221
}
198222
}
199-
200-
return &Code{
201-
Argcount: argcount,
202-
Kwonlyargcount: kwonlyargcount,
203-
Nlocals: nlocals,
204-
Stacksize: stacksize,
205-
Flags: flags,
206-
Code: code,
207-
Consts: consts,
208-
Names: names,
209-
Varnames: varnames,
210-
Freevars: freevars,
211-
Cellvars: cellvars,
212-
Cell2arg: cell2arg,
213-
Filename: filename,
214-
Name: name,
215-
Firstlineno: firstlineno,
216-
Lnotab: lnotab,
217-
Weakreflist: nil,
218-
}
223+
co.Cell2arg = cell2arg
219224
}
220225

221226
// Return number of free variables

‎vm/tests/class.py

+11-12
Original file line numberDiff line numberDiff line change
@@ -47,17 +47,16 @@ def method1(self, x):
4747
c = x()
4848
assert c.method1(1) == 2
4949

50-
# FIXME doesn't work
51-
# doc="CLASS_DEREF2"
52-
# def classderef2(x):
53-
# class DeRefTest:
54-
# VAR = x
55-
# def method1(self, x):
56-
# "method1"
57-
# return self.VAR+x
58-
# return DeRefTest
59-
# x = classderef2(1)
60-
# c = x()
61-
# assert c.method1(1) == 2
50+
doc="CLASS_DEREF2"
51+
def classderef2(x):
52+
class DeRefTest:
53+
VAR = x
54+
def method1(self, x):
55+
"method1"
56+
return self.VAR+x
57+
return DeRefTest
58+
x = classderef2(1)
59+
c = x()
60+
assert c.method1(1) == 2
6261

6362
doc="finished"

‎vm/tests/decorators.py

+327
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,327 @@
1+
# Copyright 2023 The go-python Authors. All rights reserved.
2+
# Use of this source code is governed by a BSD-style
3+
# license that can be found in the LICENSE file.
4+
5+
# Copied from Python-3.4.9\Lib\test\test_decorators.py
6+
7+
import libtest as self
8+
9+
def funcattrs(**kwds):
10+
def decorate(func):
11+
# FIXME func.__dict__.update(kwds)
12+
for k, v in kwds.items():
13+
func.__dict__[k] = v
14+
return func
15+
return decorate
16+
17+
class MiscDecorators (object):
18+
@staticmethod
19+
def author(name):
20+
def decorate(func):
21+
func.__dict__['author'] = name
22+
return func
23+
return decorate
24+
25+
# -----------------------------------------------
26+
27+
class DbcheckError (Exception):
28+
def __init__(self, exprstr, func, args, kwds):
29+
# A real version of this would set attributes here
30+
Exception.__init__(self, "dbcheck %r failed (func=%s args=%s kwds=%s)" %
31+
(exprstr, func, args, kwds))
32+
33+
34+
def dbcheck(exprstr, globals=None, locals=None):
35+
"Decorator to implement debugging assertions"
36+
def decorate(func):
37+
expr = compile(exprstr, "dbcheck-%s" % func.__name__, "eval")
38+
def check(*args, **kwds):
39+
if not eval(expr, globals, locals):
40+
raise DbcheckError(exprstr, func, args, kwds)
41+
return func(*args, **kwds)
42+
return check
43+
return decorate
44+
45+
# -----------------------------------------------
46+
47+
def countcalls(counts):
48+
"Decorator to count calls to a function"
49+
def decorate(func):
50+
func_name = func.__name__
51+
counts[func_name] = 0
52+
def call(*args, **kwds):
53+
counts[func_name] += 1
54+
return func(*args, **kwds)
55+
call.__name__ = func_name
56+
return call
57+
return decorate
58+
59+
# -----------------------------------------------
60+
61+
# FIXME: dict can only have string keys
62+
# def memoize(func):
63+
# saved = {}
64+
# def call(*args):
65+
# try:
66+
# return saved[args]
67+
# except KeyError:
68+
# res = func(*args)
69+
# saved[args] = res
70+
# return res
71+
# except TypeError:
72+
# # Unhashable argument
73+
# return func(*args)
74+
# call.__name__ = func.__name__
75+
# return call
76+
def memoize(func):
77+
saved = {}
78+
def call(*args):
79+
try:
80+
if isinstance(args[0], list):
81+
raise TypeError
82+
return saved[str(args)]
83+
except KeyError:
84+
res = func(*args)
85+
saved[str(args)] = res
86+
return res
87+
except TypeError:
88+
# Unhashable argument
89+
return func(*args)
90+
call.__name__ = func.__name__
91+
return call
92+
93+
# -----------------------------------------------
94+
95+
doc="test_single"
96+
# FIXME staticmethod
97+
# class C(object):
98+
# @staticmethod
99+
# def foo(): return 42
100+
# self.assertEqual(C.foo(), 42)
101+
# self.assertEqual(C().foo(), 42)
102+
103+
doc="test_staticmethod_function"
104+
@staticmethod
105+
def notamethod(x):
106+
return x
107+
self.assertRaises(TypeError, notamethod, 1)
108+
109+
doc="test_dotted"
110+
# FIXME class decorator
111+
# decorators = MiscDecorators()
112+
# @decorators.author('Cleese')
113+
# def foo(): return 42
114+
# self.assertEqual(foo(), 42)
115+
# self.assertEqual(foo.author, 'Cleese')
116+
117+
doc="test_argforms"
118+
def noteargs(*args, **kwds):
119+
def decorate(func):
120+
setattr(func, 'dbval', (args, kwds))
121+
return func
122+
return decorate
123+
124+
args = ( 'Now', 'is', 'the', 'time' )
125+
kwds = dict(one=1, two=2)
126+
@noteargs(*args, **kwds)
127+
def f1(): return 42
128+
self.assertEqual(f1(), 42)
129+
self.assertEqual(f1.dbval, (args, kwds))
130+
131+
@noteargs('terry', 'gilliam', eric='idle', john='cleese')
132+
def f2(): return 84
133+
self.assertEqual(f2(), 84)
134+
self.assertEqual(f2.dbval, (('terry', 'gilliam'),
135+
dict(eric='idle', john='cleese')))
136+
137+
@noteargs(1, 2,)
138+
def f3(): pass
139+
self.assertEqual(f3.dbval, ((1, 2), {}))
140+
141+
doc="test_dbcheck"
142+
# FIXME TypeError: "catching 'BaseException' that does not inherit from BaseException is not allowed"
143+
# @dbcheck('args[1] is not None')
144+
# def f(a, b):
145+
# return a + b
146+
# self.assertEqual(f(1, 2), 3)
147+
# self.assertRaises(DbcheckError, f, 1, None)
148+
149+
doc="test_memoize"
150+
counts = {}
151+
152+
@memoize
153+
@countcalls(counts)
154+
def double(x):
155+
return x * 2
156+
self.assertEqual(double.__name__, 'double')
157+
158+
self.assertEqual(counts, dict(double=0))
159+
160+
# Only the first call with a given argument bumps the call count:
161+
#
162+
# Only the first call with a given argument bumps the call count:
163+
#
164+
self.assertEqual(double(2), 4)
165+
self.assertEqual(counts['double'], 1)
166+
self.assertEqual(double(2), 4)
167+
self.assertEqual(counts['double'], 1)
168+
self.assertEqual(double(3), 6)
169+
self.assertEqual(counts['double'], 2)
170+
171+
# Unhashable arguments do not get memoized:
172+
#
173+
self.assertEqual(double([10]), [10, 10])
174+
self.assertEqual(counts['double'], 3)
175+
self.assertEqual(double([10]), [10, 10])
176+
self.assertEqual(counts['double'], 4)
177+
178+
doc="test_errors"
179+
# Test syntax restrictions - these are all compile-time errors:
180+
#
181+
for expr in [ "1+2", "x[3]", "(1, 2)" ]:
182+
# Sanity check: is expr is a valid expression by itself?
183+
compile(expr, "testexpr", "exec")
184+
185+
codestr = "@%s\ndef f(): pass" % expr
186+
self.assertRaises(SyntaxError, compile, codestr, "test", "exec")
187+
188+
# You can't put multiple decorators on a single line:
189+
#
190+
self.assertRaises(SyntaxError, compile,
191+
"@f1 @f2\ndef f(): pass", "test", "exec")
192+
193+
# Test runtime errors
194+
195+
def unimp(func):
196+
raise NotImplementedError
197+
context = dict(nullval=None, unimp=unimp)
198+
199+
for expr, exc in [ ("undef", NameError),
200+
("nullval", TypeError),
201+
("nullval.attr", NameError), # FIXME ("nullval.attr", AttributeError),
202+
("unimp", NotImplementedError)]:
203+
codestr = "@%s\ndef f(): pass\nassert f() is None" % expr
204+
code = compile(codestr, "test", "exec")
205+
self.assertRaises(exc, eval, code, context)
206+
207+
doc="test_double"
208+
class C(object):
209+
@funcattrs(abc=1, xyz="haha")
210+
@funcattrs(booh=42)
211+
def foo(self): return 42
212+
self.assertEqual(C().foo(), 42)
213+
self.assertEqual(C.foo.abc, 1)
214+
self.assertEqual(C.foo.xyz, "haha")
215+
self.assertEqual(C.foo.booh, 42)
216+
217+
218+
doc="test_order"
219+
# Test that decorators are applied in the proper order to the function
220+
# they are decorating.
221+
def callnum(num):
222+
"""Decorator factory that returns a decorator that replaces the
223+
passed-in function with one that returns the value of 'num'"""
224+
def deco(func):
225+
return lambda: num
226+
return deco
227+
@callnum(2)
228+
@callnum(1)
229+
def foo(): return 42
230+
self.assertEqual(foo(), 2,
231+
"Application order of decorators is incorrect")
232+
233+
234+
doc="test_eval_order"
235+
# Evaluating a decorated function involves four steps for each
236+
# decorator-maker (the function that returns a decorator):
237+
#
238+
# 1: Evaluate the decorator-maker name
239+
# 2: Evaluate the decorator-maker arguments (if any)
240+
# 3: Call the decorator-maker to make a decorator
241+
# 4: Call the decorator
242+
#
243+
# When there are multiple decorators, these steps should be
244+
# performed in the above order for each decorator, but we should
245+
# iterate through the decorators in the reverse of the order they
246+
# appear in the source.
247+
# FIXME class decorator
248+
# actions = []
249+
#
250+
# def make_decorator(tag):
251+
# actions.append('makedec' + tag)
252+
# def decorate(func):
253+
# actions.append('calldec' + tag)
254+
# return func
255+
# return decorate
256+
#
257+
# class NameLookupTracer (object):
258+
# def __init__(self, index):
259+
# self.index = index
260+
#
261+
# def __getattr__(self, fname):
262+
# if fname == 'make_decorator':
263+
# opname, res = ('evalname', make_decorator)
264+
# elif fname == 'arg':
265+
# opname, res = ('evalargs', str(self.index))
266+
# else:
267+
# assert False, "Unknown attrname %s" % fname
268+
# actions.append('%s%d' % (opname, self.index))
269+
# return res
270+
#
271+
# c1, c2, c3 = map(NameLookupTracer, [ 1, 2, 3 ])
272+
#
273+
# expected_actions = [ 'evalname1', 'evalargs1', 'makedec1',
274+
# 'evalname2', 'evalargs2', 'makedec2',
275+
# 'evalname3', 'evalargs3', 'makedec3',
276+
# 'calldec3', 'calldec2', 'calldec1' ]
277+
#
278+
# actions = []
279+
# @c1.make_decorator(c1.arg)
280+
# @c2.make_decorator(c2.arg)
281+
# @c3.make_decorator(c3.arg)
282+
# def foo(): return 42
283+
# self.assertEqual(foo(), 42)
284+
#
285+
# self.assertEqual(actions, expected_actions)
286+
#
287+
# # Test the equivalence claim in chapter 7 of the reference manual.
288+
# #
289+
# actions = []
290+
# def bar(): return 42
291+
# bar = c1.make_decorator(c1.arg)(c2.make_decorator(c2.arg)(c3.make_decorator(c3.arg)(bar)))
292+
# self.assertEqual(bar(), 42)
293+
# self.assertEqual(actions, expected_actions)
294+
295+
doc="test_simple"
296+
def plain(x):
297+
x.extra = 'Hello'
298+
return x
299+
@plain
300+
class C(object): pass
301+
self.assertEqual(C.extra, 'Hello')
302+
303+
doc="test_double"
304+
def ten(x):
305+
x.extra = 10
306+
return x
307+
def add_five(x):
308+
x.extra += 5
309+
return x
310+
311+
@add_five
312+
@ten
313+
class C(object): pass
314+
self.assertEqual(C.extra, 15)
315+
316+
doc="test_order"
317+
def applied_first(x):
318+
x.extra = 'first'
319+
return x
320+
def applied_second(x):
321+
x.extra = 'second'
322+
return x
323+
@applied_second
324+
@applied_first
325+
class C(object): pass
326+
self.assertEqual(C.extra, 'second')
327+
doc="finished"

‎vm/tests/functions.py

+25-13
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,32 @@ def fn2(x,y=1):
2121
assert fn2(1,y=4) == 5
2222

2323
# Closure
24+
doc="closure1"
25+
closure1 = lambda x: lambda y: x+y
26+
cf1 = closure1(1)
27+
assert cf1(1) == 2
28+
assert cf1(2) == 3
29+
30+
doc="closure2"
31+
def closure2(*args, **kwargs):
32+
def inc():
33+
kwargs['x'] += 1
34+
return kwargs['x']
35+
return inc
36+
cf2 = closure2(x=1)
37+
assert cf2() == 2
38+
assert cf2() == 3
2439

25-
# FIXME something wrong with closures over function arguments...
26-
# doc="counter3"
27-
# def counter3(x):
28-
# def inc():
29-
# nonlocal x
30-
# x += 1
31-
# return x
32-
# return inc
33-
# fn3 = counter3(1)
34-
# assert fn3() == 2
35-
# assert fn3() == 3
40+
doc="counter3"
41+
def counter3(x):
42+
def inc():
43+
nonlocal x
44+
x += 1
45+
return x
46+
return inc
47+
fn3 = counter3(1)
48+
assert fn3() == 2
49+
assert fn3() == 3
3650

3751
doc="counter4"
3852
def counter4(initial):
@@ -238,6 +252,4 @@ def fn16_6(*,a,b,c):
238252
ck(fn16_5, "fn16_5() missing 2 required keyword-only arguments: 'a' and 'b'")
239253
ck(fn16_6, "fn16_6() missing 3 required keyword-only arguments: 'a', 'b', and 'c'")
240254

241-
#FIXME decorators
242-
243255
doc="finished"

‎vm/tests/libtest.py

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Copyright 2023 The go-python Authors. All rights reserved.
2+
# Use of this source code is governed by a BSD-style
3+
# license that can be found in the LICENSE file.
4+
5+
# Imitate the calling method of unittest
6+
7+
def assertRaises(expecting, fn, *args, **kwargs):
8+
"""Check the exception was raised - don't check the text"""
9+
try:
10+
fn(*args, **kwargs)
11+
except expecting as e:
12+
pass
13+
else:
14+
assert False, "%s not raised" % (expecting,)
15+
16+
def assertEqual(first, second, msg=None):
17+
if msg:
18+
assert first == second, "%s not equal" % (msg,)
19+
else:
20+
assert first == second
21+
22+
def assertIs(expr1, expr2, msg=None):
23+
if msg:
24+
assert expr1 is expr2, "%s is not None" % (msg,)
25+
else:
26+
assert expr1 is expr2
27+
28+
def assertIsNone(obj, msg=None):
29+
if msg:
30+
assert obj is None, "%s is not None" % (msg,)
31+
else:
32+
assert obj is None
33+
34+
def assertTrue(obj, msg=None):
35+
if msg:
36+
assert obj, "%s is not True" % (msg,)
37+
else:
38+
assert obj
39+
40+
def assertRaisesText(expecting, text, fn, *args, **kwargs):
41+
"""Check the exception with text in is raised"""
42+
try:
43+
fn(*args, **kwargs)
44+
except expecting as e:
45+
assert text in e.args[0], "'%s' not found in '%s'" % (text, e.args[0])
46+
else:
47+
assert False, "%s not raised" % (expecting,)
48+
49+
def assertTypedEqual(actual, expect, msg=None):
50+
assertEqual(actual, expect, msg)
51+
def recurse(actual, expect):
52+
if isinstance(expect, (tuple, list)):
53+
for x, y in zip(actual, expect):
54+
recurse(x, y)
55+
else:
56+
assertIs(type(actual), type(expect))
57+
recurse(actual, expect)

0 commit comments

Comments
 (0)
Please sign in to comment.