Preprocessor for ULP/RTC macros

esp32_ulp/assemble.py

@@ -240,7 +240,7 @@ def d_align(self, align=4, fill=None):
             self.fill(self.section, amount, fill)
 
     def d_set(self, symbol, expr):
-        value = int(expr)  # TODO: support more than just integers
+        value = int(opcodes.eval_arg(expr))  # TODO: support more than just integers
         self.symbols.set_sym(symbol, ABS, None, value)
 
     def d_global(self, symbol):

esp32_ulp/opcodes.py

@@ -6,6 +6,7 @@
 from uctypes import struct, addressof, LITTLE_ENDIAN, UINT32, BFUINT32, BF_POS, BF_LEN
 
 from .soc import *
+from .util import split_tokens, validate_expression
 
 # XXX dirty hack: use a global for the symbol table
 symbols = None
@@ -267,6 +268,20 @@ def make_ins(layout):
 ARG = namedtuple('ARG', ('type', 'value', 'raw'))
 
 
+def eval_arg(arg):
+    parts = []
+    for token in split_tokens(arg):
+        if symbols.has_sym(token):
+            _, _, sym_value = symbols.get_sym(token)
+            parts.append(str(sym_value))
+        else:
+            parts.append(token)
+    parts = "".join(parts)
+    if not validate_expression(parts):
+        raise ValueError('Unsupported expression: %s' % parts)
+    return eval(parts)
+
+
 def arg_qualify(arg):
     """
     look at arg and qualify its type:
@@ -289,8 +304,12 @@ def arg_qualify(arg):
         return ARG(IMM, int(arg), arg)
     except ValueError:
         pass
-    entry = symbols.get_sym(arg)
-    return ARG(SYM, entry, arg)
+    try:
+        entry = symbols.get_sym(arg)
+        return ARG(SYM, entry, arg)
+    except KeyError:
+        pass
+    return ARG(IMM, int(eval_arg(arg)), arg)
 
 
 def get_reg(arg):

esp32_ulp/util.py

@@ -2,10 +2,68 @@
 
 import gc
 
+NORMAL, WHITESPACE = 0, 1
+
 
 def garbage_collect(msg, verbose=DEBUG):
     free_before = gc.mem_free()
     gc.collect()
     free_after = gc.mem_free()
     if verbose:
         print("%s: %d --gc--> %d bytes free" % (msg, free_before, free_after))
+
+
+def split_tokens(line):
+    buf = ""
+    tokens = []
+    state = NORMAL
+    for c in line:
+        if ('a' <= c <= 'z') or ('A' <= c <= 'Z') or ('0' <= c <= '9') or c == '_':
+            if state != NORMAL:
+                if len(buf) > 0:
+                    tokens.append(buf)
+                    buf = ""
+                state = NORMAL
+            buf += c
+        elif c == ' ' or c == '\t':
+            if state != WHITESPACE:
+                if len(buf) > 0:
+                    tokens.append(buf)
+                    buf = ""
+                state = WHITESPACE
+            buf += c
+        else:
+            if len(buf) > 0:
+                tokens.append(buf)
+                buf = ""
+            tokens.append(c)
+
+    if len(buf) > 0:
+        tokens.append(buf)
+
+    return tokens
+
+
+def validate_expression(param):
+    for token in split_tokens(param):
+        state = 0
+        for c in token:
+            if c not in ' \t+-*/%()<>&|~x0123456789abcdef':
+                return False
+
+            # the following allows hex digits a-f after 0x but not otherwise
+            if state == 0:
+                if c in 'abcdef':
+                    return False
+                if c == '0':
+                    state = 1
+                continue
+
+            if state == 1:
+                state = 2 if c == 'x' else 0
+                continue
+
+            if state == 2:
+                if c not in '0123456789abcdef':
+                    state = 0
+    return True

tests/00_unit_tests.sh

@@ -4,7 +4,7 @@
 
 set -e
 
-for file in opcodes assemble link ; do
+for file in opcodes assemble link util; do
     echo testing $file...
     micropython $file.py
 done

tests/assemble.py

@@ -135,6 +135,28 @@ def test_assemble_uppercase_opcode():
     assert not raised
 
 
+def test_assemble_evalulate_expressions():
+    src_w_expr = """\
+    .set shft, 2
+    .set loops, (1 << shft)
+
+entry:
+    move r0, 1+1
+    move r1, loops
+    move r2, (shft + 10) * 2
+    move r3, entry << 2
+"""
+    a = Assembler()
+    a.assemble(src_w_expr)
+
+    assert a.symbols.has_sym('shft')
+    assert a.symbols.has_sym('loops')
+    assert a.symbols.has_sym('entry')
+    assert a.symbols.get_sym('shft') == (ABS, None, 2)
+    assert a.symbols.get_sym('loops') == (ABS, None, 4)
+    assert a.symbols.get_sym('entry') == (REL, TEXT, 0)
+
+
 def test_symbols():
     st = SymbolTable({}, {}, {})
     for entry in [
@@ -195,4 +217,5 @@ def test_symbols():
 test_assemble_bss_with_value()
 test_assemble_global()
 test_assemble_uppercase_opcode()
+test_assemble_evalulate_expressions()
 test_symbols()

tests/opcodes.py

@@ -1,6 +1,8 @@
 from uctypes import UINT32, BFUINT32, BF_POS, BF_LEN
 from esp32_ulp.opcodes import make_ins, make_ins_struct_def
-from esp32_ulp.opcodes import get_reg, get_imm, get_cond, arg_qualify, ARG, REG, IMM, COND
+from esp32_ulp.opcodes import get_reg, get_imm, get_cond, arg_qualify, eval_arg, ARG, REG, IMM, SYM, COND
+from esp32_ulp.assemble import SymbolTable, ABS, REL, TEXT
+import esp32_ulp.opcodes as opcodes
 
 OPCODE_DELAY = 4
 LAYOUT_DELAY = """
@@ -43,6 +45,19 @@ def test_arg_qualify():
     assert arg_qualify('Eq') == ARG(COND, 'eq', 'Eq')
     assert arg_qualify('EQ') == ARG(COND, 'eq', 'EQ')
 
+    # for the next tests, ensure the opcodes module has a SymbolTable
+    opcodes.symbols = SymbolTable({}, {}, {})
+    opcodes.symbols.set_sym('const', ABS, None, 42)  # constant as defined by .set
+    opcodes.symbols.set_sym('entry', REL, TEXT, 4)  # label pointing to code
+
+    assert arg_qualify('1+1') == ARG(IMM, 2, '1+1')
+    assert arg_qualify('const >> 1') == ARG(IMM, 21, 'const >> 1')
+    assert arg_qualify('entry') == ARG(SYM, (REL, TEXT, 4), 'entry')  # symbols should not (yet) be evaluated
+    assert arg_qualify('entry + const') == ARG(IMM, 46, 'entry + const')
+
+    # clean up
+    opcodes.symbols = None
+
 
 def test_get_reg():
     assert get_reg('r0') == 0
@@ -57,9 +72,46 @@ def test_get_cond():
     assert get_cond('Eq') == 'eq'
 
 
+def test_eval_arg():
+    opcodes.symbols = SymbolTable({}, {}, {})
+    opcodes.symbols.set_sym('const', ABS, None, 42)  # constant
+    opcodes.symbols.set_sym('raise', ABS, None, 99)  # constant using a python keyword as name (is allowed)
+
+    assert eval_arg('1+1') == 2
+    assert eval_arg('1+const') == 43
+    assert eval_arg('raise*2/3') == 66
+    assert eval_arg('raise-const') == 57
+    assert eval_arg('(raise-const)*2') == 114
+    assert eval_arg('const    % 5') == 2
+    assert eval_arg('const + 0x19af') == 0x19af + 42
+    assert eval_arg('const & ~2') == 40
+    assert eval_arg('const << 3') == 336
+    assert eval_arg('const >> 1') == 21
+    assert eval_arg('(const|4)&0xf') == 0xe
+
+    assert_raises(ValueError, eval_arg, 'evil()')
+    assert_raises(ValueError, eval_arg, 'def cafe()')
+    assert_raises(ValueError, eval_arg, '1 ^ 2')
+    assert_raises(ValueError, eval_arg, '!100')
+
+    # clean up
+    opcodes.symbols = None
+
+
+def assert_raises(exception, func, *args):
+    try:
+        func(*args)
+    except exception:
+        raised = True
+    else:
+        raised = False
+    assert raised
+
+
 test_make_ins_struct_def()
 test_make_ins()
 test_arg_qualify()
 test_get_reg()
 test_get_imm()
 test_get_cond()
+test_eval_arg()

tests/util.py

@@ -0,0 +1,62 @@
+from esp32_ulp.util import split_tokens, validate_expression
+
+tests = []
+
+
+def test(param):
+    """
+    the @test decorator
+    """
+    tests.append(param)
+
+
+@test
+def test_split_tokens():
+    assert split_tokens("") == []
+    assert split_tokens("t") == ['t']
+    assert split_tokens("test") == ['test']
+    assert split_tokens("t t") == ['t', ' ', 't']
+    assert split_tokens("t,t") == ['t', ',', 't']
+    assert split_tokens("test(arg)") == ['test', '(', 'arg', ')']
+    assert split_tokens("test(arg,arg2)") == ['test', '(', 'arg', ',', 'arg2', ')']
+    assert split_tokens("test(arg,arg2)") == ['test', '(', 'arg', ',', 'arg2', ')']
+    assert split_tokens("  test(  arg,  arg2)") == ['  ', 'test', '(', '  ', 'arg', ',', '  ', 'arg2', ')']
+    assert split_tokens("  test(  arg )  ") == ['  ', 'test', '(', '  ', 'arg', ' ', ')', '  ']
+    assert split_tokens("\t  test  \t  ") == ['\t  ', 'test', "  \t  "]
+    assert split_tokens("test\nrow2") == ['test', "\n", "row2"]
+
+    # split_token does not support comments. should generally only be used after comments are already stripped
+    assert split_tokens("test(arg /*comment*/)") == ['test', '(', 'arg', ' ', '/', '*', 'comment', '*', '/', ')']
+    assert split_tokens("#test") == ['#', 'test']
+
+
+@test
+def test_validate_expression():
+    assert validate_expression('') is True
+    assert validate_expression('1') is True
+    assert validate_expression('1+1') is True
+    assert validate_expression('(1+1)') is True
+    assert validate_expression('(1+1)*2') is True
+    assert validate_expression('(1 + 1)') is True
+    assert validate_expression('10 % 2') is True
+    assert validate_expression('0x100 << 2') is True
+    assert validate_expression('0x100 & ~2') is True
+    assert validate_expression('0xabcdef') is True
+    assert validate_expression('0x123def') is True
+    assert validate_expression('2*3+4/5&6|7') is True
+    assert validate_expression('(((((1+1) * 2') is True  # valid characters, even if expression is not valid
+
+    assert validate_expression(':') is False
+    assert validate_expression('_') is False
+    assert validate_expression('=') is False
+    assert validate_expression('.') is False
+    assert validate_expression('!') is False
+    assert validate_expression('123 ^ 4') is False  # operator not supported for now
+    assert validate_expression('evil()') is False
+    assert validate_expression('def cafe()') is False  # valid hex digits, but potentially dangerous code
+
+
+if __name__ == '__main__':
+    # run all methods marked with @test
+    for t in tests:
+        t()