scripts: Adopted Parser class in prettyasserts.py

This ended up being a pretty in-depth rework of prettyasserts.py to
adopt the shared Parser class. But now prettyasserts.py should be both
more robust and faster.

The tricky parts:

- The Parser class eagerly munches whitespace by default. This is
  usually a good thing, but for prettyasserts.py we need to keep track
  of the whitespace somehow in order to write it to the output file.

  The solution here is a little bit hacky. Instead of complicating the
  Parser class, we implicitly add a regex group for whitespace when
  compiling our lexer.

  Unfortunately this does make last-minute patching of the lexer a bit
  messy (for things like -p/--prefix, etc), thanks to Python's
  re.Pattern class not being extendable. To work around this, the Lexer
  class keeps track of the original patterns to allow recompilation.

- Since we no longer tokenize in a separate pass, we can't use the
  None token to match any unmatched tokens.

  Fortunately this can be worked around with sufficiently ugly regex.
  See the 'STUFF' rule.

  It's a good thing Python has negative lookaheads.

  On the flip side, this means we no longer need to explicitly specify
  all possible tokens when multiple tokens overlap.

- Unlike stack.py/csv.py, prettyasserts.py needs multi-token lookahead.

  Fortunately this has a pretty straightforward solution with the
  addition of an optional stack to the Parser class.

  We can even have a bit of fun with Python's with statements (though I
  do wish with statements could have else clauses, so we wouldn't need
  double nesting to catch parser exceptions).

---

In addition to adopting the new Parser class, I also made sure to
eliminate intermediate string allocation through heavy use of Python's
io.StringIO class.

This, plus Parser's cheap shallow chomp/slice operations, gives
prettyasserts.py a much needed speed boost.

(Honestly, the original prettyasserts.py was pretty naive, with the
assumption that it wouldn't be the bottleneck during compilation. This
turned out to be wrong.)

These changes cut total compile time in ~half:

                                          real      user      sys
  before (time make test-runner -j): 0m56.202s 2m31.853s 0m2.827s
  after  (time make test-runner -j): 0m26.836s 1m51.213s 0m2.338s

Keep in mind this includes both prettyasserts.py and gcc -Os (and other
Makefile stuff).

scripts/csv.py

@@ -370,7 +370,7 @@ class RGStddev:
 # basically just because memoryview doesn't support strs
 class Parser:
     def __init__(self, data, ws='\s*', ws_flags=0):
-        self.data = data.lstrip()
+        self.data = data
         self.i = 0
         self.m = None
         # also consume whitespace
@@ -378,9 +378,10 @@ class Parser:
         self.i = self.ws.match(self.data, self.i).end()
 
     def __repr__(self):
-        return '%s(%r...)' % (
+        if len(self.data) - self.i <= 32:
-                self.__class__.__name__,
+            return repr(self.data[self.i:])
-                self.data[self.i:self.i+32])
+        else:
+            return "%s..." % repr(self.data[self.i:self.i+32])[:32]
 
     def __str__(self):
         return self.data[self.i:]
@@ -411,15 +412,36 @@ class Parser:
 
     def chompmatch(self, pattern, flags=0, *groups):
         if not self.match(pattern, flags):
-            raise Parser.Error(
+            raise Parser.Error("expected %r, found %r" % (pattern, self))
-                    "expected %r, found %r..." % (
-                        pattern, self.data[self.i:self.i+32]))
         return self.chomp(*groups)
 
     def unexpected(self):
-        raise Parser.Error(
+        raise Parser.Error("unexpected %r" % self)
-                "unexpected %r..." % (
+
-                    self.data[self.i:self.i+32]))
+    def lookahead(self):
+        # push state on the stack
+        if not hasattr(self, 'stack'):
+            self.stack = []
+        self.stack.append((self.i, self.m))
+        return self
+
+    def consume(self):
+        # pop and use new state
+        self.stack.pop()
+
+    def discard(self):
+        # pop and discard new state
+        self.i, self.m = self.stack.pop()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, et, ev, tb):
+        # keep new state if no exception occured
+        if et is None:
+            self.consume()
+        else:
+            self.discard()
 
 # a lazily-evaluated field expression
 class RExpr:

scripts/prettyasserts.py

@@ -13,12 +13,15 @@
 # prevent local imports
 __import__('sys').path.pop(0)
 
+import io
 import re
 import sys
 
 
+# default prettyassert limit
 LIMIT = 16
 
+# comparison ops
 CMP = {
     '==': 'eq',
     '!=': 'ne',
@@ -28,22 +31,50 @@ CMP = {
     '>':  'gt',
 }
 
-LEXEMES = {
+# helper class for lexical regexes
-    'ws':           [r'(?:\s|\n|#.*?(?<!\\)\n|//.*?(?<!\\)\n|/\*.*?\*/)+'],
+class Lexer:
-    'assert':       [r'\bassert\b', r'\b__builtin_assert\b'],
+    def __init__(self):
-    'unreachable':  [r'\bunreachable\b', r'\b__builtin_unreachable\b'],
+        self.patterns = {}
-    'memcmp':       [r'\bmemcmp\b', r'\b__builtin_memcmp\b'],
+
-    'strcmp':       [r'\bstrcmp\b', r'\b__builtin_strcmp\b'],
+    def lex(self, k, *patterns):
-    'arrow':        ['=>'],
+        # compile with whitespace
-    'string':       [r'"(?:\\.|[^"])*"', r"'(?:\\.|[^'])\'"],
+        l = re.compile(
-    'paren':        ['\(', '\)'],
+                '(?P<token>%s)(?P<ws>(?:%s)*)' % (
-    'cmp':          list(CMP.keys()),
+                    '|'.join(patterns) if patterns
-    'logic':        ['\&\&', '\|\|'],
+                        # force a failure if we have no patterns
-    'sep':          ['\?', ':', ','],
+                        else '(?!)',
-    'term':         [';', '\{', '\}'],
+                    '|'.join(self.patterns.get('WS', ''))),
-    # specifically ops that conflict with cmp
+                re.DOTALL)
-    'op':           ['->', '>>', '<<'],
+        # add to class members
-}
+        setattr(self, k, l)
+        # keep track of patterns
+        self.patterns[k] = patterns
+
+    def extend(self, k, *patterns):
+        self.lex(k, *self.patterns.get(k, []), *patterns)
+
+L = Lexer()
+L.lex('WS',             r'(?:\s|\n|#.*?(?<!\\)\n|//.*?(?<!\\)\n|/\*.*?\*/)+')
+L.lex('ASSERT',         r'\bassert\b', r'\b__builtin_assert\b')
+L.lex('UNREACHABLE',    r'\bunreachable\b', r'\b__builtin_unreachable\b')
+L.lex('MEMCMP',         r'\bmemcmp\b', r'\b__builtin_memcmp\b')
+L.lex('STRCMP',         r'\bstrcmp\b', r'\b__builtin_strcmp\b')
+L.lex('ARROW',          '=>')
+L.lex('STR',            r'"(?:\\.|[^"])*"', r"'(?:\\.|[^'])\'")
+L.lex('LPAREN',         '\(')
+L.lex('RPAREN',         '\)')
+L.lex('ZERO',           '\\b0\\b')
+L.lex('CMP',            *CMP.keys())
+L.lex('LOGIC',          '\&\&', '\|\|')
+L.lex('TERN',           '\?', ':')
+L.lex('COMMA',          ',')
+L.lex('TERM',           ';', '\{', '\}')
+L.lex('STUFF',          '[^;{}?:,()"\'=!<>\-&|/#]+',
+                        # these need special handling because we're only
+                        # using regex
+                        '->', '>>', '<<', '-(?!>)',
+                        '=(?![=>])', '!(?!=)', '&(?!&)', '\|(?!\|)',
+                        '/(?!/)', '/(?!\*)')
 
 
 def openio(path, mode='r', buffering=-1):
@@ -56,7 +87,7 @@ def openio(path, mode='r', buffering=-1):
     else:
         return open(path, mode, buffering)
 
-def write_header(f, limit=LIMIT):
+def mkheader(f, limit=LIMIT):
     f.writeln("// Generated by %s:" % sys.argv[0])
     f.writeln("//")
     f.writeln("// %s" % ' '.join(sys.argv))
@@ -204,225 +235,306 @@ def write_header(f, limit=LIMIT):
     f.writeln()
     f.writeln()
 
-def mkassert(type, cmp, lh, rh, size=None):
+def mkassert(f, type, cmp, lh, rh, size=None):
     if size is not None:
-        return ("__PRETTY_ASSERT_%s_%s(%s, %s, %s)" % (
+        f.write("__PRETTY_ASSERT_%s_%s(%s, %s, %s)" % (
                 type.upper(), cmp.upper(), lh, rh, size))
     else:
-        return ("__PRETTY_ASSERT_%s_%s(%s, %s)" % (
+        f.write("__PRETTY_ASSERT_%s_%s(%s, %s)" % (
                 type.upper(), cmp.upper(), lh, rh))
 
-def mkunreachable():
+def mkunreachable(f):
-    return "__PRETTY_ASSERT_UNREACHABLE()"
+    f.write("__PRETTY_ASSERT_UNREACHABLE()")
 
 
-# simple recursive descent parser
+# a simple general-purpose parser class
-class ParseFailure(Exception):
+#
-    def __init__(self, expected, found):
+# basically just because memoryview doesn't support strs
-        self.expected = expected
+class Parser:
-        self.found = found
+    def __init__(self, data, ws='\s*', ws_flags=0):
+        self.data = data
+        self.i = 0
+        self.m = None
+        # also consume whitespace
+        self.ws = re.compile(ws, ws_flags)
+        self.i = self.ws.match(self.data, self.i).end()
+
+    def __repr__(self):
+        if len(self.data) - self.i <= 32:
+            return repr(self.data[self.i:])
+        else:
+            return "%s..." % repr(self.data[self.i:self.i+32])[:32]
 
     def __str__(self):
-        return "expected %r, found %s..." % (
+        return self.data[self.i:]
-                self.expected, repr(self.found)[:70])
 
-class Parser:
+    def __len__(self):
-    def __init__(self, in_f, lexemes=LEXEMES):
+        return len(self.data) - self.i
-        p = '|'.join('(?P<%s>%s)' % (n, '|'.join(l))
-                for n, l in lexemes.items())
-        p = re.compile(p, re.DOTALL)
-        data = in_f.read()
-        tokens = []
-        line = 1
-        col = 0
-        while True:
-            m = p.search(data)
-            if m:
-                if m.start() > 0:
-                    tokens.append((None, data[:m.start()], line, col))
-                tokens.append((m.lastgroup, m.group(), line, col))
-                data = data[m.end():]
-            else:
-                tokens.append((None, data, line, col))
-                break
-        self.tokens = tokens
-        self.off = 0
 
-    def lookahead(self, *pattern):
+    def __bool__(self):
-        if self.off < len(self.tokens):
+        return self.i != len(self.data)
-            token = self.tokens[self.off]
+
-            if token[0] in pattern or token[1] in pattern:
+    def match(self, pattern, flags=0):
-                self.m = token[1]
+        # compile so we can use the pos arg, this is still cached
-                return self.m
+        self.m = re.compile(pattern, flags).match(self.data, self.i)
-        self.m = None
         return self.m
 
-    def accept(self, *patterns):
+    def group(self, *groups):
-        m = self.lookahead(*patterns)
+        return self.m.group(*groups)
-        if m is not None:
-            self.off += 1
-        return m
 
-    def expect(self, *patterns):
+    def chomp(self, *groups):
-        m = self.accept(*patterns)
+        g = self.group(*groups)
-        if not m:
+        self.i = self.m.end()
-            raise ParseFailure(patterns, self.tokens[self.off:])
+        # also consume whitespace
-        return m
+        self.i = self.ws.match(self.data, self.i).end()
+        return g
 
-    def push(self):
+    class Error(Exception):
-        return self.off
+        pass
 
-    def pop(self, state):
+    def chompmatch(self, pattern, flags=0, *groups):
-        self.off = state
+        if not self.match(pattern, flags):
+            raise Parser.Error("expected %r, found %r" % (pattern, self))
+        return self.chomp(*groups)
 
-def p_assert(p):
+    def unexpected(self):
-    state = p.push()
+        raise Parser.Error("unexpected %r" % self)
 
+    def lookahead(self):
+        # push state on the stack
+        if not hasattr(self, 'stack'):
+            self.stack = []
+        self.stack.append((self.i, self.m))
+        return self
+
+    def consume(self):
+        # pop and use new state
+        self.stack.pop()
+
+    def discard(self):
+        # pop and discard new state
+        self.i, self.m = self.stack.pop()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, et, ev, tb):
+        # keep new state if no exception occured
+        if et is None:
+            self.consume()
+        else:
+            self.discard()
+
+
+# parse rules
+
+def p_assert(p, f):
     # assert(memcmp(a,b,size) cmp 0)?
     try:
-        p.expect('assert') ; p.accept('ws')
+        with p.lookahead():
-        p.expect('(') ; p.accept('ws')
+            p.chompmatch(L.ASSERT)
-        p.expect('memcmp') ; p.accept('ws')
+            p.chompmatch(L.LPAREN)
-        p.expect('(') ; p.accept('ws')
+            p.chompmatch(L.MEMCMP)
-        lh = p_expr(p) ; p.accept('ws')
+            p.chompmatch(L.LPAREN)
-        p.expect(',') ; p.accept('ws')
+            lh = io.StringIO()
-        rh = p_expr(p) ; p.accept('ws')
+            p_expr(p, lh)
-        p.expect(',') ; p.accept('ws')
+            lh = lh.getvalue()
-        size = p_expr(p) ; p.accept('ws')
+            p.chompmatch(L.COMMA)
-        p.expect(')') ; p.accept('ws')
+            rh = io.StringIO()
-        cmp = p.expect('cmp') ; p.accept('ws')
+            p_expr(p, rh)
-        p.expect('0') ; p.accept('ws')
+            rh = rh.getvalue()
-        p.expect(')')
+            p.chompmatch(L.COMMA)
-        return mkassert('mem', CMP[cmp], lh, rh, size)
+            size = io.StringIO()
-    except ParseFailure:
+            p_expr(p, size)
-        p.pop(state)
+            size = size.getvalue()
+            p.chompmatch(L.RPAREN)
+            cmp = p.chompmatch(L.CMP, 0, 'token')
+            p.chompmatch(L.ZERO)
+            ws = p.chompmatch(L.RPAREN, 0, 'ws')
+            mkassert(f, 'mem', CMP[cmp], lh, rh, size)
+            f.write(ws)
+            return
+    except Parser.Error:
+        pass
 
     # assert(strcmp(a,b) cmp 0)?
     try:
-        p.expect('assert') ; p.accept('ws')
+        with p.lookahead():
-        p.expect('(') ; p.accept('ws')
+            p.chompmatch(L.ASSERT)
-        p.expect('strcmp') ; p.accept('ws')
+            p.chompmatch(L.LPAREN)
-        p.expect('(') ; p.accept('ws')
+            p.chompmatch(L.STRCMP)
-        lh = p_expr(p) ; p.accept('ws')
+            p.chompmatch(L.LPAREN)
-        p.expect(',') ; p.accept('ws')
+            lh = io.StringIO()
-        rh = p_expr(p) ; p.accept('ws')
+            p_expr(p, lh)
-        p.expect(')') ; p.accept('ws')
+            lh = lh.getvalue()
-        cmp = p.expect('cmp') ; p.accept('ws')
+            p.chompmatch(L.COMMA)
-        p.expect('0') ; p.accept('ws')
+            rh = io.StringIO()
-        p.expect(')')
+            p_expr(p, rh)
-        return mkassert('str', CMP[cmp], lh, rh)
+            rh = rh.getvalue()
-    except ParseFailure:
+            p.chompmatch(L.RPAREN)
-        p.pop(state)
+            cmp = p.chompmatch(L.CMP, 0, 'token')
+            p.chompmatch(L.ZERO)
+            ws = p.chompmatch(L.RPAREN, 0, 'ws')
+            mkassert(f, 'str', CMP[cmp], lh, rh)
+            f.write(ws)
+            return
+    except Parser.Error:
+        pass
 
     # assert(a cmp b)?
     try:
-        p.expect('assert') ; p.accept('ws')
+        with p.lookahead():
-        p.expect('(') ; p.accept('ws')
+            p.chompmatch(L.ASSERT)
-        lh = p_expr(p) ; p.accept('ws')
+            p.chompmatch(L.LPAREN)
-        cmp = p.expect('cmp') ; p.accept('ws')
+            lh = io.StringIO()
-        rh = p_expr(p) ; p.accept('ws')
+            p_simpleexpr(p, lh)
-        p.expect(')')
+            lh = lh.getvalue()
-        return mkassert('int', CMP[cmp], lh, rh)
+            cmp = p.chompmatch(L.CMP, 0, 'token')
-    except ParseFailure:
+            rh = io.StringIO()
-        p.pop(state)
+            p_simpleexpr(p, rh)
+            rh = rh.getvalue()
+            ws = p.chompmatch(L.RPAREN, 0, 'ws')
+            mkassert(f, 'int', CMP[cmp], lh, rh)
+            f.write(ws)
+            return
+    except Parser.Error:
+        pass
 
     # assert(a)?
-    p.expect('assert') ; p.accept('ws')
+    p.chompmatch(L.ASSERT)
-    p.expect('(') ; p.accept('ws')
+    p.chompmatch(L.LPAREN)
-    lh = p_exprs(p) ; p.accept('ws')
+    lh = io.StringIO()
-    p.expect(')')
+    p_exprs(p, lh)
-    return mkassert('bool', 'eq', lh, 'true')
+    lh = lh.getvalue()
+    ws = p.chompmatch(L.RPAREN, 0, 'ws')
+    mkassert(f, 'bool', 'eq', lh, 'true')
+    f.write(ws)
 
-def p_unreachable(p):
+def p_unreachable(p, f):
     # unreachable()?
-    p.expect('unreachable') ; p.accept('ws')
+    p.chompmatch(L.UNREACHABLE)
-    p.expect('(') ; p.accept('ws')
+    p.chompmatch(L.LPAREN)
-    p.expect(')')
+    ws = p.chompmatch(L.RPAREN, 0, 'ws')
-    return mkunreachable()
+    mkunreachable(f)
+    f.write(ws)
 
-def p_expr(p):
+def p_simpleexpr(p, f):
-    res = []
     while True:
-        if p.accept('('):
+        # parens
-            res.append(p.m)
+        if p.match(L.LPAREN):
+            f.write(p.chomp())
+            # allow terms in parens
             while True:
-                res.append(p_exprs(p))
+                p_exprs(p, f)
-                if p.accept('sep', 'term'):
+                if p.match(L.TERM):
-                    res.append(p.m)
+                    f.write(p.chomp())
                 else:
                     break
-            res.append(p.expect(')'))
+            f.write(p.chompmatch(L.RPAREN))
-        elif p.lookahead('assert'):
+        # asserts
-            state = p.push()
+        elif p.match(L.ASSERT):
             try:
-                res.append(p_assert(p))
+                with p.lookahead():
-            except ParseFailure:
+                    p_assert(p, f)
-                p.pop(state)
+            except Parser.Error:
-                res.append(p.expect('assert'))
+                f.write(p.chomp())
-        elif p.lookahead('unreachable'):
+        # unreachables
-            state = p.push()
+        elif p.match(L.UNREACHABLE):
             try:
-                res.append(p_unreachable(p))
+                with p.lookahead():
-            except ParseFailure:
+                    p_unreachable(p, f)
-                p.pop(state)
+            except Parser.Error:
-                res.append(p.expect('unreachable'))
+                f.write(p.chomp())
-        elif p.accept('memcmp', 'strcmp', 'string', 'op', 'ws', None):
+        # anything else
-            res.append(p.m)
+        elif p.match(L.STR) or p.match(L.STUFF):
+            f.write(p.chomp())
         else:
-            return ''.join(res)
+            break
 
-def p_exprs(p):
+def p_expr(p, f):
-    res = []
     while True:
-        res.append(p_expr(p))
+        p_simpleexpr(p, f)
-        if p.accept('cmp', 'logic', 'sep'):
+        # continue if we hit a complex expr
-            res.append(p.m)
+        if p.match(L.CMP) or p.match(L.LOGIC) or p.match(L.TERN):
+            f.write(p.chomp())
         else:
-            return ''.join(res)
+            break
 
-def p_stmt(p):
+def p_exprs(p, f):
-    ws = p.accept('ws') or ''
+    while True:
+        p_expr(p, f)
+        # continue if we hit a comma
+        if p.match(L.COMMA):
+            f.write(p.chomp())
+        else:
+            break
+
+def p_stmt(p, f):
+    # leading whitespace?
+    if p.match(L.WS):
+        f.write(p.chomp())
 
     # memcmp(lh,rh,size) => 0?
-    if p.lookahead('memcmp'):
+    if p.match(L.MEMCMP):
-        state = p.push()
         try:
-            p.expect('memcmp') ; p.accept('ws')
+            with p.lookahead():
-            p.expect('(') ; p.accept('ws')
+                p.chompmatch(L.MEMCMP)
-            lh = p_expr(p) ; p.accept('ws')
+                p.chompmatch(L.LPAREN)
-            p.expect(',') ; p.accept('ws')
+                lh = io.StringIO()
-            rh = p_expr(p) ; p.accept('ws')
+                p_expr(p, lh)
-            p.expect(',') ; p.accept('ws')
+                lh = lh.getvalue()
-            size = p_expr(p) ; p.accept('ws')
+                p.chompmatch(L.COMMA)
-            p.expect(')') ; p.accept('ws')
+                rh = io.StringIO()
-            p.expect('=>') ; p.accept('ws')
+                p_expr(p, rh)
-            p.expect('0') ; p.accept('ws')
+                rh = rh.getvalue()
-            return ws + mkassert('mem', 'eq', lh, rh, size)
+                p.chompmatch(L.COMMA)
-        except ParseFailure:
+                size = io.StringIO()
-            p.pop(state)
+                p_expr(p, size)
+                size = size.getvalue()
+                p.chompmatch(L.RPAREN)
+                p.chompmatch(L.ARROW)
+                ws = p.chompmatch(L.ZERO, 0, 'ws')
+                mkassert(f, 'mem', 'eq', lh, rh, size)
+                f.write(ws)
+                return
+        except Parse.Error:
+            pass
 
     # strcmp(lh,rh) => 0?
-    if p.lookahead('strcmp'):
+    if p.match(L.STRCMP):
-        state = p.push()
         try:
-            p.expect('strcmp') ; p.accept('ws') ; p.expect('(') ; p.accept('ws')
+            with p.lookahead():
-            lh = p_expr(p) ; p.accept('ws')
+                p.chompmatch(L.STRCMP)
-            p.expect(',') ; p.accept('ws')
+                p.chompmatch(L.LPAREN)
-            rh = p_expr(p) ; p.accept('ws')
+                lh = io.StringIO()
-            p.expect(')') ; p.accept('ws')
+                p_expr(p, lh)
-            p.expect('=>') ; p.accept('ws')
+                lh = lh.getvalue()
-            p.expect('0') ; p.accept('ws')
+                p.chompmatch(L.COMMA)
-            return ws + mkassert('str', 'eq', lh, rh)
+                rh = io.StringIO()
-        except ParseFailure:
+                p_expr(p, rh)
-            p.pop(state)
+                rh = rh.getvalue()
+                p.chompmatch(L.RPAREN)
+                p.chompmatch(L.ARROW)
+                ws = p.chompmatch(L.ZERO, 0, 'ws')
+                mkassert(f, 'str', 'eq', lh, rh)
+                f.write(ws)
+                return
+        except Parse.Error:
+            pass
 
     # lh => rh?
-    lh = p_exprs(p)
+    lh = io.StringIO()
-    if p.accept('=>'):
+    p_exprs(p, lh)
-        rh = p_exprs(p)
+    lh = lh.getvalue()
-        return ws + mkassert('int', 'eq', lh, rh)
+    if p.match(L.ARROW):
+        p.chomp()
+        rh = io.StringIO()
+        p_exprs(p, rh)
+        rh = rh.getvalue()
+        mkassert(f, 'int', 'eq', lh, rh)
     else:
-        return ws + lh
+        f.write(lh)
+
 
 def main(input=None, output=None, *,
         prefix=[],
@@ -435,35 +547,38 @@ def main(input=None, output=None, *,
         no_upper=False,
         no_arrows=False,
         limit=LIMIT):
+    # modify lexer rules?
+    if no_defaults:
+        L.lex('ASSERT', [])
+        L.lex('UNREACHABLE', [])
+        L.lex('MEMCMP', [])
+        L.lex('STRCMP', [])
+    for p in prefix + prefix_insensitive:
+        L.extend('ASSERT', r'\b%sassert\b' % p)
+        L.extend('UNREACHABLE', r'\b%sunreachable\b' % p)
+        L.extend('MEMCMP', r'\b%smemcmp\b' % p)
+        L.extend('STRCMP', r'\b%sstrcmp\b' % p)
+    for p in prefix_insensitive:
+        L.extend('ASSERT', r'\b%sassert\b' % p.lower())
+        L.extend('UNREACHABLE', r'\b%sunreachable\b' % p.lower())
+        L.extend('MEMCMP', r'\b%smemcmp\b' % p.lower())
+        L.extend('STRCMP', r'\b%sstrcmp\b' % p.lower())
+        L.extend('ASSERT', r'\b%sASSERT\b' % p.upper())
+        L.extend('UNREACHABLE', r'\b%sUNREACHABLE\b' % p.upper())
+        L.extend('MEMCMP', r'\b%sMEMCMP\b' % p.upper())
+        L.extend('STRCMP', r'\b%sSTRCMP\b' % p.upper())
+    if assert_:
+        L.extend('ASSERT', *[r'\b%s\b' % r for r in assert_])
+    if unreachable:
+        L.extend('UNREACHABLE', *[r'\b%s\b' % r for r in unreachable])
+    if memcmp:
+        L.extend('MEMCMP', *[r'\b%s\b' % r for r in memcmp])
+    if strcmp:
+        L.extend('STRCMP', *[r'\b%s\b' % r for r in strcmp])
+
+    # start parsing
     with openio(input or '-', 'r') as in_f:
-        # create parser
+        p = Parser(in_f.read(), '')
-        lexemes = {n: l.copy() for n, l in LEXEMES.items()}
-        if no_defaults:
-            lexemes['assert'] = []
-            lexemes['unreachable'] = []
-            lexemes['memcmp'] = []
-            lexemes['strcmp'] = []
-        if no_arrows:
-            lexemes['arrow'] = []
-        for p in prefix + prefix_insensitive:
-            lexemes['assert'].append(r'\b%sassert\b' % p)
-            lexemes['unreachable'].append(r'\b%sunreachable\b' % p)
-            lexemes['memcmp'].append(r'\b%smemcmp\b' % p)
-            lexemes['strcmp'].append(r'\b%sstrcmp\b' % p)
-        for p in prefix_insensitive:
-            lexemes['assert'].append(r'\b%sassert\b' % p.lower())
-            lexemes['unreachable'].append(r'\b%sunreachable\b' % p.lower())
-            lexemes['memcmp'].append(r'\b%smemcmp\b' % p.lower())
-            lexemes['strcmp'].append(r'\b%sstrcmp\b' % p.lower())
-            lexemes['assert'].append(r'\b%sASSERT\b' % p.upper())
-            lexemes['unreachable'].append(r'\b%sUNREACHABLE\b' % p.upper())
-            lexemes['memcmp'].append(r'\b%sMEMCMP\b' % p.upper())
-            lexemes['strcmp'].append(r'\b%sSTRCMP\b' % p.upper())
-        lexemes['assert'].extend(r'\b%s\b' % r for r in assert_)
-        lexemes['unreachable'].extend(r'\b%s\b' % r for r in unreachable)
-        lexemes['memcmp'].extend(r'\b%s\b' % r for r in memcmp)
-        lexemes['strcmp'].extend(r'\b%s\b' % r for r in strcmp)
-        p = Parser(in_f, lexemes)
 
         with openio(output or '-', 'w') as f:
             def writeln(s=''):
@@ -472,24 +587,29 @@ def main(input=None, output=None, *,
             f.writeln = writeln
 
             # write extra verbose asserts
-            write_header(f, limit=limit)
+            mkheader(f, limit=limit)
             if input is not None:
                 f.writeln("#line %d \"%s\"" % (1, input))
 
             # parse and write out stmt at a time
             try:
                 while True:
-                    f.write(p_stmt(p))
+                    p_stmt(p, f)
-                    if p.accept('term'):
+                    if p.match(L.TERM):
-                        f.write(p.m)
+                        f.write(p.chomp())
                     else:
                         break
-            except ParseFailure as e:
-                print('warning: %s' % e)
-                pass
 
-            for i in range(p.off, len(p.tokens)):
+                # trailing junk?
-                f.write(p.tokens[i][1])
+                if p:
+                    p.unexpected()
+
+            except Parser.Error as e:
+                # warn on error
+                print('warning: %s' % e)
+                # still write out the rest of the file so compiler
+                # errors can be reported, these are usually more useful
+                f.write(str(p))
 
 
 if __name__ == "__main__":

scripts/stack.py

@@ -171,7 +171,7 @@ def openio(path, mode='r', buffering=-1):
 # basically just because memoryview doesn't support strs
 class Parser:
     def __init__(self, data, ws='\s*', ws_flags=0):
-        self.data = data.lstrip()
+        self.data = data
         self.i = 0
         self.m = None
         # also consume whitespace
@@ -179,9 +179,10 @@ class Parser:
         self.i = self.ws.match(self.data, self.i).end()
 
     def __repr__(self):
-        return '%s(%r...)' % (
+        if len(self.data) - self.i <= 32:
-                self.__class__.__name__,
+            return repr(self.data[self.i:])
-                self.data[self.i:self.i+32])
+        else:
+            return "%s..." % repr(self.data[self.i:self.i+32])[:32]
 
     def __str__(self):
         return self.data[self.i:]
@@ -212,15 +213,36 @@ class Parser:
 
     def chompmatch(self, pattern, flags=0, *groups):
         if not self.match(pattern, flags):
-            raise Parser.Error(
+            raise Parser.Error("expected %r, found %r" % (pattern, self))
-                    "expected %r, found %r..." % (
-                        pattern, self.data[self.i:self.i+32]))
         return self.chomp(*groups)
 
     def unexpected(self):
-        raise Parser.Error(
+        raise Parser.Error("unexpected %r" % self)
-                "unexpected %r..." % (
+
-                    self.data[self.i:self.i+32]))
+    def lookahead(self):
+        # push state on the stack
+        if not hasattr(self, 'stack'):
+            self.stack = []
+        self.stack.append((self.i, self.m))
+        return self
+
+    def consume(self):
+        # pop and use new state
+        self.stack.pop()
+
+    def discard(self):
+        # pop and discard new state
+        self.i, self.m = self.stack.pop()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, et, ev, tb):
+        # keep new state if no exception occured
+        if et is None:
+            self.consume()
+        else:
+            self.discard()
 
 class CGNode(co.namedtuple('CGNode', [
         'name', 'file', 'size', 'qualifiers', 'calls'])):