scripts: dbgflags.py: Added -d/--diff, lineno, better find reuse

This started with adding -d/--diff support to dbgflags.py, which is very
useful for comparing flags during test failure.

The flag asserts in our tests generally look like this:

  tests/test_mount.toml:171:assert: assert failed with 33570064,
  expected eq 33570576
      assert(fsinfo.flags == (

Which can now be quickly compared with dbgflags.py:

  $ ./scripts/dbgflags.py +i 33570064 -d 33570576
   LFS3_I_GBMAP         0x02000000  Global on-disk block-map in use
   LFS3_I_REVPERTURB    0x00000010  Mounted with LFS3_M_REVPERTURB
   LFS3_I_MKCONSISTENT  0x00000100  Filesystem needs mkconsistent to write
  -LFS3_I_LOOKAHEAD     0x00000200  Lookahead buffer is not full
   LFS3_I_PREERASE      0x00000400  Blocks can be pre-erased
   LFS3_I_COMPACT       0x00000800  Filesystem may have uncompacted metadata
   LFS3_I_CKMETA        0x00001000  Metadata checksums not checked recently
   LFS3_I_CKDATA        0x00002000  Data checksums not checked recently

The assert print is a bit more annoying than it needs to be, as it only
prints in decimal. But, since our prettyasserts.py only works at the
syntax layer, it's not possible to make it any smarter.

---

To make this diffing work required a couple more features in our
self-parsing Flag class:

- Keep track of lineno, mainly for ordering things
- Moved find logic into a staticmethod on all classes
- Added _sentinel based defaults to find functions
- Allowed self to be non-class in line functions to deduplicate "Unknown
  flag" messages

I went ahead and extended these to the other self-parsing classes (Err
and Tag) in case they're useful in the future.

scripts/dbgbmap.py

@@ -167,11 +167,13 @@ TAG_GCKSUMDELTA = 0x3300    ##  v-11 --11 ++++ ++++
 
 # self-parsing tag repr
 class Tag:
-    def __init__(self, name, tag, encoding, help):
+    def __init__(self, name, tag, encoding, help, *,
+            lineno=0):
         self.name = name
         self.tag = tag
         self.encoding = encoding
         self.help = help
+        self.lineno = lineno
         # derive mask from encoding
         self.mask = sum(
                 (1 if x in 'v-01' else 0) << len(self.encoding)-1-i
@@ -240,29 +242,37 @@ class Tag:
         tag_pattern = re.compile(
             '^(?P<name>TAG_[^ ]*) *= *(?P<tag>[^#]*?) *'
                 '#+ *(?P<encoding>(?:[^ ] *?){16}) *(?P<help>.*)$')
-        for line in (inspect.getsource(
+        for i, line in enumerate(
-                    inspect.getmodule(inspect.currentframe()))
+                inspect.getsource(inspect.getmodule(inspect.currentframe()))
-                .replace('\\\n', '')
+                    .replace('\\\n', '')
-                .splitlines()):
+                    .splitlines()):
             m = tag_pattern.match(line)
             if m:
                 tags.append(Tag(
                         m.group('name'),
                         globals()[m.group('name')],
                         m.group('encoding').replace(' ', ''),
-                        m.group('help')))
+                        m.group('help'),
+                        lineno=1+i))
         return tags
 
     # find best matching tag
+    _sentinel = object()
     @staticmethod
-    def find(tag):
+    def find(tag, *, default=_sentinel):
         # find tags, note this is cached
         tags__ = Tag.tags()
 
         # find the most specific matching tag, ignoring valid bits
-        return max((t for t in tags__ if t.matches(tag & 0x7fff)),
+        t = max((t for t in tags__ if t.matches(tag & 0x7fff)),
                 key=lambda t: t.specificity(),
                 default=None)
+        if t is not None:
+            return t
+        elif default is Tag._sentinel:
+            raise KeyError(tag)
+        else:
+            return default
 
     # human readable tag repr
     @staticmethod
@@ -270,7 +280,9 @@ class Tag:
             global_=False,
             toff=None):
         # find the most specific matching tag, ignoring the shrub bit
-        t = Tag.find(tag & ~(TAG_SHRUB if tag & 0x7000 == TAG_SHRUB else 0))
+        t = Tag.find(
+                tag & ~(TAG_SHRUB if tag & 0x7000 == TAG_SHRUB else 0),
+                default=None)
 
         # build repr
         r = []

scripts/dbgbmapsvg.py

@@ -187,11 +187,13 @@ TAG_GCKSUMDELTA = 0x3300    ##  v-11 --11 ++++ ++++
 
 # self-parsing tag repr
 class Tag:
-    def __init__(self, name, tag, encoding, help):
+    def __init__(self, name, tag, encoding, help, *,
+            lineno=0):
         self.name = name
         self.tag = tag
         self.encoding = encoding
         self.help = help
+        self.lineno = lineno
         # derive mask from encoding
         self.mask = sum(
                 (1 if x in 'v-01' else 0) << len(self.encoding)-1-i
@@ -260,29 +262,37 @@ class Tag:
         tag_pattern = re.compile(
             '^(?P<name>TAG_[^ ]*) *= *(?P<tag>[^#]*?) *'
                 '#+ *(?P<encoding>(?:[^ ] *?){16}) *(?P<help>.*)$')
-        for line in (inspect.getsource(
+        for i, line in enumerate(
-                    inspect.getmodule(inspect.currentframe()))
+                inspect.getsource(inspect.getmodule(inspect.currentframe()))
-                .replace('\\\n', '')
+                    .replace('\\\n', '')
-                .splitlines()):
+                    .splitlines()):
             m = tag_pattern.match(line)
             if m:
                 tags.append(Tag(
                         m.group('name'),
                         globals()[m.group('name')],
                         m.group('encoding').replace(' ', ''),
-                        m.group('help')))
+                        m.group('help'),
+                        lineno=1+i))
         return tags
 
     # find best matching tag
+    _sentinel = object()
     @staticmethod
-    def find(tag):
+    def find(tag, *, default=_sentinel):
         # find tags, note this is cached
         tags__ = Tag.tags()
 
         # find the most specific matching tag, ignoring valid bits
-        return max((t for t in tags__ if t.matches(tag & 0x7fff)),
+        t = max((t for t in tags__ if t.matches(tag & 0x7fff)),
                 key=lambda t: t.specificity(),
                 default=None)
+        if t is not None:
+            return t
+        elif default is Tag._sentinel:
+            raise KeyError(tag)
+        else:
+            return default
 
     # human readable tag repr
     @staticmethod
@@ -290,7 +300,9 @@ class Tag:
             global_=False,
             toff=None):
         # find the most specific matching tag, ignoring the shrub bit
-        t = Tag.find(tag & ~(TAG_SHRUB if tag & 0x7000 == TAG_SHRUB else 0))
+        t = Tag.find(
+                tag & ~(TAG_SHRUB if tag & 0x7000 == TAG_SHRUB else 0),
+                default=None)
 
         # build repr
         r = []

scripts/dbgbtree.py

@@ -73,11 +73,13 @@ TAG_GCKSUMDELTA = 0x3300    ##  v-11 --11 ++++ ++++
 
 # self-parsing tag repr
 class Tag:
-    def __init__(self, name, tag, encoding, help):
+    def __init__(self, name, tag, encoding, help, *,
+            lineno=0):
         self.name = name
         self.tag = tag
         self.encoding = encoding
         self.help = help
+        self.lineno = lineno
         # derive mask from encoding
         self.mask = sum(
                 (1 if x in 'v-01' else 0) << len(self.encoding)-1-i
@@ -146,29 +148,37 @@ class Tag:
         tag_pattern = re.compile(
             '^(?P<name>TAG_[^ ]*) *= *(?P<tag>[^#]*?) *'
                 '#+ *(?P<encoding>(?:[^ ] *?){16}) *(?P<help>.*)$')
-        for line in (inspect.getsource(
+        for i, line in enumerate(
-                    inspect.getmodule(inspect.currentframe()))
+                inspect.getsource(inspect.getmodule(inspect.currentframe()))
-                .replace('\\\n', '')
+                    .replace('\\\n', '')
-                .splitlines()):
+                    .splitlines()):
             m = tag_pattern.match(line)
             if m:
                 tags.append(Tag(
                         m.group('name'),
                         globals()[m.group('name')],
                         m.group('encoding').replace(' ', ''),
-                        m.group('help')))
+                        m.group('help'),
+                        lineno=1+i))
         return tags
 
     # find best matching tag
+    _sentinel = object()
     @staticmethod
-    def find(tag):
+    def find(tag, *, default=_sentinel):
         # find tags, note this is cached
         tags__ = Tag.tags()
 
         # find the most specific matching tag, ignoring valid bits
-        return max((t for t in tags__ if t.matches(tag & 0x7fff)),
+        t = max((t for t in tags__ if t.matches(tag & 0x7fff)),
                 key=lambda t: t.specificity(),
                 default=None)
+        if t is not None:
+            return t
+        elif default is Tag._sentinel:
+            raise KeyError(tag)
+        else:
+            return default
 
     # human readable tag repr
     @staticmethod
@@ -176,7 +186,9 @@ class Tag:
             global_=False,
             toff=None):
         # find the most specific matching tag, ignoring the shrub bit
-        t = Tag.find(tag & ~(TAG_SHRUB if tag & 0x7000 == TAG_SHRUB else 0))
+        t = Tag.find(
+                tag & ~(TAG_SHRUB if tag & 0x7000 == TAG_SHRUB else 0),
+                default=None)
 
         # build repr
         r = []

scripts/dbgerr.py

@@ -30,10 +30,12 @@ ERR_RANGE       = -34   # Result out of range
 
 # self-parsing error codes
 class Err:
-    def __init__(self, name, code, help):
+    def __init__(self, name, code, help, *,
+            lineno=0):
         self.name = name
         self.code = code
         self.help = help
+        self.lineno = lineno
 
     def __repr__(self):
         return 'Err(%r, %r, %r)' % (
@@ -51,7 +53,10 @@ class Err:
         return hash(self.name)
 
     def line(self):
-        return ('LFS3_%s' % self.name, '%d' % self.code, self.help)
+        if isinstance(self, Err):
+            return ('LFS3_%s' % self.name, '%d' % self.code, self.help)
+        else:
+            return ('?', str(self), 'Unknown err code')
 
     @staticmethod
     @ft.cache
@@ -63,82 +68,74 @@ class Err:
         err_pattern = re.compile(
                 '^(?P<name>ERR_[^ ]*) *= *(?P<code>[^#]*?) *'
                     '#+ *(?P<help>.*)$')
-        for line in (inspect.getsource(
+        for i, line in enumerate(
-                    inspect.getmodule(inspect.currentframe()))
+                inspect.getsource(inspect.getmodule(inspect.currentframe()))
-                .replace('\\\n', '')
+                    .replace('\\\n', '')
-                .splitlines()):
+                    .splitlines()):
             m = err_pattern.match(line)
             if m:
                 errs.append(Err(
                         m.group('name'),
                         globals()[m.group('name')],
-                        m.group('help')))
+                        m.group('help'),
+                        lineno=1+i))
         return errs
 
+    _sentinel = object()
+    @staticmethod
+    def find(e_, *, default=_sentinel):
+        # find errs, note this is cached
+        errs__ = Err.errs()
+
+        # find by LFS3_ERR_+name
+        for e in errs__:
+            if 'LFS3_%s' % e.name.upper() == e_.upper():
+                return e
+        # find by ERR_+name
+        for e in errs__:
+            if e.name.upper() == e_.upper():
+                return e
+        # find by name
+        for e in errs__:
+            if e.name.split('_', 1)[1] == e_.upper():
+                return e
+        # find by E+name
+        for e in errs__:
+            if 'E%s' % e.name.split('_', 1)[1].upper() == e_.upper():
+                return e
+        try:
+            # find by err code
+            for e in errs__:
+                if e.code == int(e_, 0):
+                    return e
+            # find by negated err code
+            for e in errs__:
+                if e.code == -int(e_, 0):
+                    return e
+        except ValueError:
+            pass
+        # not found
+        if default is Err._sentinel:
+            raise KeyError(e_)
+        else:
+            return default
+
 
 def main(errs, *,
         list=False):
     import builtins
     list_, list = list, builtins.list
 
-    # find errs
-    errs__ = Err.errs()
-
     lines = []
     # list all known error codes
     if list_:
-        for e in errs__:
+        for e in Err.errs():
             lines.append(e.line())
 
     # find errs by name or value
     else:
         for e_ in errs:
-            found = False
+            lines.append(Err.line(Err.find(e_, default=e_)))
-            # find by LFS3_ERR_+name
-            for e in errs__:
-                if 'LFS3_%s' % e.name.upper() == e_.upper():
-                    lines.append(e.line())
-                    found = True
-            if found:
-                continue
-            # find by ERR_+name
-            for e in errs__:
-                if e.name.upper() == e_.upper():
-                    lines.append(e.line())
-                    found = True
-            if found:
-                continue
-            # find by name
-            for e in errs__:
-                if e.name.split('_', 1)[1] == e_.upper():
-                    lines.append(e.line())
-                    found = True
-            if found:
-                continue
-            # find by E+name
-            for e in errs__:
-                if 'E%s' % e.name.split('_', 1)[1].upper() == e_.upper():
-                    lines.append(e.line())
-                    found = True
-            if found:
-                continue
-            try:
-                # find by err code
-                for e in errs__:
-                    if e.code == int(e_, 0):
-                        lines.append(e.line())
-                        found = True
-                if found:
-                    continue
-                # find by negated err code
-                for e in errs__:
-                    if e.code == -int(e_, 0):
-                        lines.append(e.line())
-                        found = True
-                if found:
-                    continue
-            except ValueError:
-                lines.append(('?', e_, 'Unknown err code'))
 
     # first find widths
     w = [0, 0]

scripts/dbgflags.py

@@ -6,6 +6,7 @@ if __name__ == "__main__":
 
 import collections as co
 import functools as ft
+import math as mt
 
 
 # Flag prefixes
@@ -262,6 +263,7 @@ class Prefix:
 # self-parsing flags
 class Flag:
     def __init__(self, name, flag, help, *,
+            lineno=0,
             prefix=None,
             yes=False,
             alias=False,
@@ -271,6 +273,7 @@ class Flag:
         self.name = name
         self.flag = flag
         self.help = help
+        self.lineno = lineno
         self.prefix = prefix
         self.yes = yes
         self.alias = alias
@@ -285,20 +288,32 @@ class Flag:
                 self.help)
 
     def __eq__(self, other):
-        return self.name == other.name
+        return self.name == getattr(other, 'name', None)
 
     def __ne__(self, other):
-        return self.name != other.name
+        return self.name != getattr(other, 'name', None)
 
     def __hash__(self):
         return hash(self.name)
 
     def line(self):
-        return ('LFS3_%s' % self.name, '0x%08x' % self.flag, self.help)
+        if isinstance(self, Flag):
+            return ('LFS3_%s' % self.name, '0x%08x' % self.flag, self.help)
+        elif isinstance(self, int):
+            return ('?', '0x%08x' % self, 'Unknown flags')
+        else:
+            return ('?', str(self), 'Unknown flag')
 
     @staticmethod
     @ft.cache
-    def flags():
+    def _flags(*, filter=None):
+        # filter by prefixes
+        if filter:
+            assert isinstance(filter, frozenset)
+            # make sure to cache all flags
+            flags = Flag._flags()
+            return [f for f in flags if f.prefix in filter]
+
         # parse our script's source to figure out flags
         import inspect
         import re
@@ -315,16 +330,17 @@ class Flag:
                         '*= *(?P<flag>[^#]*?) *'
                         '#+ (?P<mode>[^ ]+) *(?P<help>.*)$'
                     % '|'.join(prefixes_.keys()))
-        for line in (inspect.getsource(
+        for i, line in enumerate(
-                    inspect.getmodule(inspect.currentframe()))
+                inspect.getsource(inspect.getmodule(inspect.currentframe()))
-                .replace('\\\n', '')
+                    .replace('\\\n', '')
-                .splitlines()):
+                    .splitlines()):
             m = flag_pattern.match(line)
             if m:
                 flags.append(Flag(
                         m.group('name'),
                         globals()[m.group('name')],
                         m.group('help'),
+                        lineno=1+i,
                         # associate flags -> prefix
                         prefix=prefixes_[
                             m.group('name').split('_', 1)[0].upper()],
@@ -341,76 +357,129 @@ class Flag:
 
         return flags
 
+    @staticmethod
+    def flags(*, filter=None):
+        if isinstance(filter, str):
+            filter = frozenset((filter,))
+        if filter is not None and not isinstance(filter, frozenset):
+            filter = frozenset(filter)
+        return Flag._flags(filter=filter)
+
+    _sentinel = object()
+    @staticmethod
+    def find(f_, *, filter=None, default=_sentinel):
+        # find flags, note this is cached
+        flags__ = Flag.flags(filter=filter)
+
+        flags_ = []
+        # find by LFS3_+prefix+_+name
+        for f in flags__:
+            if 'LFS3_%s' % f.name.upper() == f_.upper():
+                flags_.append(f)
+        if flags_:
+            return flags_
+        # find by prefix+_+name
+        for f in flags__:
+            if '%s' % f.name.upper() == f_.upper():
+                flags_.append(f)
+        if flags_:
+            return flags_
+        # find by name
+        for f in flags__:
+            if f.name.split('_', 1)[1].upper() == f_.upper():
+                flags_.append(f)
+        if flags_:
+            return flags_
+        # find by value
+        try:
+            f__ = int(f_, 0)
+            f___ = f__
+            for f in flags__:
+                # ignore aliases and type masks here
+                if f.alias or f.mask:
+                    continue
+                # matches flag?
+                if not f.type and (f__ & f.flag) == f.flag:
+                    flags_.append(f)
+                    f___ &= ~f.flag
+                # matches type?
+                elif f.type and (f__ & f.type.flag) == f.flag:
+                    flags_.append(f)
+                    f___ &= ~f.type.flag
+            if f___:
+                flags_.append(f___)
+            return flags_
+        except ValueError:
+            pass
+        # not found
+        if default is Flag._sentinel:
+            raise KeyError(f_)
+        else:
+            return default
+
 
 def main(flags, *,
         list=False,
         all=False,
+        diff=None,
+        color='auto',
         prefixes=[]):
     import builtins
     list_, list = list, builtins.list
     all_, all = all, builtins.all
 
-    # find flags
+    # figure out what color should be
-    flags__ = Flag.flags()
+    if color == 'auto':
-
+        color = sys.stdout.isatty()
-    # filter by prefixes if there are any prefixes
+    elif color == 'always':
-    if prefixes:
+        color = True
-        prefixes = set(prefixes)
+    else:
-        flags__ = [f for f in flags__ if f.prefix in prefixes]
+        color = False
 
     lines = []
     # list all known flags
     if list_:
-        for f in flags__:
+        for f in Flag.flags(filter=prefixes or None):
             if not all_ and (f.internal or f.type):
                 continue
             lines.append(f.line())
 
+    # diff flags by name or value
+    elif diff:
+        # first find flags
+        a = []
+        for f_ in flags:
+            a.extend(Flag.find(f_, filter=prefixes or None, default=[f_]))
+
+        b = Flag.find(diff, filter=prefixes or None, default=[diff])
+
+        # compute line-by-line diff
+        a_set = set(a)
+        b_set = set(b)
+        i, j = 0, 0
+        while i < len(a) or j < len(b):
+            if i < len(a) and (
+                    j >= len(b)
+                        or getattr(a[i], 'lineno', mt.inf)
+                            <= getattr(b[j], 'lineno', mt.inf)):
+                if a[i] not in b_set:
+                    l = Flag.line(a[i])
+                    lines.append(('+'+l[0], *l[1:]))
+                else:
+                    l = Flag.line(a[i])
+                    lines.append((' '+l[0], *l[1:]))
+                i += 1
+            else:
+                if b[j] not in a_set:
+                    l = Flag.line(b[j])
+                    lines.append(('-'+l[0], *l[1:]))
+                j += 1
+
     # find flags by name or value
     else:
         for f_ in flags:
-            found = False
+            for f in Flag.find(f_, filter=prefixes or None, default=[f_]):
-            # find by LFS3_+prefix+_+name
+                lines.append(Flag.line(f))
-            for f in flags__:
-                if 'LFS3_%s' % f.name.upper() == f_.upper():
-                    lines.append(f.line())
-                    found = True
-            if found:
-                continue
-            # find by prefix+_+name
-            for f in flags__:
-                if '%s' % f.name.upper() == f_.upper():
-                    lines.append(f.line())
-                    found = True
-            if found:
-                continue
-            # find by name
-            for f in flags__:
-                if f.name.split('_', 1)[1].upper() == f_.upper():
-                    lines.append(f.line())
-                    found = True
-            if found:
-                continue
-            # find by value
-            try:
-                f__ = int(f_, 0)
-                f___ = f__
-                for f in flags__:
-                    # ignore aliases and type masks here
-                    if f.alias or f.mask:
-                        continue
-                    # matches flag?
-                    if not f.type and (f__ & f.flag) == f.flag:
-                        lines.append(f.line())
-                        f___ &= ~f.flag
-                    # matches type?
-                    elif f.type and (f__ & f.type.flag) == f.flag:
-                        lines.append(f.line())
-                        f___ &= ~f.type.flag
-                if f___:
-                    lines.append(('?', '0x%08x' % f___, 'Unknown flags'))
-            except ValueError:
-                lines.append(('?', f_, 'Unknown flag'))
 
     # first find widths
     w = [0, 0]
@@ -420,10 +489,16 @@ def main(flags, *,
 
     # then print results
     for l in lines:
-        print('%-*s  %-*s  %s' % (
+        print('%s%-*s  %-*s  %s%s' % (
+                '\x1b[32m' if color and diff and l[0].startswith('+')
+                    else '\x1b[31m' if color and diff and l[0].startswith('-')
+                    else '',
                 w[0], l[0],
                 w[1], l[1],
-                l[2]))
+                l[2],
+                '\x1b[m' if color and diff and l[0].startswith('+')
+                    else '\x1b[m' if color and diff and l[0].startswith('-')
+                    else ''))
 
 
 if __name__ == "__main__":
@@ -446,6 +521,15 @@ if __name__ == "__main__":
             '-a', '--all',
             action='store_true',
             help="Also show internal flags and types.")
+    parser.add_argument(
+            '-d', '--diff',
+            help="Diff against these flags.")
+    parser.add_argument(
+            '--color',
+            choices=['never', 'always', 'auto'],
+            default='auto',
+            help="When to use terminal colors. Defaults to 'auto'.")
+    prefixes = parser.add_argument_group('prefixes')
     class AppendPrefix(argparse.Action):
         def __init__(self, nargs=None, **kwargs):
             super().__init__(nargs=0, **kwargs)
@@ -454,7 +538,7 @@ if __name__ == "__main__":
                 namespace.prefixes = []
             namespace.prefixes.append(self.const)
     for p in Prefix.prefixes():
-        parser.add_argument(
+        prefixes.add_argument(
                 *p.aliases,
                 action=AppendPrefix,
                 const=p,

scripts/dbglfs3.py

@@ -92,11 +92,13 @@ TAG_GCKSUMDELTA = 0x3300    ##  v-11 --11 ++++ ++++
 
 # self-parsing tag repr
 class Tag:
-    def __init__(self, name, tag, encoding, help):
+    def __init__(self, name, tag, encoding, help, *,
+            lineno=0):
         self.name = name
         self.tag = tag
         self.encoding = encoding
         self.help = help
+        self.lineno = lineno
         # derive mask from encoding
         self.mask = sum(
                 (1 if x in 'v-01' else 0) << len(self.encoding)-1-i
@@ -165,29 +167,37 @@ class Tag:
         tag_pattern = re.compile(
             '^(?P<name>TAG_[^ ]*) *= *(?P<tag>[^#]*?) *'
                 '#+ *(?P<encoding>(?:[^ ] *?){16}) *(?P<help>.*)$')
-        for line in (inspect.getsource(
+        for i, line in enumerate(
-                    inspect.getmodule(inspect.currentframe()))
+                inspect.getsource(inspect.getmodule(inspect.currentframe()))
-                .replace('\\\n', '')
+                    .replace('\\\n', '')
-                .splitlines()):
+                    .splitlines()):
             m = tag_pattern.match(line)
             if m:
                 tags.append(Tag(
                         m.group('name'),
                         globals()[m.group('name')],
                         m.group('encoding').replace(' ', ''),
-                        m.group('help')))
+                        m.group('help'),
+                        lineno=1+i))
         return tags
 
     # find best matching tag
+    _sentinel = object()
     @staticmethod
-    def find(tag):
+    def find(tag, *, default=_sentinel):
         # find tags, note this is cached
         tags__ = Tag.tags()
 
         # find the most specific matching tag, ignoring valid bits
-        return max((t for t in tags__ if t.matches(tag & 0x7fff)),
+        t = max((t for t in tags__ if t.matches(tag & 0x7fff)),
                 key=lambda t: t.specificity(),
                 default=None)
+        if t is not None:
+            return t
+        elif default is Tag._sentinel:
+            raise KeyError(tag)
+        else:
+            return default
 
     # human readable tag repr
     @staticmethod
@@ -195,7 +205,9 @@ class Tag:
             global_=False,
             toff=None):
         # find the most specific matching tag, ignoring the shrub bit
-        t = Tag.find(tag & ~(TAG_SHRUB if tag & 0x7000 == TAG_SHRUB else 0))
+        t = Tag.find(
+                tag & ~(TAG_SHRUB if tag & 0x7000 == TAG_SHRUB else 0),
+                default=None)
 
         # build repr
         r = []

scripts/dbgmtree.py

@@ -73,11 +73,13 @@ TAG_GCKSUMDELTA = 0x3300    ##  v-11 --11 ++++ ++++
 
 # self-parsing tag repr
 class Tag:
-    def __init__(self, name, tag, encoding, help):
+    def __init__(self, name, tag, encoding, help, *,
+            lineno=0):
         self.name = name
         self.tag = tag
         self.encoding = encoding
         self.help = help
+        self.lineno = lineno
         # derive mask from encoding
         self.mask = sum(
                 (1 if x in 'v-01' else 0) << len(self.encoding)-1-i
@@ -146,29 +148,37 @@ class Tag:
         tag_pattern = re.compile(
             '^(?P<name>TAG_[^ ]*) *= *(?P<tag>[^#]*?) *'
                 '#+ *(?P<encoding>(?:[^ ] *?){16}) *(?P<help>.*)$')
-        for line in (inspect.getsource(
+        for i, line in enumerate(
-                    inspect.getmodule(inspect.currentframe()))
+                inspect.getsource(inspect.getmodule(inspect.currentframe()))
-                .replace('\\\n', '')
+                    .replace('\\\n', '')
-                .splitlines()):
+                    .splitlines()):
             m = tag_pattern.match(line)
             if m:
                 tags.append(Tag(
                         m.group('name'),
                         globals()[m.group('name')],
                         m.group('encoding').replace(' ', ''),
-                        m.group('help')))
+                        m.group('help'),
+                        lineno=1+i))
         return tags
 
     # find best matching tag
+    _sentinel = object()
     @staticmethod
-    def find(tag):
+    def find(tag, *, default=_sentinel):
         # find tags, note this is cached
         tags__ = Tag.tags()
 
         # find the most specific matching tag, ignoring valid bits
-        return max((t for t in tags__ if t.matches(tag & 0x7fff)),
+        t = max((t for t in tags__ if t.matches(tag & 0x7fff)),
                 key=lambda t: t.specificity(),
                 default=None)
+        if t is not None:
+            return t
+        elif default is Tag._sentinel:
+            raise KeyError(tag)
+        else:
+            return default
 
     # human readable tag repr
     @staticmethod
@@ -176,7 +186,9 @@ class Tag:
             global_=False,
             toff=None):
         # find the most specific matching tag, ignoring the shrub bit
-        t = Tag.find(tag & ~(TAG_SHRUB if tag & 0x7000 == TAG_SHRUB else 0))
+        t = Tag.find(
+                tag & ~(TAG_SHRUB if tag & 0x7000 == TAG_SHRUB else 0),
+                default=None)
 
         # build repr
         r = []

scripts/dbgrbyd.py

@@ -83,11 +83,13 @@ TAG_GCKSUMDELTA = 0x3300    ##  v-11 --11 ++++ ++++
 
 # self-parsing tag repr
 class Tag:
-    def __init__(self, name, tag, encoding, help):
+    def __init__(self, name, tag, encoding, help, *,
+            lineno=0):
         self.name = name
         self.tag = tag
         self.encoding = encoding
         self.help = help
+        self.lineno = lineno
         # derive mask from encoding
         self.mask = sum(
                 (1 if x in 'v-01' else 0) << len(self.encoding)-1-i
@@ -156,29 +158,37 @@ class Tag:
         tag_pattern = re.compile(
             '^(?P<name>TAG_[^ ]*) *= *(?P<tag>[^#]*?) *'
                 '#+ *(?P<encoding>(?:[^ ] *?){16}) *(?P<help>.*)$')
-        for line in (inspect.getsource(
+        for i, line in enumerate(
-                    inspect.getmodule(inspect.currentframe()))
+                inspect.getsource(inspect.getmodule(inspect.currentframe()))
-                .replace('\\\n', '')
+                    .replace('\\\n', '')
-                .splitlines()):
+                    .splitlines()):
             m = tag_pattern.match(line)
             if m:
                 tags.append(Tag(
                         m.group('name'),
                         globals()[m.group('name')],
                         m.group('encoding').replace(' ', ''),
-                        m.group('help')))
+                        m.group('help'),
+                        lineno=1+i))
         return tags
 
     # find best matching tag
+    _sentinel = object()
     @staticmethod
-    def find(tag):
+    def find(tag, *, default=_sentinel):
         # find tags, note this is cached
         tags__ = Tag.tags()
 
         # find the most specific matching tag, ignoring valid bits
-        return max((t for t in tags__ if t.matches(tag & 0x7fff)),
+        t = max((t for t in tags__ if t.matches(tag & 0x7fff)),
                 key=lambda t: t.specificity(),
                 default=None)
+        if t is not None:
+            return t
+        elif default is Tag._sentinel:
+            raise KeyError(tag)
+        else:
+            return default
 
     # human readable tag repr
     @staticmethod
@@ -186,7 +196,9 @@ class Tag:
             global_=False,
             toff=None):
         # find the most specific matching tag, ignoring the shrub bit
-        t = Tag.find(tag & ~(TAG_SHRUB if tag & 0x7000 == TAG_SHRUB else 0))
+        t = Tag.find(
+                tag & ~(TAG_SHRUB if tag & 0x7000 == TAG_SHRUB else 0),
+                default=None)
 
         # build repr
         r = []

scripts/dbgtag.py

@@ -67,11 +67,13 @@ TAG_GCKSUMDELTA = 0x3300    ##  v-11 --11 ++++ ++++
 
 # self-parsing tag repr
 class Tag:
-    def __init__(self, name, tag, encoding, help):
+    def __init__(self, name, tag, encoding, help, *,
+            lineno=0):
         self.name = name
         self.tag = tag
         self.encoding = encoding
         self.help = help
+        self.lineno = lineno
         # derive mask from encoding
         self.mask = sum(
                 (1 if x in 'v-01' else 0) << len(self.encoding)-1-i
@@ -140,29 +142,37 @@ class Tag:
         tag_pattern = re.compile(
             '^(?P<name>TAG_[^ ]*) *= *(?P<tag>[^#]*?) *'
                 '#+ *(?P<encoding>(?:[^ ] *?){16}) *(?P<help>.*)$')
-        for line in (inspect.getsource(
+        for i, line in enumerate(
-                    inspect.getmodule(inspect.currentframe()))
+                inspect.getsource(inspect.getmodule(inspect.currentframe()))
-                .replace('\\\n', '')
+                    .replace('\\\n', '')
-                .splitlines()):
+                    .splitlines()):
             m = tag_pattern.match(line)
             if m:
                 tags.append(Tag(
                         m.group('name'),
                         globals()[m.group('name')],
                         m.group('encoding').replace(' ', ''),
-                        m.group('help')))
+                        m.group('help'),
+                        lineno=1+i))
         return tags
 
     # find best matching tag
+    _sentinel = object()
     @staticmethod
-    def find(tag):
+    def find(tag, *, default=_sentinel):
         # find tags, note this is cached
         tags__ = Tag.tags()
 
         # find the most specific matching tag, ignoring valid bits
-        return max((t for t in tags__ if t.matches(tag & 0x7fff)),
+        t = max((t for t in tags__ if t.matches(tag & 0x7fff)),
                 key=lambda t: t.specificity(),
                 default=None)
+        if t is not None:
+            return t
+        elif default is Tag._sentinel:
+            raise KeyError(tag)
+        else:
+            return default
 
     # human readable tag repr
     @staticmethod
@@ -170,7 +180,9 @@ class Tag:
             global_=False,
             toff=None):
         # find the most specific matching tag, ignoring the shrub bit
-        t = Tag.find(tag & ~(TAG_SHRUB if tag & 0x7000 == TAG_SHRUB else 0))
+        t = Tag.find(
+                tag & ~(TAG_SHRUB if tag & 0x7000 == TAG_SHRUB else 0),
+                default=None)
 
         # build repr
         r = []