scripts: Added -i/--internal to ctx.py/structs.py, re-limiting structs.py

This adds -i/--internal to ctx.py and structs.py, which has proven
useful for introspection/debugging. Being able to view the ctx/args of
internal functions is nice, even if they don't actually contribute to
the high-level cost.

This also reverts structs.py to limit to .h files by default, to match
ctx.py, once again relying on dwarf file info. This has been a bit
unreliable in the past, but there's not much else that determines if a
struct is part of the "public interface" in C.

But that's what ctx.py is for.

---

Also fixed an issue where structs appearing in multiple files would have
their sizes added together, which ends up with some pretty confusing
results (sizeof(uint32_t) => 8?).

scripts/ctx.py

@@ -457,6 +457,7 @@ def collect_dwarf_info(obj_path, tags=None, *,
     return DwarfInfo(info)
 
 def collect_ctx(obj_paths, *,
+        internal=False,
         everything=False,
         no_strip=False,
         depth=1,
@@ -466,7 +467,8 @@ def collect_ctx(obj_paths, *,
         # find global symbols
         syms = collect_syms(obj_path,
                 sections=['.text'],
-                global_=not everything,
+                # only include internal symbols if explicitly requested
+                global_=not internal and not everything,
                 **args)
 
         # find dwarf info
@@ -1509,6 +1511,11 @@ if __name__ == "__main__":
             '--prefix',
             help="Prefix to use for fields in CSV/JSON output. Defaults "
                 "to %r." % ("%s_" % CtxResult._prefix))
+    parser.add_argument(
+            '-i', '--internal',
+            action='store_true',
+            help="Include internal symbols. Useful for introspection, but "
+                "usually you don't care about these.")
     parser.add_argument(
             '--everything',
             action='store_true',

scripts/structs.py

@@ -153,7 +153,7 @@ class StructResult(co.namedtuple('StructResult', [
     def __add__(self, other):
         return StructResult(self.z, self.i, self.file, self.struct,
                 min(self.off, other.off),
-                self.size + other.size,
+                max(self.size, other.size),
                 max(self.align, other.align),
                 self.children + other.children)
 
@@ -168,6 +168,61 @@ def openio(path, mode='r', buffering=-1):
     else:
         return open(path, mode, buffering)
 
+def collect_dwarf_files(obj_path, *,
+        objdump_path=OBJDUMP_PATH,
+        **args):
+    line_pattern = re.compile(
+            '^\s*(?P<no>[0-9]+)'
+                '(?:\s+(?P<dir>[0-9]+))?'
+                '.*\s+(?P<path>[^\s]+)\s*$')
+
+    # find source paths
+    dirs = co.OrderedDict()
+    files = co.OrderedDict()
+    # note objdump-path may contain extra args
+    cmd = objdump_path + ['--dwarf=rawline', obj_path]
+    if args.get('verbose'):
+        print(' '.join(shlex.quote(c) for c in cmd))
+    proc = sp.Popen(cmd,
+            stdout=sp.PIPE,
+            universal_newlines=True,
+            errors='replace',
+            close_fds=False)
+    for line in proc.stdout:
+        # note that files contain references to dirs, which we
+        # dereference as soon as we see them as each file table
+        # follows a dir table
+        m = line_pattern.match(line)
+        if m:
+            if not m.group('dir'):
+                # found a directory entry
+                dirs[int(m.group('no'))] = m.group('path')
+            else:
+                # found a file entry
+                dir = int(m.group('dir'))
+                if dir in dirs:
+                    files[int(m.group('no'))] = os.path.join(
+                            dirs[dir],
+                            m.group('path'))
+                else:
+                    files[int(m.group('no'))] = m.group('path')
+    proc.wait()
+    if proc.returncode != 0:
+        raise sp.CalledProcessError(proc.returncode, proc.args)
+
+    # simplify paths
+    files_ = co.OrderedDict()
+    for no, file in files.items():
+        if os.path.commonpath([
+                    os.getcwd(),
+                    os.path.abspath(file)]) == os.getcwd():
+            files_[no] = os.path.relpath(file)
+        else:
+            files_[no] = os.path.abspath(file)
+    files = files_
+
+    return files
+
 # each dwarf entry can have attrs and children entries
 class DwarfEntry:
     def __init__(self, level, off, tag, ats={}, children=[]):
@@ -316,6 +371,7 @@ def collect_dwarf_info(obj_path, tags=None, *,
     return DwarfInfo(info)
 
 def collect_structs(obj_paths, *,
+        internal=False,
         everything=False,
         depth=1,
         **args):
@@ -324,6 +380,9 @@ def collect_structs(obj_paths, *,
         # find dwarf info
         info = collect_dwarf_info(obj_path, **args)
 
+        # find related file info
+        files = collect_dwarf_files(obj_path, **args)
+
         # find source file from dwarf info
         for entry in info:
             if (entry.tag == 'DW_TAG_compile_unit'
@@ -490,6 +549,16 @@ def collect_structs(obj_paths, *,
             if not everything and entry.name.startswith('__'):
                 continue
 
+            # find source file
+            if 'DW_AT_decl_file' in entry:
+                src = files.get(int(entry['DW_AT_decl_file']), '?')
+                # discard internal/stdlib types
+                if not everything and src.startswith('/usr/include'):
+                    continue
+                # limit to .h files unless explicitly requested
+                if not everything and not internal and not src.endswith('.h'):
+                    continue
+
             # find name
             name = entry.name
 
@@ -1323,6 +1392,11 @@ if __name__ == "__main__":
             '--prefix',
             help="Prefix to use for fields in CSV/JSON output. Defaults "
                 "to %r." % ("%s_" % StructResult._prefix))
+    parser.add_argument(
+            '-i', '--internal',
+            action='store_true',
+            help="Include internal symbols. Useful for introspection, but "
+                "usually you don't care about these.")
     parser.add_argument(
             '--everything',
             action='store_true',