From 748815bb46bac6274e53dc2a2922db00444e96be Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 27 Feb 2025 14:55:46 -0600 Subject: [PATCH] scripts: Disentangled -r/--hot and -i/--enumerate This removes most of the special behavior around how -r/--hot and -i/--enumerate interact. This does mean -r/--hot risks folding results if -i/--enumerate is not specified, but this is _technically_ a valid operation. For most of the recursive result scripts, I've replaced the "i" field with separate "z" and "i" fields for depth and field number, which I think is a bit more informative/useful. I've also added a default-hidden "off" field to structs.py/ctx.py, since we have that info available. I considered replacing "i" with this, but decided against it since non-zero offsets for union members would risk being confusing/mistake prone. --- scripts/code.py | 12 +++-- scripts/cov.py | 12 +++-- scripts/csv.py | 112 +++++++++++++++++++++------------------------ scripts/ctx.py | 87 +++++++++++++++++++++-------------- scripts/data.py | 12 +++-- scripts/perf.py | 59 ++++++++++++------------ scripts/perfbd.py | 62 ++++++++++++------------- scripts/stack.py | 65 ++++++++++++++------------ scripts/structs.py | 82 +++++++++++++++++++-------------- 9 files changed, 268 insertions(+), 235 deletions(-) diff --git a/scripts/code.py b/scripts/code.py index cca700ce..3398c274 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -922,11 +922,13 @@ def write_csv(path, Result, results, *, with openio(path, 'w') as f: # write csv? if not json: - writer = csv.DictWriter(f, - (by if by is not None else Result._by) - + [k for k in (fields - if fields is not None - else Result._fields)]) + writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain( + by + if by is not None + else Result._by, + fields + if fields is not None + else Result._fields)).keys())) writer.writeheader() for r in results: # note this allows by/fields to overlap diff --git a/scripts/cov.py b/scripts/cov.py index a89a24d8..b9b4b948 100755 --- a/scripts/cov.py +++ b/scripts/cov.py @@ -783,11 +783,13 @@ def write_csv(path, Result, results, *, with openio(path, 'w') as f: # write csv? if not json: - writer = csv.DictWriter(f, - (by if by is not None else Result._by) - + [k for k in (fields - if fields is not None - else Result._fields)]) + writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain( + by + if by is not None + else Result._by, + fields + if fields is not None + else Result._fields)).keys())) writer.writeheader() for r in results: # note this allows by/fields to overlap diff --git a/scripts/csv.py b/scripts/csv.py index fadaafc6..78b71ab7 100755 --- a/scripts/csv.py +++ b/scripts/csv.py @@ -1399,20 +1399,16 @@ def compile(fields_, results, sort=None, enumerate=None, children=None, - notes=None, - hot=None): + hot=None, + notes=None): import builtins enumerate_, enumerate = enumerate, builtins.enumerate by = by.copy() fields = fields.copy() - # we need _something_ to order hot results by, so default to - # i if no enumerate field is specified - if hot is not None and enumerate_ is None: - enumerate_ = 'i' # make sure enumerate fields are included - if enumerate_ is not None or hot is not None: + if enumerate_ is not None: if enumerate_ not in by: by.insert(0, enumerate_) # make sure define fields are included @@ -1543,15 +1539,12 @@ def compile(fields_, results, _types={k: t for k, (_, t) in folds.items()}, _mods=mods, _exprs=exprs, - **{'_i': enumerate_} if enumerate_ is not None else {}, **{'_children': children} if children is not None else {}, **{'_notes': notes} if notes is not None else {})) def homogenize(Result, results, *, - depth=1, - hot=None, enumerate=None, - children=None): + depth=1): import builtins enumerate_, enumerate = enumerate, builtins.enumerate @@ -1562,19 +1555,17 @@ def homogenize(Result, results, *, results_.append(Result(**( r # enumerate? - | ({Result._i: RInt(i)} + | ({enumerate_: RInt(i)} if enumerate_ is not None else {}) # recurse? - | ({children: homogenize( - Result, r[children], - depth=depth-1, - # only enumerate top-level if hotifying - enumerate=(enumerate_ if hot is None else None), - children=children)} - if children is not None - and children in r - and r[children] is not None + | ({Result._children: homogenize( + Result, r[Result._children], + enumerate=enumerate_, + depth=depth-1)} + if hasattr(Result, '_children') + and Result._children in r + and r[Result._children] is not None and depth > 1 else {})))) return results_ @@ -1663,17 +1654,17 @@ def fold(Result, results, *, return folded def hotify(Result, results, *, - fields=None, - sort=None, + enumerate=None, depth=1, hot=None, **_): - # hotify only makes sense for recursive results - assert hasattr(Result, '_i') - assert hasattr(Result, '_children') + # note! hotifying risks confusion if you don't enumerate/have a z + # field, since it will allow folding across recursive boundaries + import builtins + enumerate_, enumerate = enumerate, builtins.enumerate - if fields is None: - fields = Result._fields + # hotify only makes sense for recursive results + assert hasattr(Result, '_children') results_ = [] for r in results: @@ -1694,9 +1685,10 @@ def hotify(Result, results, *, for k_ in ([k] if k else Result._sort))) for k, reverse in it.chain(hot, [(None, False)]))) - hot_.append(r._replace(**{ - Result._i: RInt(len(hot_)), - Result._children: []})) + hot_.append(r._replace(**( + ({enumerate_: len(hot_)} + if enumerate_ is not None else {}) + | {Result._children: []}))) # recurse? if depth_ > 1: @@ -1704,8 +1696,7 @@ def hotify(Result, results, *, depth_-1) recurse(getattr(r, Result._children), depth-1) - results_.append(r._replace(**{ - Result._children: hot_})) + results_.append(r._replace(**{Result._children: hot_})) return results_ @@ -2052,11 +2043,13 @@ def write_csv(path, Result, results, *, with openio(path, 'w') as f: # write csv? if not json: - writer = csv.DictWriter(f, - (by if by is not None else Result._by) - + [k for k in (fields - if fields is not None - else Result._fields)]) + writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain( + by + if by is not None + else Result._by, + fields + if fields is not None + else Result._fields)).keys())) writer.writeheader() for r in results: # note this allows by/fields to overlap @@ -2113,11 +2106,11 @@ def main(csv_paths, *, fields=None, defines=[], sort=None, - depth=None, enumerate=None, + depth=None, children=None, - notes=None, hot=None, + notes=None, **args): import builtins enumerate_, enumerate = enumerate, builtins.enumerate @@ -2194,8 +2187,8 @@ def main(csv_paths, *, and not any(k == k_ for k_, _ in (sort or [])) and k != enumerate_ and k != children - and k != notes and not any(k == k_ for k_, _ in (hot or [])) + and k != notes and not any(k == k_ for _, expr in exprs for k_ in expr.fields())] @@ -2208,8 +2201,8 @@ def main(csv_paths, *, and not any(k == k_ for k_, _ in (sort or [])) and k != enumerate_ and k != children - and k != notes and not any(k == k_ for k_, _ in (hot or [])) + and k != notes and not any(k == k_ for _, expr in exprs for k_ in expr.fields())] @@ -2224,14 +2217,13 @@ def main(csv_paths, *, sort=sort, enumerate=enumerate_, children=children, - notes=notes, - hot=hot) + hot=hot, + notes=notes) # homogenize results = homogenize(Result, results, - depth=depth, enumerate=enumerate_, - children=children) + depth=depth) # fold results = fold(Result, results, @@ -2242,10 +2234,9 @@ def main(csv_paths, *, # hotify? if hot: results = hotify(Result, results, - fields=fields, + enumerate=enumerate_, depth=depth, - hot=hot, - **args) + hot=hot) # write results to CSV/JSON if args.get('output'): @@ -2394,6 +2385,12 @@ if __name__ == "__main__": const=(None, None), help="Sort by this field, but backwards. Can include an expression " "of the form field=expr.") + parser.add_argument( + '-i', '--enumerate', + nargs='?', + const='i', + help="Field to use for enumerating results. This will prevent " + "result folding.") parser.add_argument( '-z', '--depth', nargs='?', @@ -2401,24 +2398,12 @@ if __name__ == "__main__": const=0, help="Depth of function calls to show. 0 shows all calls unless " "we find a cycle. Defaults to 0.") - parser.add_argument( - '-i', '--enumerate', - nargs='?', - const='i', - help="Field to use for enumerating results. This will prevent " - "result folding. This can also be used to override which " - "field -r/--hot uses to order results.") parser.add_argument( '-Z', '--children', nargs='?', const='children', help="Field to use for recursive results. This expects a list " "and really only works with JSON input.") - parser.add_argument( - '-N', '--notes', - nargs='?', - const='notes', - help="Field to use for notes.") class AppendHot(argparse.Action): def __call__(self, parser, namespace, value, option): if namespace.hot is None: @@ -2448,6 +2433,11 @@ if __name__ == "__main__": )(*x.split('=', 1)), const=(None, None), help="Like -r/--hot, but backwards.") + parser.add_argument( + '-N', '--notes', + nargs='?', + const='notes', + help="Field to use for notes.") parser.add_argument( '--no-header', action='store_true', diff --git a/scripts/ctx.py b/scripts/ctx.py index c3ed120b..b1cf4054 100755 --- a/scripts/ctx.py +++ b/scripts/ctx.py @@ -133,27 +133,27 @@ class RInt(co.namedtuple('RInt', 'x')): # ctx size results class CtxResult(co.namedtuple('CtxResult', [ - 'i', 'file', 'function', - 'size', + 'z', 'i', 'file', 'function', + 'off', 'size', 'children', 'notes'])): - _by = ['i', 'file', 'function'] - _fields = ['size'] + _by = ['z', 'i', 'file', 'function'] + _fields = ['off', 'size'] _sort = ['size'] - _types = {'size': RInt} - _i = 'i' + _types = {'off': RInt, 'size': RInt} _children = 'children' _notes = 'notes' __slots__ = () - def __new__(cls, i=None, file='', function='', size=0, + def __new__(cls, z=0, i=0, file='', function='', off=0, size=0, children=None, notes=None): - return super().__new__(cls, i, file, function, - RInt(size), + return super().__new__(cls, z, i, file, function, + RInt(off), RInt(size), children if children is not None else [], notes if notes is not None else set()) def __add__(self, other): - return CtxResult(self.i, self.file, self.function, + return CtxResult(self.z, self.i, self.file, self.function, + min(self.off, other.off), max(self.size, other.size), self.children + other.children, self.notes | other.notes) @@ -596,11 +596,13 @@ def collect_ctx(obj_paths, *, type = info[int(type['DW_AT_type'].strip('<>'), 0)] if (type.name is not None and type.tag != 'DW_TAG_subroutine_type'): + # find size, etc name_ = type.name size_ = sizeof(type, seen | {entry.off}) children_, notes_, dirty_ = childrenof( type, depth-1, seen | {entry.off}) - children.append(CtxResult(0, file, name_, size_, + children.append(CtxResult( + 0, 0, file, name_, 0, size_, children=children_, notes=notes_)) dirty = dirty or dirty_ @@ -608,15 +610,22 @@ def collect_ctx(obj_paths, *, elif entry.tag in { 'DW_TAG_structure_type', 'DW_TAG_union_type'}: + # iterate over children in struct/union children, notes, dirty = [], set(), False for child in entry.children: if child.tag != 'DW_TAG_member': continue + # find name name_ = child.name + # try to find offset for struct members, note this + # is _not_ the same as the dwarf entry offset + off_ = int(child.get('DW_AT_data_member_location', 0)) + # find size, children, etc size_ = sizeof(child, seen | {entry.off}) children_, notes_, dirty_ = childrenof( child, depth-1, seen | {entry.off}) - children.append(CtxResult(child.off, file, name_, size_, + children.append(CtxResult( + 0, len(children), file, name_, off_, size_, children=children_, notes=notes_)) dirty = dirty or dirty_ @@ -678,16 +687,24 @@ def collect_ctx(obj_paths, *, # find children, recursing if necessary children_, notes_, _ = childrenof(param, depth-2) - params.append(CtxResult(param.off, file, name_, size_, + params.append(CtxResult( + 0, len(params), file, name_, 0, size_, children=children_, notes=notes_)) # context = sum of params name = entry.name size = sum((param.size for param in params), start=RInt(0)) - results.append(CtxResult(None, file, name, size, + results.append(CtxResult( + 0, 0, file, name, 0, size, children=params)) + # assign z at the end to avoid issues with caching + def zed(results, z): + return [r._replace(z=z, children=zed(r.children, z+1)) + for r in results] + results = zed(results, 0) + return results @@ -774,17 +791,17 @@ def fold(Result, results, *, return folded def hotify(Result, results, *, - fields=None, - sort=None, + enumerate=None, depth=1, hot=None, **_): - # hotify only makes sense for recursive results - assert hasattr(Result, '_i') - assert hasattr(Result, '_children') + # note! hotifying risks confusion if you don't enumerate/have a z + # field, since it will allow folding across recursive boundaries + import builtins + enumerate_, enumerate = enumerate, builtins.enumerate - if fields is None: - fields = Result._fields + # hotify only makes sense for recursive results + assert hasattr(Result, '_children') results_ = [] for r in results: @@ -805,9 +822,10 @@ def hotify(Result, results, *, for k_ in ([k] if k else Result._sort))) for k, reverse in it.chain(hot, [(None, False)]))) - hot_.append(r._replace(**{ - Result._i: RInt(len(hot_)), - Result._children: []})) + hot_.append(r._replace(**( + ({enumerate_: len(hot_)} + if enumerate_ is not None else {}) + | {Result._children: []}))) # recurse? if depth_ > 1: @@ -815,8 +833,7 @@ def hotify(Result, results, *, depth_-1) recurse(getattr(r, Result._children), depth-1) - results_.append(r._replace(**{ - Result._children: hot_})) + results_.append(r._replace(**{Result._children: hot_})) return results_ @@ -1163,11 +1180,13 @@ def write_csv(path, Result, results, *, with openio(path, 'w') as f: # write csv? if not json: - writer = csv.DictWriter(f, - (by if by is not None else Result._by) - + [k for k in (fields - if fields is not None - else Result._fields)]) + writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain( + by + if by is not None + else Result._by, + fields + if fields is not None + else Result._fields)).keys())) writer.writeheader() for r in results: # note this allows by/fields to overlap @@ -1260,10 +1279,8 @@ def main(obj_paths, *, # hotify? if hot: results = hotify(CtxResult, results, - fields=fields, depth=depth, - hot=hot, - **args) + hot=hot) # write results to CSV/JSON if args.get('output'): @@ -1302,7 +1319,7 @@ def main(obj_paths, *, if not args.get('quiet'): table(CtxResult, results, diff_results, by=by if by is not None else ['function'], - fields=fields, + fields=fields if fields is not None else ['size'], sort=sort, depth=depth, **args) diff --git a/scripts/data.py b/scripts/data.py index decadf21..e8959c4e 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -922,11 +922,13 @@ def write_csv(path, Result, results, *, with openio(path, 'w') as f: # write csv? if not json: - writer = csv.DictWriter(f, - (by if by is not None else Result._by) - + [k for k in (fields - if fields is not None - else Result._fields)]) + writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain( + by + if by is not None + else Result._by, + fields + if fields is not None + else Result._fields)).keys())) writer.writeheader() for r in results: # note this allows by/fields to overlap diff --git a/scripts/perf.py b/scripts/perf.py index 3195c8e2..0d46303f 100755 --- a/scripts/perf.py +++ b/scripts/perf.py @@ -147,31 +147,30 @@ class RInt(co.namedtuple('RInt', 'x')): # perf results class PerfResult(co.namedtuple('PerfResult', [ - 'i', 'file', 'function', 'line', + 'z', 'file', 'function', 'line', 'cycles', 'bmisses', 'branches', 'cmisses', 'caches', 'children'])): - _by = ['i', 'file', 'function', 'line'] + _by = ['z', 'file', 'function', 'line'] _fields = ['cycles', 'bmisses', 'branches', 'cmisses', 'caches'] _sort = ['cycles', 'bmisses', 'cmisses', 'branches', 'caches'] _types = { 'cycles': RInt, 'bmisses': RInt, 'branches': RInt, 'cmisses': RInt, 'caches': RInt} - _i = 'i' _children = 'children' __slots__ = () - def __new__(cls, i=None, file='', function='', line=0, + def __new__(cls, z=0, file='', function='', line=0, cycles=0, bmisses=0, branches=0, cmisses=0, caches=0, children=None): - return super().__new__(cls, i, file, function, int(RInt(line)), + return super().__new__(cls, z, file, function, int(RInt(line)), RInt(cycles), RInt(bmisses), RInt(branches), RInt(cmisses), RInt(caches), children if children is not None else []) def __add__(self, other): - return PerfResult(self.i, self.file, self.function, self.line, + return PerfResult(self.z, self.file, self.function, self.line, self.cycles + other.cycles, self.bmisses + other.bmisses, self.branches + other.branches, @@ -756,15 +755,15 @@ def collect_decompressed(path, *, raise sp.CalledProcessError(proc.returncode, proc.args) # rearrange results into result type - def to_results(results): + def to_results(results, z): results_ = [] for name, (r, children) in results.items(): - results_.append(PerfResult(None, *name, + results_.append(PerfResult(z, *name, **{events[k]: v for k, v in r.items()}, - children=to_results(children))) + children=to_results(children, z+1))) return results_ - return to_results(results) + return to_results(results, 0) def collect_job(path, i, **args): # decompress into a temporary file, this is to work around @@ -896,17 +895,17 @@ def fold(Result, results, *, return folded def hotify(Result, results, *, - fields=None, - sort=None, + enumerate=None, depth=1, hot=None, **_): - # hotify only makes sense for recursive results - assert hasattr(Result, '_i') - assert hasattr(Result, '_children') + # note! hotifying risks confusion if you don't enumerate/have a z + # field, since it will allow folding across recursive boundaries + import builtins + enumerate_, enumerate = enumerate, builtins.enumerate - if fields is None: - fields = Result._fields + # hotify only makes sense for recursive results + assert hasattr(Result, '_children') results_ = [] for r in results: @@ -927,9 +926,10 @@ def hotify(Result, results, *, for k_ in ([k] if k else Result._sort))) for k, reverse in it.chain(hot, [(None, False)]))) - hot_.append(r._replace(**{ - Result._i: RInt(len(hot_)), - Result._children: []})) + hot_.append(r._replace(**( + ({enumerate_: len(hot_)} + if enumerate_ is not None else {}) + | {Result._children: []}))) # recurse? if depth_ > 1: @@ -937,8 +937,7 @@ def hotify(Result, results, *, depth_-1) recurse(getattr(r, Result._children), depth-1) - results_.append(r._replace(**{ - Result._children: hot_})) + results_.append(r._replace(**{Result._children: hot_})) return results_ @@ -1285,11 +1284,13 @@ def write_csv(path, Result, results, *, with openio(path, 'w') as f: # write csv? if not json: - writer = csv.DictWriter(f, - (by if by is not None else Result._by) - + [k for k in (fields - if fields is not None - else Result._fields)]) + writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain( + by + if by is not None + else Result._by, + fields + if fields is not None + else Result._fields)).keys())) writer.writeheader() for r in results: # note this allows by/fields to overlap @@ -1491,10 +1492,8 @@ def report(perf_paths, *, # hotify? if hot: results = hotify(PerfResult, results, - fields=fields, depth=depth, - hot=hot, - **args) + hot=hot) # write results to CSV/JSON if args.get('output'): diff --git a/scripts/perfbd.py b/scripts/perfbd.py index 05ead106..e3f884e9 100755 --- a/scripts/perfbd.py +++ b/scripts/perfbd.py @@ -138,26 +138,25 @@ class RInt(co.namedtuple('RInt', 'x')): # perf results class PerfBdResult(co.namedtuple('PerfBdResult', [ - 'i', 'file', 'function', 'line', + 'z', 'file', 'function', 'line', 'readed', 'proged', 'erased', 'children'])): - _by = ['i', 'file', 'function', 'line'] + _by = ['z', 'file', 'function', 'line'] _fields = ['readed', 'proged', 'erased'] _sort = ['erased', 'proged', 'readed'] _types = {'readed': RInt, 'proged': RInt, 'erased': RInt} - _i = 'i' _children = 'children' __slots__ = () - def __new__(cls, i=None, file='', function='', line=0, + def __new__(cls, z=0, file='', function='', line=0, readed=0, proged=0, erased=0, children=None): - return super().__new__(cls, i, file, function, int(RInt(line)), + return super().__new__(cls, z, file, function, int(RInt(line)), RInt(readed), RInt(proged), RInt(erased), children if children is not None else []) def __add__(self, other): - return PerfBdResult(self.i, self.file, self.function, self.line, + return PerfBdResult(self.z, self.file, self.function, self.line, self.readed + other.readed, self.proged + other.proged, self.erased + other.erased, @@ -715,15 +714,15 @@ def collect_job(path, start, stop, syms, lines, *, commit() # rearrange results into result type - def to_results(results): + def to_results(results, z): results_ = [] for name, (r, p, e, children) in results.items(): - results_.append(PerfBdResult(None, *name, + results_.append(PerfBdResult(z, *name, r, p, e, - children=to_results(children))) + children=to_results(children, z+1))) return results_ - return to_results(results) + return to_results(results, 0) def starapply(args): f, args, kwargs = args @@ -866,17 +865,17 @@ def fold(Result, results, *, return folded def hotify(Result, results, *, - fields=None, - sort=None, + enumerate=None, depth=1, hot=None, **_): - # hotify only makes sense for recursive results - assert hasattr(Result, '_i') - assert hasattr(Result, '_children') + # note! hotifying risks confusion if you don't enumerate/have a z + # field, since it will allow folding across recursive boundaries + import builtins + enumerate_, enumerate = enumerate, builtins.enumerate - if fields is None: - fields = Result._fields + # hotify only makes sense for recursive results + assert hasattr(Result, '_children') results_ = [] for r in results: @@ -897,9 +896,10 @@ def hotify(Result, results, *, for k_ in ([k] if k else Result._sort))) for k, reverse in it.chain(hot, [(None, False)]))) - hot_.append(r._replace(**{ - Result._i: RInt(len(hot_)), - Result._children: []})) + hot_.append(r._replace(**( + ({enumerate_: len(hot_)} + if enumerate_ is not None else {}) + | {Result._children: []}))) # recurse? if depth_ > 1: @@ -907,8 +907,7 @@ def hotify(Result, results, *, depth_-1) recurse(getattr(r, Result._children), depth-1) - results_.append(r._replace(**{ - Result._children: hot_})) + results_.append(r._replace(**{Result._children: hot_})) return results_ @@ -1255,11 +1254,13 @@ def write_csv(path, Result, results, *, with openio(path, 'w') as f: # write csv? if not json: - writer = csv.DictWriter(f, - (by if by is not None else Result._by) - + [k for k in (fields - if fields is not None - else Result._fields)]) + writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain( + by + if by is not None + else Result._by, + fields + if fields is not None + else Result._fields)).keys())) writer.writeheader() for r in results: # note this allows by/fields to overlap @@ -1490,10 +1491,8 @@ def report(paths, *, # hotify? if hot: results = hotify(PerfBdResult, results, - fields=fields, depth=depth, - hot=hot, - **args) + hot=hot) # write results to CSV/JSON if args.get('output'): @@ -1541,7 +1540,8 @@ def report(paths, *, # print table table(PerfBdResult, results, diff_results, by=by if by is not None else ['function'], - fields=fields, + fields=fields if fields is not None + else ['readed', 'proged', 'erased'], sort=sort, depth=depth, **args) diff --git a/scripts/stack.py b/scripts/stack.py index d5fc0183..85561523 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -133,27 +133,26 @@ class RInt(co.namedtuple('RInt', 'x')): # stack size results class StackResult(co.namedtuple('StackResult', [ - 'i', 'file', 'function', + 'z', 'file', 'function', 'frame', 'limit', 'children', 'notes'])): - _by = ['i', 'file', 'function'] + _by = ['z', 'file', 'function'] _fields = ['frame', 'limit'] _sort = ['limit', 'frame'] _types = {'frame': RInt, 'limit': RInt} - _i = 'i' _children = 'children' _notes = 'notes' __slots__ = () - def __new__(cls, i=None, file='', function='', frame=0, limit=0, + def __new__(cls, z=0, file='', function='', frame=0, limit=0, children=None, notes=None): - return super().__new__(cls, i, file, function, + return super().__new__(cls, z, file, function, RInt(frame), RInt(limit), children if children is not None else [], notes if notes is not None else set()) def __add__(self, other): - return StackResult(self.i, self.file, self.function, + return StackResult(self.z, self.file, self.function, self.frame + other.frame, max(self.limit, other.limit), self.children + other.children, @@ -417,7 +416,8 @@ def collect_stack(ci_paths, *, limit_ = limitof(node_, seen | {node.name}) children_, notes_, dirty_ = childrenof( node_, depth-1, seen | {node.name}) - children.append(StackResult(None, file_, name_, frame_, limit_, + children.append(StackResult( + file=file_, function=name_, frame=frame_, limit=limit_, children=children_, notes=notes_)) dirty = dirty or dirty_ @@ -440,10 +440,17 @@ def collect_stack(ci_paths, *, frame = frameof(node) limit = limitof(node) children, notes, _ = childrenof(node, depth-1) - results.append(StackResult(None, file, name, frame, limit, + results.append(StackResult( + file=file, function=name, frame=frame, limit=limit, children=children, notes=notes)) + # assign z at the end to avoid issues with caching + def zed(results, z): + return [r._replace(z=z, children=zed(r.children, z+1)) + for r in results] + results = zed(results, 0) + return results @@ -530,17 +537,17 @@ def fold(Result, results, *, return folded def hotify(Result, results, *, - fields=None, - sort=None, + enumerate=None, depth=1, hot=None, **_): - # hotify only makes sense for recursive results - assert hasattr(Result, '_i') - assert hasattr(Result, '_children') + # note! hotifying risks confusion if you don't enumerate/have a z + # field, since it will allow folding across recursive boundaries + import builtins + enumerate_, enumerate = enumerate, builtins.enumerate - if fields is None: - fields = Result._fields + # hotify only makes sense for recursive results + assert hasattr(Result, '_children') results_ = [] for r in results: @@ -561,9 +568,10 @@ def hotify(Result, results, *, for k_ in ([k] if k else Result._sort))) for k, reverse in it.chain(hot, [(None, False)]))) - hot_.append(r._replace(**{ - Result._i: RInt(len(hot_)), - Result._children: []})) + hot_.append(r._replace(**( + ({enumerate_: len(hot_)} + if enumerate_ is not None else {}) + | {Result._children: []}))) # recurse? if depth_ > 1: @@ -571,8 +579,7 @@ def hotify(Result, results, *, depth_-1) recurse(getattr(r, Result._children), depth-1) - results_.append(r._replace(**{ - Result._children: hot_})) + results_.append(r._replace(**{Result._children: hot_})) return results_ @@ -919,11 +926,13 @@ def write_csv(path, Result, results, *, with openio(path, 'w') as f: # write csv? if not json: - writer = csv.DictWriter(f, - (by if by is not None else Result._by) - + [k for k in (fields - if fields is not None - else Result._fields)]) + writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain( + by + if by is not None + else Result._by, + fields + if fields is not None + else Result._fields)).keys())) writer.writeheader() for r in results: # note this allows by/fields to overlap @@ -1016,10 +1025,8 @@ def main(ci_paths, # hotify? if hot: results = hotify(StackResult, results, - fields=fields, depth=depth, - hot=hot, - **args) + hot=hot) # write results to CSV/JSON if args.get('output'): @@ -1058,7 +1065,7 @@ def main(ci_paths, if not args.get('quiet'): table(StackResult, results, diff_results, by=by if by is not None else ['function'], - fields=fields, + fields=fields if fields is not None else ['frame', 'limit'], sort=sort, depth=depth, **args) diff --git a/scripts/structs.py b/scripts/structs.py index bb4e486a..a9623537 100755 --- a/scripts/structs.py +++ b/scripts/structs.py @@ -133,25 +133,25 @@ class RInt(co.namedtuple('RInt', 'x')): # struct size results class StructResult(co.namedtuple('StructResult', [ - 'i', 'file', 'struct', - 'size', 'align', + 'z', 'i', 'file', 'struct', + 'off', 'size', 'align', 'children'])): - _by = ['i', 'file', 'struct'] - _fields = ['size', 'align'] + _by = ['z', 'i', 'file', 'struct'] + _fields = ['off', 'size', 'align'] _sort = ['size', 'align'] - _types = {'size': RInt, 'align': RInt} - _i = 'i' + _types = {'off': RInt, 'size': RInt, 'align': RInt} _children = 'children' __slots__ = () - def __new__(cls, i=None, file='', struct='', size=0, align=0, + def __new__(cls, z=0, i=0, file='', struct='', off=0, size=0, align=0, children=None): - return super().__new__(cls, i, file, struct, - RInt(size), RInt(align), + return super().__new__(cls, z, i, file, struct, + RInt(off), RInt(size), RInt(align), children if children is not None else []) def __add__(self, other): - return StructResult(self.i, self.file, self.struct, + return StructResult(self.z, self.i, self.file, self.struct, + min(self.off, other.off), self.size + other.size, max(self.align, other.align), self.children + other.children) @@ -437,14 +437,22 @@ def collect_structs(obj_paths, *, elif entry.tag in { 'DW_TAG_structure_type', 'DW_TAG_union_type'}: + # iterate over children in struct/union children = [] for child in entry.children: + if child.tag != 'DW_TAG_member': + continue + # find name name_ = child.name + # try to find offset for struct members, note this + # is _not_ the same as the dwarf entry offset + off_ = int(child.get('DW_AT_data_member_location', 0)) + # find size, align, children, etc size_ = sizeof(child) align_ = alignof(child) children_ = childrenof(child, depth-1) children.append(StructResult( - child.off, file, name_, size_, align_, + 0, len(children), file, name_, off_, size_, align_, children=children_)) # indirect type? elif entry.tag in { @@ -497,12 +505,12 @@ def collect_structs(obj_paths, *, # these separately if entry.tag == 'DW_TAG_typedef': typedefs[entry.off] = StructResult( - None, file, name, size, align, + 0, 0, file, name, 0, size, align, children=children) typedefed.add(int(entry['DW_AT_type'].strip('<>'), 0)) else: types[entry.off] = StructResult( - None, file, name, size, align, + 0, 0, file, name, 0, size, align, children=children) # let typedefs take priority @@ -511,6 +519,12 @@ def collect_structs(obj_paths, *, for off, type in types.items() if off not in typedefed) + # assign z at the end to avoid issues with caching + def zed(results, z): + return [r._replace(z=z, children=zed(r.children, z+1)) + for r in results] + results = zed(results, 0) + return results @@ -597,17 +611,17 @@ def fold(Result, results, *, return folded def hotify(Result, results, *, - fields=None, - sort=None, + enumerate=None, depth=1, hot=None, **_): - # hotify only makes sense for recursive results - assert hasattr(Result, '_i') - assert hasattr(Result, '_children') + # note! hotifying risks confusion if you don't enumerate/have a z + # field, since it will allow folding across recursive boundaries + import builtins + enumerate_, enumerate = enumerate, builtins.enumerate - if fields is None: - fields = Result._fields + # hotify only makes sense for recursive results + assert hasattr(Result, '_children') results_ = [] for r in results: @@ -628,9 +642,10 @@ def hotify(Result, results, *, for k_ in ([k] if k else Result._sort))) for k, reverse in it.chain(hot, [(None, False)]))) - hot_.append(r._replace(**{ - Result._i: RInt(len(hot_)), - Result._children: []})) + hot_.append(r._replace(**( + ({enumerate_: len(hot_)} + if enumerate_ is not None else {}) + | {Result._children: []}))) # recurse? if depth_ > 1: @@ -638,8 +653,7 @@ def hotify(Result, results, *, depth_-1) recurse(getattr(r, Result._children), depth-1) - results_.append(r._replace(**{ - Result._children: hot_})) + results_.append(r._replace(**{Result._children: hot_})) return results_ @@ -986,11 +1000,13 @@ def write_csv(path, Result, results, *, with openio(path, 'w') as f: # write csv? if not json: - writer = csv.DictWriter(f, - (by if by is not None else Result._by) - + [k for k in (fields - if fields is not None - else Result._fields)]) + writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain( + by + if by is not None + else Result._by, + fields + if fields is not None + else Result._fields)).keys())) writer.writeheader() for r in results: # note this allows by/fields to overlap @@ -1083,10 +1099,8 @@ def main(obj_paths, *, # hotify? if hot: results = hotify(StructResult, results, - fields=fields, depth=depth, - hot=hot, - **args) + hot=hot) # write results to CSV/JSON if args.get('output'): @@ -1125,7 +1139,7 @@ def main(obj_paths, *, if not args.get('quiet'): table(StructResult, results, diff_results, by=by if by is not None else ['struct'], - fields=fields, + fields=fields if fields is not None else ['size', 'align'], sort=sort, depth=depth, **args)