diff --git a/scripts/bench.py b/scripts/bench.py index e106432b..80a52cae 100755 --- a/scripts/bench.py +++ b/scripts/bench.py @@ -667,7 +667,7 @@ def find_runner(runner, id=None, main=True, **args): # run under perf? if args.get('perf'): cmd[:0] = args['perf_script'] + list(filter(None, [ - '-R', + '--record', '--perf-freq=%s' % args['perf_freq'] if args.get('perf_freq') else None, '--perf-period=%s' % args['perf_period'] diff --git a/scripts/code.py b/scripts/code.py index 18cefdfd..cca700ce 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -18,8 +18,9 @@ if __name__ == "__main__": import collections as co import csv -import itertools as it import functools as ft +import io +import itertools as it import math as mt import os import re @@ -449,7 +450,7 @@ def collect_dwarf_info(obj_path, tags=None, *, return DwarfInfo(info) -def collect(obj_paths, *, +def collect_code(obj_paths, *, everything=False, **args): results = [] @@ -496,7 +497,31 @@ def collect(obj_paths, *, return results -def fold(Result, results, by=None, defines=[]): +# common folding/tabling/read/write code + +class Rev(co.namedtuple('Rev', 'x')): + __slots__ = () + # yes we need all of these because we're a namedtuple + def __lt__(self, other): + return self.x > other.x + def __gt__(self, other): + return self.x < other.x + def __le__(self, other): + return self.x >= other.x + def __ge__(self, other): + return self.x <= other.x + +def fold(Result, results, *, + by=None, + defines=[], + sort=None, + depth=1, + **_): + # stop when depth hits zero + if depth == 0: + return [] + + # organize by by if by is None: by = Result._by @@ -510,7 +535,7 @@ def fold(Result, results, by=None, defines=[]): if defines: results_ = [] for r in results: - if all(getattr(r, k) in vs for k, vs in defines): + if all(str(getattr(r, k)) in vs for k, vs in defines): results_.append(r) results = results_ @@ -527,6 +552,31 @@ def fold(Result, results, by=None, defines=[]): for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) + # sort, note that python's sort is stable + folded.sort(key=lambda r: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(r, k_),) + if getattr(r, k_) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by result + r)) + + # recurse if we have recursive results + if hasattr(Result, '_children'): + folded = [r._replace(**{ + Result._children: fold( + Result, getattr(r, Result._children), + by=by, + defines=defines, + sort=sort, + depth=depth-1)}) + for r in folded] + return folded def table(Result, results, diff_results=None, *, @@ -554,124 +604,32 @@ def table(Result, results, diff_results=None, *, fields = Result._fields types = Result._types - # fold again - results = fold(Result, results, by=by) + # fold again, otherwise results risk being hidden + results = fold(Result, results, + by=by, + depth=depth) if diff_results is not None: - diff_results = fold(Result, diff_results, by=by) - - # reduce children to hot paths? only used by some scripts - if hot: - # subclass to reintroduce __dict__ - Result_ = Result - class HotResult(Result_): - _i = '_hot_i' - _children = '_hot_children' - - def __new__(cls, r, i=None, children=None, notes=None): - self = HotResult._make(r) - self._hot_i = i - self._hot_children = children if children is not None else [] - return self - - def __add__(self, other): - return HotResult( - Result_.__add__(self, other), - self._hot_i if other._hot_i is None - else other._hot_i if self._hot_i is None - else min(self._hot_i, other._hot_i), - self._hot_children + other._hot_children) - - results_ = [] - for r in results: - hot_ = [] - def recurse(results_, depth_): - nonlocal hot_ - if not results_: - return - - # find the hottest result - r = max(results_, - key=lambda r: tuple( - tuple((getattr(r, k),) - if getattr(r, k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields]) - if k in fields) - for k in it.chain(hot, [None]))) - hot_.append(HotResult(r, i=len(hot_))) - - # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), - depth_-1) - - recurse(getattr(r, Result._children), depth-1) - results_.append(HotResult(r, children=hot_)) - - Result = HotResult - results = results_ + diff_results = fold(Result, diff_results, + by=by, + depth=depth) # organize by name table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results} diff_table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in diff_results or []} - names = [name - for name in table.keys() | diff_table.keys() - if diff_results is None - or all_ - or any( - types[k].ratio( - getattr(table.get(name), k, None), - getattr(diff_table.get(name), k, None)) - for k in fields)] # find compare entry if there is one if compare: - compare_result = table.get(','.join(str(k) for k in compare)) - - # sort again, now with diff info, note that python's sort is stable - names.sort() - if compare: - names.sort( - key=lambda n: ( - # move compare entry to the top, note this can be - # overridden by explicitly sorting by fields - table.get(n) == compare_result, - # sort by ratio if comparing - tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(compare_result, k, None)) - for k in fields)), - reverse=True) - if diff or percent: - names.sort( - # sort by ratio if diffing - key=lambda n: tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(diff_table.get(n), k, None)) - for k in fields), - reverse=True) - if sort: - for k, reverse in reversed(sort): - names.sort( - key=lambda n: tuple( - (getattr(table[n], k),) - if getattr(table.get(n), k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) - + compare_r = table.get(','.join(str(k) for k in compare)) # build up our lines lines = [] @@ -698,11 +656,16 @@ def table(Result, results, diff_results=None, *, header.append('d'+k) lines.append(header) + # delete these to try to catch typos below, we need to rebuild + # these tables at each recursive layer + del table + del diff_table + # entry helper def table_entry(name, r, diff_r=None): entry = [name] # normal entry? - if ((compare is None or r == compare_result) + if ((compare is None or r == compare_r) and not percent and not diff): for k in fields: @@ -723,7 +686,7 @@ def table(Result, results, diff_results=None, *, else ['%+.1f%%' % (100*t)])( types[k].ratio( getattr(r, k, None), - getattr(compare_result, k, None))))) + getattr(compare_r, k, None))))) # percent entry? elif not diff: for k in fields: @@ -769,71 +732,94 @@ def table(Result, results, diff_results=None, *, return entry - # recursive entry helper, only used by some scripts - def recurse(results_, depth_, + # recursive entry helper + def table_recurse(results_, diff_results_, + depth_, prefixes=('', '', '', '')): # build the children table at each layer - results_ = fold(Result, results_, by=by) table_ = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results_} - names_ = list(table_.keys()) + diff_table_ = { + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r + for r in diff_results_ or []} + names_ = [n + for n in table_.keys() | diff_table_.keys() + if diff_results_ is None + or all_ + or any( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)] - # sort the children layer - names_.sort() - if hasattr(Result, '_i'): - names_.sort(key=lambda n: getattr(table_[n], Result._i)) - if sort: - for k, reverse in reversed(sort): - names_.sort( - key=lambda n: tuple( - (getattr(table_[n], k),) - if getattr(table_.get(n), k, None) - is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) + # sort again, now with diff info, note that python's sort is stable + names_.sort(key=lambda n: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(table_[n], k_),) + if getattr(table_.get(n), k_, None) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by ratio if diffing + Rev(tuple(types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)) + if diff or percent + else (), + # move compare entry to the top, note this can be + # overridden by explicitly sorting by fields + (table_.get(n) != compare_r, + # sort by ratio if comparing + Rev(tuple( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(compare_r, k, None)) + for k in fields))) + if compare + else (), + # sort by result + (table_[n],) if n in table_ else (), + # and finally by name (diffs may be missing results) + n)) - for i, name in enumerate(names_): - r = table_[name] - is_last = (i == len(names_)-1) + for i, n in enumerate(names_): + # find comparable results + r = table_.get(n) + diff_r = diff_table_.get(n) + + # build line + line = table_entry(n, r, diff_r) - line = table_entry(name, r) - line = [x if isinstance(x, tuple) else (x, []) for x in line] # add prefixes - line[0] = (prefixes[0+is_last] + line[0][0], line[0][1]) + line = [x if isinstance(x, tuple) else (x, []) for x in line] + line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) lines.append(line) # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), + if n in table_ and depth_ > 1: + table_recurse( + getattr(r, Result._children), + getattr(diff_r, Result._children, None) or [], depth_-1, - (prefixes[2+is_last] + "|-> ", - prefixes[2+is_last] + "'-> ", - prefixes[2+is_last] + "| ", - prefixes[2+is_last] + " ")) + (prefixes[2+(i==len(names_)-1)] + "|-> ", + prefixes[2+(i==len(names_)-1)] + "'-> ", + prefixes[2+(i==len(names_)-1)] + "| ", + prefixes[2+(i==len(names_)-1)] + " ")) - # entries + # build entries if not summary: - for name in names: - r = table.get(name) - if diff_results is None: - diff_r = None - else: - diff_r = diff_table.get(name) - lines.append(table_entry(name, r, diff_r)) - - # recursive entries - if name in table and depth > 1: - recurse(getattr(table[name], Result._children), - depth-1, - ("|-> ", - "'-> ", - "| ", - " ")) + table_recurse(results, diff_results, depth) # total if not no_total and not (small_table and not summary): @@ -845,9 +831,8 @@ def table(Result, results, diff_results=None, *, lines.append(table_entry('TOTAL', r, diff_r)) # homogenize - lines = [ - [x if isinstance(x, tuple) else (x, []) for x in line] - for line in lines] + lines = [[x if isinstance(x, tuple) else (x, []) for x in line] + for line in lines] # find the best widths, note that column 0 contains the names and is # handled a bit differently @@ -868,6 +853,130 @@ def table(Result, results, diff_results=None, *, nwidths[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') for i, x in enumerate(line[1:], 1)))) +def read_csv(path, Result, *, + depth=1, + **_): + with openio(path, 'r') as f: + # csv or json? assume json starts with [ + json = (f.buffer.peek(1)[:1] == b'[') + + # read csv? + if not json: + results = [] + reader = csv.DictReader(f, restval='') + for r in reader: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k].strip()} + | {k: r[k] for k in Result._fields + if k in r and r[k].strip()}))) + except TypeError: + pass + return results + + # read json? + else: + import json + def unjsonify(results, depth_): + results_ = [] + for r in results: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results_.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k] is not None} + | {k: r[k] for k in Result._fields + if k in r and r[k] is not None} + | ({Result._children: unjsonify( + r[Result._children], + depth_-1)} + if hasattr(Result, '_children') + and Result._children in r + and r[Result._children] is not None + and depth_ > 1 + else {}) + | ({Result._notes: set(r[Result._notes])} + if hasattr(Result, '_notes') + and Result._notes in r + and r[Result._notes] is not None + else {})))) + except TypeError: + pass + return results_ + return unjsonify(json.load(f), depth) + +def write_csv(path, Result, results, *, + json=False, + by=None, + fields=None, + depth=1, + **_): + with openio(path, 'w') as f: + # write csv? + if not json: + writer = csv.DictWriter(f, + (by if by is not None else Result._by) + + [k for k in (fields + if fields is not None + else Result._fields)]) + writer.writeheader() + for r in results: + # note this allows by/fields to overlap + writer.writerow( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None}) + + # write json? + else: + import json + # the neat thing about json is we can include recursive results + def jsonify(results, depth_): + results_ = [] + for r in results: + # note this allows by/fields to overlap + results_.append( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None} + | ({Result._children: jsonify( + getattr(r, Result._children), + depth_-1)} + if hasattr(Result, '_children') + and getattr(r, Result._children) + and depth_ > 1 + else {}) + | ({Result._notes: list( + getattr(r, Result._notes))} + if hasattr(Result, '_notes') + and getattr(r, Result._notes) + else {})) + return results_ + json.dump(jsonify(results, depth), f, + separators=(',', ':')) + def main(obj_paths, *, by=None, @@ -884,87 +993,46 @@ def main(obj_paths, *, sys.exit(1) # collect info - results = collect(obj_paths, **args) + results = collect_code(obj_paths, + **args) else: - results = [] - with openio(args['use']) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - if not any(k in r and r[k].strip() - for k in CodeResult._fields): - continue - try: - results.append(CodeResult( - **{k: r[k] for k in CodeResult._by - if k in r and r[k].strip()}, - **{k: r[k] for k in CodeResult._fields - if k in r and r[k].strip()})) - except TypeError: - pass + results = read_csv(args['use'], CodeResult, + **args) # fold - results = fold(CodeResult, results, by=by, defines=defines) + results = fold(CodeResult, results, + by=by, + defines=defines) - # sort, note that python's sort is stable - results.sort() - if sort: - for k, reverse in reversed(sort): - results.sort( - key=lambda r: tuple( - (getattr(r, k),) if getattr(r, k) is not None else () - for k in ([k] if k else CodeResult._sort)), - reverse=reverse ^ (not k or k in CodeResult._fields)) - - # write results to CSV + # write results to CSV/JSON if args.get('output'): - with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, - (by if by is not None else CodeResult._by) - + [k for k in ( - fields if fields is not None - else CodeResult._fields)]) - writer.writeheader() - for r in results: - writer.writerow( - {k: getattr(r, k) for k in ( - by if by is not None else CodeResult._by)} - | {k: getattr(r, k) for k in ( - fields if fields is not None - else CodeResult._fields)}) + write_csv(args['output'], CodeResult, results, + by=by, + fields=fields, + **args) + if args.get('output_json'): + write_csv(args['output_json'], CodeResult, results, + json=True, + by=by, + fields=fields, + **args) # find previous results? diff_results = None if args.get('diff') or args.get('percent'): - diff_results = [] try: - with openio(args.get('diff') or args.get('percent')) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - if not any(k in r and r[k].strip() - for k in CodeResult._fields): - continue - try: - diff_results.append(CodeResult( - **{k: r[k] for k in CodeResult._by - if k in r and r[k].strip()}, - **{k: r[k] for k in CodeResult._fields - if k in r and r[k].strip()})) - except TypeError: - pass + diff_results = read_csv( + args.get('diff') or args.get('percent'), + CodeResult, + **args) except FileNotFoundError: - pass + diff_results = [] # fold - diff_results = fold(CodeResult, diff_results, by=by, defines=defines) + diff_results = fold(CodeResult, diff_results, + by=by, + defines=defines) # print table if not args.get('quiet'): @@ -996,24 +1064,28 @@ if __name__ == "__main__": parser.add_argument( '-o', '--output', help="Specify CSV file to store results.") + parser.add_argument( + '-O', '--output-json', + help="Specify JSON file to store results. This may contain " + "recursive info.") parser.add_argument( '-u', '--use', - help="Don't parse anything, use this CSV file.") + help="Don't parse anything, use this CSV/JSON file.") parser.add_argument( '-d', '--diff', - help="Specify CSV file to diff against.") + help="Specify CSV/JSON file to diff against.") parser.add_argument( '-p', '--percent', - help="Specify CSV file to diff against, but only show precentage " - "change, not a full diff.") - parser.add_argument( - '-a', '--all', - action='store_true', - help="Show all, not just the ones that changed.") + help="Specify CSV/JSON file to diff against, but only show " + "percentage change, not a full diff.") parser.add_argument( '-c', '--compare', type=lambda x: tuple(v.strip() for v in x.split(',')), help="Compare results to the row matching this by pattern.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") parser.add_argument( '-b', '--by', action='append', @@ -1039,7 +1111,7 @@ if __name__ == "__main__": def __call__(self, parser, namespace, value, option): if namespace.sort is None: namespace.sort = [] - namespace.sort.append((value, True if option == '-S' else False)) + namespace.sort.append((value, option in {'-S', '--reverse-sort'})) parser.add_argument( '-s', '--sort', nargs='?', diff --git a/scripts/cov.py b/scripts/cov.py index d11a0e43..a89a24d8 100755 --- a/scripts/cov.py +++ b/scripts/cov.py @@ -18,6 +18,7 @@ if __name__ == "__main__": import collections as co import csv +import io import itertools as it import json import math as mt @@ -270,7 +271,7 @@ def openio(path, mode='r', buffering=-1): else: return open(path, mode, buffering) -def collect_cov(gcda_path, *, +def collect_gcov(gcda_path, *, gcov_path=GCOV_PATH, **args): # get coverage info through gcov's json output @@ -290,14 +291,14 @@ def collect_cov(gcda_path, *, return cov -def collect(gcda_paths, *, +def collect_cov(gcda_paths, *, sources=None, everything=False, **args): results = [] for gcda_path in gcda_paths: # find coverage info - cov = collect_cov(gcda_path, **args) + cov = collect_gcov(gcda_path, **args) # collect line/branch coverage for file in cov['files']: @@ -357,7 +358,31 @@ def collect(gcda_paths, *, return results -def fold(Result, results, by=None, defines=[]): +# common folding/tabling/read/write code + +class Rev(co.namedtuple('Rev', 'x')): + __slots__ = () + # yes we need all of these because we're a namedtuple + def __lt__(self, other): + return self.x > other.x + def __gt__(self, other): + return self.x < other.x + def __le__(self, other): + return self.x >= other.x + def __ge__(self, other): + return self.x <= other.x + +def fold(Result, results, *, + by=None, + defines=[], + sort=None, + depth=1, + **_): + # stop when depth hits zero + if depth == 0: + return [] + + # organize by by if by is None: by = Result._by @@ -371,7 +396,7 @@ def fold(Result, results, by=None, defines=[]): if defines: results_ = [] for r in results: - if all(getattr(r, k) in vs for k, vs in defines): + if all(str(getattr(r, k)) in vs for k, vs in defines): results_.append(r) results = results_ @@ -388,6 +413,31 @@ def fold(Result, results, by=None, defines=[]): for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) + # sort, note that python's sort is stable + folded.sort(key=lambda r: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(r, k_),) + if getattr(r, k_) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by result + r)) + + # recurse if we have recursive results + if hasattr(Result, '_children'): + folded = [r._replace(**{ + Result._children: fold( + Result, getattr(r, Result._children), + by=by, + defines=defines, + sort=sort, + depth=depth-1)}) + for r in folded] + return folded def table(Result, results, diff_results=None, *, @@ -415,124 +465,32 @@ def table(Result, results, diff_results=None, *, fields = Result._fields types = Result._types - # fold again - results = fold(Result, results, by=by) + # fold again, otherwise results risk being hidden + results = fold(Result, results, + by=by, + depth=depth) if diff_results is not None: - diff_results = fold(Result, diff_results, by=by) - - # reduce children to hot paths? only used by some scripts - if hot: - # subclass to reintroduce __dict__ - Result_ = Result - class HotResult(Result_): - _i = '_hot_i' - _children = '_hot_children' - - def __new__(cls, r, i=None, children=None, notes=None): - self = HotResult._make(r) - self._hot_i = i - self._hot_children = children if children is not None else [] - return self - - def __add__(self, other): - return HotResult( - Result_.__add__(self, other), - self._hot_i if other._hot_i is None - else other._hot_i if self._hot_i is None - else min(self._hot_i, other._hot_i), - self._hot_children + other._hot_children) - - results_ = [] - for r in results: - hot_ = [] - def recurse(results_, depth_): - nonlocal hot_ - if not results_: - return - - # find the hottest result - r = max(results_, - key=lambda r: tuple( - tuple((getattr(r, k),) - if getattr(r, k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields]) - if k in fields) - for k in it.chain(hot, [None]))) - hot_.append(HotResult(r, i=len(hot_))) - - # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), - depth_-1) - - recurse(getattr(r, Result._children), depth-1) - results_.append(HotResult(r, children=hot_)) - - Result = HotResult - results = results_ + diff_results = fold(Result, diff_results, + by=by, + depth=depth) # organize by name table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results} diff_table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in diff_results or []} - names = [name - for name in table.keys() | diff_table.keys() - if diff_results is None - or all_ - or any( - types[k].ratio( - getattr(table.get(name), k, None), - getattr(diff_table.get(name), k, None)) - for k in fields)] # find compare entry if there is one if compare: - compare_result = table.get(','.join(str(k) for k in compare)) - - # sort again, now with diff info, note that python's sort is stable - names.sort() - if compare: - names.sort( - key=lambda n: ( - # move compare entry to the top, note this can be - # overridden by explicitly sorting by fields - table.get(n) == compare_result, - # sort by ratio if comparing - tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(compare_result, k, None)) - for k in fields)), - reverse=True) - if diff or percent: - names.sort( - # sort by ratio if diffing - key=lambda n: tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(diff_table.get(n), k, None)) - for k in fields), - reverse=True) - if sort: - for k, reverse in reversed(sort): - names.sort( - key=lambda n: tuple( - (getattr(table[n], k),) - if getattr(table.get(n), k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) - + compare_r = table.get(','.join(str(k) for k in compare)) # build up our lines lines = [] @@ -559,11 +517,16 @@ def table(Result, results, diff_results=None, *, header.append('d'+k) lines.append(header) + # delete these to try to catch typos below, we need to rebuild + # these tables at each recursive layer + del table + del diff_table + # entry helper def table_entry(name, r, diff_r=None): entry = [name] # normal entry? - if ((compare is None or r == compare_result) + if ((compare is None or r == compare_r) and not percent and not diff): for k in fields: @@ -584,7 +547,7 @@ def table(Result, results, diff_results=None, *, else ['%+.1f%%' % (100*t)])( types[k].ratio( getattr(r, k, None), - getattr(compare_result, k, None))))) + getattr(compare_r, k, None))))) # percent entry? elif not diff: for k in fields: @@ -630,71 +593,94 @@ def table(Result, results, diff_results=None, *, return entry - # recursive entry helper, only used by some scripts - def recurse(results_, depth_, + # recursive entry helper + def table_recurse(results_, diff_results_, + depth_, prefixes=('', '', '', '')): # build the children table at each layer - results_ = fold(Result, results_, by=by) table_ = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results_} - names_ = list(table_.keys()) + diff_table_ = { + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r + for r in diff_results_ or []} + names_ = [n + for n in table_.keys() | diff_table_.keys() + if diff_results_ is None + or all_ + or any( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)] - # sort the children layer - names_.sort() - if hasattr(Result, '_i'): - names_.sort(key=lambda n: getattr(table_[n], Result._i)) - if sort: - for k, reverse in reversed(sort): - names_.sort( - key=lambda n: tuple( - (getattr(table_[n], k),) - if getattr(table_.get(n), k, None) - is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) + # sort again, now with diff info, note that python's sort is stable + names_.sort(key=lambda n: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(table_[n], k_),) + if getattr(table_.get(n), k_, None) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by ratio if diffing + Rev(tuple(types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)) + if diff or percent + else (), + # move compare entry to the top, note this can be + # overridden by explicitly sorting by fields + (table_.get(n) != compare_r, + # sort by ratio if comparing + Rev(tuple( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(compare_r, k, None)) + for k in fields))) + if compare + else (), + # sort by result + (table_[n],) if n in table_ else (), + # and finally by name (diffs may be missing results) + n)) - for i, name in enumerate(names_): - r = table_[name] - is_last = (i == len(names_)-1) + for i, n in enumerate(names_): + # find comparable results + r = table_.get(n) + diff_r = diff_table_.get(n) + + # build line + line = table_entry(n, r, diff_r) - line = table_entry(name, r) - line = [x if isinstance(x, tuple) else (x, []) for x in line] # add prefixes - line[0] = (prefixes[0+is_last] + line[0][0], line[0][1]) + line = [x if isinstance(x, tuple) else (x, []) for x in line] + line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) lines.append(line) # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), + if n in table_ and depth_ > 1: + table_recurse( + getattr(r, Result._children), + getattr(diff_r, Result._children, None) or [], depth_-1, - (prefixes[2+is_last] + "|-> ", - prefixes[2+is_last] + "'-> ", - prefixes[2+is_last] + "| ", - prefixes[2+is_last] + " ")) + (prefixes[2+(i==len(names_)-1)] + "|-> ", + prefixes[2+(i==len(names_)-1)] + "'-> ", + prefixes[2+(i==len(names_)-1)] + "| ", + prefixes[2+(i==len(names_)-1)] + " ")) - # entries + # build entries if not summary: - for name in names: - r = table.get(name) - if diff_results is None: - diff_r = None - else: - diff_r = diff_table.get(name) - lines.append(table_entry(name, r, diff_r)) - - # recursive entries - if name in table and depth > 1: - recurse(getattr(table[name], Result._children), - depth-1, - ("|-> ", - "'-> ", - "| ", - " ")) + table_recurse(results, diff_results, depth) # total if not no_total and not (small_table and not summary): @@ -706,9 +692,8 @@ def table(Result, results, diff_results=None, *, lines.append(table_entry('TOTAL', r, diff_r)) # homogenize - lines = [ - [x if isinstance(x, tuple) else (x, []) for x in line] - for line in lines] + lines = [[x if isinstance(x, tuple) else (x, []) for x in line] + for line in lines] # find the best widths, note that column 0 contains the names and is # handled a bit differently @@ -729,6 +714,130 @@ def table(Result, results, diff_results=None, *, nwidths[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') for i, x in enumerate(line[1:], 1)))) +def read_csv(path, Result, *, + depth=1, + **_): + with openio(path, 'r') as f: + # csv or json? assume json starts with [ + json = (f.buffer.peek(1)[:1] == b'[') + + # read csv? + if not json: + results = [] + reader = csv.DictReader(f, restval='') + for r in reader: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k].strip()} + | {k: r[k] for k in Result._fields + if k in r and r[k].strip()}))) + except TypeError: + pass + return results + + # read json? + else: + import json + def unjsonify(results, depth_): + results_ = [] + for r in results: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results_.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k] is not None} + | {k: r[k] for k in Result._fields + if k in r and r[k] is not None} + | ({Result._children: unjsonify( + r[Result._children], + depth_-1)} + if hasattr(Result, '_children') + and Result._children in r + and r[Result._children] is not None + and depth_ > 1 + else {}) + | ({Result._notes: set(r[Result._notes])} + if hasattr(Result, '_notes') + and Result._notes in r + and r[Result._notes] is not None + else {})))) + except TypeError: + pass + return results_ + return unjsonify(json.load(f), depth) + +def write_csv(path, Result, results, *, + json=False, + by=None, + fields=None, + depth=1, + **_): + with openio(path, 'w') as f: + # write csv? + if not json: + writer = csv.DictWriter(f, + (by if by is not None else Result._by) + + [k for k in (fields + if fields is not None + else Result._fields)]) + writer.writeheader() + for r in results: + # note this allows by/fields to overlap + writer.writerow( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None}) + + # write json? + else: + import json + # the neat thing about json is we can include recursive results + def jsonify(results, depth_): + results_ = [] + for r in results: + # note this allows by/fields to overlap + results_.append( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None} + | ({Result._children: jsonify( + getattr(r, Result._children), + depth_-1)} + if hasattr(Result, '_children') + and getattr(r, Result._children) + and depth_ > 1 + else {}) + | ({Result._notes: list( + getattr(r, Result._notes))} + if hasattr(Result, '_notes') + and getattr(r, Result._notes) + else {})) + return results_ + json.dump(jsonify(results, depth), f, + separators=(',', ':')) + def annotate(Result, results, *, annotate=False, @@ -829,88 +938,46 @@ def main(gcda_paths, *, sys.exit(1) # collect info - results = collect(gcda_paths, **args) + results = collect_cov(gcda_paths, + **args) else: - results = [] - with openio(args['use']) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - if not any(k in r and r[k].strip() - for k in CovResult._fields): - continue - try: - results.append(CovResult( - **{k: r[k] for k in CovResult._by - if k in r and r[k].strip()}, - **{k: r[k] - for k in CovResult._fields - if k in r and r[k].strip()})) - except TypeError: - pass + results = read_csv(args['use'], CovResult, + **args) # fold - results = fold(CovResult, results, by=by, defines=defines) + results = fold(CovResult, results, + by=by, + defines=defines) - # sort, note that python's sort is stable - results.sort() - if sort: - for k, reverse in reversed(sort): - results.sort( - key=lambda r: tuple( - (getattr(r, k),) if getattr(r, k) is not None else () - for k in ([k] if k else CovResult._sort)), - reverse=reverse ^ (not k or k in CovResult._fields)) - - # write results to CSV + # write results to CSV/JSON if args.get('output'): - with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, - (by if by is not None else CovResult._by) - + [k for k in ( - fields if fields is not None - else CovResult._fields)]) - writer.writeheader() - for r in results: - writer.writerow( - {k: getattr(r, k) for k in ( - by if by is not None else CovResult._by)} - | {k: getattr(r, k) for k in ( - fields if fields is not None - else CovResult._fields)}) + write_csv(args['output'], CovResult, results, + by=by, + fields=fields, + **args) + if args.get('output_json'): + write_csv(args['output_json'], CovResult, results, + json=True, + by=by, + fields=fields, + **args) # find previous results? diff_results = None if args.get('diff') or args.get('percent'): - diff_results = [] try: - with openio(args.get('diff') or args.get('percent')) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - if not any(k in r and r[k].strip() - for k in CovResult._fields): - continue - try: - diff_results.append(CovResult( - **{k: r[k] for k in CovResult._by - if k in r and r[k].strip()}, - **{k: r[k] for k in CovResult._fields - if k in r and r[k].strip()})) - except TypeError: - pass + diff_results = read_csv( + args.get('diff') or args.get('percent'), + CovResult, + **args) except FileNotFoundError: - pass + diff_results = [] # fold - diff_results = fold(CovResult, diff_results, by=by, defines=defines) + diff_results = fold(CovResult, diff_results, + by=by, + defines=defines) # print table if not args.get('quiet'): @@ -959,24 +1026,28 @@ if __name__ == "__main__": parser.add_argument( '-o', '--output', help="Specify CSV file to store results.") + parser.add_argument( + '-O', '--output-json', + help="Specify JSON file to store results. This may contain " + "recursive info.") parser.add_argument( '-u', '--use', - help="Don't parse anything, use this CSV file.") + help="Don't parse anything, use this CSV/JSON file.") parser.add_argument( '-d', '--diff', - help="Specify CSV file to diff against.") + help="Specify CSV/JSON file to diff against.") parser.add_argument( '-p', '--percent', - help="Specify CSV file to diff against, but only show precentage " - "change, not a full diff.") - parser.add_argument( - '-a', '--all', - action='store_true', - help="Show all, not just the ones that changed.") + help="Specify CSV/JSON file to diff against, but only show " + "percentage change, not a full diff.") parser.add_argument( '-c', '--compare', type=lambda x: tuple(v.strip() for v in x.split(',')), help="Compare results to the row matching this by pattern.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") parser.add_argument( '-b', '--by', action='append', @@ -1002,7 +1073,7 @@ if __name__ == "__main__": def __call__(self, parser, namespace, value, option): if namespace.sort is None: namespace.sort = [] - namespace.sort.append((value, True if option == '-S' else False)) + namespace.sort.append((value, option in {'-S', '--reverse-sort'})) parser.add_argument( '-s', '--sort', nargs='?', diff --git a/scripts/csv.py b/scripts/csv.py index 16bcab6a..fadaafc6 100755 --- a/scripts/csv.py +++ b/scripts/csv.py @@ -1321,27 +1321,74 @@ def openio(path, mode='r', buffering=-1): else: return open(path, mode, buffering) -def collect(csv_paths, defines=[]): - # collect results from CSV files - fields = [] +def collect(csv_paths, *, + depth=1, + children=None, + notes=None, + **_): + # collect both results and fields from CSV files + fields = co.OrderedDict() results = [] for path in csv_paths: try: with openio(path) as f: - reader = csv.DictReader(f, restval='') - fields.extend( - k for k in reader.fieldnames - if k not in fields) - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue + # csv or json? assume json starts with [ + json = (f.buffer.peek(1)[:1] == b'[') + + # read csv? + if not json: + reader = csv.DictReader(f, restval='') + # collect fields + fields.update((k, True) for k in reader.fieldnames) + for r in reader: + # strip and drop empty fields + r_ = {k: v.strip() + for k, v in r.items() + if k not in {'notes'} + and v.strip()} + # special handling for notes field + if notes is not None and notes in r: + r_[notes] = set(r[notes].split(',')) + results.append(r_) + + # read json? + else: + import json + def unjsonify(results, depth_): + results_ = [] + for r in results: + # collect fields + fields.update((k, True) for k in r.keys()) + # convert to strings, we'll reparse these later + # + # this may seem a bit backwards, but it keeps + # the rest of the script simpler if we pretend + # everything came from a csv + r_ = {k: str(v).strip() + for k, v in r.items() + if k not in {'children', 'notes'} + and str(v).strip()} + # special handling for children field + if (children is not None + and children in r + and r[children] is not None + and depth_ > 1): + r_[children] = unjsonify( + r[children], + depth_-1) + # special handling for notes field + if (notes is not None + and notes in r + and r[notes] is not None): + r_[notes] = set(r[notes]) + results_.append(r_) + return results_ + results.extend(unjsonify(json.load(f), depth)) - results.append(r) except FileNotFoundError: pass - return fields, results + return list(fields.keys()), results def compile(fields_, results, by=None, @@ -1349,15 +1396,33 @@ def compile(fields_, results, mods=[], exprs=[], defines=[], - sort=None): - # deduplicate by/fields - by = list(co.OrderedDict.fromkeys(by).keys()) - fields = list(co.OrderedDict.fromkeys(fields).keys()) + sort=None, + enumerate=None, + children=None, + notes=None, + hot=None): + import builtins + enumerate_, enumerate = enumerate, builtins.enumerate - # make sure sort fields are included - if sort is not None: - fields.extend(k for k, reverse in sort - if k and k not in fields) + by = by.copy() + fields = fields.copy() + + # we need _something_ to order hot results by, so default to + # i if no enumerate field is specified + if hot is not None and enumerate_ is None: + enumerate_ = 'i' + # make sure enumerate fields are included + if enumerate_ is not None or hot is not None: + if enumerate_ not in by: + by.insert(0, enumerate_) + # make sure define fields are included + for k, _ in defines: + if k not in by and k not in fields: + by.append(k) + # make sure sort/hot fields are included + for k, reverse in it.chain(sort or [], hot or []): + if k and k not in by and k not in fields: + fields.append(k) # we only really care about the last mod/expr for each field mods = {k: mod for k, mod in mods} @@ -1419,10 +1484,14 @@ def compile(fields_, results, r_ = r__ # return result - return cls.__mro__[1].__new__(cls, - **{k: r_.get(k, '') for k in by}, - **{k: ([r_[k]], 1) if k in r_ else ([], 0) - for k in fields}) + return cls.__mro__[1].__new__(cls, **( + {k: r_.get(k, '') for k in by} + | {k: ([r_[k]], 1) if k in r_ else ([], 0) + for k in fields} + | ({children: r[children] if children in r else []} + if children is not None else {}) + | ({notes: r[notes] if notes in r else set()} + if notes is not None else {}))) def __add__(self, other): # reuse lists if possible @@ -1434,12 +1503,16 @@ def compile(fields_, results, return (a[0][:a[1]] + b[0][:b[1]], a[1] + b[1]) # lazily fold results - return self.__class__.__mro__[1].__new__(self.__class__, - **{k: getattr(self, k) for k in by}, - **{k: extend( - object.__getattribute__(self, k), - object.__getattribute__(other, k)) - for k in fields}) + return self.__class__.__mro__[1].__new__(self.__class__, **( + {k: getattr(self, k) for k in by} + | {k: extend( + object.__getattribute__(self, k), + object.__getattribute__(other, k)) + for k in fields} + | ({children: self.children + other.children} + if children is not None else {}) + | ({notes: self.notes | other.notes} + if notes is not None else {}))) def __getattribute__(self, k): # lazily fold results on demand, this avoids issues with fold @@ -1452,21 +1525,86 @@ def compile(fields_, results, return None return object.__getattribute__(self, k) - return type('Result', (co.namedtuple('Result', by + fields),), { - '__slots__': (), - '__new__': __new__, - '__add__': __add__, - '__getattribute__': __getattribute__, - '_by': by, - '_fields': fields, - '_sort': fields, - '_types': {k: t for k, (_, t) in folds.items()}, - '_mods': mods, - '_exprs': exprs, - }) + return type( + 'Result', + (co.namedtuple('Result', list(co.OrderedDict.fromkeys(it.chain( + by, + fields, + [children] if children is not None else [], + [notes] if notes is not None else [])).keys())),), + dict( + __slots__=(), + __new__=__new__, + __add__=__add__, + __getattribute__=__getattribute__, + _by=by, + _fields=fields, + _sort=fields, + _types={k: t for k, (_, t) in folds.items()}, + _mods=mods, + _exprs=exprs, + **{'_i': enumerate_} if enumerate_ is not None else {}, + **{'_children': children} if children is not None else {}, + **{'_notes': notes} if notes is not None else {})) + +def homogenize(Result, results, *, + depth=1, + hot=None, + enumerate=None, + children=None): + import builtins + enumerate_, enumerate = enumerate, builtins.enumerate + + # this just converts all (possibly recursive) results to our + # result type + results_ = [] + for i, r in enumerate(results): + results_.append(Result(**( + r + # enumerate? + | ({Result._i: RInt(i)} + if enumerate_ is not None + else {}) + # recurse? + | ({children: homogenize( + Result, r[children], + depth=depth-1, + # only enumerate top-level if hotifying + enumerate=(enumerate_ if hot is None else None), + children=children)} + if children is not None + and children in r + and r[children] is not None + and depth > 1 + else {})))) + return results_ -def fold(Result, results, by=None, defines=[]): +# common folding/tabling/read/write code + +class Rev(co.namedtuple('Rev', 'x')): + __slots__ = () + # yes we need all of these because we're a namedtuple + def __lt__(self, other): + return self.x > other.x + def __gt__(self, other): + return self.x < other.x + def __le__(self, other): + return self.x >= other.x + def __ge__(self, other): + return self.x <= other.x + +def fold(Result, results, *, + by=None, + defines=[], + sort=None, + depth=1, + **_): + # stop when depth hits zero + if depth == 0: + return [] + + # organize by by if by is None: by = Result._by @@ -1480,7 +1618,7 @@ def fold(Result, results, by=None, defines=[]): if defines: results_ = [] for r in results: - if all(getattr(r, k) in vs for k, vs in defines): + if all(str(getattr(r, k)) in vs for k, vs in defines): results_.append(r) results = results_ @@ -1497,8 +1635,80 @@ def fold(Result, results, by=None, defines=[]): for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) + # sort, note that python's sort is stable + folded.sort(key=lambda r: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(r, k_),) + if getattr(r, k_) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by result + r)) + + # recurse if we have recursive results + if hasattr(Result, '_children'): + folded = [r._replace(**{ + Result._children: fold( + Result, getattr(r, Result._children), + by=by, + defines=defines, + sort=sort, + depth=depth-1)}) + for r in folded] + return folded +def hotify(Result, results, *, + fields=None, + sort=None, + depth=1, + hot=None, + **_): + # hotify only makes sense for recursive results + assert hasattr(Result, '_i') + assert hasattr(Result, '_children') + + if fields is None: + fields = Result._fields + + results_ = [] + for r in results: + hot_ = [] + def recurse(results_, depth_): + nonlocal hot_ + if not results_: + return + + # find the hottest result + r = min(results_, key=lambda r: + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(r, k_),) + if getattr(r, k_) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in it.chain(hot, [(None, False)]))) + + hot_.append(r._replace(**{ + Result._i: RInt(len(hot_)), + Result._children: []})) + + # recurse? + if depth_ > 1: + recurse(getattr(r, Result._children), + depth_-1) + + recurse(getattr(r, Result._children), depth-1) + results_.append(r._replace(**{ + Result._children: hot_})) + + return results_ + def table(Result, results, diff_results=None, *, by=None, fields=None, @@ -1524,124 +1734,32 @@ def table(Result, results, diff_results=None, *, fields = Result._fields types = Result._types - # fold again - results = fold(Result, results, by=by) + # fold again, otherwise results risk being hidden + results = fold(Result, results, + by=by, + depth=depth) if diff_results is not None: - diff_results = fold(Result, diff_results, by=by) - - # reduce children to hot paths? only used by some scripts - if hot: - # subclass to reintroduce __dict__ - Result_ = Result - class HotResult(Result_): - _i = '_hot_i' - _children = '_hot_children' - - def __new__(cls, r, i=None, children=None, notes=None): - self = HotResult._make(r) - self._hot_i = i - self._hot_children = children if children is not None else [] - return self - - def __add__(self, other): - return HotResult( - Result_.__add__(self, other), - self._hot_i if other._hot_i is None - else other._hot_i if self._hot_i is None - else min(self._hot_i, other._hot_i), - self._hot_children + other._hot_children) - - results_ = [] - for r in results: - hot_ = [] - def recurse(results_, depth_): - nonlocal hot_ - if not results_: - return - - # find the hottest result - r = max(results_, - key=lambda r: tuple( - tuple((getattr(r, k),) - if getattr(r, k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields]) - if k in fields) - for k in it.chain(hot, [None]))) - hot_.append(HotResult(r, i=len(hot_))) - - # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), - depth_-1) - - recurse(getattr(r, Result._children), depth-1) - results_.append(HotResult(r, children=hot_)) - - Result = HotResult - results = results_ + diff_results = fold(Result, diff_results, + by=by, + depth=depth) # organize by name table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results} diff_table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in diff_results or []} - names = [name - for name in table.keys() | diff_table.keys() - if diff_results is None - or all_ - or any( - types[k].ratio( - getattr(table.get(name), k, None), - getattr(diff_table.get(name), k, None)) - for k in fields)] # find compare entry if there is one if compare: - compare_result = table.get(','.join(str(k) for k in compare)) - - # sort again, now with diff info, note that python's sort is stable - names.sort() - if compare: - names.sort( - key=lambda n: ( - # move compare entry to the top, note this can be - # overridden by explicitly sorting by fields - table.get(n) == compare_result, - # sort by ratio if comparing - tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(compare_result, k, None)) - for k in fields)), - reverse=True) - if diff or percent: - names.sort( - # sort by ratio if diffing - key=lambda n: tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(diff_table.get(n), k, None)) - for k in fields), - reverse=True) - if sort: - for k, reverse in reversed(sort): - names.sort( - key=lambda n: tuple( - (getattr(table[n], k),) - if getattr(table.get(n), k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) - + compare_r = table.get(','.join(str(k) for k in compare)) # build up our lines lines = [] @@ -1668,11 +1786,16 @@ def table(Result, results, diff_results=None, *, header.append('d'+k) lines.append(header) + # delete these to try to catch typos below, we need to rebuild + # these tables at each recursive layer + del table + del diff_table + # entry helper def table_entry(name, r, diff_r=None): entry = [name] # normal entry? - if ((compare is None or r == compare_result) + if ((compare is None or r == compare_r) and not percent and not diff): for k in fields: @@ -1693,7 +1816,7 @@ def table(Result, results, diff_results=None, *, else ['%+.1f%%' % (100*t)])( types[k].ratio( getattr(r, k, None), - getattr(compare_result, k, None))))) + getattr(compare_r, k, None))))) # percent entry? elif not diff: for k in fields: @@ -1739,71 +1862,94 @@ def table(Result, results, diff_results=None, *, return entry - # recursive entry helper, only used by some scripts - def recurse(results_, depth_, + # recursive entry helper + def table_recurse(results_, diff_results_, + depth_, prefixes=('', '', '', '')): # build the children table at each layer - results_ = fold(Result, results_, by=by) table_ = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results_} - names_ = list(table_.keys()) + diff_table_ = { + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r + for r in diff_results_ or []} + names_ = [n + for n in table_.keys() | diff_table_.keys() + if diff_results_ is None + or all_ + or any( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)] - # sort the children layer - names_.sort() - if hasattr(Result, '_i'): - names_.sort(key=lambda n: getattr(table_[n], Result._i)) - if sort: - for k, reverse in reversed(sort): - names_.sort( - key=lambda n: tuple( - (getattr(table_[n], k),) - if getattr(table_.get(n), k, None) - is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) + # sort again, now with diff info, note that python's sort is stable + names_.sort(key=lambda n: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(table_[n], k_),) + if getattr(table_.get(n), k_, None) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by ratio if diffing + Rev(tuple(types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)) + if diff or percent + else (), + # move compare entry to the top, note this can be + # overridden by explicitly sorting by fields + (table_.get(n) != compare_r, + # sort by ratio if comparing + Rev(tuple( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(compare_r, k, None)) + for k in fields))) + if compare + else (), + # sort by result + (table_[n],) if n in table_ else (), + # and finally by name (diffs may be missing results) + n)) - for i, name in enumerate(names_): - r = table_[name] - is_last = (i == len(names_)-1) + for i, n in enumerate(names_): + # find comparable results + r = table_.get(n) + diff_r = diff_table_.get(n) + + # build line + line = table_entry(n, r, diff_r) - line = table_entry(name, r) - line = [x if isinstance(x, tuple) else (x, []) for x in line] # add prefixes - line[0] = (prefixes[0+is_last] + line[0][0], line[0][1]) + line = [x if isinstance(x, tuple) else (x, []) for x in line] + line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) lines.append(line) # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), + if n in table_ and depth_ > 1: + table_recurse( + getattr(r, Result._children), + getattr(diff_r, Result._children, None) or [], depth_-1, - (prefixes[2+is_last] + "|-> ", - prefixes[2+is_last] + "'-> ", - prefixes[2+is_last] + "| ", - prefixes[2+is_last] + " ")) + (prefixes[2+(i==len(names_)-1)] + "|-> ", + prefixes[2+(i==len(names_)-1)] + "'-> ", + prefixes[2+(i==len(names_)-1)] + "| ", + prefixes[2+(i==len(names_)-1)] + " ")) - # entries + # build entries if not summary: - for name in names: - r = table.get(name) - if diff_results is None: - diff_r = None - else: - diff_r = diff_table.get(name) - lines.append(table_entry(name, r, diff_r)) - - # recursive entries - if name in table and depth > 1: - recurse(getattr(table[name], Result._children), - depth-1, - ("|-> ", - "'-> ", - "| ", - " ")) + table_recurse(results, diff_results, depth) # total if not no_total and not (small_table and not summary): @@ -1815,9 +1961,8 @@ def table(Result, results, diff_results=None, *, lines.append(table_entry('TOTAL', r, diff_r)) # homogenize - lines = [ - [x if isinstance(x, tuple) else (x, []) for x in line] - for line in lines] + lines = [[x if isinstance(x, tuple) else (x, []) for x in line] + for line in lines] # find the best widths, note that column 0 contains the names and is # handled a bit differently @@ -1838,27 +1983,176 @@ def table(Result, results, diff_results=None, *, nwidths[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') for i, x in enumerate(line[1:], 1)))) +def read_csv(path, Result, *, + depth=1, + **_): + with openio(path, 'r') as f: + # csv or json? assume json starts with [ + json = (f.buffer.peek(1)[:1] == b'[') + + # read csv? + if not json: + results = [] + reader = csv.DictReader(f, restval='') + for r in reader: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k].strip()} + | {k: r[k] for k in Result._fields + if k in r and r[k].strip()}))) + except TypeError: + pass + return results + + # read json? + else: + import json + def unjsonify(results, depth_): + results_ = [] + for r in results: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results_.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k] is not None} + | {k: r[k] for k in Result._fields + if k in r and r[k] is not None} + | ({Result._children: unjsonify( + r[Result._children], + depth_-1)} + if hasattr(Result, '_children') + and Result._children in r + and r[Result._children] is not None + and depth_ > 1 + else {}) + | ({Result._notes: set(r[Result._notes])} + if hasattr(Result, '_notes') + and Result._notes in r + and r[Result._notes] is not None + else {})))) + except TypeError: + pass + return results_ + return unjsonify(json.load(f), depth) + +def write_csv(path, Result, results, *, + json=False, + by=None, + fields=None, + depth=1, + **_): + with openio(path, 'w') as f: + # write csv? + if not json: + writer = csv.DictWriter(f, + (by if by is not None else Result._by) + + [k for k in (fields + if fields is not None + else Result._fields)]) + writer.writeheader() + for r in results: + # note this allows by/fields to overlap + writer.writerow( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None}) + + # write json? + else: + import json + # the neat thing about json is we can include recursive results + def jsonify(results, depth_): + results_ = [] + for r in results: + # note this allows by/fields to overlap + results_.append( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None} + | ({Result._children: jsonify( + getattr(r, Result._children), + depth_-1)} + if hasattr(Result, '_children') + and getattr(r, Result._children) + and depth_ > 1 + else {}) + | ({Result._notes: list( + getattr(r, Result._notes))} + if hasattr(Result, '_notes') + and getattr(r, Result._notes) + else {})) + return results_ + json.dump(jsonify(results, depth), f, + separators=(',', ':')) + def main(csv_paths, *, by=None, fields=None, defines=[], sort=None, + depth=None, + enumerate=None, + children=None, + notes=None, + hot=None, **args): + import builtins + enumerate_, enumerate = enumerate, builtins.enumerate + # show expr help text? if args.get('help_exprs'): return RExpr.help() + if by is None and fields is None: + print("error: needs --by or --fields to figure out fields", + file=sys.stderr) + sys.exit(-1) + + # recursive results imply --children + if (depth is not None or hot is not None) and children is None: + children = 'children' + + # figure out depth + if depth is None: + depth = mt.inf if hot else 1 + elif depth == 0: + depth = mt.inf + # separate out mods/exprs + # # by supports mods => -ba=%(b)s - # fields/sort support exprs => -fa=b+c + # fields/sort/hot support exprs => -fa=b+c mods = [(k, v) for k, v in (by or []) if v is not None] exprs = [(k, v) for k, v in it.chain( fields or [], - ((k, v) for (k, v), reverse in sort or [])) + ((k, v) for (k, v), reverse in (sort or [])), + ((k, v) for (k, v), reverse in (hot or []))) if v is not None] if by is not None: by = [k for k, _ in by] @@ -1866,41 +2160,56 @@ def main(csv_paths, *, fields = [k for k, _ in fields] if sort is not None: sort = [(k, reverse) for (k, v), reverse in sort] + if hot is not None: + hot = [(k, reverse) for (k, v), reverse in hot] - if by is None and fields is None: - print("error: needs --by or --fields to figure out fields", - file=sys.stderr) - sys.exit(-1) + # find results + if not args.get('use', None): + # not enough info? + if not csv_paths: + print("error: no *.csv files?", + file=sys.stderr) + sys.exit(1) - # use is just an alias - if args.get('use'): - csv_paths = csv_paths + [args['use']] + # collect info + fields_, results = collect(csv_paths, + depth=depth, + children=children, + notes=notes, + **args) - # not enough info? - if not csv_paths: - print("error: no *.csv files?", - file=sys.stderr) - sys.exit(1) + else: + # use is just an alias but takes priority + fields_, results = collect([args['use']], + depth=depth, + children=children, + notes=notes, + **args) - # collect info - fields_, results = collect(csv_paths, defines) - - # if by not specified, guess it's anything not in fields/defines/sort/exprs + # if by not specified, guess it's anything not in fields/defines/exprs/etc if not by: by = [k for k in fields_ if k not in (fields or []) and not any(k == k_ for k_, _ in defines) and not any(k == k_ for k_, _ in (sort or [])) + and k != enumerate_ + and k != children + and k != notes + and not any(k == k_ for k_, _ in (hot or [])) and not any(k == k_ for _, expr in exprs for k_ in expr.fields())] - # if fields not specified, guess it's anything not in by/defines/sort/exprs + # if fields not specified, guess it's anything not in by/defines/exprs/etc if not fields: fields = [k for k in fields_ if k not in (by or []) and not any(k == k_ for k_, _ in defines) and not any(k == k_ for k_, _ in (sort or [])) + and k != enumerate_ + and k != children + and k != notes + and not any(k == k_ for k_, _ in (hot or [])) and not any(k == k_ for _, expr in exprs for k_ in expr.fields())] @@ -1912,60 +2221,64 @@ def main(csv_paths, *, mods=mods, exprs=exprs, defines=defines, - sort=sort) + sort=sort, + enumerate=enumerate_, + children=children, + notes=notes, + hot=hot) # homogenize - results_ = [] - for r in results: - results_.append(Result(**{ - k: v for k, v in r.items() if v.strip()})) - results = results_ + results = homogenize(Result, results, + depth=depth, + enumerate=enumerate_, + children=children) # fold - results = fold(Result, results, by=by) + results = fold(Result, results, + by=by, + defines=defines, + depth=depth) - # sort, note that python's sort is stable - results.sort() - if sort: - for k, reverse in reversed(sort): - results.sort( - key=lambda r: tuple( - (getattr(r, k),) if getattr(r, k) is not None else () - for k in ([k] if k else Result._sort)), - reverse=reverse ^ (not k or k in Result._fields)) + # hotify? + if hot: + results = hotify(Result, results, + fields=fields, + depth=depth, + hot=hot, + **args) - # write results to CSV + # write results to CSV/JSON if args.get('output'): - with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, Result._by + Result._fields) - writer.writeheader() - for r in results: - # note we need to go through getattr to resolve lazy fields - writer.writerow({ - k: getattr(r, k) - for k in Result._by + Result._fields}) + write_csv(args['output'], Result, results, + by=by, + fields=fields, + depth=depth, + **args) + if args.get('output_json'): + write_csv(args['output_json'], Result, results, + json=True, + by=by, + fields=fields, + depth=depth, + **args) # find previous results? diff_results = None if args.get('diff') or args.get('percent'): - _, diff_results = collect( - [args.get('diff') or args.get('percent')], - defines) - diff_results_ = [] - for r in diff_results: - if not any(k in r and r[k].strip() - for k in Result._fields): - continue - try: - diff_results_.append(Result(**{ - k: r[k] for k in Result._by + Result._fields - if k in r and r[k].strip()})) - except TypeError: - pass - diff_results = diff_results_ + try: + diff_results = read_csv( + args.get('diff') or args.get('percent'), + Result, + depth=depth, + **args) + except FileNotFoundError: + diff_results = [] # fold - diff_results = fold(Result, diff_results, by=by) + diff_results = fold(Result, diff_results, + by=by, + defines=defines, + depth=depth) # print table if not args.get('quiet'): @@ -1973,6 +2286,7 @@ def main(csv_paths, *, by=by, fields=fields, sort=sort, + depth=depth, **args) @@ -1997,24 +2311,28 @@ if __name__ == "__main__": parser.add_argument( '-o', '--output', help="Specify CSV file to store results.") + parser.add_argument( + '-O', '--output-json', + help="Specify JSON file to store results. This may contain " + "recursive info.") parser.add_argument( '-u', '--use', - help="Don't parse anything, use this CSV file.") + help="Don't parse anything, use this CSV/JSON file.") parser.add_argument( '-d', '--diff', - help="Specify CSV file to diff against.") + help="Specify CSV/JSON file to diff against.") parser.add_argument( '-p', '--percent', - help="Specify CSV file to diff against, but only show precentage " - "change, not a full diff.") - parser.add_argument( - '-a', '--all', - action='store_true', - help="Show all, not just the ones that changed.") + help="Specify CSV/JSON file to diff against, but only show " + "percentage change, not a full diff.") parser.add_argument( '-c', '--compare', type=lambda x: tuple(v.strip() for v in x.split(',')), help="Compare results to the row matching this by pattern.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") parser.add_argument( '-b', '--by', action='append', @@ -2051,7 +2369,7 @@ if __name__ == "__main__": def __call__(self, parser, namespace, value, option): if namespace.sort is None: namespace.sort = [] - namespace.sort.append((value, True if option == '-S' else False)) + namespace.sort.append((value, option in {'-S', '--reverse-sort'})) parser.add_argument( '-s', '--sort', nargs='?', @@ -2076,6 +2394,60 @@ if __name__ == "__main__": const=(None, None), help="Sort by this field, but backwards. Can include an expression " "of the form field=expr.") + parser.add_argument( + '-z', '--depth', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Depth of function calls to show. 0 shows all calls unless " + "we find a cycle. Defaults to 0.") + parser.add_argument( + '-i', '--enumerate', + nargs='?', + const='i', + help="Field to use for enumerating results. This will prevent " + "result folding. This can also be used to override which " + "field -r/--hot uses to order results.") + parser.add_argument( + '-Z', '--children', + nargs='?', + const='children', + help="Field to use for recursive results. This expects a list " + "and really only works with JSON input.") + parser.add_argument( + '-N', '--notes', + nargs='?', + const='notes', + help="Field to use for notes.") + class AppendHot(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.hot is None: + namespace.hot = [] + namespace.hot.append((value, option in {'-R', '--reverse-hot'})) + parser.add_argument( + '-r', '--hot', + nargs='?', + action=AppendHot, + type=lambda x: ( + lambda k, v=None: ( + k.strip(), + RExpr(v) if v is not None else None) + )(*x.split('=', 1)), + const=(None, None), + help="Show only the hot path for each function call. Can " + "optionally provide fields like sort. Can include an " + "expression in the form of field=expr.") + parser.add_argument( + '-R', '--reverse-hot', + nargs='?', + action=AppendHot, + type=lambda x: ( + lambda k, v=None: ( + k.strip(), + RExpr(v) if v is not None else None) + )(*x.split('=', 1)), + const=(None, None), + help="Like -r/--hot, but backwards.") parser.add_argument( '--no-header', action='store_true', diff --git a/scripts/ctx.py b/scripts/ctx.py index ef64cd49..c3ed120b 100755 --- a/scripts/ctx.py +++ b/scripts/ctx.py @@ -15,8 +15,9 @@ if __name__ == "__main__": import collections as co import csv -import itertools as it import functools as ft +import io +import itertools as it import math as mt import os import re @@ -130,12 +131,12 @@ class RInt(co.namedtuple('RInt', 'x')): def __mod__(self, other): return self.__class__(self.x % other.x) -# struct size results +# ctx size results class CtxResult(co.namedtuple('CtxResult', [ - 'file', 'function', + 'i', 'file', 'function', 'size', - 'i', 'children', 'notes'])): - _by = ['file', 'function'] + 'children', 'notes'])): + _by = ['i', 'file', 'function'] _fields = ['size'] _sort = ['size'] _types = {'size': RInt} @@ -144,20 +145,16 @@ class CtxResult(co.namedtuple('CtxResult', [ _notes = 'notes' __slots__ = () - def __new__(cls, file='', function='', size=0, - i=None, children=None, notes=None): - return super().__new__(cls, file, function, + def __new__(cls, i=None, file='', function='', size=0, + children=None, notes=None): + return super().__new__(cls, i, file, function, RInt(size), - i, children if children is not None else [], notes if notes is not None else set()) def __add__(self, other): - return CtxResult(self.file, self.function, + return CtxResult(self.i, self.file, self.function, max(self.size, other.size), - self.i if other.i is None - else other.i if self.i is None - else min(self.i, other.i), self.children + other.children, self.notes | other.notes) @@ -458,7 +455,7 @@ def collect_dwarf_info(obj_path, tags=None, *, return DwarfInfo(info) -def collect(obj_paths, *, +def collect_ctx(obj_paths, *, everything=False, depth=1, **args): @@ -603,7 +600,7 @@ def collect(obj_paths, *, size_ = sizeof(type, seen | {entry.off}) children_, notes_, dirty_ = childrenof( type, depth-1, seen | {entry.off}) - children.append(CtxResult(file, name_, size_, + children.append(CtxResult(0, file, name_, size_, children=children_, notes=notes_)) dirty = dirty or dirty_ @@ -619,8 +616,7 @@ def collect(obj_paths, *, size_ = sizeof(child, seen | {entry.off}) children_, notes_, dirty_ = childrenof( child, depth-1, seen | {entry.off}) - children.append(CtxResult(file, name_, size_, - i=child.off, + children.append(CtxResult(child.off, file, name_, size_, children=children_, notes=notes_)) dirty = dirty or dirty_ @@ -682,21 +678,44 @@ def collect(obj_paths, *, # find children, recursing if necessary children_, notes_, _ = childrenof(param, depth-2) - params.append(CtxResult(file, name_, size_, - i=param.off, + params.append(CtxResult(param.off, file, name_, size_, children=children_, notes=notes_)) # context = sum of params name = entry.name size = sum((param.size for param in params), start=RInt(0)) - results.append(CtxResult(file, name, size, + results.append(CtxResult(None, file, name, size, children=params)) return results -def fold(Result, results, by=None, defines=[]): +# common folding/tabling/read/write code + +class Rev(co.namedtuple('Rev', 'x')): + __slots__ = () + # yes we need all of these because we're a namedtuple + def __lt__(self, other): + return self.x > other.x + def __gt__(self, other): + return self.x < other.x + def __le__(self, other): + return self.x >= other.x + def __ge__(self, other): + return self.x <= other.x + +def fold(Result, results, *, + by=None, + defines=[], + sort=None, + depth=1, + **_): + # stop when depth hits zero + if depth == 0: + return [] + + # organize by by if by is None: by = Result._by @@ -710,7 +729,7 @@ def fold(Result, results, by=None, defines=[]): if defines: results_ = [] for r in results: - if all(getattr(r, k) in vs for k, vs in defines): + if all(str(getattr(r, k)) in vs for k, vs in defines): results_.append(r) results = results_ @@ -727,8 +746,80 @@ def fold(Result, results, by=None, defines=[]): for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) + # sort, note that python's sort is stable + folded.sort(key=lambda r: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(r, k_),) + if getattr(r, k_) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by result + r)) + + # recurse if we have recursive results + if hasattr(Result, '_children'): + folded = [r._replace(**{ + Result._children: fold( + Result, getattr(r, Result._children), + by=by, + defines=defines, + sort=sort, + depth=depth-1)}) + for r in folded] + return folded +def hotify(Result, results, *, + fields=None, + sort=None, + depth=1, + hot=None, + **_): + # hotify only makes sense for recursive results + assert hasattr(Result, '_i') + assert hasattr(Result, '_children') + + if fields is None: + fields = Result._fields + + results_ = [] + for r in results: + hot_ = [] + def recurse(results_, depth_): + nonlocal hot_ + if not results_: + return + + # find the hottest result + r = min(results_, key=lambda r: + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(r, k_),) + if getattr(r, k_) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in it.chain(hot, [(None, False)]))) + + hot_.append(r._replace(**{ + Result._i: RInt(len(hot_)), + Result._children: []})) + + # recurse? + if depth_ > 1: + recurse(getattr(r, Result._children), + depth_-1) + + recurse(getattr(r, Result._children), depth-1) + results_.append(r._replace(**{ + Result._children: hot_})) + + return results_ + def table(Result, results, diff_results=None, *, by=None, fields=None, @@ -754,124 +845,32 @@ def table(Result, results, diff_results=None, *, fields = Result._fields types = Result._types - # fold again - results = fold(Result, results, by=by) + # fold again, otherwise results risk being hidden + results = fold(Result, results, + by=by, + depth=depth) if diff_results is not None: - diff_results = fold(Result, diff_results, by=by) - - # reduce children to hot paths? only used by some scripts - if hot: - # subclass to reintroduce __dict__ - Result_ = Result - class HotResult(Result_): - _i = '_hot_i' - _children = '_hot_children' - - def __new__(cls, r, i=None, children=None, notes=None): - self = HotResult._make(r) - self._hot_i = i - self._hot_children = children if children is not None else [] - return self - - def __add__(self, other): - return HotResult( - Result_.__add__(self, other), - self._hot_i if other._hot_i is None - else other._hot_i if self._hot_i is None - else min(self._hot_i, other._hot_i), - self._hot_children + other._hot_children) - - results_ = [] - for r in results: - hot_ = [] - def recurse(results_, depth_): - nonlocal hot_ - if not results_: - return - - # find the hottest result - r = max(results_, - key=lambda r: tuple( - tuple((getattr(r, k),) - if getattr(r, k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields]) - if k in fields) - for k in it.chain(hot, [None]))) - hot_.append(HotResult(r, i=len(hot_))) - - # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), - depth_-1) - - recurse(getattr(r, Result._children), depth-1) - results_.append(HotResult(r, children=hot_)) - - Result = HotResult - results = results_ + diff_results = fold(Result, diff_results, + by=by, + depth=depth) # organize by name table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results} diff_table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in diff_results or []} - names = [name - for name in table.keys() | diff_table.keys() - if diff_results is None - or all_ - or any( - types[k].ratio( - getattr(table.get(name), k, None), - getattr(diff_table.get(name), k, None)) - for k in fields)] # find compare entry if there is one if compare: - compare_result = table.get(','.join(str(k) for k in compare)) - - # sort again, now with diff info, note that python's sort is stable - names.sort() - if compare: - names.sort( - key=lambda n: ( - # move compare entry to the top, note this can be - # overridden by explicitly sorting by fields - table.get(n) == compare_result, - # sort by ratio if comparing - tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(compare_result, k, None)) - for k in fields)), - reverse=True) - if diff or percent: - names.sort( - # sort by ratio if diffing - key=lambda n: tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(diff_table.get(n), k, None)) - for k in fields), - reverse=True) - if sort: - for k, reverse in reversed(sort): - names.sort( - key=lambda n: tuple( - (getattr(table[n], k),) - if getattr(table.get(n), k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) - + compare_r = table.get(','.join(str(k) for k in compare)) # build up our lines lines = [] @@ -898,11 +897,16 @@ def table(Result, results, diff_results=None, *, header.append('d'+k) lines.append(header) + # delete these to try to catch typos below, we need to rebuild + # these tables at each recursive layer + del table + del diff_table + # entry helper def table_entry(name, r, diff_r=None): entry = [name] # normal entry? - if ((compare is None or r == compare_result) + if ((compare is None or r == compare_r) and not percent and not diff): for k in fields: @@ -923,7 +927,7 @@ def table(Result, results, diff_results=None, *, else ['%+.1f%%' % (100*t)])( types[k].ratio( getattr(r, k, None), - getattr(compare_result, k, None))))) + getattr(compare_r, k, None))))) # percent entry? elif not diff: for k in fields: @@ -969,71 +973,94 @@ def table(Result, results, diff_results=None, *, return entry - # recursive entry helper, only used by some scripts - def recurse(results_, depth_, + # recursive entry helper + def table_recurse(results_, diff_results_, + depth_, prefixes=('', '', '', '')): # build the children table at each layer - results_ = fold(Result, results_, by=by) table_ = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results_} - names_ = list(table_.keys()) + diff_table_ = { + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r + for r in diff_results_ or []} + names_ = [n + for n in table_.keys() | diff_table_.keys() + if diff_results_ is None + or all_ + or any( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)] - # sort the children layer - names_.sort() - if hasattr(Result, '_i'): - names_.sort(key=lambda n: getattr(table_[n], Result._i)) - if sort: - for k, reverse in reversed(sort): - names_.sort( - key=lambda n: tuple( - (getattr(table_[n], k),) - if getattr(table_.get(n), k, None) - is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) + # sort again, now with diff info, note that python's sort is stable + names_.sort(key=lambda n: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(table_[n], k_),) + if getattr(table_.get(n), k_, None) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by ratio if diffing + Rev(tuple(types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)) + if diff or percent + else (), + # move compare entry to the top, note this can be + # overridden by explicitly sorting by fields + (table_.get(n) != compare_r, + # sort by ratio if comparing + Rev(tuple( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(compare_r, k, None)) + for k in fields))) + if compare + else (), + # sort by result + (table_[n],) if n in table_ else (), + # and finally by name (diffs may be missing results) + n)) - for i, name in enumerate(names_): - r = table_[name] - is_last = (i == len(names_)-1) + for i, n in enumerate(names_): + # find comparable results + r = table_.get(n) + diff_r = diff_table_.get(n) + + # build line + line = table_entry(n, r, diff_r) - line = table_entry(name, r) - line = [x if isinstance(x, tuple) else (x, []) for x in line] # add prefixes - line[0] = (prefixes[0+is_last] + line[0][0], line[0][1]) + line = [x if isinstance(x, tuple) else (x, []) for x in line] + line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) lines.append(line) # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), + if n in table_ and depth_ > 1: + table_recurse( + getattr(r, Result._children), + getattr(diff_r, Result._children, None) or [], depth_-1, - (prefixes[2+is_last] + "|-> ", - prefixes[2+is_last] + "'-> ", - prefixes[2+is_last] + "| ", - prefixes[2+is_last] + " ")) + (prefixes[2+(i==len(names_)-1)] + "|-> ", + prefixes[2+(i==len(names_)-1)] + "'-> ", + prefixes[2+(i==len(names_)-1)] + "| ", + prefixes[2+(i==len(names_)-1)] + " ")) - # entries + # build entries if not summary: - for name in names: - r = table.get(name) - if diff_results is None: - diff_r = None - else: - diff_r = diff_table.get(name) - lines.append(table_entry(name, r, diff_r)) - - # recursive entries - if name in table and depth > 1: - recurse(getattr(table[name], Result._children), - depth-1, - ("|-> ", - "'-> ", - "| ", - " ")) + table_recurse(results, diff_results, depth) # total if not no_total and not (small_table and not summary): @@ -1045,9 +1072,8 @@ def table(Result, results, diff_results=None, *, lines.append(table_entry('TOTAL', r, diff_r)) # homogenize - lines = [ - [x if isinstance(x, tuple) else (x, []) for x in line] - for line in lines] + lines = [[x if isinstance(x, tuple) else (x, []) for x in line] + for line in lines] # find the best widths, note that column 0 contains the names and is # handled a bit differently @@ -1068,18 +1094,144 @@ def table(Result, results, diff_results=None, *, nwidths[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') for i, x in enumerate(line[1:], 1)))) +def read_csv(path, Result, *, + depth=1, + **_): + with openio(path, 'r') as f: + # csv or json? assume json starts with [ + json = (f.buffer.peek(1)[:1] == b'[') + + # read csv? + if not json: + results = [] + reader = csv.DictReader(f, restval='') + for r in reader: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k].strip()} + | {k: r[k] for k in Result._fields + if k in r and r[k].strip()}))) + except TypeError: + pass + return results + + # read json? + else: + import json + def unjsonify(results, depth_): + results_ = [] + for r in results: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results_.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k] is not None} + | {k: r[k] for k in Result._fields + if k in r and r[k] is not None} + | ({Result._children: unjsonify( + r[Result._children], + depth_-1)} + if hasattr(Result, '_children') + and Result._children in r + and r[Result._children] is not None + and depth_ > 1 + else {}) + | ({Result._notes: set(r[Result._notes])} + if hasattr(Result, '_notes') + and Result._notes in r + and r[Result._notes] is not None + else {})))) + except TypeError: + pass + return results_ + return unjsonify(json.load(f), depth) + +def write_csv(path, Result, results, *, + json=False, + by=None, + fields=None, + depth=1, + **_): + with openio(path, 'w') as f: + # write csv? + if not json: + writer = csv.DictWriter(f, + (by if by is not None else Result._by) + + [k for k in (fields + if fields is not None + else Result._fields)]) + writer.writeheader() + for r in results: + # note this allows by/fields to overlap + writer.writerow( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None}) + + # write json? + else: + import json + # the neat thing about json is we can include recursive results + def jsonify(results, depth_): + results_ = [] + for r in results: + # note this allows by/fields to overlap + results_.append( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None} + | ({Result._children: jsonify( + getattr(r, Result._children), + depth_-1)} + if hasattr(Result, '_children') + and getattr(r, Result._children) + and depth_ > 1 + else {}) + | ({Result._notes: list( + getattr(r, Result._notes))} + if hasattr(Result, '_notes') + and getattr(r, Result._notes) + else {})) + return results_ + json.dump(jsonify(results, depth), f, + separators=(',', ':')) + def main(obj_paths, *, by=None, fields=None, defines=[], sort=None, + depth=None, + hot=None, **args): # figure out depth - if args.get('depth') is None: - args['depth'] = mt.inf if args.get('hot') else 1 - elif args.get('depth') == 0: - args['depth'] = mt.inf + if depth is None: + depth = mt.inf if hot else 1 + elif depth == 0: + depth = mt.inf # find sizes if not args.get('use', None): @@ -1090,89 +1242,61 @@ def main(obj_paths, *, sys.exit(1) # collect info - results = collect(obj_paths, **args) + results = collect_ctx(obj_paths, + depth=depth, + **args) else: - results = [] - with openio(args['use']) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - if not any(k in r and r[k].strip() - for k in CtxResult._fields): - continue - try: - results.append(CtxResult( - **{k: r[k] for k in CtxResult._by - if k in r and r[k].strip()}, - **{k: r[k] - for k in CtxResult._fields - if k in r and r[k].strip()})) - except TypeError: - pass + results = read_csv(args['use'], CtxResult, + depth=depth, + **args) # fold - results = fold(CtxResult, results, by=by, defines=defines) + results = fold(CtxResult, results, + by=by, + defines=defines, + depth=depth) - # sort, note that python's sort is stable - results.sort() - if sort: - for k, reverse in reversed(sort): - results.sort( - key=lambda r: tuple( - (getattr(r, k),) if getattr(r, k) is not None else () - for k in ([k] if k else CtxResult._sort)), - reverse=reverse ^ (not k or k in CtxResult._fields)) + # hotify? + if hot: + results = hotify(CtxResult, results, + fields=fields, + depth=depth, + hot=hot, + **args) - # write results to CSV + # write results to CSV/JSON if args.get('output'): - with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, - (by if by is not None else CtxResult._by) - + [k for k in ( - fields if fields is not None - else CtxResult._fields)]) - writer.writeheader() - for r in results: - writer.writerow( - {k: getattr(r, k) for k in ( - by if by is not None else CtxResult._by)} - | {k: getattr(r, k) for k in ( - fields if fields is not None - else CtxResult._fields)}) + write_csv(args['output'], CtxResult, results, + by=by, + fields=fields, + depth=depth, + **args) + if args.get('output_json'): + write_csv(args['output_json'], CtxResult, results, + json=True, + by=by, + fields=fields, + depth=depth, + **args) # find previous results? diff_results = None if args.get('diff') or args.get('percent'): - diff_results = [] try: - with openio(args.get('diff') or args.get('percent')) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - if not any(k in r and r[k].strip() - for k in CtxResult._fields): - continue - try: - diff_results.append(CtxResult( - **{k: r[k] for k in CtxResult._by - if k in r and r[k].strip()}, - **{k: r[k] - for k in CtxResult._fields - if k in r and r[k].strip()})) - except TypeError: - pass + diff_results = read_csv( + args.get('diff') or args.get('percent'), + CtxResult, + depth=depth, + **args) except FileNotFoundError: - pass + diff_results = [] # fold - diff_results = fold(CtxResult, diff_results, by=by, defines=defines) + diff_results = fold(CtxResult, diff_results, + by=by, + defines=defines, + depth=depth) # print table if not args.get('quiet'): @@ -1180,6 +1304,7 @@ def main(obj_paths, *, by=by if by is not None else ['function'], fields=fields, sort=sort, + depth=depth, **args) @@ -1204,24 +1329,28 @@ if __name__ == "__main__": parser.add_argument( '-o', '--output', help="Specify CSV file to store results.") + parser.add_argument( + '-O', '--output-json', + help="Specify JSON file to store results. This may contain " + "recursive info.") parser.add_argument( '-u', '--use', - help="Don't parse anything, use this CSV file.") + help="Don't parse anything, use this CSV/JSON file.") parser.add_argument( '-d', '--diff', - help="Specify CSV file to diff against.") + help="Specify CSV/JSON file to diff against.") parser.add_argument( '-p', '--percent', - help="Specify CSV file to diff against, but only show precentage " - "change, not a full diff.") - parser.add_argument( - '-a', '--all', - action='store_true', - help="Show all, not just the ones that changed.") + help="Specify CSV/JSON file to diff against, but only show " + "percentage change, not a full diff.") parser.add_argument( '-c', '--compare', type=lambda x: tuple(v.strip() for v in x.split(',')), help="Compare results to the row matching this by pattern.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") parser.add_argument( '-b', '--by', action='append', @@ -1247,7 +1376,7 @@ if __name__ == "__main__": def __call__(self, parser, namespace, value, option): if namespace.sort is None: namespace.sort = [] - namespace.sort.append((value, True if option == '-S' else False)) + namespace.sort.append((value, option in {'-S', '--reverse-sort'})) parser.add_argument( '-s', '--sort', nargs='?', @@ -1258,6 +1387,29 @@ if __name__ == "__main__": nargs='?', action=AppendSort, help="Sort by this field, but backwards.") + parser.add_argument( + '-z', '--depth', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Depth of function calls to show. 0 shows all calls unless " + "we find a cycle. Defaults to 0.") + class AppendHot(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.hot is None: + namespace.hot = [] + namespace.hot.append((value, option in {'-R', '--reverse-hot'})) + parser.add_argument( + '-r', '--hot', + nargs='?', + action=AppendHot, + help="Show only the hot path for each function call. Can " + "optionally provide fields like sort.") + parser.add_argument( + '-R', '--reverse-hot', + nargs='?', + action=AppendHot, + help="Like -r/--hot, but backwards.") parser.add_argument( '--no-header', action='store_true', @@ -1282,18 +1434,6 @@ if __name__ == "__main__": '--everything', action='store_true', help="Include builtin and libc specific symbols.") - parser.add_argument( - '-z', '--depth', - nargs='?', - type=lambda x: int(x, 0), - const=0, - help="Depth of function calls to show. 0 shows all calls unless " - "we find a cycle. Defaults to 0.") - parser.add_argument( - '-t', '--hot', - nargs='?', - action='append', - help="Show only the hot path for each function call.") parser.add_argument( '--objdump-path', type=lambda x: x.split(), diff --git a/scripts/data.py b/scripts/data.py index de979fdf..decadf21 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -18,8 +18,9 @@ if __name__ == "__main__": import collections as co import csv -import itertools as it import functools as ft +import io +import itertools as it import math as mt import os import re @@ -449,7 +450,7 @@ def collect_dwarf_info(obj_path, tags=None, *, return DwarfInfo(info) -def collect(obj_paths, *, +def collect_data(obj_paths, *, everything=False, **args): results = [] @@ -496,7 +497,31 @@ def collect(obj_paths, *, return results -def fold(Result, results, by=None, defines=[]): +# common folding/tabling/read/write code + +class Rev(co.namedtuple('Rev', 'x')): + __slots__ = () + # yes we need all of these because we're a namedtuple + def __lt__(self, other): + return self.x > other.x + def __gt__(self, other): + return self.x < other.x + def __le__(self, other): + return self.x >= other.x + def __ge__(self, other): + return self.x <= other.x + +def fold(Result, results, *, + by=None, + defines=[], + sort=None, + depth=1, + **_): + # stop when depth hits zero + if depth == 0: + return [] + + # organize by by if by is None: by = Result._by @@ -510,7 +535,7 @@ def fold(Result, results, by=None, defines=[]): if defines: results_ = [] for r in results: - if all(getattr(r, k) in vs for k, vs in defines): + if all(str(getattr(r, k)) in vs for k, vs in defines): results_.append(r) results = results_ @@ -527,6 +552,31 @@ def fold(Result, results, by=None, defines=[]): for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) + # sort, note that python's sort is stable + folded.sort(key=lambda r: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(r, k_),) + if getattr(r, k_) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by result + r)) + + # recurse if we have recursive results + if hasattr(Result, '_children'): + folded = [r._replace(**{ + Result._children: fold( + Result, getattr(r, Result._children), + by=by, + defines=defines, + sort=sort, + depth=depth-1)}) + for r in folded] + return folded def table(Result, results, diff_results=None, *, @@ -554,124 +604,32 @@ def table(Result, results, diff_results=None, *, fields = Result._fields types = Result._types - # fold again - results = fold(Result, results, by=by) + # fold again, otherwise results risk being hidden + results = fold(Result, results, + by=by, + depth=depth) if diff_results is not None: - diff_results = fold(Result, diff_results, by=by) - - # reduce children to hot paths? only used by some scripts - if hot: - # subclass to reintroduce __dict__ - Result_ = Result - class HotResult(Result_): - _i = '_hot_i' - _children = '_hot_children' - - def __new__(cls, r, i=None, children=None, notes=None): - self = HotResult._make(r) - self._hot_i = i - self._hot_children = children if children is not None else [] - return self - - def __add__(self, other): - return HotResult( - Result_.__add__(self, other), - self._hot_i if other._hot_i is None - else other._hot_i if self._hot_i is None - else min(self._hot_i, other._hot_i), - self._hot_children + other._hot_children) - - results_ = [] - for r in results: - hot_ = [] - def recurse(results_, depth_): - nonlocal hot_ - if not results_: - return - - # find the hottest result - r = max(results_, - key=lambda r: tuple( - tuple((getattr(r, k),) - if getattr(r, k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields]) - if k in fields) - for k in it.chain(hot, [None]))) - hot_.append(HotResult(r, i=len(hot_))) - - # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), - depth_-1) - - recurse(getattr(r, Result._children), depth-1) - results_.append(HotResult(r, children=hot_)) - - Result = HotResult - results = results_ + diff_results = fold(Result, diff_results, + by=by, + depth=depth) # organize by name table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results} diff_table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in diff_results or []} - names = [name - for name in table.keys() | diff_table.keys() - if diff_results is None - or all_ - or any( - types[k].ratio( - getattr(table.get(name), k, None), - getattr(diff_table.get(name), k, None)) - for k in fields)] # find compare entry if there is one if compare: - compare_result = table.get(','.join(str(k) for k in compare)) - - # sort again, now with diff info, note that python's sort is stable - names.sort() - if compare: - names.sort( - key=lambda n: ( - # move compare entry to the top, note this can be - # overridden by explicitly sorting by fields - table.get(n) == compare_result, - # sort by ratio if comparing - tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(compare_result, k, None)) - for k in fields)), - reverse=True) - if diff or percent: - names.sort( - # sort by ratio if diffing - key=lambda n: tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(diff_table.get(n), k, None)) - for k in fields), - reverse=True) - if sort: - for k, reverse in reversed(sort): - names.sort( - key=lambda n: tuple( - (getattr(table[n], k),) - if getattr(table.get(n), k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) - + compare_r = table.get(','.join(str(k) for k in compare)) # build up our lines lines = [] @@ -698,11 +656,16 @@ def table(Result, results, diff_results=None, *, header.append('d'+k) lines.append(header) + # delete these to try to catch typos below, we need to rebuild + # these tables at each recursive layer + del table + del diff_table + # entry helper def table_entry(name, r, diff_r=None): entry = [name] # normal entry? - if ((compare is None or r == compare_result) + if ((compare is None or r == compare_r) and not percent and not diff): for k in fields: @@ -723,7 +686,7 @@ def table(Result, results, diff_results=None, *, else ['%+.1f%%' % (100*t)])( types[k].ratio( getattr(r, k, None), - getattr(compare_result, k, None))))) + getattr(compare_r, k, None))))) # percent entry? elif not diff: for k in fields: @@ -769,71 +732,94 @@ def table(Result, results, diff_results=None, *, return entry - # recursive entry helper, only used by some scripts - def recurse(results_, depth_, + # recursive entry helper + def table_recurse(results_, diff_results_, + depth_, prefixes=('', '', '', '')): # build the children table at each layer - results_ = fold(Result, results_, by=by) table_ = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results_} - names_ = list(table_.keys()) + diff_table_ = { + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r + for r in diff_results_ or []} + names_ = [n + for n in table_.keys() | diff_table_.keys() + if diff_results_ is None + or all_ + or any( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)] - # sort the children layer - names_.sort() - if hasattr(Result, '_i'): - names_.sort(key=lambda n: getattr(table_[n], Result._i)) - if sort: - for k, reverse in reversed(sort): - names_.sort( - key=lambda n: tuple( - (getattr(table_[n], k),) - if getattr(table_.get(n), k, None) - is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) + # sort again, now with diff info, note that python's sort is stable + names_.sort(key=lambda n: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(table_[n], k_),) + if getattr(table_.get(n), k_, None) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by ratio if diffing + Rev(tuple(types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)) + if diff or percent + else (), + # move compare entry to the top, note this can be + # overridden by explicitly sorting by fields + (table_.get(n) != compare_r, + # sort by ratio if comparing + Rev(tuple( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(compare_r, k, None)) + for k in fields))) + if compare + else (), + # sort by result + (table_[n],) if n in table_ else (), + # and finally by name (diffs may be missing results) + n)) - for i, name in enumerate(names_): - r = table_[name] - is_last = (i == len(names_)-1) + for i, n in enumerate(names_): + # find comparable results + r = table_.get(n) + diff_r = diff_table_.get(n) + + # build line + line = table_entry(n, r, diff_r) - line = table_entry(name, r) - line = [x if isinstance(x, tuple) else (x, []) for x in line] # add prefixes - line[0] = (prefixes[0+is_last] + line[0][0], line[0][1]) + line = [x if isinstance(x, tuple) else (x, []) for x in line] + line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) lines.append(line) # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), + if n in table_ and depth_ > 1: + table_recurse( + getattr(r, Result._children), + getattr(diff_r, Result._children, None) or [], depth_-1, - (prefixes[2+is_last] + "|-> ", - prefixes[2+is_last] + "'-> ", - prefixes[2+is_last] + "| ", - prefixes[2+is_last] + " ")) + (prefixes[2+(i==len(names_)-1)] + "|-> ", + prefixes[2+(i==len(names_)-1)] + "'-> ", + prefixes[2+(i==len(names_)-1)] + "| ", + prefixes[2+(i==len(names_)-1)] + " ")) - # entries + # build entries if not summary: - for name in names: - r = table.get(name) - if diff_results is None: - diff_r = None - else: - diff_r = diff_table.get(name) - lines.append(table_entry(name, r, diff_r)) - - # recursive entries - if name in table and depth > 1: - recurse(getattr(table[name], Result._children), - depth-1, - ("|-> ", - "'-> ", - "| ", - " ")) + table_recurse(results, diff_results, depth) # total if not no_total and not (small_table and not summary): @@ -845,9 +831,8 @@ def table(Result, results, diff_results=None, *, lines.append(table_entry('TOTAL', r, diff_r)) # homogenize - lines = [ - [x if isinstance(x, tuple) else (x, []) for x in line] - for line in lines] + lines = [[x if isinstance(x, tuple) else (x, []) for x in line] + for line in lines] # find the best widths, note that column 0 contains the names and is # handled a bit differently @@ -868,6 +853,130 @@ def table(Result, results, diff_results=None, *, nwidths[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') for i, x in enumerate(line[1:], 1)))) +def read_csv(path, Result, *, + depth=1, + **_): + with openio(path, 'r') as f: + # csv or json? assume json starts with [ + json = (f.buffer.peek(1)[:1] == b'[') + + # read csv? + if not json: + results = [] + reader = csv.DictReader(f, restval='') + for r in reader: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k].strip()} + | {k: r[k] for k in Result._fields + if k in r and r[k].strip()}))) + except TypeError: + pass + return results + + # read json? + else: + import json + def unjsonify(results, depth_): + results_ = [] + for r in results: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results_.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k] is not None} + | {k: r[k] for k in Result._fields + if k in r and r[k] is not None} + | ({Result._children: unjsonify( + r[Result._children], + depth_-1)} + if hasattr(Result, '_children') + and Result._children in r + and r[Result._children] is not None + and depth_ > 1 + else {}) + | ({Result._notes: set(r[Result._notes])} + if hasattr(Result, '_notes') + and Result._notes in r + and r[Result._notes] is not None + else {})))) + except TypeError: + pass + return results_ + return unjsonify(json.load(f), depth) + +def write_csv(path, Result, results, *, + json=False, + by=None, + fields=None, + depth=1, + **_): + with openio(path, 'w') as f: + # write csv? + if not json: + writer = csv.DictWriter(f, + (by if by is not None else Result._by) + + [k for k in (fields + if fields is not None + else Result._fields)]) + writer.writeheader() + for r in results: + # note this allows by/fields to overlap + writer.writerow( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None}) + + # write json? + else: + import json + # the neat thing about json is we can include recursive results + def jsonify(results, depth_): + results_ = [] + for r in results: + # note this allows by/fields to overlap + results_.append( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None} + | ({Result._children: jsonify( + getattr(r, Result._children), + depth_-1)} + if hasattr(Result, '_children') + and getattr(r, Result._children) + and depth_ > 1 + else {}) + | ({Result._notes: list( + getattr(r, Result._notes))} + if hasattr(Result, '_notes') + and getattr(r, Result._notes) + else {})) + return results_ + json.dump(jsonify(results, depth), f, + separators=(',', ':')) + def main(obj_paths, *, by=None, @@ -884,84 +993,46 @@ def main(obj_paths, *, sys.exit(1) # collect info - results = collect(obj_paths, **args) + results = collect_data(obj_paths, + **args) else: - results = [] - with openio(args['use']) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - try: - results.append(DataResult( - **{k: r[k] for k in DataResult._by - if k in r and r[k].strip()}, - **{k: r[k] for k in DataResult._fields - if k in r and r[k].strip()})) - except TypeError: - pass + results = read_csv(args['use'], DataResult, + **args) # fold - results = fold(DataResult, results, by=by, defines=defines) + results = fold(DataResult, results, + by=by, + defines=defines) - # sort, note that python's sort is stable - results.sort() - if sort: - for k, reverse in reversed(sort): - results.sort( - key=lambda r: tuple( - (getattr(r, k),) if getattr(r, k) is not None else () - for k in ([k] if k else DataResult._sort)), - reverse=reverse ^ (not k or k in DataResult._fields)) - - # write results to CSV + # write results to CSV/JSON if args.get('output'): - with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, - (by if by is not None else DataResult._by) - + [k for k in ( - fields if fields is not None - else DataResult._fields)]) - writer.writeheader() - for r in results: - writer.writerow( - {k: getattr(r, k) for k in ( - by if by is not None else DataResult._by)} - | {k: getattr(r, k) for k in ( - fields if fields is not None - else DataResult._fields)}) + write_csv(args['output'], DataResult, results, + by=by, + fields=fields, + **args) + if args.get('output_json'): + write_csv(args['output_json'], DataResult, results, + json=True, + by=by, + fields=fields, + **args) # find previous results? diff_results = None if args.get('diff') or args.get('percent'): - diff_results = [] try: - with openio(args.get('diff') or args.get('percent')) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - if not any(k in r and r[k].strip() - for k in DataResult._fields): - continue - try: - diff_results.append(DataResult( - **{k: r[k] for k in DataResult._by - if k in r and r[k].strip()}, - **{k: r[k] for k in DataResult._fields - if k in r and r[k].strip()})) - except TypeError: - pass + diff_results = read_csv( + args.get('diff') or args.get('percent'), + DataResult, + **args) except FileNotFoundError: - pass + diff_results = [] # fold - diff_results = fold(DataResult, diff_results, by=by, defines=defines) + diff_results = fold(DataResult, diff_results, + by=by, + defines=defines) # print table if not args.get('quiet'): @@ -993,24 +1064,28 @@ if __name__ == "__main__": parser.add_argument( '-o', '--output', help="Specify CSV file to store results.") + parser.add_argument( + '-O', '--output-json', + help="Specify JSON file to store results. This may contain " + "recursive info.") parser.add_argument( '-u', '--use', - help="Don't parse anything, use this CSV file.") + help="Don't parse anything, use this CSV/JSON file.") parser.add_argument( '-d', '--diff', - help="Specify CSV file to diff against.") + help="Specify CSV/JSON file to diff against.") parser.add_argument( '-p', '--percent', - help="Specify CSV file to diff against, but only show precentage " - "change, not a full diff.") - parser.add_argument( - '-a', '--all', - action='store_true', - help="Show all, not just the ones that changed.") + help="Specify CSV/JSON file to diff against, but only show " + "percentage change, not a full diff.") parser.add_argument( '-c', '--compare', type=lambda x: tuple(v.strip() for v in x.split(',')), help="Compare results to the row matching this by pattern.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") parser.add_argument( '-b', '--by', action='append', @@ -1036,7 +1111,7 @@ if __name__ == "__main__": def __call__(self, parser, namespace, value, option): if namespace.sort is None: namespace.sort = [] - namespace.sort.append((value, True if option == '-S' else False)) + namespace.sort.append((value, option in {'-S', '--reverse-sort'})) parser.add_argument( '-s', '--sort', nargs='?', diff --git a/scripts/perf.py b/scripts/perf.py index 42ce12c2..3195c8e2 100755 --- a/scripts/perf.py +++ b/scripts/perf.py @@ -3,7 +3,7 @@ # Script to aggregate and report Linux perf results. # # Example: -# ./scripts/perf.py -R -obench.perf ./runners/bench_runner +# ./scripts/perf.py --record -obench.perf ./runners/bench_runner # ./scripts/perf.py bench.perf -j -Flfs.c -Flfs_util.c -Scycles # # Copyright (c) 2022, The littlefs authors. @@ -20,6 +20,7 @@ import csv import errno import fcntl import functools as ft +import io import itertools as it import math as mt import multiprocessing as mp @@ -146,30 +147,31 @@ class RInt(co.namedtuple('RInt', 'x')): # perf results class PerfResult(co.namedtuple('PerfResult', [ - 'file', 'function', 'line', + 'i', 'file', 'function', 'line', 'cycles', 'bmisses', 'branches', 'cmisses', 'caches', 'children'])): - _by = ['file', 'function', 'line'] + _by = ['i', 'file', 'function', 'line'] _fields = ['cycles', 'bmisses', 'branches', 'cmisses', 'caches'] _sort = ['cycles', 'bmisses', 'cmisses', 'branches', 'caches'] _types = { 'cycles': RInt, 'bmisses': RInt, 'branches': RInt, 'cmisses': RInt, 'caches': RInt} + _i = 'i' _children = 'children' __slots__ = () - def __new__(cls, file='', function='', line=0, + def __new__(cls, i=None, file='', function='', line=0, cycles=0, bmisses=0, branches=0, cmisses=0, caches=0, children=None): - return super().__new__(cls, file, function, int(RInt(line)), + return super().__new__(cls, i, file, function, int(RInt(line)), RInt(cycles), RInt(bmisses), RInt(branches), RInt(cmisses), RInt(caches), children if children is not None else []) def __add__(self, other): - return PerfResult(self.file, self.function, self.line, + return PerfResult(self.i, self.file, self.function, self.line, self.cycles + other.cycles, self.bmisses + other.bmisses, self.branches + other.branches, @@ -757,7 +759,7 @@ def collect_decompressed(path, *, def to_results(results): results_ = [] for name, (r, children) in results.items(): - results_.append(PerfResult(*name, + results_.append(PerfResult(None, *name, **{events[k]: v for k, v in r.items()}, children=to_results(children))) return results_ @@ -779,7 +781,7 @@ def starapply(args): f, args, kwargs = args return f(*args, **kwargs) -def collect(perf_paths, *, +def collect_perf(perf_paths, *, jobs=None, **args): # automatic job detection? @@ -811,7 +813,31 @@ def collect(perf_paths, *, return results -def fold(Result, results, by=None, defines=[]): +# common folding/tabling/read/write code + +class Rev(co.namedtuple('Rev', 'x')): + __slots__ = () + # yes we need all of these because we're a namedtuple + def __lt__(self, other): + return self.x > other.x + def __gt__(self, other): + return self.x < other.x + def __le__(self, other): + return self.x >= other.x + def __ge__(self, other): + return self.x <= other.x + +def fold(Result, results, *, + by=None, + defines=[], + sort=None, + depth=1, + **_): + # stop when depth hits zero + if depth == 0: + return [] + + # organize by by if by is None: by = Result._by @@ -825,7 +851,7 @@ def fold(Result, results, by=None, defines=[]): if defines: results_ = [] for r in results: - if all(getattr(r, k) in vs for k, vs in defines): + if all(str(getattr(r, k)) in vs for k, vs in defines): results_.append(r) results = results_ @@ -842,8 +868,80 @@ def fold(Result, results, by=None, defines=[]): for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) + # sort, note that python's sort is stable + folded.sort(key=lambda r: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(r, k_),) + if getattr(r, k_) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by result + r)) + + # recurse if we have recursive results + if hasattr(Result, '_children'): + folded = [r._replace(**{ + Result._children: fold( + Result, getattr(r, Result._children), + by=by, + defines=defines, + sort=sort, + depth=depth-1)}) + for r in folded] + return folded +def hotify(Result, results, *, + fields=None, + sort=None, + depth=1, + hot=None, + **_): + # hotify only makes sense for recursive results + assert hasattr(Result, '_i') + assert hasattr(Result, '_children') + + if fields is None: + fields = Result._fields + + results_ = [] + for r in results: + hot_ = [] + def recurse(results_, depth_): + nonlocal hot_ + if not results_: + return + + # find the hottest result + r = min(results_, key=lambda r: + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(r, k_),) + if getattr(r, k_) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in it.chain(hot, [(None, False)]))) + + hot_.append(r._replace(**{ + Result._i: RInt(len(hot_)), + Result._children: []})) + + # recurse? + if depth_ > 1: + recurse(getattr(r, Result._children), + depth_-1) + + recurse(getattr(r, Result._children), depth-1) + results_.append(r._replace(**{ + Result._children: hot_})) + + return results_ + def table(Result, results, diff_results=None, *, by=None, fields=None, @@ -869,124 +967,32 @@ def table(Result, results, diff_results=None, *, fields = Result._fields types = Result._types - # fold again - results = fold(Result, results, by=by) + # fold again, otherwise results risk being hidden + results = fold(Result, results, + by=by, + depth=depth) if diff_results is not None: - diff_results = fold(Result, diff_results, by=by) - - # reduce children to hot paths? only used by some scripts - if hot: - # subclass to reintroduce __dict__ - Result_ = Result - class HotResult(Result_): - _i = '_hot_i' - _children = '_hot_children' - - def __new__(cls, r, i=None, children=None, notes=None): - self = HotResult._make(r) - self._hot_i = i - self._hot_children = children if children is not None else [] - return self - - def __add__(self, other): - return HotResult( - Result_.__add__(self, other), - self._hot_i if other._hot_i is None - else other._hot_i if self._hot_i is None - else min(self._hot_i, other._hot_i), - self._hot_children + other._hot_children) - - results_ = [] - for r in results: - hot_ = [] - def recurse(results_, depth_): - nonlocal hot_ - if not results_: - return - - # find the hottest result - r = max(results_, - key=lambda r: tuple( - tuple((getattr(r, k),) - if getattr(r, k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields]) - if k in fields) - for k in it.chain(hot, [None]))) - hot_.append(HotResult(r, i=len(hot_))) - - # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), - depth_-1) - - recurse(getattr(r, Result._children), depth-1) - results_.append(HotResult(r, children=hot_)) - - Result = HotResult - results = results_ + diff_results = fold(Result, diff_results, + by=by, + depth=depth) # organize by name table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results} diff_table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in diff_results or []} - names = [name - for name in table.keys() | diff_table.keys() - if diff_results is None - or all_ - or any( - types[k].ratio( - getattr(table.get(name), k, None), - getattr(diff_table.get(name), k, None)) - for k in fields)] # find compare entry if there is one if compare: - compare_result = table.get(','.join(str(k) for k in compare)) - - # sort again, now with diff info, note that python's sort is stable - names.sort() - if compare: - names.sort( - key=lambda n: ( - # move compare entry to the top, note this can be - # overridden by explicitly sorting by fields - table.get(n) == compare_result, - # sort by ratio if comparing - tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(compare_result, k, None)) - for k in fields)), - reverse=True) - if diff or percent: - names.sort( - # sort by ratio if diffing - key=lambda n: tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(diff_table.get(n), k, None)) - for k in fields), - reverse=True) - if sort: - for k, reverse in reversed(sort): - names.sort( - key=lambda n: tuple( - (getattr(table[n], k),) - if getattr(table.get(n), k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) - + compare_r = table.get(','.join(str(k) for k in compare)) # build up our lines lines = [] @@ -1013,11 +1019,16 @@ def table(Result, results, diff_results=None, *, header.append('d'+k) lines.append(header) + # delete these to try to catch typos below, we need to rebuild + # these tables at each recursive layer + del table + del diff_table + # entry helper def table_entry(name, r, diff_r=None): entry = [name] # normal entry? - if ((compare is None or r == compare_result) + if ((compare is None or r == compare_r) and not percent and not diff): for k in fields: @@ -1038,7 +1049,7 @@ def table(Result, results, diff_results=None, *, else ['%+.1f%%' % (100*t)])( types[k].ratio( getattr(r, k, None), - getattr(compare_result, k, None))))) + getattr(compare_r, k, None))))) # percent entry? elif not diff: for k in fields: @@ -1084,71 +1095,94 @@ def table(Result, results, diff_results=None, *, return entry - # recursive entry helper, only used by some scripts - def recurse(results_, depth_, + # recursive entry helper + def table_recurse(results_, diff_results_, + depth_, prefixes=('', '', '', '')): # build the children table at each layer - results_ = fold(Result, results_, by=by) table_ = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results_} - names_ = list(table_.keys()) + diff_table_ = { + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r + for r in diff_results_ or []} + names_ = [n + for n in table_.keys() | diff_table_.keys() + if diff_results_ is None + or all_ + or any( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)] - # sort the children layer - names_.sort() - if hasattr(Result, '_i'): - names_.sort(key=lambda n: getattr(table_[n], Result._i)) - if sort: - for k, reverse in reversed(sort): - names_.sort( - key=lambda n: tuple( - (getattr(table_[n], k),) - if getattr(table_.get(n), k, None) - is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) + # sort again, now with diff info, note that python's sort is stable + names_.sort(key=lambda n: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(table_[n], k_),) + if getattr(table_.get(n), k_, None) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by ratio if diffing + Rev(tuple(types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)) + if diff or percent + else (), + # move compare entry to the top, note this can be + # overridden by explicitly sorting by fields + (table_.get(n) != compare_r, + # sort by ratio if comparing + Rev(tuple( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(compare_r, k, None)) + for k in fields))) + if compare + else (), + # sort by result + (table_[n],) if n in table_ else (), + # and finally by name (diffs may be missing results) + n)) - for i, name in enumerate(names_): - r = table_[name] - is_last = (i == len(names_)-1) + for i, n in enumerate(names_): + # find comparable results + r = table_.get(n) + diff_r = diff_table_.get(n) + + # build line + line = table_entry(n, r, diff_r) - line = table_entry(name, r) - line = [x if isinstance(x, tuple) else (x, []) for x in line] # add prefixes - line[0] = (prefixes[0+is_last] + line[0][0], line[0][1]) + line = [x if isinstance(x, tuple) else (x, []) for x in line] + line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) lines.append(line) # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), + if n in table_ and depth_ > 1: + table_recurse( + getattr(r, Result._children), + getattr(diff_r, Result._children, None) or [], depth_-1, - (prefixes[2+is_last] + "|-> ", - prefixes[2+is_last] + "'-> ", - prefixes[2+is_last] + "| ", - prefixes[2+is_last] + " ")) + (prefixes[2+(i==len(names_)-1)] + "|-> ", + prefixes[2+(i==len(names_)-1)] + "'-> ", + prefixes[2+(i==len(names_)-1)] + "| ", + prefixes[2+(i==len(names_)-1)] + " ")) - # entries + # build entries if not summary: - for name in names: - r = table.get(name) - if diff_results is None: - diff_r = None - else: - diff_r = diff_table.get(name) - lines.append(table_entry(name, r, diff_r)) - - # recursive entries - if name in table and depth > 1: - recurse(getattr(table[name], Result._children), - depth-1, - ("|-> ", - "'-> ", - "| ", - " ")) + table_recurse(results, diff_results, depth) # total if not no_total and not (small_table and not summary): @@ -1160,9 +1194,8 @@ def table(Result, results, diff_results=None, *, lines.append(table_entry('TOTAL', r, diff_r)) # homogenize - lines = [ - [x if isinstance(x, tuple) else (x, []) for x in line] - for line in lines] + lines = [[x if isinstance(x, tuple) else (x, []) for x in line] + for line in lines] # find the best widths, note that column 0 contains the names and is # handled a bit differently @@ -1183,6 +1216,130 @@ def table(Result, results, diff_results=None, *, nwidths[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') for i, x in enumerate(line[1:], 1)))) +def read_csv(path, Result, *, + depth=1, + **_): + with openio(path, 'r') as f: + # csv or json? assume json starts with [ + json = (f.buffer.peek(1)[:1] == b'[') + + # read csv? + if not json: + results = [] + reader = csv.DictReader(f, restval='') + for r in reader: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k].strip()} + | {k: r[k] for k in Result._fields + if k in r and r[k].strip()}))) + except TypeError: + pass + return results + + # read json? + else: + import json + def unjsonify(results, depth_): + results_ = [] + for r in results: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results_.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k] is not None} + | {k: r[k] for k in Result._fields + if k in r and r[k] is not None} + | ({Result._children: unjsonify( + r[Result._children], + depth_-1)} + if hasattr(Result, '_children') + and Result._children in r + and r[Result._children] is not None + and depth_ > 1 + else {}) + | ({Result._notes: set(r[Result._notes])} + if hasattr(Result, '_notes') + and Result._notes in r + and r[Result._notes] is not None + else {})))) + except TypeError: + pass + return results_ + return unjsonify(json.load(f), depth) + +def write_csv(path, Result, results, *, + json=False, + by=None, + fields=None, + depth=1, + **_): + with openio(path, 'w') as f: + # write csv? + if not json: + writer = csv.DictWriter(f, + (by if by is not None else Result._by) + + [k for k in (fields + if fields is not None + else Result._fields)]) + writer.writeheader() + for r in results: + # note this allows by/fields to overlap + writer.writerow( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None}) + + # write json? + else: + import json + # the neat thing about json is we can include recursive results + def jsonify(results, depth_): + results_ = [] + for r in results: + # note this allows by/fields to overlap + results_.append( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None} + | ({Result._children: jsonify( + getattr(r, Result._children), + depth_-1)} + if hasattr(Result, '_children') + and getattr(r, Result._children) + and depth_ > 1 + else {}) + | ({Result._notes: list( + getattr(r, Result._notes))} + if hasattr(Result, '_notes') + and getattr(r, Result._notes) + else {})) + return results_ + json.dump(jsonify(results, depth), f, + separators=(',', ':')) + def annotate(Result, results, *, annotate=None, @@ -1290,6 +1447,8 @@ def report(perf_paths, *, sort=None, branches=False, caches=False, + depth=None, + hot=None, **args): # figure out what color should be if args.get('color') == 'auto': @@ -1300,10 +1459,10 @@ def report(perf_paths, *, args['color'] = False # figure out depth - if args.get('depth') is None: - args['depth'] = mt.inf if args.get('hot') else 1 - elif args.get('depth') == 0: - args['depth'] = mt.inf + if depth is None: + depth = mt.inf if hot else 1 + elif depth == 0: + depth = mt.inf # find sizes if not args.get('use', None): @@ -1314,87 +1473,61 @@ def report(perf_paths, *, sys.exit(1) # collect info - results = collect(perf_paths, **args) + results = collect_perf(perf_paths, + depth=depth, + **args) else: - results = [] - with openio(args['use']) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - if not any(k in r and r[k].strip() - for k in PerfResult._fields): - continue - try: - results.append(PerfResult( - **{k: r[k] for k in PerfResult._by - if k in r and r[k].strip()}, - **{k: r[k] for k in PerfResult._fields - if k in r and r[k].strip()})) - except TypeError: - pass + results = read_csv(args['use'], PerfResult, + depth=depth, + **args) # fold - results = fold(PerfResult, results, by=by, defines=defines) + results = fold(PerfResult, results, + by=by, + defines=defines, + depth=depth) - # sort, note that python's sort is stable - results.sort() - if sort: - for k, reverse in reversed(sort): - results.sort( - key=lambda r: tuple( - (getattr(r, k),) if getattr(r, k) is not None else () - for k in ([k] if k else PerfResult._sort)), - reverse=reverse ^ (not k or k in PerfResult._fields)) + # hotify? + if hot: + results = hotify(PerfResult, results, + fields=fields, + depth=depth, + hot=hot, + **args) - # write results to CSV + # write results to CSV/JSON if args.get('output'): - with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, - (by if by is not None else PerfResult._by) - + [k for k in ( - fields if fields is not None - else PerfResult._fields)]) - writer.writeheader() - for r in results: - writer.writerow( - {k: getattr(r, k) for k in ( - by if by is not None else PerfResult._by)} - | {k: getattr(r, k) for k in ( - fields if fields is not None - else PerfResult._fields)}) + write_csv(args['output'], PerfResult, results, + by=by, + fields=fields, + depth=depth, + **args) + if args.get('output_json'): + write_csv(args['output_json'], PerfResult, results, + json=True, + by=by, + fields=fields, + depth=depth, + **args) # find previous results? diff_results = None if args.get('diff') or args.get('percent'): - diff_results = [] try: - with openio(args.get('diff') or args.get('percent')) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - if not any(k in r and r[k].strip() - for k in PerfResult._fields): - continue - try: - diff_results.append(PerfResult( - **{k: r[k] for k in PerfResult._by - if k in r and r[k].strip()}, - **{k: r[k] for k in PerfResult._fields - if k in r and r[k].strip()})) - except TypeError: - pass + diff_results = read_csv( + args.get('diff') or args.get('percent'), + PerfResult, + depth=depth, + **args) except FileNotFoundError: - pass + diff_results = [] # fold - diff_results = fold(PerfResult, diff_results, by=by, defines=defines) + diff_results = fold(PerfResult, diff_results, + by=by, + defines=defines, + depth=depth) # print table if not args.get('quiet'): @@ -1413,6 +1546,7 @@ def report(perf_paths, *, else ['bmisses', 'branches'] if branches else ['cmisses', 'caches'], sort=sort, + depth=depth, **args) @@ -1430,7 +1564,7 @@ if __name__ == "__main__": # bit of a hack, but parse_intermixed_args and REMAINDER are # incompatible, so we need to figure out what we want before running # argparse - if '-R' in sys.argv or '--record' in sys.argv: + if '--record' in sys.argv: nargs = argparse.REMAINDER else: nargs = '*' @@ -1456,24 +1590,28 @@ if __name__ == "__main__": parser.add_argument( '-o', '--output', help="Specify CSV file to store results.") + parser.add_argument( + '-O', '--output-json', + help="Specify JSON file to store results. This may contain " + "recursive info.") parser.add_argument( '-u', '--use', - help="Don't parse anything, use this CSV file.") + help="Don't parse anything, use this CSV/JSON file.") parser.add_argument( '-d', '--diff', - help="Specify CSV file to diff against.") + help="Specify CSV/JSON file to diff against.") parser.add_argument( '-p', '--percent', - help="Specify CSV file to diff against, but only show precentage " - "change, not a full diff.") - parser.add_argument( - '-a', '--all', - action='store_true', - help="Show all, not just the ones that changed.") + help="Specify CSV/JSON file to diff against, but only show " + "percentage change, not a full diff.") parser.add_argument( '-c', '--compare', type=lambda x: tuple(v.strip() for v in x.split(',')), help="Compare results to the row matching this by pattern.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") parser.add_argument( '-b', '--by', action='append', @@ -1499,7 +1637,7 @@ if __name__ == "__main__": def __call__(self, parser, namespace, value, option): if namespace.sort is None: namespace.sort = [] - namespace.sort.append((value, True if option == '-S' else False)) + namespace.sort.append((value, option in {'-S', '--reverse-sort'})) parser.add_argument( '-s', '--sort', nargs='?', @@ -1510,6 +1648,34 @@ if __name__ == "__main__": nargs='?', action=AppendSort, help="Sort by this field, but backwards.") + parser.add_argument( + '-z', '--depth', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Depth of function calls to show. 0 shows all calls unless " + "we find a cycle. Defaults to 0.") + parser.add_argument( + '-g', '--propagate', + type=lambda x: int(x, 0), + help="Depth to propagate samples up the call-stack. 0 propagates " + "up to the entry point, 1 does no propagation. Defaults to 0.") + class AppendHot(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.hot is None: + namespace.hot = [] + namespace.hot.append((value, option in {'-R', '--reverse-hot'})) + parser.add_argument( + '-r', '--hot', + nargs='?', + action=AppendHot, + help="Show only the hot path for each function call. Can " + "optionally provide fields like sort.") + parser.add_argument( + '-R', '--reverse-hot', + nargs='?', + action=AppendHot, + help="Like -r/--hot, but backwards.") parser.add_argument( '--no-header', action='store_true', @@ -1548,23 +1714,6 @@ if __name__ == "__main__": '--caches', action='store_true', help="Show cache accesses and cache misses.") - parser.add_argument( - '-g', '--propagate', - type=lambda x: int(x, 0), - help="Depth to propagate samples up the call-stack. 0 propagates " - "up to the entry point, 1 does no propagation. Defaults to 0.") - parser.add_argument( - '-z', '--depth', - nargs='?', - type=lambda x: int(x, 0), - const=0, - help="Depth of function calls to show. 0 shows all calls unless " - "we find a cycle. Defaults to 0.") - parser.add_argument( - '-t', '--hot', - nargs='?', - action='append', - help="Show only the hot path for each function call.") parser.add_argument( '-A', '--annotate', action='store_true', @@ -1618,7 +1767,7 @@ if __name__ == "__main__": nargs=nargs, help="Command to run.") record_parser.add_argument( - '-R', '--record', + '--record', action='store_true', help="Run a command and aggregate perf measurements.") record_parser.add_argument( diff --git a/scripts/perfbd.py b/scripts/perfbd.py index 7f471a41..05ead106 100755 --- a/scripts/perfbd.py +++ b/scripts/perfbd.py @@ -19,6 +19,7 @@ import bisect import collections as co import csv import functools as ft +import io import itertools as it import math as mt import multiprocessing as mp @@ -137,25 +138,26 @@ class RInt(co.namedtuple('RInt', 'x')): # perf results class PerfBdResult(co.namedtuple('PerfBdResult', [ - 'file', 'function', 'line', + 'i', 'file', 'function', 'line', 'readed', 'proged', 'erased', 'children'])): - _by = ['file', 'function', 'line'] + _by = ['i', 'file', 'function', 'line'] _fields = ['readed', 'proged', 'erased'] _sort = ['erased', 'proged', 'readed'] _types = {'readed': RInt, 'proged': RInt, 'erased': RInt} + _i = 'i' _children = 'children' __slots__ = () - def __new__(cls, file='', function='', line=0, + def __new__(cls, i=None, file='', function='', line=0, readed=0, proged=0, erased=0, children=None): - return super().__new__(cls, file, function, int(RInt(line)), + return super().__new__(cls, i, file, function, int(RInt(line)), RInt(readed), RInt(proged), RInt(erased), children if children is not None else []) def __add__(self, other): - return PerfBdResult(self.file, self.function, self.line, + return PerfBdResult(self.i, self.file, self.function, self.line, self.readed + other.readed, self.proged + other.proged, self.erased + other.erased, @@ -716,7 +718,7 @@ def collect_job(path, start, stop, syms, lines, *, def to_results(results): results_ = [] for name, (r, p, e, children) in results.items(): - results_.append(PerfBdResult(*name, + results_.append(PerfBdResult(None, *name, r, p, e, children=to_results(children))) return results_ @@ -727,7 +729,7 @@ def starapply(args): f, args, kwargs = args return f(*args, **kwargs) -def collect(elf_path, trace_paths, *, +def collect_perfbd(elf_path, trace_paths, *, jobs=None, **args): # automatic job detection? @@ -781,7 +783,31 @@ def collect(elf_path, trace_paths, *, return results -def fold(Result, results, by=None, defines=[]): +# common folding/tabling/read/write code + +class Rev(co.namedtuple('Rev', 'x')): + __slots__ = () + # yes we need all of these because we're a namedtuple + def __lt__(self, other): + return self.x > other.x + def __gt__(self, other): + return self.x < other.x + def __le__(self, other): + return self.x >= other.x + def __ge__(self, other): + return self.x <= other.x + +def fold(Result, results, *, + by=None, + defines=[], + sort=None, + depth=1, + **_): + # stop when depth hits zero + if depth == 0: + return [] + + # organize by by if by is None: by = Result._by @@ -795,7 +821,7 @@ def fold(Result, results, by=None, defines=[]): if defines: results_ = [] for r in results: - if all(getattr(r, k) in vs for k, vs in defines): + if all(str(getattr(r, k)) in vs for k, vs in defines): results_.append(r) results = results_ @@ -812,8 +838,80 @@ def fold(Result, results, by=None, defines=[]): for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) + # sort, note that python's sort is stable + folded.sort(key=lambda r: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(r, k_),) + if getattr(r, k_) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by result + r)) + + # recurse if we have recursive results + if hasattr(Result, '_children'): + folded = [r._replace(**{ + Result._children: fold( + Result, getattr(r, Result._children), + by=by, + defines=defines, + sort=sort, + depth=depth-1)}) + for r in folded] + return folded +def hotify(Result, results, *, + fields=None, + sort=None, + depth=1, + hot=None, + **_): + # hotify only makes sense for recursive results + assert hasattr(Result, '_i') + assert hasattr(Result, '_children') + + if fields is None: + fields = Result._fields + + results_ = [] + for r in results: + hot_ = [] + def recurse(results_, depth_): + nonlocal hot_ + if not results_: + return + + # find the hottest result + r = min(results_, key=lambda r: + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(r, k_),) + if getattr(r, k_) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in it.chain(hot, [(None, False)]))) + + hot_.append(r._replace(**{ + Result._i: RInt(len(hot_)), + Result._children: []})) + + # recurse? + if depth_ > 1: + recurse(getattr(r, Result._children), + depth_-1) + + recurse(getattr(r, Result._children), depth-1) + results_.append(r._replace(**{ + Result._children: hot_})) + + return results_ + def table(Result, results, diff_results=None, *, by=None, fields=None, @@ -839,124 +937,32 @@ def table(Result, results, diff_results=None, *, fields = Result._fields types = Result._types - # fold again - results = fold(Result, results, by=by) + # fold again, otherwise results risk being hidden + results = fold(Result, results, + by=by, + depth=depth) if diff_results is not None: - diff_results = fold(Result, diff_results, by=by) - - # reduce children to hot paths? only used by some scripts - if hot: - # subclass to reintroduce __dict__ - Result_ = Result - class HotResult(Result_): - _i = '_hot_i' - _children = '_hot_children' - - def __new__(cls, r, i=None, children=None, notes=None): - self = HotResult._make(r) - self._hot_i = i - self._hot_children = children if children is not None else [] - return self - - def __add__(self, other): - return HotResult( - Result_.__add__(self, other), - self._hot_i if other._hot_i is None - else other._hot_i if self._hot_i is None - else min(self._hot_i, other._hot_i), - self._hot_children + other._hot_children) - - results_ = [] - for r in results: - hot_ = [] - def recurse(results_, depth_): - nonlocal hot_ - if not results_: - return - - # find the hottest result - r = max(results_, - key=lambda r: tuple( - tuple((getattr(r, k),) - if getattr(r, k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields]) - if k in fields) - for k in it.chain(hot, [None]))) - hot_.append(HotResult(r, i=len(hot_))) - - # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), - depth_-1) - - recurse(getattr(r, Result._children), depth-1) - results_.append(HotResult(r, children=hot_)) - - Result = HotResult - results = results_ + diff_results = fold(Result, diff_results, + by=by, + depth=depth) # organize by name table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results} diff_table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in diff_results or []} - names = [name - for name in table.keys() | diff_table.keys() - if diff_results is None - or all_ - or any( - types[k].ratio( - getattr(table.get(name), k, None), - getattr(diff_table.get(name), k, None)) - for k in fields)] # find compare entry if there is one if compare: - compare_result = table.get(','.join(str(k) for k in compare)) - - # sort again, now with diff info, note that python's sort is stable - names.sort() - if compare: - names.sort( - key=lambda n: ( - # move compare entry to the top, note this can be - # overridden by explicitly sorting by fields - table.get(n) == compare_result, - # sort by ratio if comparing - tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(compare_result, k, None)) - for k in fields)), - reverse=True) - if diff or percent: - names.sort( - # sort by ratio if diffing - key=lambda n: tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(diff_table.get(n), k, None)) - for k in fields), - reverse=True) - if sort: - for k, reverse in reversed(sort): - names.sort( - key=lambda n: tuple( - (getattr(table[n], k),) - if getattr(table.get(n), k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) - + compare_r = table.get(','.join(str(k) for k in compare)) # build up our lines lines = [] @@ -983,11 +989,16 @@ def table(Result, results, diff_results=None, *, header.append('d'+k) lines.append(header) + # delete these to try to catch typos below, we need to rebuild + # these tables at each recursive layer + del table + del diff_table + # entry helper def table_entry(name, r, diff_r=None): entry = [name] # normal entry? - if ((compare is None or r == compare_result) + if ((compare is None or r == compare_r) and not percent and not diff): for k in fields: @@ -1008,7 +1019,7 @@ def table(Result, results, diff_results=None, *, else ['%+.1f%%' % (100*t)])( types[k].ratio( getattr(r, k, None), - getattr(compare_result, k, None))))) + getattr(compare_r, k, None))))) # percent entry? elif not diff: for k in fields: @@ -1054,71 +1065,94 @@ def table(Result, results, diff_results=None, *, return entry - # recursive entry helper, only used by some scripts - def recurse(results_, depth_, + # recursive entry helper + def table_recurse(results_, diff_results_, + depth_, prefixes=('', '', '', '')): # build the children table at each layer - results_ = fold(Result, results_, by=by) table_ = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results_} - names_ = list(table_.keys()) + diff_table_ = { + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r + for r in diff_results_ or []} + names_ = [n + for n in table_.keys() | diff_table_.keys() + if diff_results_ is None + or all_ + or any( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)] - # sort the children layer - names_.sort() - if hasattr(Result, '_i'): - names_.sort(key=lambda n: getattr(table_[n], Result._i)) - if sort: - for k, reverse in reversed(sort): - names_.sort( - key=lambda n: tuple( - (getattr(table_[n], k),) - if getattr(table_.get(n), k, None) - is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) + # sort again, now with diff info, note that python's sort is stable + names_.sort(key=lambda n: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(table_[n], k_),) + if getattr(table_.get(n), k_, None) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by ratio if diffing + Rev(tuple(types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)) + if diff or percent + else (), + # move compare entry to the top, note this can be + # overridden by explicitly sorting by fields + (table_.get(n) != compare_r, + # sort by ratio if comparing + Rev(tuple( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(compare_r, k, None)) + for k in fields))) + if compare + else (), + # sort by result + (table_[n],) if n in table_ else (), + # and finally by name (diffs may be missing results) + n)) - for i, name in enumerate(names_): - r = table_[name] - is_last = (i == len(names_)-1) + for i, n in enumerate(names_): + # find comparable results + r = table_.get(n) + diff_r = diff_table_.get(n) + + # build line + line = table_entry(n, r, diff_r) - line = table_entry(name, r) - line = [x if isinstance(x, tuple) else (x, []) for x in line] # add prefixes - line[0] = (prefixes[0+is_last] + line[0][0], line[0][1]) + line = [x if isinstance(x, tuple) else (x, []) for x in line] + line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) lines.append(line) # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), + if n in table_ and depth_ > 1: + table_recurse( + getattr(r, Result._children), + getattr(diff_r, Result._children, None) or [], depth_-1, - (prefixes[2+is_last] + "|-> ", - prefixes[2+is_last] + "'-> ", - prefixes[2+is_last] + "| ", - prefixes[2+is_last] + " ")) + (prefixes[2+(i==len(names_)-1)] + "|-> ", + prefixes[2+(i==len(names_)-1)] + "'-> ", + prefixes[2+(i==len(names_)-1)] + "| ", + prefixes[2+(i==len(names_)-1)] + " ")) - # entries + # build entries if not summary: - for name in names: - r = table.get(name) - if diff_results is None: - diff_r = None - else: - diff_r = diff_table.get(name) - lines.append(table_entry(name, r, diff_r)) - - # recursive entries - if name in table and depth > 1: - recurse(getattr(table[name], Result._children), - depth-1, - ("|-> ", - "'-> ", - "| ", - " ")) + table_recurse(results, diff_results, depth) # total if not no_total and not (small_table and not summary): @@ -1130,9 +1164,8 @@ def table(Result, results, diff_results=None, *, lines.append(table_entry('TOTAL', r, diff_r)) # homogenize - lines = [ - [x if isinstance(x, tuple) else (x, []) for x in line] - for line in lines] + lines = [[x if isinstance(x, tuple) else (x, []) for x in line] + for line in lines] # find the best widths, note that column 0 contains the names and is # handled a bit differently @@ -1153,6 +1186,130 @@ def table(Result, results, diff_results=None, *, nwidths[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') for i, x in enumerate(line[1:], 1)))) +def read_csv(path, Result, *, + depth=1, + **_): + with openio(path, 'r') as f: + # csv or json? assume json starts with [ + json = (f.buffer.peek(1)[:1] == b'[') + + # read csv? + if not json: + results = [] + reader = csv.DictReader(f, restval='') + for r in reader: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k].strip()} + | {k: r[k] for k in Result._fields + if k in r and r[k].strip()}))) + except TypeError: + pass + return results + + # read json? + else: + import json + def unjsonify(results, depth_): + results_ = [] + for r in results: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results_.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k] is not None} + | {k: r[k] for k in Result._fields + if k in r and r[k] is not None} + | ({Result._children: unjsonify( + r[Result._children], + depth_-1)} + if hasattr(Result, '_children') + and Result._children in r + and r[Result._children] is not None + and depth_ > 1 + else {}) + | ({Result._notes: set(r[Result._notes])} + if hasattr(Result, '_notes') + and Result._notes in r + and r[Result._notes] is not None + else {})))) + except TypeError: + pass + return results_ + return unjsonify(json.load(f), depth) + +def write_csv(path, Result, results, *, + json=False, + by=None, + fields=None, + depth=1, + **_): + with openio(path, 'w') as f: + # write csv? + if not json: + writer = csv.DictWriter(f, + (by if by is not None else Result._by) + + [k for k in (fields + if fields is not None + else Result._fields)]) + writer.writeheader() + for r in results: + # note this allows by/fields to overlap + writer.writerow( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None}) + + # write json? + else: + import json + # the neat thing about json is we can include recursive results + def jsonify(results, depth_): + results_ = [] + for r in results: + # note this allows by/fields to overlap + results_.append( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None} + | ({Result._children: jsonify( + getattr(r, Result._children), + depth_-1)} + if hasattr(Result, '_children') + and getattr(r, Result._children) + and depth_ > 1 + else {}) + | ({Result._notes: list( + getattr(r, Result._notes))} + if hasattr(Result, '_notes') + and getattr(r, Result._notes) + else {})) + return results_ + json.dump(jsonify(results, depth), f, + separators=(',', ':')) + def annotate(Result, results, *, annotate=None, @@ -1272,6 +1429,8 @@ def report(paths, *, fields=None, defines=[], sort=None, + depth=None, + hot=None, **args): # figure out what color should be if args.get('color') == 'auto': @@ -1282,10 +1441,10 @@ def report(paths, *, args['color'] = False # figure out depth - if args.get('depth') is None: - args['depth'] = mt.inf if args.get('hot') else 1 - elif args.get('depth') == 0: - args['depth'] = mt.inf + if depth is None: + depth = mt.inf if hot else 1 + elif depth == 0: + depth = mt.inf # find sizes if not args.get('use', None): @@ -1313,87 +1472,61 @@ def report(paths, *, sys.exit(1) # collect info - results = collect(elf_paths[0], trace_paths, **args) + results = collect_perfbd(elf_paths[0], trace_paths, + depth=depth, + **args) else: - results = [] - with openio(args['use']) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - if not any(k in r and r[k].strip() - for k in PerfBdResult._fields): - continue - try: - results.append(PerfBdResult( - **{k: r[k] for k in PerfBdResult._by - if k in r and r[k].strip()}, - **{k: r[k] for k in PerfBdResult._fields - if k in r and r[k].strip()})) - except TypeError: - pass + results = read_csv(args['use'], PerfBdResult, + depth=depth, + **args) # fold - results = fold(PerfBdResult, results, by=by, defines=defines) + results = fold(PerfBdResult, results, + by=by, + defines=defines, + depth=depth) - # sort, note that python's sort is stable - results.sort() - if sort: - for k, reverse in reversed(sort): - results.sort( - key=lambda r: tuple( - (getattr(r, k),) if getattr(r, k) is not None else () - for k in ([k] if k else PerfBdResult._sort)), - reverse=reverse ^ (not k or k in PerfBdResult._fields)) + # hotify? + if hot: + results = hotify(PerfBdResult, results, + fields=fields, + depth=depth, + hot=hot, + **args) - # write results to CSV + # write results to CSV/JSON if args.get('output'): - with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, - (by if by is not None else PerfBdResult._by) - + [k for k in ( - fields if fields is not None - else PerfBdResult._fields)]) - writer.writeheader() - for r in results: - writer.writerow( - {k: getattr(r, k) for k in ( - by if by is not None else PerfBdResult._by)} - | {k: getattr(r, k) for k in ( - fields if fields is not None - else PerfBdResult._fields)}) + write_csv(args['output'], PerfBdResult, results, + by=by, + fields=fields, + depth=depth, + **args) + if args.get('output_json'): + write_csv(args['output_json'], PerfBdResult, results, + json=True, + by=by, + fields=fields, + depth=depth, + **args) # find previous results? diff_results = None if args.get('diff') or args.get('percent'): - diff_results = [] try: - with openio(args.get('diff') or args.get('percent')) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - if not any(k in r and r[k].strip() - for k in PerfBdResult._fields): - continue - try: - diff_results.append(PerfBdResult( - **{k: r[k] for k in PerfBdResult._by - if k in r and r[k].strip()}, - **{k: r[k] for k in PerfBdResult._fields - if k in r and r[k].strip()})) - except TypeError: - pass + diff_results = read_csv( + args.get('diff') or args.get('percent'), + PerfBdResult, + depth=depth, + **args) except FileNotFoundError: - pass + diff_results = [] # fold - diff_results = fold(PerfBdResult, diff_results, by=by, defines=defines) + diff_results = fold(PerfBdResult, diff_results, + by=by, + defines=defines, + depth=depth) # print table if not args.get('quiet'): @@ -1410,6 +1543,7 @@ def report(paths, *, by=by if by is not None else ['function'], fields=fields, sort=sort, + depth=depth, **args) @@ -1458,24 +1592,28 @@ if __name__ == "__main__": parser.add_argument( '-o', '--output', help="Specify CSV file to store results.") + parser.add_argument( + '-O', '--output-json', + help="Specify JSON file to store results. This may contain " + "recursive info.") parser.add_argument( '-u', '--use', - help="Don't parse anything, use this CSV file.") + help="Don't parse anything, use this CSV/JSON file.") parser.add_argument( '-d', '--diff', - help="Specify CSV file to diff against.") + help="Specify CSV/JSON file to diff against.") parser.add_argument( '-p', '--percent', - help="Specify CSV file to diff against, but only show precentage " - "change, not a full diff.") - parser.add_argument( - '-a', '--all', - action='store_true', - help="Show all, not just the ones that changed.") + help="Specify CSV/JSON file to diff against, but only show " + "percentage change, not a full diff.") parser.add_argument( '-c', '--compare', type=lambda x: tuple(v.strip() for v in x.split(',')), help="Compare results to the row matching this by pattern.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") parser.add_argument( '-b', '--by', action='append', @@ -1501,7 +1639,7 @@ if __name__ == "__main__": def __call__(self, parser, namespace, value, option): if namespace.sort is None: namespace.sort = [] - namespace.sort.append((value, True if option == '-S' else False)) + namespace.sort.append((value, option in {'-S', '--reverse-sort'})) parser.add_argument( '-s', '--sort', nargs='?', @@ -1512,6 +1650,34 @@ if __name__ == "__main__": nargs='?', action=AppendSort, help="Sort by this field, but backwards.") + parser.add_argument( + '-z', '--depth', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Depth of function calls to show. 0 shows all calls unless " + "we find a cycle. Defaults to 0.") + parser.add_argument( + '-g', '--propagate', + type=lambda x: int(x, 0), + help="Depth to propagate samples up the call-stack. 0 propagates " + "up to the entry point, 1 does no propagation. Defaults to 0.") + class AppendHot(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.hot is None: + namespace.hot = [] + namespace.hot.append((value, option in {'-R', '--reverse-hot'})) + parser.add_argument( + '-r', '--hot', + nargs='?', + action=AppendHot, + help="Show only the hot path for each function call. Can " + "optionally provide fields like sort.") + parser.add_argument( + '-R', '--reverse-hot', + nargs='?', + action=AppendHot, + help="Like -r/--hot, but backwards.") parser.add_argument( '--no-header', action='store_true', @@ -1542,23 +1708,6 @@ if __name__ == "__main__": '--everything', action='store_true', help="Include builtin and libc specific symbols.") - parser.add_argument( - '-g', '--propagate', - type=lambda x: int(x, 0), - help="Depth to propagate samples up the call-stack. 0 propagates " - "up to the entry point, 1 does no propagation. Defaults to 0.") - parser.add_argument( - '-z', '--depth', - nargs='?', - type=lambda x: int(x, 0), - const=0, - help="Depth of function calls to show. 0 shows all calls unless " - "we find a cycle. Defaults to 0.") - parser.add_argument( - '-t', '--hot', - nargs='?', - action='append', - help="Show only the hot path for each function call.") parser.add_argument( '-A', '--annotate', action='store_true', diff --git a/scripts/stack.py b/scripts/stack.py index a16c23a6..d5fc0183 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -16,8 +16,9 @@ if __name__ == "__main__": import collections as co import csv -import itertools as it import functools as ft +import io +import itertools as it import math as mt import os import re @@ -130,28 +131,29 @@ class RInt(co.namedtuple('RInt', 'x')): def __mod__(self, other): return self.__class__(self.x % other.x) -# size results +# stack size results class StackResult(co.namedtuple('StackResult', [ - 'file', 'function', + 'i', 'file', 'function', 'frame', 'limit', 'children', 'notes'])): - _by = ['file', 'function'] + _by = ['i', 'file', 'function'] _fields = ['frame', 'limit'] _sort = ['limit', 'frame'] _types = {'frame': RInt, 'limit': RInt} + _i = 'i' _children = 'children' _notes = 'notes' __slots__ = () - def __new__(cls, file='', function='', frame=0, limit=0, + def __new__(cls, i=None, file='', function='', frame=0, limit=0, children=None, notes=None): - return super().__new__(cls, file, function, + return super().__new__(cls, i, file, function, RInt(frame), RInt(limit), children if children is not None else [], notes if notes is not None else set()) def __add__(self, other): - return StackResult(self.file, self.function, + return StackResult(self.i, self.file, self.function, self.frame + other.frame, max(self.limit, other.limit), self.children + other.children, @@ -319,7 +321,7 @@ def collect_callgraph(ci_path, return cg_ -def collect(ci_paths, *, +def collect_stack(ci_paths, *, everything=False, depth=1, **args): @@ -415,7 +417,7 @@ def collect(ci_paths, *, limit_ = limitof(node_, seen | {node.name}) children_, notes_, dirty_ = childrenof( node_, depth-1, seen | {node.name}) - children.append(StackResult(file_, name_, frame_, limit_, + children.append(StackResult(None, file_, name_, frame_, limit_, children=children_, notes=notes_)) dirty = dirty or dirty_ @@ -438,14 +440,38 @@ def collect(ci_paths, *, frame = frameof(node) limit = limitof(node) children, notes, _ = childrenof(node, depth-1) - results.append(StackResult(file, name, frame, limit, + results.append(StackResult(None, file, name, frame, limit, children=children, notes=notes)) return results -def fold(Result, results, by=None, defines=[]): +# common folding/tabling/read/write code + +class Rev(co.namedtuple('Rev', 'x')): + __slots__ = () + # yes we need all of these because we're a namedtuple + def __lt__(self, other): + return self.x > other.x + def __gt__(self, other): + return self.x < other.x + def __le__(self, other): + return self.x >= other.x + def __ge__(self, other): + return self.x <= other.x + +def fold(Result, results, *, + by=None, + defines=[], + sort=None, + depth=1, + **_): + # stop when depth hits zero + if depth == 0: + return [] + + # organize by by if by is None: by = Result._by @@ -459,7 +485,7 @@ def fold(Result, results, by=None, defines=[]): if defines: results_ = [] for r in results: - if all(getattr(r, k) in vs for k, vs in defines): + if all(str(getattr(r, k)) in vs for k, vs in defines): results_.append(r) results = results_ @@ -476,8 +502,80 @@ def fold(Result, results, by=None, defines=[]): for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) + # sort, note that python's sort is stable + folded.sort(key=lambda r: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(r, k_),) + if getattr(r, k_) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by result + r)) + + # recurse if we have recursive results + if hasattr(Result, '_children'): + folded = [r._replace(**{ + Result._children: fold( + Result, getattr(r, Result._children), + by=by, + defines=defines, + sort=sort, + depth=depth-1)}) + for r in folded] + return folded +def hotify(Result, results, *, + fields=None, + sort=None, + depth=1, + hot=None, + **_): + # hotify only makes sense for recursive results + assert hasattr(Result, '_i') + assert hasattr(Result, '_children') + + if fields is None: + fields = Result._fields + + results_ = [] + for r in results: + hot_ = [] + def recurse(results_, depth_): + nonlocal hot_ + if not results_: + return + + # find the hottest result + r = min(results_, key=lambda r: + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(r, k_),) + if getattr(r, k_) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in it.chain(hot, [(None, False)]))) + + hot_.append(r._replace(**{ + Result._i: RInt(len(hot_)), + Result._children: []})) + + # recurse? + if depth_ > 1: + recurse(getattr(r, Result._children), + depth_-1) + + recurse(getattr(r, Result._children), depth-1) + results_.append(r._replace(**{ + Result._children: hot_})) + + return results_ + def table(Result, results, diff_results=None, *, by=None, fields=None, @@ -503,124 +601,32 @@ def table(Result, results, diff_results=None, *, fields = Result._fields types = Result._types - # fold again - results = fold(Result, results, by=by) + # fold again, otherwise results risk being hidden + results = fold(Result, results, + by=by, + depth=depth) if diff_results is not None: - diff_results = fold(Result, diff_results, by=by) - - # reduce children to hot paths? only used by some scripts - if hot: - # subclass to reintroduce __dict__ - Result_ = Result - class HotResult(Result_): - _i = '_hot_i' - _children = '_hot_children' - - def __new__(cls, r, i=None, children=None, notes=None): - self = HotResult._make(r) - self._hot_i = i - self._hot_children = children if children is not None else [] - return self - - def __add__(self, other): - return HotResult( - Result_.__add__(self, other), - self._hot_i if other._hot_i is None - else other._hot_i if self._hot_i is None - else min(self._hot_i, other._hot_i), - self._hot_children + other._hot_children) - - results_ = [] - for r in results: - hot_ = [] - def recurse(results_, depth_): - nonlocal hot_ - if not results_: - return - - # find the hottest result - r = max(results_, - key=lambda r: tuple( - tuple((getattr(r, k),) - if getattr(r, k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields]) - if k in fields) - for k in it.chain(hot, [None]))) - hot_.append(HotResult(r, i=len(hot_))) - - # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), - depth_-1) - - recurse(getattr(r, Result._children), depth-1) - results_.append(HotResult(r, children=hot_)) - - Result = HotResult - results = results_ + diff_results = fold(Result, diff_results, + by=by, + depth=depth) # organize by name table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results} diff_table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in diff_results or []} - names = [name - for name in table.keys() | diff_table.keys() - if diff_results is None - or all_ - or any( - types[k].ratio( - getattr(table.get(name), k, None), - getattr(diff_table.get(name), k, None)) - for k in fields)] # find compare entry if there is one if compare: - compare_result = table.get(','.join(str(k) for k in compare)) - - # sort again, now with diff info, note that python's sort is stable - names.sort() - if compare: - names.sort( - key=lambda n: ( - # move compare entry to the top, note this can be - # overridden by explicitly sorting by fields - table.get(n) == compare_result, - # sort by ratio if comparing - tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(compare_result, k, None)) - for k in fields)), - reverse=True) - if diff or percent: - names.sort( - # sort by ratio if diffing - key=lambda n: tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(diff_table.get(n), k, None)) - for k in fields), - reverse=True) - if sort: - for k, reverse in reversed(sort): - names.sort( - key=lambda n: tuple( - (getattr(table[n], k),) - if getattr(table.get(n), k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) - + compare_r = table.get(','.join(str(k) for k in compare)) # build up our lines lines = [] @@ -647,11 +653,16 @@ def table(Result, results, diff_results=None, *, header.append('d'+k) lines.append(header) + # delete these to try to catch typos below, we need to rebuild + # these tables at each recursive layer + del table + del diff_table + # entry helper def table_entry(name, r, diff_r=None): entry = [name] # normal entry? - if ((compare is None or r == compare_result) + if ((compare is None or r == compare_r) and not percent and not diff): for k in fields: @@ -672,7 +683,7 @@ def table(Result, results, diff_results=None, *, else ['%+.1f%%' % (100*t)])( types[k].ratio( getattr(r, k, None), - getattr(compare_result, k, None))))) + getattr(compare_r, k, None))))) # percent entry? elif not diff: for k in fields: @@ -718,71 +729,94 @@ def table(Result, results, diff_results=None, *, return entry - # recursive entry helper, only used by some scripts - def recurse(results_, depth_, + # recursive entry helper + def table_recurse(results_, diff_results_, + depth_, prefixes=('', '', '', '')): # build the children table at each layer - results_ = fold(Result, results_, by=by) table_ = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results_} - names_ = list(table_.keys()) + diff_table_ = { + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r + for r in diff_results_ or []} + names_ = [n + for n in table_.keys() | diff_table_.keys() + if diff_results_ is None + or all_ + or any( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)] - # sort the children layer - names_.sort() - if hasattr(Result, '_i'): - names_.sort(key=lambda n: getattr(table_[n], Result._i)) - if sort: - for k, reverse in reversed(sort): - names_.sort( - key=lambda n: tuple( - (getattr(table_[n], k),) - if getattr(table_.get(n), k, None) - is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) + # sort again, now with diff info, note that python's sort is stable + names_.sort(key=lambda n: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(table_[n], k_),) + if getattr(table_.get(n), k_, None) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by ratio if diffing + Rev(tuple(types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)) + if diff or percent + else (), + # move compare entry to the top, note this can be + # overridden by explicitly sorting by fields + (table_.get(n) != compare_r, + # sort by ratio if comparing + Rev(tuple( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(compare_r, k, None)) + for k in fields))) + if compare + else (), + # sort by result + (table_[n],) if n in table_ else (), + # and finally by name (diffs may be missing results) + n)) - for i, name in enumerate(names_): - r = table_[name] - is_last = (i == len(names_)-1) + for i, n in enumerate(names_): + # find comparable results + r = table_.get(n) + diff_r = diff_table_.get(n) + + # build line + line = table_entry(n, r, diff_r) - line = table_entry(name, r) - line = [x if isinstance(x, tuple) else (x, []) for x in line] # add prefixes - line[0] = (prefixes[0+is_last] + line[0][0], line[0][1]) + line = [x if isinstance(x, tuple) else (x, []) for x in line] + line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) lines.append(line) # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), + if n in table_ and depth_ > 1: + table_recurse( + getattr(r, Result._children), + getattr(diff_r, Result._children, None) or [], depth_-1, - (prefixes[2+is_last] + "|-> ", - prefixes[2+is_last] + "'-> ", - prefixes[2+is_last] + "| ", - prefixes[2+is_last] + " ")) + (prefixes[2+(i==len(names_)-1)] + "|-> ", + prefixes[2+(i==len(names_)-1)] + "'-> ", + prefixes[2+(i==len(names_)-1)] + "| ", + prefixes[2+(i==len(names_)-1)] + " ")) - # entries + # build entries if not summary: - for name in names: - r = table.get(name) - if diff_results is None: - diff_r = None - else: - diff_r = diff_table.get(name) - lines.append(table_entry(name, r, diff_r)) - - # recursive entries - if name in table and depth > 1: - recurse(getattr(table[name], Result._children), - depth-1, - ("|-> ", - "'-> ", - "| ", - " ")) + table_recurse(results, diff_results, depth) # total if not no_total and not (small_table and not summary): @@ -794,9 +828,8 @@ def table(Result, results, diff_results=None, *, lines.append(table_entry('TOTAL', r, diff_r)) # homogenize - lines = [ - [x if isinstance(x, tuple) else (x, []) for x in line] - for line in lines] + lines = [[x if isinstance(x, tuple) else (x, []) for x in line] + for line in lines] # find the best widths, note that column 0 contains the names and is # handled a bit differently @@ -817,18 +850,144 @@ def table(Result, results, diff_results=None, *, nwidths[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') for i, x in enumerate(line[1:], 1)))) +def read_csv(path, Result, *, + depth=1, + **_): + with openio(path, 'r') as f: + # csv or json? assume json starts with [ + json = (f.buffer.peek(1)[:1] == b'[') + + # read csv? + if not json: + results = [] + reader = csv.DictReader(f, restval='') + for r in reader: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k].strip()} + | {k: r[k] for k in Result._fields + if k in r and r[k].strip()}))) + except TypeError: + pass + return results + + # read json? + else: + import json + def unjsonify(results, depth_): + results_ = [] + for r in results: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results_.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k] is not None} + | {k: r[k] for k in Result._fields + if k in r and r[k] is not None} + | ({Result._children: unjsonify( + r[Result._children], + depth_-1)} + if hasattr(Result, '_children') + and Result._children in r + and r[Result._children] is not None + and depth_ > 1 + else {}) + | ({Result._notes: set(r[Result._notes])} + if hasattr(Result, '_notes') + and Result._notes in r + and r[Result._notes] is not None + else {})))) + except TypeError: + pass + return results_ + return unjsonify(json.load(f), depth) + +def write_csv(path, Result, results, *, + json=False, + by=None, + fields=None, + depth=1, + **_): + with openio(path, 'w') as f: + # write csv? + if not json: + writer = csv.DictWriter(f, + (by if by is not None else Result._by) + + [k for k in (fields + if fields is not None + else Result._fields)]) + writer.writeheader() + for r in results: + # note this allows by/fields to overlap + writer.writerow( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None}) + + # write json? + else: + import json + # the neat thing about json is we can include recursive results + def jsonify(results, depth_): + results_ = [] + for r in results: + # note this allows by/fields to overlap + results_.append( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None} + | ({Result._children: jsonify( + getattr(r, Result._children), + depth_-1)} + if hasattr(Result, '_children') + and getattr(r, Result._children) + and depth_ > 1 + else {}) + | ({Result._notes: list( + getattr(r, Result._notes))} + if hasattr(Result, '_notes') + and getattr(r, Result._notes) + else {})) + return results_ + json.dump(jsonify(results, depth), f, + separators=(',', ':')) + def main(ci_paths, by=None, fields=None, defines=[], sort=None, + depth=None, + hot=None, **args): # figure out depth - if args.get('depth') is None: - args['depth'] = mt.inf if args.get('hot') else 1 - elif args.get('depth') == 0: - args['depth'] = mt.inf + if depth is None: + depth = mt.inf if hot else 1 + elif depth == 0: + depth = mt.inf # find sizes if not args.get('use', None): @@ -839,87 +998,61 @@ def main(ci_paths, sys.exit(1) # collect info - results = collect(ci_paths, **args) + results = collect_stack(ci_paths, + depth=depth, + **args) else: - results = [] - with openio(args['use']) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - if not any(k in r and r[k].strip() - for k in StackResult._fields): - continue - try: - results.append(StackResult( - **{k: r[k] for k in StackResult._by - if k in r and r[k].strip()}, - **{k: r[k] for k in StackResult._fields - if k in r and r[k].strip()})) - except TypeError: - pass + results = read_csv(args['use'], StackResult, + depth=depth, + **args) # fold - results = fold(StackResult, results, by=by, defines=defines) + results = fold(StackResult, results, + by=by, + defines=defines, + depth=depth) - # sort, note that python's sort is stable - results.sort() - if sort: - for k, reverse in reversed(sort): - results.sort( - key=lambda r: tuple( - (getattr(r, k),) if getattr(r, k) is not None else () - for k in ([k] if k else StackResult._sort)), - reverse=reverse ^ (not k or k in StackResult._fields)) + # hotify? + if hot: + results = hotify(StackResult, results, + fields=fields, + depth=depth, + hot=hot, + **args) - # write results to CSV + # write results to CSV/JSON if args.get('output'): - with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, - (by if by is not None else StackResult._by) - + [k for k in ( - fields if fields is not None - else StackResult._fields)]) - writer.writeheader() - for r in results: - writer.writerow( - {k: getattr(r, k) for k in ( - by if by is not None else StackResult._by)} - | {k: getattr(r, k) for k in ( - fields if fields is not None - else StackResult._fields)}) + write_csv(args['output'], StackResult, results, + by=by, + fields=fields, + depth=depth, + **args) + if args.get('output_json'): + write_csv(args['output_json'], StackResult, results, + json=True, + by=by, + fields=fields, + depth=depth, + **args) # find previous results? diff_results = None if args.get('diff') or args.get('percent'): - diff_results = [] try: - with openio(args.get('diff') or args.get('percent')) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - if not any(k in r and r[k].strip() - for k in StackResult._fields): - continue - try: - diff_results.append(StackResult( - **{k: r[k] for k in StackResult._by - if k in r and r[k].strip()}, - **{k: r[k] for k in StackResult._fields - if k in r and r[k].strip()})) - except TypeError: - raise + diff_results = read_csv( + args.get('diff') or args.get('percent'), + StackResult, + depth=depth, + **args) except FileNotFoundError: - pass + diff_results = [] # fold - diff_results = fold(StackResult, diff_results, by=by, defines=defines) + diff_results = fold(StackResult, diff_results, + by=by, + defines=defines, + depth=depth) # print table if not args.get('quiet'): @@ -927,6 +1060,7 @@ def main(ci_paths, by=by if by is not None else ['function'], fields=fields, sort=sort, + depth=depth, **args) # error on recursion @@ -956,24 +1090,28 @@ if __name__ == "__main__": parser.add_argument( '-o', '--output', help="Specify CSV file to store results.") + parser.add_argument( + '-O', '--output-json', + help="Specify JSON file to store results. This may contain " + "recursive info.") parser.add_argument( '-u', '--use', - help="Don't parse anything, use this CSV file.") + help="Don't parse anything, use this CSV/JSON file.") parser.add_argument( '-d', '--diff', - help="Specify CSV file to diff against.") + help="Specify CSV/JSON file to diff against.") parser.add_argument( '-p', '--percent', - help="Specify CSV file to diff against, but only show precentage " - "change, not a full diff.") - parser.add_argument( - '-a', '--all', - action='store_true', - help="Show all, not just the ones that changed.") + help="Specify CSV/JSON file to diff against, but only show " + "percentage change, not a full diff.") parser.add_argument( '-c', '--compare', type=lambda x: tuple(v.strip() for v in x.split(',')), help="Compare results to the row matching this by pattern.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") parser.add_argument( '-b', '--by', action='append', @@ -999,7 +1137,7 @@ if __name__ == "__main__": def __call__(self, parser, namespace, value, option): if namespace.sort is None: namespace.sort = [] - namespace.sort.append((value, True if option == '-S' else False)) + namespace.sort.append((value, option in {'-S', '--reverse-sort'})) parser.add_argument( '-s', '--sort', nargs='?', @@ -1010,6 +1148,29 @@ if __name__ == "__main__": nargs='?', action=AppendSort, help="Sort by this field, but backwards.") + parser.add_argument( + '-z', '--depth', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Depth of function calls to show. 0 shows all calls unless " + "we find a cycle. Defaults to 0.") + class AppendHot(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.hot is None: + namespace.hot = [] + namespace.hot.append((value, option in {'-R', '--reverse-hot'})) + parser.add_argument( + '-r', '--hot', + nargs='?', + action=AppendHot, + help="Show only the hot path for each function call. Can " + "optionally provide fields like sort.") + parser.add_argument( + '-R', '--reverse-hot', + nargs='?', + action=AppendHot, + help="Like -r/--hot, but backwards.") parser.add_argument( '--no-header', action='store_true', @@ -1034,18 +1195,6 @@ if __name__ == "__main__": '--everything', action='store_true', help="Include builtin and libc specific symbols.") - parser.add_argument( - '-z', '--depth', - nargs='?', - type=lambda x: int(x, 0), - const=0, - help="Depth of function calls to show. 0 shows all calls unless " - "we find a cycle. Defaults to 0.") - parser.add_argument( - '-t', '--hot', - nargs='?', - action='append', - help="Show only the hot path for each function call.") parser.add_argument( '-e', '--error-on-recursion', action='store_true', diff --git a/scripts/structs.py b/scripts/structs.py index f66abbb3..bb4e486a 100755 --- a/scripts/structs.py +++ b/scripts/structs.py @@ -16,6 +16,7 @@ if __name__ == "__main__": import collections as co import csv import itertools as it +import io import functools as ft import math as mt import os @@ -132,10 +133,10 @@ class RInt(co.namedtuple('RInt', 'x')): # struct size results class StructResult(co.namedtuple('StructResult', [ - 'file', 'struct', + 'i', 'file', 'struct', 'size', 'align', - 'i', 'children'])): - _by = ['file', 'struct'] + 'children'])): + _by = ['i', 'file', 'struct'] _fields = ['size', 'align'] _sort = ['size', 'align'] _types = {'size': RInt, 'align': RInt} @@ -143,20 +144,16 @@ class StructResult(co.namedtuple('StructResult', [ _children = 'children' __slots__ = () - def __new__(cls, file='', struct='', size=0, align=0, - i=None, children=None): - return super().__new__(cls, file, struct, + def __new__(cls, i=None, file='', struct='', size=0, align=0, + children=None): + return super().__new__(cls, i, file, struct, RInt(size), RInt(align), - i, children if children is not None else []) def __add__(self, other): - return StructResult(self.file, self.struct, + return StructResult(self.i, self.file, self.struct, self.size + other.size, max(self.align, other.align), - self.i if other.i is None - else other.i if self.i is None - else min(self.i, other.i), self.children + other.children) @@ -317,8 +314,9 @@ def collect_dwarf_info(obj_path, tags=None, *, return DwarfInfo(info) -def collect(obj_paths, *, +def collect_structs(obj_paths, *, everything=False, + depth=1, **args): results = [] for obj_path in obj_paths: @@ -420,7 +418,10 @@ def collect(obj_paths, *, return align # recursive+cached children finder - def childrenof(entry): + def childrenof(entry, depth): + # stop here? + if depth < 1: + return [] # cached? if not hasattr(childrenof, 'cache'): childrenof.cache = {} @@ -441,10 +442,9 @@ def collect(obj_paths, *, name_ = child.name size_ = sizeof(child) align_ = alignof(child) - children_ = childrenof(child) + children_ = childrenof(child, depth-1) children.append(StructResult( - file, name_, size_, align_, - i=child.off, + child.off, file, name_, size_, align_, children=children_)) # indirect type? elif entry.tag in { @@ -456,7 +456,7 @@ def collect(obj_paths, *, 'DW_TAG_volatile_type', 'DW_TAG_restrict_type'}: type = int(entry['DW_AT_type'].strip('<>'), 0) - children = childrenof(info[type]) + children = childrenof(info[type], depth) else: assert False, "Unknown dwarf entry? %r" % entry.tag @@ -491,18 +491,18 @@ def collect(obj_paths, *, align = alignof(entry) # find children, recursing if necessary - children = childrenof(entry) + children = childrenof(entry, depth-1) # typdefs exist in a separate namespace, so we need to track # these separately if entry.tag == 'DW_TAG_typedef': typedefs[entry.off] = StructResult( - file, name, size, align, + None, file, name, size, align, children=children) typedefed.add(int(entry['DW_AT_type'].strip('<>'), 0)) else: types[entry.off] = StructResult( - file, name, size, align, + None, file, name, size, align, children=children) # let typedefs take priority @@ -514,7 +514,31 @@ def collect(obj_paths, *, return results -def fold(Result, results, by=None, defines=[]): +# common folding/tabling/read/write code + +class Rev(co.namedtuple('Rev', 'x')): + __slots__ = () + # yes we need all of these because we're a namedtuple + def __lt__(self, other): + return self.x > other.x + def __gt__(self, other): + return self.x < other.x + def __le__(self, other): + return self.x >= other.x + def __ge__(self, other): + return self.x <= other.x + +def fold(Result, results, *, + by=None, + defines=[], + sort=None, + depth=1, + **_): + # stop when depth hits zero + if depth == 0: + return [] + + # organize by by if by is None: by = Result._by @@ -528,7 +552,7 @@ def fold(Result, results, by=None, defines=[]): if defines: results_ = [] for r in results: - if all(getattr(r, k) in vs for k, vs in defines): + if all(str(getattr(r, k)) in vs for k, vs in defines): results_.append(r) results = results_ @@ -545,8 +569,80 @@ def fold(Result, results, by=None, defines=[]): for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) + # sort, note that python's sort is stable + folded.sort(key=lambda r: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(r, k_),) + if getattr(r, k_) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by result + r)) + + # recurse if we have recursive results + if hasattr(Result, '_children'): + folded = [r._replace(**{ + Result._children: fold( + Result, getattr(r, Result._children), + by=by, + defines=defines, + sort=sort, + depth=depth-1)}) + for r in folded] + return folded +def hotify(Result, results, *, + fields=None, + sort=None, + depth=1, + hot=None, + **_): + # hotify only makes sense for recursive results + assert hasattr(Result, '_i') + assert hasattr(Result, '_children') + + if fields is None: + fields = Result._fields + + results_ = [] + for r in results: + hot_ = [] + def recurse(results_, depth_): + nonlocal hot_ + if not results_: + return + + # find the hottest result + r = min(results_, key=lambda r: + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(r, k_),) + if getattr(r, k_) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in it.chain(hot, [(None, False)]))) + + hot_.append(r._replace(**{ + Result._i: RInt(len(hot_)), + Result._children: []})) + + # recurse? + if depth_ > 1: + recurse(getattr(r, Result._children), + depth_-1) + + recurse(getattr(r, Result._children), depth-1) + results_.append(r._replace(**{ + Result._children: hot_})) + + return results_ + def table(Result, results, diff_results=None, *, by=None, fields=None, @@ -572,124 +668,32 @@ def table(Result, results, diff_results=None, *, fields = Result._fields types = Result._types - # fold again - results = fold(Result, results, by=by) + # fold again, otherwise results risk being hidden + results = fold(Result, results, + by=by, + depth=depth) if diff_results is not None: - diff_results = fold(Result, diff_results, by=by) - - # reduce children to hot paths? only used by some scripts - if hot: - # subclass to reintroduce __dict__ - Result_ = Result - class HotResult(Result_): - _i = '_hot_i' - _children = '_hot_children' - - def __new__(cls, r, i=None, children=None, notes=None): - self = HotResult._make(r) - self._hot_i = i - self._hot_children = children if children is not None else [] - return self - - def __add__(self, other): - return HotResult( - Result_.__add__(self, other), - self._hot_i if other._hot_i is None - else other._hot_i if self._hot_i is None - else min(self._hot_i, other._hot_i), - self._hot_children + other._hot_children) - - results_ = [] - for r in results: - hot_ = [] - def recurse(results_, depth_): - nonlocal hot_ - if not results_: - return - - # find the hottest result - r = max(results_, - key=lambda r: tuple( - tuple((getattr(r, k),) - if getattr(r, k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields]) - if k in fields) - for k in it.chain(hot, [None]))) - hot_.append(HotResult(r, i=len(hot_))) - - # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), - depth_-1) - - recurse(getattr(r, Result._children), depth-1) - results_.append(HotResult(r, children=hot_)) - - Result = HotResult - results = results_ + diff_results = fold(Result, diff_results, + by=by, + depth=depth) # organize by name table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results} diff_table = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in diff_results or []} - names = [name - for name in table.keys() | diff_table.keys() - if diff_results is None - or all_ - or any( - types[k].ratio( - getattr(table.get(name), k, None), - getattr(diff_table.get(name), k, None)) - for k in fields)] # find compare entry if there is one if compare: - compare_result = table.get(','.join(str(k) for k in compare)) - - # sort again, now with diff info, note that python's sort is stable - names.sort() - if compare: - names.sort( - key=lambda n: ( - # move compare entry to the top, note this can be - # overridden by explicitly sorting by fields - table.get(n) == compare_result, - # sort by ratio if comparing - tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(compare_result, k, None)) - for k in fields)), - reverse=True) - if diff or percent: - names.sort( - # sort by ratio if diffing - key=lambda n: tuple( - types[k].ratio( - getattr(table.get(n), k, None), - getattr(diff_table.get(n), k, None)) - for k in fields), - reverse=True) - if sort: - for k, reverse in reversed(sort): - names.sort( - key=lambda n: tuple( - (getattr(table[n], k),) - if getattr(table.get(n), k, None) is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) - + compare_r = table.get(','.join(str(k) for k in compare)) # build up our lines lines = [] @@ -716,11 +720,16 @@ def table(Result, results, diff_results=None, *, header.append('d'+k) lines.append(header) + # delete these to try to catch typos below, we need to rebuild + # these tables at each recursive layer + del table + del diff_table + # entry helper def table_entry(name, r, diff_r=None): entry = [name] # normal entry? - if ((compare is None or r == compare_result) + if ((compare is None or r == compare_r) and not percent and not diff): for k in fields: @@ -741,7 +750,7 @@ def table(Result, results, diff_results=None, *, else ['%+.1f%%' % (100*t)])( types[k].ratio( getattr(r, k, None), - getattr(compare_result, k, None))))) + getattr(compare_r, k, None))))) # percent entry? elif not diff: for k in fields: @@ -787,71 +796,94 @@ def table(Result, results, diff_results=None, *, return entry - # recursive entry helper, only used by some scripts - def recurse(results_, depth_, + # recursive entry helper + def table_recurse(results_, diff_results_, + depth_, prefixes=('', '', '', '')): # build the children table at each layer - results_ = fold(Result, results_, by=by) table_ = { - ','.join(str(getattr(r, k) or '') for k in by): r + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r for r in results_} - names_ = list(table_.keys()) + diff_table_ = { + ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by): r + for r in diff_results_ or []} + names_ = [n + for n in table_.keys() | diff_table_.keys() + if diff_results_ is None + or all_ + or any( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)] - # sort the children layer - names_.sort() - if hasattr(Result, '_i'): - names_.sort(key=lambda n: getattr(table_[n], Result._i)) - if sort: - for k, reverse in reversed(sort): - names_.sort( - key=lambda n: tuple( - (getattr(table_[n], k),) - if getattr(table_.get(n), k, None) - is not None - else () - for k in ( - [k] if k else [ - k for k in Result._sort - if k in fields])), - reverse=reverse ^ (not k or k in Result._fields)) + # sort again, now with diff info, note that python's sort is stable + names_.sort(key=lambda n: ( + # sort by explicit sort fields + tuple((Rev + if reverse ^ (not k or k in Result._fields) + else lambda x: x)( + tuple((getattr(table_[n], k_),) + if getattr(table_.get(n), k_, None) is not None + else () + for k_ in ([k] if k else Result._sort))) + for k, reverse in (sort or [])), + # sort by ratio if diffing + Rev(tuple(types[k].ratio( + getattr(table_.get(n), k, None), + getattr(diff_table_.get(n), k, None)) + for k in fields)) + if diff or percent + else (), + # move compare entry to the top, note this can be + # overridden by explicitly sorting by fields + (table_.get(n) != compare_r, + # sort by ratio if comparing + Rev(tuple( + types[k].ratio( + getattr(table_.get(n), k, None), + getattr(compare_r, k, None)) + for k in fields))) + if compare + else (), + # sort by result + (table_[n],) if n in table_ else (), + # and finally by name (diffs may be missing results) + n)) - for i, name in enumerate(names_): - r = table_[name] - is_last = (i == len(names_)-1) + for i, n in enumerate(names_): + # find comparable results + r = table_.get(n) + diff_r = diff_table_.get(n) + + # build line + line = table_entry(n, r, diff_r) - line = table_entry(name, r) - line = [x if isinstance(x, tuple) else (x, []) for x in line] # add prefixes - line[0] = (prefixes[0+is_last] + line[0][0], line[0][1]) + line = [x if isinstance(x, tuple) else (x, []) for x in line] + line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) lines.append(line) # recurse? - if depth_ > 1: - recurse(getattr(r, Result._children), + if n in table_ and depth_ > 1: + table_recurse( + getattr(r, Result._children), + getattr(diff_r, Result._children, None) or [], depth_-1, - (prefixes[2+is_last] + "|-> ", - prefixes[2+is_last] + "'-> ", - prefixes[2+is_last] + "| ", - prefixes[2+is_last] + " ")) + (prefixes[2+(i==len(names_)-1)] + "|-> ", + prefixes[2+(i==len(names_)-1)] + "'-> ", + prefixes[2+(i==len(names_)-1)] + "| ", + prefixes[2+(i==len(names_)-1)] + " ")) - # entries + # build entries if not summary: - for name in names: - r = table.get(name) - if diff_results is None: - diff_r = None - else: - diff_r = diff_table.get(name) - lines.append(table_entry(name, r, diff_r)) - - # recursive entries - if name in table and depth > 1: - recurse(getattr(table[name], Result._children), - depth-1, - ("|-> ", - "'-> ", - "| ", - " ")) + table_recurse(results, diff_results, depth) # total if not no_total and not (small_table and not summary): @@ -863,9 +895,8 @@ def table(Result, results, diff_results=None, *, lines.append(table_entry('TOTAL', r, diff_r)) # homogenize - lines = [ - [x if isinstance(x, tuple) else (x, []) for x in line] - for line in lines] + lines = [[x if isinstance(x, tuple) else (x, []) for x in line] + for line in lines] # find the best widths, note that column 0 contains the names and is # handled a bit differently @@ -886,18 +917,144 @@ def table(Result, results, diff_results=None, *, nwidths[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') for i, x in enumerate(line[1:], 1)))) +def read_csv(path, Result, *, + depth=1, + **_): + with openio(path, 'r') as f: + # csv or json? assume json starts with [ + json = (f.buffer.peek(1)[:1] == b'[') + + # read csv? + if not json: + results = [] + reader = csv.DictReader(f, restval='') + for r in reader: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k].strip()} + | {k: r[k] for k in Result._fields + if k in r and r[k].strip()}))) + except TypeError: + pass + return results + + # read json? + else: + import json + def unjsonify(results, depth_): + results_ = [] + for r in results: + if not any(k in r and r[k].strip() + for k in Result._fields): + continue + try: + # note this allows by/fields to overlap + results_.append(Result(**( + {k: r[k] for k in Result._by + if k in r and r[k] is not None} + | {k: r[k] for k in Result._fields + if k in r and r[k] is not None} + | ({Result._children: unjsonify( + r[Result._children], + depth_-1)} + if hasattr(Result, '_children') + and Result._children in r + and r[Result._children] is not None + and depth_ > 1 + else {}) + | ({Result._notes: set(r[Result._notes])} + if hasattr(Result, '_notes') + and Result._notes in r + and r[Result._notes] is not None + else {})))) + except TypeError: + pass + return results_ + return unjsonify(json.load(f), depth) + +def write_csv(path, Result, results, *, + json=False, + by=None, + fields=None, + depth=1, + **_): + with openio(path, 'w') as f: + # write csv? + if not json: + writer = csv.DictWriter(f, + (by if by is not None else Result._by) + + [k for k in (fields + if fields is not None + else Result._fields)]) + writer.writeheader() + for r in results: + # note this allows by/fields to overlap + writer.writerow( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None}) + + # write json? + else: + import json + # the neat thing about json is we can include recursive results + def jsonify(results, depth_): + results_ = [] + for r in results: + # note this allows by/fields to overlap + results_.append( + {k: getattr(r, k) + for k in (by + if by is not None + else Result._by) + if getattr(r, k) is not None} + | {k: str(getattr(r, k)) + for k in (fields + if fields is not None + else Result._fields) + if getattr(r, k) is not None} + | ({Result._children: jsonify( + getattr(r, Result._children), + depth_-1)} + if hasattr(Result, '_children') + and getattr(r, Result._children) + and depth_ > 1 + else {}) + | ({Result._notes: list( + getattr(r, Result._notes))} + if hasattr(Result, '_notes') + and getattr(r, Result._notes) + else {})) + return results_ + json.dump(jsonify(results, depth), f, + separators=(',', ':')) + def main(obj_paths, *, by=None, fields=None, defines=[], sort=None, + depth=None, + hot=None, **args): # figure out depth - if args.get('depth') is None: - args['depth'] = mt.inf if args.get('hot') else 1 - elif args.get('depth') == 0: - args['depth'] = mt.inf + if depth is None: + depth = mt.inf if hot else 1 + elif depth == 0: + depth = mt.inf # find sizes if not args.get('use', None): @@ -908,89 +1065,61 @@ def main(obj_paths, *, sys.exit(1) # collect info - results = collect(obj_paths, **args) + results = collect_structs(obj_paths, + depth=depth, + **args) else: - results = [] - with openio(args['use']) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - if not any(k in r and r[k].strip() - for k in StructResult._fields): - continue - try: - results.append(StructResult( - **{k: r[k] for k in StructResult._by - if k in r and r[k].strip()}, - **{k: r[k] - for k in StructResult._fields - if k in r and r[k].strip()})) - except TypeError: - pass + results = read_csv(args['use'], StructResult, + depth=depth, + **args) # fold - results = fold(StructResult, results, by=by, defines=defines) + results = fold(StructResult, results, + by=by, + defines=defines, + depth=depth) - # sort, note that python's sort is stable - results.sort() - if sort: - for k, reverse in reversed(sort): - results.sort( - key=lambda r: tuple( - (getattr(r, k),) if getattr(r, k) is not None else () - for k in ([k] if k else StructResult._sort)), - reverse=reverse ^ (not k or k in StructResult._fields)) + # hotify? + if hot: + results = hotify(StructResult, results, + fields=fields, + depth=depth, + hot=hot, + **args) - # write results to CSV + # write results to CSV/JSON if args.get('output'): - with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, - (by if by is not None else StructResult._by) - + [k for k in ( - fields if fields is not None - else StructResult._fields)]) - writer.writeheader() - for r in results: - writer.writerow( - {k: getattr(r, k) for k in ( - by if by is not None else StructResult._by)} - | {k: getattr(r, k) for k in ( - fields if fields is not None - else StructResult._fields)}) + write_csv(args['output'], StructResult, results, + by=by, + fields=fields, + depth=depth, + **args) + if args.get('output_json'): + write_csv(args['output_json'], StructResult, results, + json=True, + by=by, + fields=fields, + depth=depth, + **args) # find previous results? diff_results = None if args.get('diff') or args.get('percent'): - diff_results = [] try: - with openio(args.get('diff') or args.get('percent')) as f: - reader = csv.DictReader(f, restval='') - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - if not any(k in r and r[k].strip() - for k in StructResult._fields): - continue - try: - diff_results.append(StructResult( - **{k: r[k] for k in StructResult._by - if k in r and r[k].strip()}, - **{k: r[k] - for k in StructResult._fields - if k in r and r[k].strip()})) - except TypeError: - pass + diff_results = read_csv( + args.get('diff') or args.get('percent'), + StructResult, + depth=depth, + **args) except FileNotFoundError: - pass + diff_results = [] # fold - diff_results = fold(StructResult, diff_results, by=by, defines=defines) + diff_results = fold(StructResult, diff_results, + by=by, + defines=defines, + depth=depth) # print table if not args.get('quiet'): @@ -998,6 +1127,7 @@ def main(obj_paths, *, by=by if by is not None else ['struct'], fields=fields, sort=sort, + depth=depth, **args) @@ -1022,24 +1152,28 @@ if __name__ == "__main__": parser.add_argument( '-o', '--output', help="Specify CSV file to store results.") + parser.add_argument( + '-O', '--output-json', + help="Specify JSON file to store results. This may contain " + "recursive info.") parser.add_argument( '-u', '--use', help="Don't parse anything, use this CSV file.") parser.add_argument( '-d', '--diff', - help="Specify CSV file to diff against.") + help="Specify CSV/JSON file to diff against.") parser.add_argument( '-p', '--percent', - help="Specify CSV file to diff against, but only show precentage " - "change, not a full diff.") - parser.add_argument( - '-a', '--all', - action='store_true', - help="Show all, not just the ones that changed.") + help="Specify CSV/JSON file to diff against, but only show " + "percentage change, not a full diff.") parser.add_argument( '-c', '--compare', type=lambda x: tuple(v.strip() for v in x.split(',')), help="Compare results to the row matching this by pattern.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") parser.add_argument( '-b', '--by', action='append', @@ -1065,7 +1199,7 @@ if __name__ == "__main__": def __call__(self, parser, namespace, value, option): if namespace.sort is None: namespace.sort = [] - namespace.sort.append((value, True if option == '-S' else False)) + namespace.sort.append((value, option in {'-S', '--reverse-sort'})) parser.add_argument( '-s', '--sort', nargs='?', @@ -1076,6 +1210,29 @@ if __name__ == "__main__": nargs='?', action=AppendSort, help="Sort by this field, but backwards.") + parser.add_argument( + '-z', '--depth', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Depth of function calls to show. 0 shows all calls unless " + "we find a cycle. Defaults to 0.") + class AppendHot(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.hot is None: + namespace.hot = [] + namespace.hot.append((value, option in {'-R', '--reverse-hot'})) + parser.add_argument( + '-r', '--hot', + nargs='?', + action=AppendHot, + help="Show only the hot path for each function call. Can " + "optionally provide fields like sort.") + parser.add_argument( + '-R', '--reverse-hot', + nargs='?', + action=AppendHot, + help="Like -r/--hot, but backwards.") parser.add_argument( '--no-header', action='store_true', @@ -1100,18 +1257,6 @@ if __name__ == "__main__": '--everything', action='store_true', help="Include builtin and libc specific symbols.") - parser.add_argument( - '-z', '--depth', - nargs='?', - type=lambda x: int(x, 0), - const=0, - help="Depth of function calls to show. 0 shows all calls unless " - "we find a cycle. Defaults to 0.") - parser.add_argument( - '-t', '--hot', - nargs='?', - action='append', - help="Show only the hot path for each function call.") parser.add_argument( '--objdump-path', type=lambda x: x.split(), diff --git a/scripts/test.py b/scripts/test.py index b6f169bd..c5d79f9b 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -685,7 +685,7 @@ def find_runner(runner, id=None, main=True, **args): # run under perf? if args.get('perf'): cmd[:0] = args['perf_script'] + list(filter(None, [ - '-R', + '--record', '--perf-freq=%s' % args['perf_freq'] if args.get('perf_freq') else None, '--perf-period=%s' % args['perf_period']