From ca669938125370bbc93c4189739c55220e099b1c Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 2 Oct 2022 03:07:30 -0500 Subject: [PATCH] Tweaked scripts to share more code, added coverage calls/hits The main change is requiring field names for -b/-f/-s/-S, this is a bit more powerful, and supports hidden extra fields, but can require a bit more typing in some cases. --- Makefile | 10 +- scripts/code.py | 421 ++++++++++++++--------- scripts/coverage.py | 634 ++++++++++++++++++---------------- scripts/data.py | 424 ++++++++++++++--------- scripts/plot.py | 36 +- scripts/stack.py | 668 ++++++++++++++++++++---------------- scripts/struct_.py | 425 ++++++++++++++--------- scripts/summary.py | 808 ++++++++++++++++++++++++-------------------- 8 files changed, 1972 insertions(+), 1454 deletions(-) diff --git a/Makefile b/Makefile index e848e9c4..1c6a4584 100644 --- a/Makefile +++ b/Makefile @@ -149,23 +149,23 @@ bench-list: bench-runner .PHONY: code code: $(OBJ) - ./scripts/code.py $^ -S $(CODEFLAGS) + ./scripts/code.py $^ -Ssize $(CODEFLAGS) .PHONY: data data: $(OBJ) - ./scripts/data.py $^ -S $(DATAFLAGS) + ./scripts/data.py $^ -Ssize $(DATAFLAGS) .PHONY: stack stack: $(CI) - ./scripts/stack.py $^ -S $(STACKFLAGS) + ./scripts/stack.py $^ -Slimit -Sframe $(STACKFLAGS) .PHONY: struct struct: $(OBJ) - ./scripts/struct_.py $^ -S $(STRUCTFLAGS) + ./scripts/struct_.py $^ -Ssize $(STRUCTFLAGS) .PHONY: coverage coverage: $(GCDA) - ./scripts/coverage.py $^ -s $(COVERAGEFLAGS) + ./scripts/coverage.py $^ -slines -sbranches $(COVERAGEFLAGS) .PHONY: summary sizes summary sizes: $(BUILDDIR)lfs.csv diff --git a/scripts/code.py b/scripts/code.py index df16ece6..6b373fcf 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -29,10 +29,10 @@ TYPE = 'tTrRdD' # integer fields -class IntField(co.namedtuple('IntField', 'x')): +class Int(co.namedtuple('Int', 'x')): __slots__ = () def __new__(cls, x=0): - if isinstance(x, IntField): + if isinstance(x, Int): return x if isinstance(x, str): try: @@ -98,35 +98,30 @@ class IntField(co.namedtuple('IntField', 'x')): return (new-old) / old def __add__(self, other): - return IntField(self.x + other.x) + return self.__class__(self.x + other.x) def __sub__(self, other): - return IntField(self.x - other.x) + return self.__class__(self.x - other.x) def __mul__(self, other): - return IntField(self.x * other.x) - - def __lt__(self, other): - return self.x < other.x - - def __gt__(self, other): - return self.__class__.__lt__(other, self) - - def __le__(self, other): - return not self.__gt__(other) - - def __ge__(self, other): - return not self.__lt__(other) + return self.__class__(self.x * other.x) # code size results -class CodeResult(co.namedtuple('CodeResult', 'file,function,code_size')): +class CodeResult(co.namedtuple('CodeResult', [ + 'file', 'function', + 'size'])): + _by = ['file', 'function'] + _fields = ['size'] + _types = {'size': Int} + __slots__ = () - def __new__(cls, file, function, code_size): - return super().__new__(cls, file, function, IntField(code_size)) + def __new__(cls, file='', function='', size=0): + return super().__new__(cls, file, function, + Int(size)) def __add__(self, other): return CodeResult(self.file, self.function, - self.code_size + other.code_size) + self.size + other.size) def openio(path, mode='r'): @@ -188,9 +183,27 @@ def collect(paths, *, return results -def fold(results, *, - by=['file', 'function'], +def fold(Result, results, *, + by=None, + defines=None, **_): + if by is None: + by = Result._by + + for k in it.chain(by or [], (k for k, _ in defines or [])): + if k not in Result._by and k not in Result._fields: + print("error: could not find field %r?" % k) + sys.exit(-1) + + # filter by matching defines + if defines is not None: + results_ = [] + for r in results: + if all(getattr(r, k) in vs for k, vs in defines): + results_.append(r) + results = results_ + + # organize results into conflicts folding = co.OrderedDict() for r in results: name = tuple(getattr(r, k) for k in by) @@ -198,157 +211,220 @@ def fold(results, *, folding[name] = [] folding[name].append(r) + # merge conflicts folded = [] - for rs in folding.values(): + for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) return folded - -def table(results, diff_results=None, *, - by_file=False, - size_sort=False, - reverse_size_sort=False, +def table(Result, results, diff_results=None, *, + by=None, + fields=None, + sort=None, summary=False, all=False, percent=False, **_): all_, all = all, __builtins__.all - # fold - results = fold(results, by=['file' if by_file else 'function']) - if diff_results is not None: - diff_results = fold(diff_results, - by=['file' if by_file else 'function']) + if by is None: + by = Result._by + if fields is None: + fields = Result._fields + types = Result._types + # fold again + results = fold(Result, results, by=by) + if diff_results is not None: + diff_results = fold(Result, diff_results, by=by) + + # organize by name table = { - r.file if by_file else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in results} diff_table = { - r.file if by_file else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in diff_results or []} - - # sort, note that python's sort is stable names = list(table.keys() | diff_table.keys()) + + # sort again, now with diff info, note that python's sort is stable names.sort() if diff_results is not None: - names.sort(key=lambda n: -IntField.ratio( - table[n].code_size if n in table else None, - diff_table[n].code_size if n in diff_table else None)) - if size_sort: - names.sort(key=lambda n: (table[n].code_size,) if n in table else (), + names.sort(key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), reverse=True) - elif reverse_size_sort: - names.sort(key=lambda n: (table[n].code_size,) if n in table else (), - reverse=False) + if sort: + for k, reverse in reversed(sort): + names.sort(key=lambda n: (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None else (), + reverse=reverse ^ (not k or k in Result._fields)) - # print header - if not summary: - title = '%s%s' % ( - 'file' if by_file else 'function', - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - name_width = max(it.chain([23, len(title)], (len(n) for n in names))) - else: - title = '' - name_width = 23 - name_width = 4*((name_width+1+4-1)//4)-1 - print('%-*s ' % (name_width, title), end='') + # build up our lines + lines = [] + + # header + line = [] + line.append('%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else '') if diff_results is None: - print(' %s' % ('size'.rjust(len(IntField.none)))) + for k in fields: + line.append(k) elif percent: - print(' %s' % ('size'.rjust(len(IntField.diff_none)))) + for k in fields: + line.append(k) else: - print(' %s %s %s' % ( - 'old'.rjust(len(IntField.diff_none)), - 'new'.rjust(len(IntField.diff_none)), - 'diff'.rjust(len(IntField.diff_none)))) + for k in fields: + line.append('o'+k) + for k in fields: + line.append('n'+k) + for k in fields: + line.append('d'+k) + line.append('') + lines.append(line) - # print entries + # entries if not summary: for name in names: r = table.get(name) if diff_results is not None: diff_r = diff_table.get(name) - ratio = IntField.ratio( - r.code_size if r else None, - diff_r.code_size if diff_r else None) - if not ratio and not all_: + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + if not any(ratios) and not all_: continue - print('%-*s ' % (name_width, name), end='') + line = [] + line.append(name) if diff_results is None: - print(' %s' % ( - r.code_size.table() - if r else IntField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - r.code_size.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - diff_r.code_size.diff_table() - if diff_r else IntField.diff_none, - r.code_size.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.code_size if r else None, - diff_r.code_size if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) - # print total - total = fold(results, by=[]) - r = total[0] if total else None + # total + r = next(iter(fold(Result, results, by=[])), None) if diff_results is not None: - diff_total = fold(diff_results, by=[]) - diff_r = diff_total[0] if diff_total else None - ratio = IntField.ratio( - r.code_size if r else None, - diff_r.code_size if diff_r else None) + diff_r = next(iter(fold(Result, diff_results, by=[])), None) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] - print('%-*s ' % (name_width, 'TOTAL'), end='') + line = [] + line.append('TOTAL') if diff_results is None: - print(' %s' % ( - r.code_size.table() - if r else IntField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - r.code_size.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - diff_r.code_size.diff_table() - if diff_r else IntField.diff_none, - r.code_size.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.code_size if r else None, - diff_r.code_size if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) + + # find the best widths, note that column 0 contains the names and column -1 + # the ratios, so those are handled a bit differently + widths = [ + ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1 + for w, i in zip( + it.chain([23], it.repeat(7)), + range(len(lines[0])-1))] + + # print our table + for line in lines: + print('%-*s %s%s' % ( + widths[0], line[0], + ' '.join('%*s' % (w, x) + for w, x in zip(widths[1:], line[1:-1])), + line[-1])) -def main(obj_paths, **args): +def main(obj_paths, *, + by=None, + fields=None, + defines=None, + sort=None, + **args): # find sizes if not args.get('use', None): # find .o files @@ -361,7 +437,7 @@ def main(obj_paths, **args): paths.append(path) if not paths: - print('no .obj files found in %r?' % obj_paths) + print("error: no .obj files found in %r?" % obj_paths) sys.exit(-1) results = collect(paths, **args) @@ -371,25 +447,35 @@ def main(obj_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - results.append(CodeResult(**{ - k: v for k, v in r.items() - if k in CodeResult._fields})) + results.append(CodeResult( + **{k: r[k] for k in CodeResult._by + if k in r and r[k].strip()}, + **{k: r['code_'+k] for k in CodeResult._fields + if 'code_'+k in r and r['code_'+k].strip()})) except TypeError: pass - # fold to remove duplicates - results = fold(results) + # fold + results = fold(CodeResult, results, by=by, defines=defines) - # sort because why not + # sort, note that python's sort is stable results.sort() + if sort: + for k, reverse in reversed(sort): + results.sort(key=lambda r: (getattr(r, k),) + if getattr(r, k) is not None else (), + reverse=reverse ^ (not k or k in CodeResult._fields)) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, CodeResult._fields) + writer = csv.DictWriter(f, CodeResult._by + + ['code_'+k for k in CodeResult._fields]) writer.writeheader() for r in results: - writer.writerow(r._asdict()) + writer.writerow( + {k: getattr(r, k) for k in CodeResult._by} + | {'code_'+k: getattr(r, k) for k in CodeResult._fields}) # find previous results? if args.get('diff'): @@ -399,22 +485,26 @@ def main(obj_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - diff_results.append(CodeResult(**{ - k: v for k, v in r.items() - if k in CodeResult._fields})) + diff_results.append(CodeResult( + **{k: r[k] for k in CodeResult._by + if k in r and r[k].strip()}, + **{k: r['code_'+k] for k in CodeResult._fields + if 'code_'+k in r and r['code_'+k].strip()})) except TypeError: pass except FileNotFoundError: pass - # fold to remove duplicates - diff_results = fold(diff_results) + # fold + diff_results = fold(CodeResult, diff_results, by=by, defines=defines) # print table if not args.get('quiet'): - table( - results, + table(CodeResult, results, diff_results if args.get('diff') else None, + by=by if by is not None else ['function'], + fields=fields, + sort=sort, **args) @@ -455,22 +545,39 @@ if __name__ == "__main__": action='store_true', help="Only show percentage change, not a full diff.") parser.add_argument( - '-b', '--by-file', - action='store_true', - help="Group by file. Note this does not include padding " - "so sizes may differ from other tools.") + '-b', '--by', + action='append', + choices=CodeResult._by, + help="Group by this field.") parser.add_argument( - '-s', '--size-sort', - action='store_true', - help="Sort by size.") + '-f', '--field', + dest='fields', + action='append', + choices=CodeResult._fields, + help="Show this field.") parser.add_argument( - '-S', '--reverse-size-sort', - action='store_true', - help="Sort by size, but backwards.") + '-D', '--define', + dest='defines', + action='append', + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), + help="Only include results where this field is this value.") + class AppendSort(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.sort is None: + namespace.sort = [] + namespace.sort.append((value, True if option == '-S' else False)) + parser.add_argument( + '-s', '--sort', + action=AppendSort, + help="Sort by this fields.") + parser.add_argument( + '-S', '--reverse-sort', + action=AppendSort, + help="Sort by this fields, but backwards.") parser.add_argument( '-Y', '--summary', action='store_true', - help="Only show the total size.") + help="Only show the total.") parser.add_argument( '-A', '--everything', action='store_true', diff --git a/scripts/coverage.py b/scripts/coverage.py index 4e4541ac..7d36a47e 100755 --- a/scripts/coverage.py +++ b/scripts/coverage.py @@ -30,10 +30,10 @@ GCOV_TOOL = ['gcov'] # integer fields -class IntField(co.namedtuple('IntField', 'x')): +class Int(co.namedtuple('Int', 'x')): __slots__ = () def __new__(cls, x=0): - if isinstance(x, IntField): + if isinstance(x, Int): return x if isinstance(x, str): try: @@ -99,37 +99,25 @@ class IntField(co.namedtuple('IntField', 'x')): return (new-old) / old def __add__(self, other): - return IntField(self.x + other.x) + return self.__class__(self.x + other.x) def __sub__(self, other): - return IntField(self.x - other.x) + return self.__class__(self.x - other.x) def __mul__(self, other): - return IntField(self.x * other.x) - - def __lt__(self, other): - return self.x < other.x - - def __gt__(self, other): - return self.__class__.__lt__(other, self) - - def __le__(self, other): - return not self.__gt__(other) - - def __ge__(self, other): - return not self.__lt__(other) + return self.__class__(self.x * other.x) # fractional fields, a/b -class FracField(co.namedtuple('FracField', 'a,b')): +class Frac(co.namedtuple('Frac', 'a,b')): __slots__ = () def __new__(cls, a=0, b=None): - if isinstance(a, FracField) and b is None: + if isinstance(a, Frac) and b is None: return a if isinstance(a, str) and b is None: a, b = a.split('/', 1) if b is None: b = a - return super().__new__(cls, IntField(a), IntField(b)) + return super().__new__(cls, Int(a), Int(b)) def __str__(self): return '%s/%s' % (self.a, self.b) @@ -139,10 +127,7 @@ class FracField(co.namedtuple('FracField', 'a,b')): none = '%11s %7s' % ('-', '-') def table(self): - if not self.b.x: - return self.none - - t = self.a.x/self.b.x + t = self.a.x/self.b.x if self.b.x else 1.0 return '%11s %7s' % ( self, '∞%' if t == +m.inf @@ -151,38 +136,35 @@ class FracField(co.namedtuple('FracField', 'a,b')): diff_none = '%11s' % '-' def diff_table(self): - if not self.b.x: - return self.diff_none - return '%11s' % (self,) def diff_diff(self, other): - new_a, new_b = self if self else (IntField(0), IntField(0)) - old_a, old_b = other if other else (IntField(0), IntField(0)) + new_a, new_b = self if self else (Int(0), Int(0)) + old_a, old_b = other if other else (Int(0), Int(0)) return '%11s' % ('%s/%s' % ( new_a.diff_diff(old_a).strip(), new_b.diff_diff(old_b).strip())) def ratio(self, other): - new_a, new_b = self if self else (IntField(0), IntField(0)) - old_a, old_b = other if other else (IntField(0), IntField(0)) + new_a, new_b = self if self else (Int(0), Int(0)) + old_a, old_b = other if other else (Int(0), Int(0)) new = new_a.x/new_b.x if new_b.x else 1.0 old = old_a.x/old_b.x if old_b.x else 1.0 return new - old def __add__(self, other): - return FracField(self.a + other.a, self.b + other.b) + return self.__class__(self.a + other.a, self.b + other.b) def __sub__(self, other): - return FracField(self.a - other.a, self.b - other.b) + return self.__class__(self.a - other.a, self.b - other.b) def __mul__(self, other): - return FracField(self.a * other.a, self.b + other.b) + return self.__class__(self.a * other.a, self.b + other.b) def __lt__(self, other): - self_r = self.a.x/self.b.x if self.b.x else -m.inf - other_r = other.a.x/other.b.x if other.b.x else -m.inf - return self_r < other_r + self_t = self.a.x/self.b.x if self.b.x else 1.0 + other_t = other.a.x/other.b.x if other.b.x else 1.0 + return (self_t, self.a.x) < (other_t, other.a.x) def __gt__(self, other): return self.__class__.__lt__(other, self) @@ -194,22 +176,28 @@ class FracField(co.namedtuple('FracField', 'a,b')): return not self.__lt__(other) # coverage results -class CoverageResult(co.namedtuple('CoverageResult', - 'file,function,line,' - 'coverage_hits,coverage_lines,coverage_branches')): +class CoverageResult(co.namedtuple('CoverageResult', [ + 'file', 'function', 'line', + 'calls', 'hits', 'funcs', 'lines', 'branches'])): + _by = ['file', 'function', 'line'] + _fields = ['calls', 'hits', 'funcs', 'lines', 'branches'] + _types = { + 'calls': Int, 'hits': Int, + 'funcs': Frac, 'lines': Frac, 'branches': Frac} + __slots__ = () - def __new__(cls, file, function, line, - coverage_hits, coverage_lines, coverage_branches): - return super().__new__(cls, file, function, int(IntField(line)), - IntField(coverage_hits), - FracField(coverage_lines), - FracField(coverage_branches)) + def __new__(cls, file='', function='', line=0, + calls=0, hits=0, funcs=0, lines=0, branches=0): + return super().__new__(cls, file, function, int(Int(line)), + Int(calls), Int(hits), Frac(funcs), Frac(lines), Frac(branches)) def __add__(self, other): return CoverageResult(self.file, self.function, self.line, - max(self.coverage_hits, other.coverage_hits), - self.coverage_lines + other.coverage_lines, - self.coverage_branches + other.coverage_branches) + max(self.calls, other.calls), + max(self.hits, other.hits), + self.funcs + other.funcs, + self.lines + other.lines, + self.branches + other.branches) def openio(path, mode='r'): @@ -257,20 +245,37 @@ def collect(paths, *, if file['file'] != src_path: continue - for line in file['lines']: - func = line.get('function_name', '(inlined)') + for func in file['functions']: + func_name = func.get('name', '(inlined)') # discard internal function (this includes injected test cases) if not everything: - if func.startswith('__'): + if func_name.startswith('__'): continue + # go ahead and add functions, later folding will merge this if + # there are other hits on this line results.append(CoverageResult( - src_path, func, line['line_number'], - line['count'], - FracField( - 1 if line['count'] > 0 else 0, - 1), - FracField( + src_path, func_name, func['start_line'], + func['execution_count'], 0, + Frac(1 if func['execution_count'] > 0 else 0, 1), + 0, + 0)) + + for line in file['lines']: + func_name = line.get('function_name', '(inlined)') + # discard internal function (this includes injected test cases) + if not everything: + if func_name.startswith('__'): + continue + + # go ahead and add lines, later folding will merge this if + # there are other hits on this line + results.append(CoverageResult( + src_path, func_name, line['line_number'], + 0, line['count'], + 0, + Frac(1 if line['count'] > 0 else 0, 1), + Frac( sum(1 if branch['count'] > 0 else 0 for branch in line['branches']), len(line['branches'])))) @@ -278,9 +283,27 @@ def collect(paths, *, return results -def fold(results, *, - by=['file', 'function', 'line'], +def fold(Result, results, *, + by=None, + defines=None, **_): + if by is None: + by = Result._by + + for k in it.chain(by or [], (k for k, _ in defines or [])): + if k not in Result._by and k not in Result._fields: + print("error: could not find field %r?" % k) + sys.exit(-1) + + # filter by matching defines + if defines is not None: + results_ = [] + for r in results: + if all(getattr(r, k) in vs for k, vs in defines): + results_.append(r) + results = results_ + + # organize results into conflicts folding = co.OrderedDict() for r in results: name = tuple(getattr(r, k) for k in by) @@ -288,231 +311,224 @@ def fold(results, *, folding[name] = [] folding[name].append(r) + # merge conflicts folded = [] - for rs in folding.values(): + for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) return folded - -def table(results, diff_results=None, *, - by_file=False, - by_line=False, - line_sort=False, - reverse_line_sort=False, - branch_sort=False, - reverse_branch_sort=False, +def table(Result, results, diff_results=None, *, + by=None, + fields=None, + sort=None, summary=False, all=False, percent=False, **_): all_, all = all, __builtins__.all - # fold - results = fold(results, - by=['file', 'line'] if by_line - else ['file'] if by_file - else ['function']) - if diff_results is not None: - diff_results = fold(diff_results, - by=['file', 'line'] if by_line - else ['file'] if by_file - else ['function']) + if by is None: + by = Result._by + if fields is None: + fields = Result._fields + types = Result._types + # fold again + results = fold(Result, results, by=by) + if diff_results is not None: + diff_results = fold(Result, diff_results, by=by) + + # organize by name table = { - '%s:%s' % (r.file, r.line) if by_line - else r.file if by_file - else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in results} diff_table = { - '%s:%s' % (r.file, r.line) if by_line - else r.file if by_file - else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in diff_results or []} - - # sort, note that python's sort is stable names = list(table.keys() | diff_table.keys()) + + # sort again, now with diff info, note that python's sort is stable names.sort() if diff_results is not None: - names.sort(key=lambda n: ( - -FracField.ratio( - table[n].coverage_lines if n in table else None, - diff_table[n].coverage_lines if n in diff_table else None), - -FracField.ratio( - table[n].coverage_branches if n in table else None, - diff_table[n].coverage_branches if n in diff_table else None))) - if line_sort: - names.sort(key=lambda n: (table[n].coverage_lines,) - if n in table else (), + names.sort(key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), reverse=True) - elif reverse_line_sort: - names.sort(key=lambda n: (table[n].coverage_lines,) - if n in table else (), - reverse=False) - elif branch_sort: - names.sort(key=lambda n: (table[n].coverage_branches,) - if n in table else (), - reverse=True) - elif reverse_branch_sort: - names.sort(key=lambda n: (table[n].coverage_branches,) - if n in table else (), - reverse=False) + if sort: + for k, reverse in reversed(sort): + names.sort(key=lambda n: (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None else (), + reverse=reverse ^ (not k or k in Result._fields)) - # print header - if not summary: - title = '%s%s' % ( - 'line' if by_line else 'file' if by_file else 'function', - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - name_width = max(it.chain([23, len(title)], (len(n) for n in names))) - else: - title = '' - name_width = 23 - name_width = 4*((name_width+1+4-1)//4)-1 - print('%-*s ' % (name_width, title), end='') + # build up our lines + lines = [] + + # header + line = [] + line.append('%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else '') if diff_results is None: - print(' %s %s' % ( - 'hits/line'.rjust(len(FracField.none)), - 'hits/branch'.rjust(len(FracField.none)))) + for k in fields: + line.append(k) elif percent: - print(' %s %s' % ( - 'hits/line'.rjust(len(FracField.diff_none)), - 'hits/branch'.rjust(len(FracField.diff_none)))) + for k in fields: + line.append(k) else: - print(' %s %s %s %s %s %s' % ( - 'oh/line'.rjust(len(FracField.diff_none)), - 'oh/branch'.rjust(len(FracField.diff_none)), - 'nh/line'.rjust(len(FracField.diff_none)), - 'nh/branch'.rjust(len(FracField.diff_none)), - 'dh/line'.rjust(len(FracField.diff_none)), - 'dh/branch'.rjust(len(FracField.diff_none)))) + for k in fields: + line.append('o'+k) + for k in fields: + line.append('n'+k) + for k in fields: + line.append('d'+k) + line.append('') + lines.append(line) - # print entries + # entries if not summary: for name in names: r = table.get(name) if diff_results is not None: diff_r = diff_table.get(name) - line_ratio = FracField.ratio( - r.coverage_lines if r else None, - diff_r.coverage_lines if diff_r else None) - branch_ratio = FracField.ratio( - r.coverage_branches if r else None, - diff_r.coverage_branches if diff_r else None) - if not line_ratio and not branch_ratio and not all_: + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + if not any(ratios) and not all_: continue - print('%-*s ' % (name_width, name), end='') + line = [] + line.append(name) if diff_results is None: - print(' %s %s' % ( - r.coverage_lines.table() - if r else FracField.none, - r.coverage_branches.table() - if r else FracField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s %s%s' % ( - r.coverage_lines.diff_table() - if r else FracField.diff_none, - r.coverage_branches.diff_table() - if r else FracField.diff_none, - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in [line_ratio, branch_ratio]))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s %s %s %s%s' % ( - diff_r.coverage_lines.diff_table() - if diff_r else FracField.diff_none, - diff_r.coverage_branches.diff_table() - if diff_r else FracField.diff_none, - r.coverage_lines.diff_table() - if r else FracField.diff_none, - r.coverage_branches.diff_table() - if r else FracField.diff_none, - FracField.diff_diff( - r.coverage_lines if r else None, - diff_r.coverage_lines if diff_r else None) - if r or diff_r else FracField.diff_none, - FracField.diff_diff( - r.coverage_branches if r else None, - diff_r.coverage_branches if diff_r else None) - if r or diff_r else FracField.diff_none, - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in [line_ratio, branch_ratio] - if t) - if line_ratio or branch_ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) - # print total - total = fold(results, by=[]) - r = total[0] if total else None + # total + r = next(iter(fold(Result, results, by=[])), None) if diff_results is not None: - diff_total = fold(diff_results, by=[]) - diff_r = diff_total[0] if diff_total else None - line_ratio = FracField.ratio( - r.coverage_lines if r else None, - diff_r.coverage_lines if diff_r else None) - branch_ratio = FracField.ratio( - r.coverage_branches if r else None, - diff_r.coverage_branches if diff_r else None) + diff_r = next(iter(fold(Result, diff_results, by=[])), None) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] - print('%-*s ' % (name_width, 'TOTAL'), end='') + line = [] + line.append('TOTAL') if diff_results is None: - print(' %s %s' % ( - r.coverage_lines.table() - if r else FracField.none, - r.coverage_branches.table() - if r else FracField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s %s%s' % ( - r.coverage_lines.diff_table() - if r else FracField.diff_none, - r.coverage_branches.diff_table() - if r else FracField.diff_none, - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in [line_ratio, branch_ratio]))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s %s %s %s%s' % ( - diff_r.coverage_lines.diff_table() - if diff_r else FracField.diff_none, - diff_r.coverage_branches.diff_table() - if diff_r else FracField.diff_none, - r.coverage_lines.diff_table() - if r else FracField.diff_none, - r.coverage_branches.diff_table() - if r else FracField.diff_none, - FracField.diff_diff( - r.coverage_lines if r else None, - diff_r.coverage_lines if diff_r else None) - if r or diff_r else FracField.diff_none, - FracField.diff_diff( - r.coverage_branches if r else None, - diff_r.coverage_branches if diff_r else None) - if r or diff_r else FracField.diff_none, - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in [line_ratio, branch_ratio] - if t) - if line_ratio or branch_ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) + + # find the best widths, note that column 0 contains the names and column -1 + # the ratios, so those are handled a bit differently + widths = [ + ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1 + for w, i in zip( + it.chain([23], it.repeat(7)), + range(len(lines[0])-1))] + + # print our table + for line in lines: + print('%-*s %s%s' % ( + widths[0], line[0], + ' '.join('%*s' % (w, x) + for w, x in zip(widths[1:], line[1:-1])), + line[-1])) -def annotate(paths, results, *, +def annotate(Result, results, paths, *, annotate=False, lines=False, branches=False, build_dir=None, **args): + # if neither branches/lines specified, color both + if annotate and not lines and not branches: + lines, branches = True, True + for path in paths: # map to source file src_path = re.sub('\.t\.a\.gcda$', '.c', path) @@ -521,7 +537,7 @@ def annotate(paths, results, *, src_path) # flatten to line info - results = fold(results, by=['file', 'line']) + results = fold(Result, results, by=['file', 'line']) table = {r.line: r for r in results if r.file == src_path} # calculate spans to show @@ -529,10 +545,8 @@ def annotate(paths, results, *, spans = [] last = None for line, r in sorted(table.items()): - if ((lines and int(r.coverage_hits) == 0) - or (branches - and r.coverage_branches.a - < r.coverage_branches.b)): + if ((lines and int(r.hits) == 0) + or (branches and r.branches.a < r.branches.b)): if last is not None and line - last.stop <= args['context']: last = range( last.start, @@ -568,24 +582,29 @@ def annotate(paths, results, *, if i+1 in table: r = table[i+1] - line = '%-*s // %s hits, %s branches' % ( + line = '%-*s // %s hits%s' % ( args['width'], line, - r.coverage_hits, - r.coverage_branches) + r.hits, + ', %s branches' % (r.branches,) + if int(r.branches.b) else '') if args['color']: - if lines and int(r.coverage_hits) == 0: + if lines and int(r.hits) == 0: line = '\x1b[1;31m%s\x1b[m' % line - elif (branches - and r.coverage_branches.a - < r.coverage_branches.b): + elif branches and r.branches.a < r.branches.b: line = '\x1b[35m%s\x1b[m' % line print(line) -def main(gcda_paths, **args): +def main(gcda_paths, *, + by=None, + fields=None, + defines=None, + sort=None, + hits=False, + **args): # figure out what color should be if args.get('color') == 'auto': args['color'] = sys.stdout.isatty() @@ -606,7 +625,7 @@ def main(gcda_paths, **args): paths.append(path) if not paths: - print('no .gcda files found in %r?' % gcda_paths) + print("error: no .gcda files found in %r?" % gcda_paths) sys.exit(-1) results = collect(paths, **args) @@ -616,25 +635,38 @@ def main(gcda_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - results.append(CoverageResult(**{ - k: v for k, v in r.items() - if k in CoverageResult._fields})) + results.append(CoverageResult( + **{k: r[k] for k in CoverageResult._by + if k in r and r[k].strip()}, + **{k: r['coverage_'+k] + for k in CoverageResult._fields + if 'coverage_'+k in r + and r['coverage_'+k].strip()})) except TypeError: pass - # fold to remove duplicates - results = fold(results) + # fold + results = fold(CoverageResult, results, by=by, defines=defines) - # sort because why not + # sort, note that python's sort is stable results.sort() + if sort: + for k, reverse in reversed(sort): + results.sort(key=lambda r: (getattr(r, k),) + if getattr(r, k) is not None else (), + reverse=reverse ^ (not k or k in CoverageResult._fields)) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, CoverageResult._fields) + writer = csv.DictWriter(f, CoverageResult._by + + ['coverage_'+k for k in CoverageResult._fields]) writer.writeheader() for r in results: - writer.writerow(r._asdict()) + writer.writerow( + {k: getattr(r, k) for k in CoverageResult._by} + | {'coverage_'+k: getattr(r, k) + for k in CoverageResult._fields}) # find previous results? if args.get('diff'): @@ -644,31 +676,39 @@ def main(gcda_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - diff_results.append(CoverageResult(**{ - k: v for k, v in r.items() - if k in CoverageResult._fields})) + diff_results.append(CoverageResult( + **{k: r[k] for k in CoverageResult._by + if k in r and r[k].strip()}, + **{k: r['coverage_'+k] + for k in CoverageResult._fields + if 'coverage_'+k in r + and r['coverage_'+k].strip()})) except TypeError: pass except FileNotFoundError: pass - # fold to remove duplicates - diff_results = fold(diff_results) + # fold + diff_results = fold(CoverageResult, diff_results, + by=by, defines=defines) + # print table if not args.get('quiet'): if (args.get('annotate') or args.get('lines') or args.get('branches')): # annotate sources - annotate( - paths, - results, + annotate(CoverageResult, results, paths, **args) else: # print table - table( - results, + table(CoverageResult, results, diff_results if args.get('diff') else None, + by=by if by is not None else ['function'], + fields=fields if fields is not None + else ['lines', 'branches'] if not hits + else ['calls', 'hits'], + sort=sort, **args) # catch lack of coverage @@ -717,33 +757,47 @@ if __name__ == "__main__": action='store_true', help="Only show percentage change, not a full diff.") parser.add_argument( - '-b', '--by-file', - action='store_true', - help="Group by file.") + '-b', '--by', + action='append', + choices=CoverageResult._by, + help="Group by this field.") parser.add_argument( - '--by-line', - action='store_true', - help="Group by line.") + '-f', '--field', + dest='fields', + action='append', + choices=CoverageResult._fields, + help="Show this field.") parser.add_argument( - '-s', '--line-sort', - action='store_true', - help="Sort by line coverage.") + '-D', '--define', + dest='defines', + action='append', + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), + help="Only include results where this field is this value.") + class AppendSort(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.sort is None: + namespace.sort = [] + namespace.sort.append((value, True if option == '-S' else False)) parser.add_argument( - '-S', '--reverse-line-sort', - action='store_true', - help="Sort by line coverage, but backwards.") + '-s', '--sort', + action=AppendSort, + help="Sort by this field.") parser.add_argument( - '--branch-sort', - action='store_true', - help="Sort by branch coverage.") - parser.add_argument( - '--reverse-branch-sort', - action='store_true', - help="Sort by branch coverage, but backwards.") + '-S', '--reverse-sort', + action=AppendSort, + help="Sort by this field, but backwards.") parser.add_argument( '-Y', '--summary', action='store_true', - help="Only show the total size.") + help="Only show the total.") + parser.add_argument( + '-A', '--everything', + action='store_true', + help="Include builtin and libc specific symbols.") + parser.add_argument( + '-H', '--hits', + action='store_true', + help="Show total hits instead of coverage.") parser.add_argument( '-l', '--annotate', action='store_true', @@ -779,10 +833,6 @@ if __name__ == "__main__": '-E', '--error-on-branches', action='store_true', help="Error if any branches are not covered.") - parser.add_argument( - '-A', '--everything', - action='store_true', - help="Include builtin and libc specific symbols.") parser.add_argument( '--gcov-tool', default=GCOV_TOOL, diff --git a/scripts/data.py b/scripts/data.py index 60707567..05ef8681 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Script to find data size at the function level. Basically just a bit wrapper +# Script to find data size at the function level. Basically just a big wrapper # around nm with some extra conveniences for comparing builds. Heavily inspired # by Linux's Bloat-O-Meter. # @@ -28,12 +28,11 @@ NM_TOOL = ['nm'] TYPE = 'dDbB' - # integer fields -class IntField(co.namedtuple('IntField', 'x')): +class Int(co.namedtuple('Int', 'x')): __slots__ = () def __new__(cls, x=0): - if isinstance(x, IntField): + if isinstance(x, Int): return x if isinstance(x, str): try: @@ -99,35 +98,30 @@ class IntField(co.namedtuple('IntField', 'x')): return (new-old) / old def __add__(self, other): - return IntField(self.x + other.x) + return self.__class__(self.x + other.x) def __sub__(self, other): - return IntField(self.x - other.x) + return self.__class__(self.x - other.x) def __mul__(self, other): - return IntField(self.x * other.x) - - def __lt__(self, other): - return self.x < other.x - - def __gt__(self, other): - return self.__class__.__lt__(other, self) - - def __le__(self, other): - return not self.__gt__(other) - - def __ge__(self, other): - return not self.__lt__(other) + return self.__class__(self.x * other.x) # data size results -class DataResult(co.namedtuple('DataResult', 'file,function,data_size')): +class DataResult(co.namedtuple('DataResult', [ + 'file', 'function', + 'size'])): + _by = ['file', 'function'] + _fields = ['size'] + _types = {'size': Int} + __slots__ = () - def __new__(cls, file, function, data_size): - return super().__new__(cls, file, function, IntField(data_size)) + def __new__(cls, file='', function='', size=0): + return super().__new__(cls, file, function, + Int(size)) def __add__(self, other): return DataResult(self.file, self.function, - self.data_size + other.data_size) + self.size + other.size) def openio(path, mode='r'): @@ -189,9 +183,27 @@ def collect(paths, *, return results -def fold(results, *, - by=['file', 'function'], +def fold(Result, results, *, + by=None, + defines=None, **_): + if by is None: + by = Result._by + + for k in it.chain(by or [], (k for k, _ in defines or [])): + if k not in Result._by and k not in Result._fields: + print("error: could not find field %r?" % k) + sys.exit(-1) + + # filter by matching defines + if defines is not None: + results_ = [] + for r in results: + if all(getattr(r, k) in vs for k, vs in defines): + results_.append(r) + results = results_ + + # organize results into conflicts folding = co.OrderedDict() for r in results: name = tuple(getattr(r, k) for k in by) @@ -199,157 +211,220 @@ def fold(results, *, folding[name] = [] folding[name].append(r) + # merge conflicts folded = [] - for rs in folding.values(): + for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) return folded - -def table(results, diff_results=None, *, - by_file=False, - size_sort=False, - reverse_size_sort=False, +def table(Result, results, diff_results=None, *, + by=None, + fields=None, + sort=None, summary=False, all=False, percent=False, **_): all_, all = all, __builtins__.all - # fold - results = fold(results, by=['file' if by_file else 'function']) - if diff_results is not None: - diff_results = fold(diff_results, - by=['file' if by_file else 'function']) + if by is None: + by = Result._by + if fields is None: + fields = Result._fields + types = Result._types + # fold again + results = fold(Result, results, by=by) + if diff_results is not None: + diff_results = fold(Result, diff_results, by=by) + + # organize by name table = { - r.file if by_file else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in results} diff_table = { - r.file if by_file else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in diff_results or []} - - # sort, note that python's sort is stable names = list(table.keys() | diff_table.keys()) + + # sort again, now with diff info, note that python's sort is stable names.sort() if diff_results is not None: - names.sort(key=lambda n: -IntField.ratio( - table[n].data_size if n in table else None, - diff_table[n].data_size if n in diff_table else None)) - if size_sort: - names.sort(key=lambda n: (table[n].data_size,) if n in table else (), + names.sort(key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), reverse=True) - elif reverse_size_sort: - names.sort(key=lambda n: (table[n].data_size,) if n in table else (), - reverse=False) + if sort: + for k, reverse in reversed(sort): + names.sort(key=lambda n: (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None else (), + reverse=reverse ^ (not k or k in Result._fields)) - # print header - if not summary: - title = '%s%s' % ( - 'file' if by_file else 'function', - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - name_width = max(it.chain([23, len(title)], (len(n) for n in names))) - else: - title = '' - name_width = 23 - name_width = 4*((name_width+1+4-1)//4)-1 - print('%-*s ' % (name_width, title), end='') + # build up our lines + lines = [] + + # header + line = [] + line.append('%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else '') if diff_results is None: - print(' %s' % ('size'.rjust(len(IntField.none)))) + for k in fields: + line.append(k) elif percent: - print(' %s' % ('size'.rjust(len(IntField.diff_none)))) + for k in fields: + line.append(k) else: - print(' %s %s %s' % ( - 'old'.rjust(len(IntField.diff_none)), - 'new'.rjust(len(IntField.diff_none)), - 'diff'.rjust(len(IntField.diff_none)))) + for k in fields: + line.append('o'+k) + for k in fields: + line.append('n'+k) + for k in fields: + line.append('d'+k) + line.append('') + lines.append(line) - # print entries + # entries if not summary: for name in names: r = table.get(name) if diff_results is not None: diff_r = diff_table.get(name) - ratio = IntField.ratio( - r.data_size if r else None, - diff_r.data_size if diff_r else None) - if not ratio and not all_: + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + if not any(ratios) and not all_: continue - print('%-*s ' % (name_width, name), end='') + line = [] + line.append(name) if diff_results is None: - print(' %s' % ( - r.data_size.table() - if r else IntField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - r.data_size.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - diff_r.data_size.diff_table() - if diff_r else IntField.diff_none, - r.data_size.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.data_size if r else None, - diff_r.data_size if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) - # print total - total = fold(results, by=[]) - r = total[0] if total else None + # total + r = next(iter(fold(Result, results, by=[])), None) if diff_results is not None: - diff_total = fold(diff_results, by=[]) - diff_r = diff_total[0] if diff_total else None - ratio = IntField.ratio( - r.data_size if r else None, - diff_r.data_size if diff_r else None) + diff_r = next(iter(fold(Result, diff_results, by=[])), None) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] - print('%-*s ' % (name_width, 'TOTAL'), end='') + line = [] + line.append('TOTAL') if diff_results is None: - print(' %s' % ( - r.data_size.table() - if r else IntField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - r.data_size.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - diff_r.data_size.diff_table() - if diff_r else IntField.diff_none, - r.data_size.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.data_size if r else None, - diff_r.data_size if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) + + # find the best widths, note that column 0 contains the names and column -1 + # the ratios, so those are handled a bit differently + widths = [ + ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1 + for w, i in zip( + it.chain([23], it.repeat(7)), + range(len(lines[0])-1))] + + # print our table + for line in lines: + print('%-*s %s%s' % ( + widths[0], line[0], + ' '.join('%*s' % (w, x) + for w, x in zip(widths[1:], line[1:-1])), + line[-1])) -def main(obj_paths, **args): +def main(obj_paths, *, + by=None, + fields=None, + defines=None, + sort=None, + **args): # find sizes if not args.get('use', None): # find .o files @@ -362,7 +437,7 @@ def main(obj_paths, **args): paths.append(path) if not paths: - print('no .obj files found in %r?' % obj_paths) + print("error: no .obj files found in %r?" % obj_paths) sys.exit(-1) results = collect(paths, **args) @@ -372,25 +447,35 @@ def main(obj_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - results.append(DataResult(**{ - k: v for k, v in r.items() - if k in DataResult._fields})) + results.append(DataResult( + **{k: r[k] for k in DataResult._by + if k in r and r[k].strip()}, + **{k: r['data_'+k] for k in DataResult._fields + if 'data_'+k in r and r['data_'+k].strip()})) except TypeError: pass - # fold to remove duplicates - results = fold(results) + # fold + results = fold(DataResult, results, by=by, defines=defines) - # sort because why not + # sort, note that python's sort is stable results.sort() + if sort: + for k, reverse in reversed(sort): + results.sort(key=lambda r: (getattr(r, k),) + if getattr(r, k) is not None else (), + reverse=reverse ^ (not k or k in DataResult._fields)) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, DataResult._fields) + writer = csv.DictWriter(f, DataResult._by + + ['data_'+k for k in DataResult._fields]) writer.writeheader() for r in results: - writer.writerow(r._asdict()) + writer.writerow( + {k: getattr(r, k) for k in DataResult._by} + | {'data_'+k: getattr(r, k) for k in DataResult._fields}) # find previous results? if args.get('diff'): @@ -400,22 +485,26 @@ def main(obj_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - diff_results.append(DataResult(**{ - k: v for k, v in r.items() - if k in DataResult._fields})) + diff_results.append(DataResult( + **{k: r[k] for k in DataResult._by + if k in r and r[k].strip()}, + **{k: r['data_'+k] for k in DataResult._fields + if 'data_'+k in r and r['data_'+k].strip()})) except TypeError: pass except FileNotFoundError: pass - # fold to remove duplicates - diff_results = fold(diff_results) + # fold + diff_results = fold(DataResult, diff_results, by=by, defines=defines) # print table if not args.get('quiet'): - table( - results, + table(DataResult, results, diff_results if args.get('diff') else None, + by=by if by is not None else ['function'], + fields=fields, + sort=sort, **args) @@ -456,22 +545,39 @@ if __name__ == "__main__": action='store_true', help="Only show percentage change, not a full diff.") parser.add_argument( - '-b', '--by-file', - action='store_true', - help="Group by file. Note this does not include padding " - "so sizes may differ from other tools.") + '-b', '--by', + action='append', + choices=DataResult._by, + help="Group by this field.") parser.add_argument( - '-s', '--size-sort', - action='store_true', - help="Sort by size.") + '-f', '--field', + dest='fields', + action='append', + choices=DataResult._fields, + help="Show this field.") parser.add_argument( - '-S', '--reverse-size-sort', - action='store_true', - help="Sort by size, but backwards.") + '-D', '--define', + dest='defines', + action='append', + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), + help="Only include results where this field is this value.") + class AppendSort(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.sort is None: + namespace.sort = [] + namespace.sort.append((value, True if option == '-S' else False)) + parser.add_argument( + '-s', '--sort', + action=AppendSort, + help="Sort by this fields.") + parser.add_argument( + '-S', '--reverse-sort', + action=AppendSort, + help="Sort by this fields, but backwards.") parser.add_argument( '-Y', '--summary', action='store_true', - help="Only show the total size.") + help="Only show the total.") parser.add_argument( '-A', '--everything', action='store_true', diff --git a/scripts/plot.py b/scripts/plot.py index 9aef2aee..15ec8462 100755 --- a/scripts/plot.py +++ b/scripts/plot.py @@ -438,9 +438,7 @@ def datasets(results, by=None, x=None, y=None, define=[]): y = co.OrderedDict() for r in results: for k, v in r.items(): - if by is not None and k in by: - continue - if y.get(k, True): + if (by is None or k not in by) and v.strip(): try: dat(v) y[k] = True @@ -462,7 +460,7 @@ def datasets(results, by=None, x=None, y=None, define=[]): for y_ in y: # hide x/y if there is only one field k_x = x_ if len(x or []) > 1 else '' - k_y = y_ if len(y or []) > 1 else '' + k_y = y_ if len(y or []) > 1 or (not ks_ and not k_x) else '' datasets[ks_ + (k_x, k_y)] = dataset( results, @@ -509,15 +507,15 @@ def main(csv_paths, *, ylim = (0, ylim[0]) # separate out renames - renames = [k.split('=', 1) - for k in it.chain(by or [], x or [], y or []) - if '=' in k] + renames = list(it.chain.from_iterable( + ((k, v) for v in vs) + for k, vs in it.chain(by or [], x or [], y or []))) if by is not None: - by = [k.split('=', 1)[0] for k in by] + by = [k for k, _ in by] if x is not None: - x = [k.split('=', 1)[0] for k in x] + x = [k for k, _ in x] if y is not None: - y = [k.split('=', 1)[0] for k in y] + y = [k for k, _ in y] def draw(f): def writeln(s=''): @@ -739,23 +737,31 @@ if __name__ == "__main__": parser.add_argument( '-b', '--by', action='append', - help="Fields to render as separate plots. All other fields will be " - "summed as needed. Can rename fields with new_name=old_name.") + type=lambda x: ( + lambda k,v=None: (k, v.split(',') if v is not None else ()) + )(*x.split('=', 1)), + help="Group by this field. Can rename fields with new_name=old_name.") parser.add_argument( '-x', action='append', - help="Fields to use for the x-axis. Can rename fields with " + type=lambda x: ( + lambda k,v=None: (k, v.split(',') if v is not None else ()) + )(*x.split('=', 1)), + help="Field to use for the x-axis. Can rename fields with " "new_name=old_name.") parser.add_argument( '-y', action='append', - help="Fields to use for the y-axis. Can rename fields with " + type=lambda x: ( + lambda k,v=None: (k, v.split(',') if v is not None else ()) + )(*x.split('=', 1)), + help="Field to use for the y-axis. Can rename fields with " "new_name=old_name.") parser.add_argument( '-D', '--define', type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), action='append', - help="Only include rows where this field is this value. May include " + help="Only include results where this field is this value. May include " "comma-separated options.") parser.add_argument( '--color', diff --git a/scripts/stack.py b/scripts/stack.py index ed5f1fce..6cb20ffa 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -23,10 +23,10 @@ CI_PATHS = ['*.ci'] # integer fields -class IntField(co.namedtuple('IntField', 'x')): +class Int(co.namedtuple('Int', 'x')): __slots__ = () def __new__(cls, x=0): - if isinstance(x, IntField): + if isinstance(x, Int): return x if isinstance(x, str): try: @@ -92,38 +92,33 @@ class IntField(co.namedtuple('IntField', 'x')): return (new-old) / old def __add__(self, other): - return IntField(self.x + other.x) + return self.__class__(self.x + other.x) def __sub__(self, other): - return IntField(self.x - other.x) + return self.__class__(self.x - other.x) def __mul__(self, other): - return IntField(self.x * other.x) - - def __lt__(self, other): - return self.x < other.x - - def __gt__(self, other): - return self.__class__.__lt__(other, self) - - def __le__(self, other): - return not self.__gt__(other) - - def __ge__(self, other): - return not self.__lt__(other) + return self.__class__(self.x * other.x) # size results -class StackResult(co.namedtuple('StackResult', - 'file,function,stack_frame,stack_limit')): +class StackResult(co.namedtuple('StackResult', [ + 'file', 'function', 'frame', 'limit', 'calls'])): + _by = ['file', 'function'] + _fields = ['frame', 'limit'] + _types = {'frame': Int, 'limit': Int} + __slots__ = () - def __new__(cls, file, function, stack_frame, stack_limit): + def __new__(cls, file='', function='', + frame=0, limit=0, calls=set()): return super().__new__(cls, file, function, - IntField(stack_frame), IntField(stack_limit)) + Int(frame), Int(limit), + calls) def __add__(self, other): return StackResult(self.file, self.function, - self.stack_frame + other.stack_frame, - max(self.stack_limit, other.stack_limit)) + self.frame + other.frame, + max(self.limit, other.limit), + self.calls | other.calls) def openio(path, mode='r'): @@ -135,7 +130,6 @@ def openio(path, mode='r'): else: return open(path, mode) - def collect(paths, *, everything=False, **args): @@ -147,10 +141,10 @@ def collect(paths, *, node = [] while True: rest = rest.lstrip() - m = k_pattern.match(rest) - if not m: + m_ = k_pattern.match(rest) + if not m_: return (node, rest) - k, rest = m.group(1), rest[m.end(0):] + k, rest = m_.group(1), rest[m_.end(0):] rest = rest.lstrip() if rest.startswith('{'): @@ -159,9 +153,9 @@ def collect(paths, *, rest = rest[1:] node.append((k, v)) else: - m = v_pattern.match(rest) - assert m, "unexpected %r" % rest[0:1] - v, rest = m.group(1) or m.group(2), rest[m.end(0):] + m_ = v_pattern.match(rest) + assert m_, "unexpected %r" % rest[0:1] + v, rest = m_.group(1) or m_.group(2), rest[m_.end(0):] node.append((k, v)) node, rest = parse_vcg(rest) @@ -181,13 +175,13 @@ def collect(paths, *, for k, info in graph: if k == 'node': info = dict(info) - m = f_pattern.match(info['label']) - if m: - function, file, size, type = m.groups() + m_ = f_pattern.match(info['label']) + if m_: + function, file, size, type = m_.groups() if (not args.get('quiet') and 'static' not in type and 'bounded' not in type): - print('warning: found non-static stack for %s (%s)' + print("warning: found non-static stack for %s (%s)" % (function, type, size)) _, _, _, targets = callgraph[info['title']] callgraph[info['title']] = ( @@ -217,7 +211,7 @@ def collect(paths, *, for target in targets: if target in seen: # found a cycle - return float('inf') + return m.inf limit_ = find_limit(target, seen | {target}) limit = max(limit, limit_) @@ -233,19 +227,35 @@ def collect(paths, *, # build results results = [] - calls = {} for source, (s_file, s_function, frame, targets) in callgraph.items(): limit = find_limit(source) - cs = find_calls(targets) - results.append(StackResult(s_file, s_function, frame, limit)) - calls[(s_file, s_function)] = cs + calls = find_calls(targets) + results.append(StackResult(s_file, s_function, frame, limit, calls)) - return results, calls + return results -def fold(results, *, - by=['file', 'function'], +def fold(Result, results, *, + by=None, + defines=None, **_): + if by is None: + by = Result._by + + for k in it.chain(by or [], (k for k, _ in defines or [])): + if k not in Result._by and k not in Result._fields: + print("error: could not find field %r?" % k) + sys.exit(-1) + + # filter by matching defines + if defines is not None: + results_ = [] + for r in results: + if all(getattr(r, k) in vs for k, vs in defines): + results_.append(r) + results = results_ + + # organize results into conflicts folding = co.OrderedDict() for r in results: name = tuple(getattr(r, k) for k in by) @@ -253,36 +263,17 @@ def fold(results, *, folding[name] = [] folding[name].append(r) + # merge conflicts folded = [] - for rs in folding.values(): + for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) return folded -def fold_calls(calls, *, - by=['file', 'function'], - **_): - def by_(name): - file, function = name - return (((file,) if 'file' in by else ()) - + ((function,) if 'function' in by else ())) - - folded = {} - for name, cs in calls.items(): - name = by_(name) - if name not in folded: - folded[name] = set() - folded[name] |= {by_(c) for c in cs} - - return folded - - -def table(results, calls, diff_results=None, *, - by_file=False, - limit_sort=False, - reverse_limit_sort=False, - frame_sort=False, - reverse_frame_sort=False, +def table(Result, results, diff_results=None, *, + by=None, + fields=None, + sort=None, summary=False, all=False, percent=False, @@ -291,209 +282,268 @@ def table(results, calls, diff_results=None, *, **_): all_, all = all, __builtins__.all - # tree doesn't really make sense with depth=0, assume depth=inf - if depth is None: - depth = float('inf') if tree else 0 + if by is None: + by = Result._by + if fields is None: + fields = Result._fields + types = Result._types - # fold - results = fold(results, by=['file' if by_file else 'function']) - calls = fold_calls(calls, by=['file' if by_file else 'function']) + # fold again + results = fold(Result, results, by=by) if diff_results is not None: - diff_results = fold(diff_results, - by=['file' if by_file else 'function']) + diff_results = fold(Result, diff_results, by=by) + # organize by name table = { - r.file if by_file else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in results} diff_table = { - r.file if by_file else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in diff_results or []} - - # sort, note that python's sort is stable names = list(table.keys() | diff_table.keys()) + + # sort again, now with diff info, note that python's sort is stable names.sort() if diff_results is not None: - names.sort(key=lambda n: -IntField.ratio( - table[n].stack_frame if n in table else None, - diff_table[n].stack_frame if n in diff_table else None)) - if limit_sort: - names.sort(key=lambda n: (table[n].stack_limit,) if n in table else (), + names.sort(key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), reverse=True) - elif reverse_limit_sort: - names.sort(key=lambda n: (table[n].stack_limit,) if n in table else (), - reverse=False) - elif frame_sort: - names.sort(key=lambda n: (table[n].stack_frame,) if n in table else (), - reverse=True) - elif reverse_frame_sort: - names.sort(key=lambda n: (table[n].stack_frame,) if n in table else (), - reverse=False) + if sort: + for k, reverse in reversed(sort): + names.sort(key=lambda n: (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None else (), + reverse=reverse ^ (not k or k in Result._fields)) - # print header - if not summary: - title = '%s%s' % ( - 'file' if by_file else 'function', - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - name_width = max(it.chain([23, len(title)], (len(n) for n in names))) + + # build up our lines + lines = [] + + # header + line = [] + line.append('%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else '') + if diff_results is None: + for k in fields: + line.append(k) + elif percent: + for k in fields: + line.append(k) else: - title = '' - name_width = 23 - name_width = 4*((name_width+1+4-1)//4)-1 + for k in fields: + line.append('o'+k) + for k in fields: + line.append('n'+k) + for k in fields: + line.append('d'+k) + line.append('') + lines.append(line) - # adjust the name width based on the expected call depth, note that we - # can't always find the depth due to recursion - if not m.isinf(depth): - name_width += 4*depth - - if not tree: - print('%-*s ' % (name_width, title), end='') - if diff_results is None: - print(' %s %s' % ( - 'frame'.rjust(len(IntField.none)), - 'limit'.rjust(len(IntField.none)))) - elif percent: - print(' %s %s' % ( - 'frame'.rjust(len(IntField.diff_none)), - 'limit'.rjust(len(IntField.diff_none)))) - else: - print(' %s %s %s %s %s %s' % ( - 'oframe'.rjust(len(IntField.diff_none)), - 'olimit'.rjust(len(IntField.diff_none)), - 'nframe'.rjust(len(IntField.diff_none)), - 'nlimit'.rjust(len(IntField.diff_none)), - 'dframe'.rjust(len(IntField.diff_none)), - 'dlimit'.rjust(len(IntField.diff_none)))) - - # print entries + # entries if not summary: - # print the tree recursively - def table_calls(names_, depth, - prefixes=('', '', '', '')): - for i, name in enumerate(names_): - r = table.get(name) - if diff_results is not None: - diff_r = diff_table.get(name) - ratio = IntField.ratio( - r.stack_limit if r else None, - diff_r.stack_limit if diff_r else None) - if not ratio and not all_: - continue + for name in names: + r = table.get(name) + if diff_results is not None: + diff_r = diff_table.get(name) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + if not any(ratios) and not all_: + continue + line = [] + line.append(name) + if diff_results is None: + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) + elif percent: + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + else: + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) + + # total + r = next(iter(fold(Result, results, by=[])), None) + if diff_results is not None: + diff_r = next(iter(fold(Result, diff_results, by=[])), None) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + + line = [] + line.append('TOTAL') + if diff_results is None: + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) + elif percent: + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + else: + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) + + # find the best widths, note that column 0 contains the names and column -1 + # the ratios, so those are handled a bit differently + widths = [ + ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1 + for w, i in zip( + it.chain([23], it.repeat(7)), + range(len(lines[0])-1))] + + # adjust the name width based on the expected call depth, though + # note this doesn't really work with unbounded recursion + if not summary: + # it doesn't really make sense to not have a depth with tree, + # so assume depth=inf if tree by default + if depth is None: + depth = m.inf if tree else 0 + elif depth == 0: + depth = m.inf + + if not m.isinf(depth): + widths[0] += 4*depth + + # print our table with optional call info + # + # note we try to adjust our name width based on expected call depth, but + # this doesn't work if there's unbounded recursion + if not tree: + print('%-*s %s%s' % ( + widths[0], lines[0][0], + ' '.join('%*s' % (w, x) + for w, x, in zip(widths[1:], lines[0][1:-1])), + lines[0][-1])) + + # print the tree recursively + if not summary: + line_table = {n: l for n, l in zip(names, lines[1:-1])} + + def recurse(names_, depth_, prefixes=('', '', '', '')): + for i, name in enumerate(names_): + if name not in line_table: + continue + line = line_table[name] is_last = (i == len(names_)-1) - print('%-*s ' % (name_width, prefixes[0+is_last]+name), end='') - if tree: - print() - elif diff_results is None: - print(' %s %s' % ( - r.stack_frame.table() - if r else IntField.none, - r.stack_limit.table() - if r else IntField.none)) - elif percent: - print(' %s %s%s' % ( - r.stack_frame.diff_table() - if r else IntField.diff_none, - r.stack_limit.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) - else: - print(' %s %s %s %s %s %s%s' % ( - diff_r.stack_frame.diff_table() - if diff_r else IntField.diff_none, - diff_r.stack_limit.diff_table() - if diff_r else IntField.diff_none, - r.stack_frame.diff_table() - if r else IntField.diff_none, - r.stack_limit.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.stack_frame if r else None, - diff_r.stack_frame if diff_r else None) - if r or diff_r else IntField.diff_none, - IntField.diff_diff( - r.stack_limit if r else None, - diff_r.stack_limit if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + + print('%s%-*s ' % ( + prefixes[0+is_last], + widths[0] - ( + len(prefixes[0+is_last]) + if not m.isinf(depth) else 0), + line[0]), + end='') + if not tree: + print(' %s%s' % ( + ' '.join('%*s' % (w, x) + for w, x, in zip(widths[1:], line[1:-1])), + line[-1]), + end='') + print() # recurse? - if depth > 0: - cs = calls.get((name,), set()) - table_calls( - [n for n in names if (n,) in cs], - depth-1, - ( prefixes[2+is_last] + "|-> ", - prefixes[2+is_last] + "'-> ", - prefixes[2+is_last] + "| ", - prefixes[2+is_last] + " ")) + if name in table and depth_ > 0: + calls = { + ','.join(str(getattr(Result(*c), k) or '') for k in by) + for c in table[name].calls} + recurse( + # note we're maintaining sort order + [n for n in names if n in calls], + depth_-1, + (prefixes[2+is_last] + "|-> ", + prefixes[2+is_last] + "'-> ", + prefixes[2+is_last] + "| ", + prefixes[2+is_last] + " ")) + recurse(names, depth) - table_calls(names, depth) - - # print total if not tree: - total = fold(results, by=[]) - r = total[0] if total else None - if diff_results is not None: - diff_total = fold(diff_results, by=[]) - diff_r = diff_total[0] if diff_total else None - ratio = IntField.ratio( - r.stack_limit if r else None, - diff_r.stack_limit if diff_r else None) - - print('%-*s ' % (name_width, 'TOTAL'), end='') - if diff_results is None: - print(' %s %s' % ( - r.stack_frame.table() - if r else IntField.none, - r.stack_limit.table() - if r else IntField.none)) - elif percent: - print(' %s %s%s' % ( - r.stack_frame.diff_table() - if r else IntField.diff_none, - r.stack_limit.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) - else: - print(' %s %s %s %s %s %s%s' % ( - diff_r.stack_frame.diff_table() - if diff_r else IntField.diff_none, - diff_r.stack_limit.diff_table() - if diff_r else IntField.diff_none, - r.stack_frame.diff_table() - if r else IntField.diff_none, - r.stack_limit.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.stack_frame if r else None, - diff_r.stack_frame if diff_r else None) - if r or diff_r else IntField.diff_none, - IntField.diff_diff( - r.stack_limit if r else None, - diff_r.stack_limit if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + print('%-*s %s%s' % ( + widths[0], lines[-1][0], + ' '.join('%*s' % (w, x) + for w, x, in zip(widths[1:], lines[-1][1:-1])), + lines[-1][-1])) -def main(ci_paths, **args): +def main(ci_paths, + by=None, + fields=None, + defines=None, + sort=None, + **args): # find sizes if not args.get('use', None): # find .ci files @@ -506,37 +556,45 @@ def main(ci_paths, **args): paths.append(path) if not paths: - print('no .ci files found in %r?' % ci_paths) + print("error: no .ci files found in %r?" % ci_paths) sys.exit(-1) - results, calls = collect(paths, **args) + results = collect(paths, **args) else: results = [] with openio(args['use']) as f: reader = csv.DictReader(f, restval='') for r in reader: try: - results.append(StackResult(**{ - k: v for k, v in r.items() - if k in StackResult._fields})) + results.append(StackResult( + **{k: r[k] for k in StackResult._by + if k in r and r[k].strip()}, + **{k: r['stack_'+k] for k in StackResult._fields + if 'stack_'+k in r and r['stack_'+k].strip()})) except TypeError: pass - calls = {} + # fold + results = fold(StackResult, results, by=by, defines=defines) - # fold to remove duplicates - results = fold(results) - - # sort because why not + # sort, note that python's sort is stable results.sort() + if sort: + for k, reverse in reversed(sort): + results.sort(key=lambda r: (getattr(r, k),) + if getattr(r, k) is not None else (), + reverse=reverse ^ (not k or k in StackResult._fields)) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, StackResult._fields) + writer = csv.DictWriter(f, StackResult._by + + ['stack_'+k for k in StackResult._fields]) writer.writeheader() for r in results: - writer.writerow(r._asdict()) + writer.writerow( + {k: getattr(r, k) for k in StackResult._by} + | {'stack_'+k: getattr(r, k) for k in StackResult._fields}) # find previous results? if args.get('diff'): @@ -546,28 +604,31 @@ def main(ci_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - diff_results.append(StackResult(**{ - k: v for k, v in r.items() - if k in StackResult._fields})) + diff_results.append(StackResult( + **{k: r[k] for k in StackResult._by + if k in r and r[k].strip()}, + **{k: r['stack_'+k] for k in StackResult._fields + if 'stack_'+k in r and r['stack_'+k].strip()})) except TypeError: - pass + raise except FileNotFoundError: pass - # fold to remove duplicates - diff_results = fold(diff_results) + # fold + diff_results = fold(StackResult, diff_results, by=by, defines=defines) # print table if not args.get('quiet'): - table( - results, - calls, + table(StackResult, results, diff_results if args.get('diff') else None, + by=by if by is not None else ['function'], + fields=fields, + sort=sort, **args) # error on recursion if args.get('error_on_recursion') and any( - m.isinf(float(r.stack_limit)) for r in results): + m.isinf(float(r.limit)) for r in results): sys.exit(2) @@ -608,47 +669,58 @@ if __name__ == "__main__": action='store_true', help="Only show percentage change, not a full diff.") parser.add_argument( - '-t', '--tree', - action='store_true', - help="Only show the function call tree.") + '-b', '--by', + action='append', + choices=StackResult._by, + help="Group by this field.") parser.add_argument( - '-b', '--by-file', - action='store_true', - help="Group by file.") + '-f', '--field', + dest='fields', + action='append', + choices=StackResult._fields, + help="Show this field.") parser.add_argument( - '-s', '--limit-sort', - action='store_true', - help="Sort by stack limit.") + '-D', '--define', + dest='defines', + action='append', + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), + help="Only include results where this field is this value.") + class AppendSort(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.sort is None: + namespace.sort = [] + namespace.sort.append((value, True if option == '-S' else False)) parser.add_argument( - '-S', '--reverse-limit-sort', - action='store_true', - help="Sort by stack limit, but backwards.") + '-s', '--sort', + action=AppendSort, + help="Sort by this fields.") parser.add_argument( - '--frame-sort', - action='store_true', - help="Sort by stack frame.") - parser.add_argument( - '--reverse-frame-sort', - action='store_true', - help="Sort by stack frame, but backwards.") + '-S', '--reverse-sort', + action=AppendSort, + help="Sort by this fields, but backwards.") parser.add_argument( '-Y', '--summary', action='store_true', - help="Only show the total size.") - parser.add_argument( - '-L', '--depth', - nargs='?', - type=lambda x: int(x, 0), - const=float('inf'), - help="Depth of function calls to show.") - parser.add_argument( - '-e', '--error-on-recursion', - action='store_true', - help="Error if any functions are recursive.") + help="Only show the total.") parser.add_argument( '-A', '--everything', action='store_true', help="Include builtin and libc specific symbols.") + parser.add_argument( + '--tree', + action='store_true', + help="Only show the function call tree.") + parser.add_argument( + '-L', '--depth', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Depth of function calls to show. 0 show all calls but may not " + "terminate!") + parser.add_argument( + '-e', '--error-on-recursion', + action='store_true', + help="Error if any functions are recursive.") parser.add_argument( '--build-dir', help="Specify the relative build directory. Used to map object files " diff --git a/scripts/struct_.py b/scripts/struct_.py index bdb98f73..ed6584cb 100755 --- a/scripts/struct_.py +++ b/scripts/struct_.py @@ -24,11 +24,12 @@ OBJ_PATHS = ['*.o'] OBJDUMP_TOOL = ['objdump'] + # integer fields -class IntField(co.namedtuple('IntField', 'x')): +class Int(co.namedtuple('Int', 'x')): __slots__ = () def __new__(cls, x=0): - if isinstance(x, IntField): + if isinstance(x, Int): return x if isinstance(x, str): try: @@ -94,35 +95,28 @@ class IntField(co.namedtuple('IntField', 'x')): return (new-old) / old def __add__(self, other): - return IntField(self.x + other.x) + return self.__class__(self.x + other.x) def __sub__(self, other): - return IntField(self.x - other.x) + return self.__class__(self.x - other.x) def __mul__(self, other): - return IntField(self.x * other.x) - - def __lt__(self, other): - return self.x < other.x - - def __gt__(self, other): - return self.__class__.__lt__(other, self) - - def __le__(self, other): - return not self.__gt__(other) - - def __ge__(self, other): - return not self.__lt__(other) + return self.__class__(self.x * other.x) # struct size results -class StructResult(co.namedtuple('StructResult', 'file,struct,struct_size')): +class StructResult(co.namedtuple('StructResult', ['file', 'struct', 'size'])): + _by = ['file', 'struct'] + _fields = ['size'] + _types = {'size': Int} + __slots__ = () - def __new__(cls, file, struct, struct_size): - return super().__new__(cls, file, struct, IntField(struct_size)) + def __new__(cls, file='', struct='', size=0): + return super().__new__(cls, file, struct, + Int(size)) def __add__(self, other): return StructResult(self.file, self.struct, - self.struct_size + other.struct_size) + self.size + other.size) def openio(path, mode='r'): @@ -231,9 +225,27 @@ def collect(paths, *, return results -def fold(results, *, - by=['file', 'struct'], +def fold(Result, results, *, + by=None, + defines=None, **_): + if by is None: + by = Result._by + + for k in it.chain(by or [], (k for k, _ in defines or [])): + if k not in Result._by and k not in Result._fields: + print("error: could not find field %r?" % k) + sys.exit(-1) + + # filter by matching defines + if defines is not None: + results_ = [] + for r in results: + if all(getattr(r, k) in vs for k, vs in defines): + results_.append(r) + results = results_ + + # organize results into conflicts folding = co.OrderedDict() for r in results: name = tuple(getattr(r, k) for k in by) @@ -241,157 +253,220 @@ def fold(results, *, folding[name] = [] folding[name].append(r) + # merge conflicts folded = [] - for rs in folding.values(): + for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) return folded - -def table(results, diff_results=None, *, - by_file=False, - size_sort=False, - reverse_size_sort=False, +def table(Result, results, diff_results=None, *, + by=None, + fields=None, + sort=None, summary=False, all=False, percent=False, **_): all_, all = all, __builtins__.all - # fold - results = fold(results, by=['file' if by_file else 'struct']) - if diff_results is not None: - diff_results = fold(diff_results, - by=['file' if by_file else 'struct']) + if by is None: + by = Result._by + if fields is None: + fields = Result._fields + types = Result._types + # fold again + results = fold(Result, results, by=by) + if diff_results is not None: + diff_results = fold(Result, diff_results, by=by) + + # organize by name table = { - r.file if by_file else r.struct: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in results} diff_table = { - r.file if by_file else r.struct: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in diff_results or []} - - # sort, note that python's sort is stable names = list(table.keys() | diff_table.keys()) + + # sort again, now with diff info, note that python's sort is stable names.sort() if diff_results is not None: - names.sort(key=lambda n: -IntField.ratio( - table[n].struct_size if n in table else None, - diff_table[n].struct_size if n in diff_table else None)) - if size_sort: - names.sort(key=lambda n: (table[n].struct_size,) if n in table else (), + names.sort(key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), reverse=True) - elif reverse_size_sort: - names.sort(key=lambda n: (table[n].struct_size,) if n in table else (), - reverse=False) + if sort: + for k, reverse in reversed(sort): + names.sort(key=lambda n: (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None else (), + reverse=reverse ^ (not k or k in Result._fields)) - # print header - if not summary: - title = '%s%s' % ( - 'file' if by_file else 'struct', - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - name_width = max(it.chain([23, len(title)], (len(n) for n in names))) - else: - title = '' - name_width = 23 - name_width = 4*((name_width+1+4-1)//4)-1 - print('%-*s ' % (name_width, title), end='') + # build up our lines + lines = [] + + # header + line = [] + line.append('%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else '') if diff_results is None: - print(' %s' % ('size'.rjust(len(IntField.none)))) + for k in fields: + line.append(k) elif percent: - print(' %s' % ('size'.rjust(len(IntField.diff_none)))) + for k in fields: + line.append(k) else: - print(' %s %s %s' % ( - 'old'.rjust(len(IntField.diff_none)), - 'new'.rjust(len(IntField.diff_none)), - 'diff'.rjust(len(IntField.diff_none)))) + for k in fields: + line.append('o'+k) + for k in fields: + line.append('n'+k) + for k in fields: + line.append('d'+k) + line.append('') + lines.append(line) - # print entries + # entries if not summary: for name in names: r = table.get(name) if diff_results is not None: diff_r = diff_table.get(name) - ratio = IntField.ratio( - r.struct_size if r else None, - diff_r.struct_size if diff_r else None) - if not ratio and not all_: + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + if not any(ratios) and not all_: continue - print('%-*s ' % (name_width, name), end='') + line = [] + line.append(name) if diff_results is None: - print(' %s' % ( - r.struct_size.table() - if r else IntField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - r.struct_size.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - diff_r.struct_size.diff_table() - if diff_r else IntField.diff_none, - r.struct_size.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.struct_size if r else None, - diff_r.struct_size if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) - # print total - total = fold(results, by=[]) - r = total[0] if total else None + # total + r = next(iter(fold(Result, results, by=[])), None) if diff_results is not None: - diff_total = fold(diff_results, by=[]) - diff_r = diff_total[0] if diff_total else None - ratio = IntField.ratio( - r.struct_size if r else None, - diff_r.struct_size if diff_r else None) + diff_r = next(iter(fold(Result, diff_results, by=[])), None) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] - print('%-*s ' % (name_width, 'TOTAL'), end='') + line = [] + line.append('TOTAL') if diff_results is None: - print(' %s' % ( - r.struct_size.table() - if r else IntField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - r.struct_size.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - diff_r.struct_size.diff_table() - if diff_r else IntField.diff_none, - r.struct_size.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.struct_size if r else None, - diff_r.struct_size if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) + + # find the best widths, note that column 0 contains the names and column -1 + # the ratios, so those are handled a bit differently + widths = [ + ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1 + for w, i in zip( + it.chain([23], it.repeat(7)), + range(len(lines[0])-1))] + + # print our table + for line in lines: + print('%-*s %s%s' % ( + widths[0], line[0], + ' '.join('%*s' % (w, x) + for w, x in zip(widths[1:], line[1:-1])), + line[-1])) -def main(obj_paths, **args): +def main(obj_paths, *, + by=None, + fields=None, + defines=None, + sort=None, + **args): # find sizes if not args.get('use', None): # find .o files @@ -404,7 +479,7 @@ def main(obj_paths, **args): paths.append(path) if not paths: - print('no .obj files found in %r?' % obj_paths) + print("error: no .obj files found in %r?" % obj_paths) sys.exit(-1) results = collect(paths, **args) @@ -414,25 +489,38 @@ def main(obj_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - results.append(StructResult(**{ - k: v for k, v in r.items() - if k in StructResult._fields})) + results.append(StructResult( + **{k: r[k] for k in StructResult._by + if k in r and r[k].strip()}, + **{k: r['struct_'+k] + for k in StructResult._fields + if 'struct_'+k in r + and r['struct_'+k].strip()})) except TypeError: pass - # fold to remove duplicates - results = fold(results) + # fold + results = fold(StructResult, results, by=by, defines=defines) - # sort because why not + # sort, note that python's sort is stable results.sort() + if sort: + for k, reverse in reversed(sort): + results.sort(key=lambda r: (getattr(r, k),) + if getattr(r, k) is not None else (), + reverse=reverse ^ (not k or k in StructResult._fields)) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, StructResult._fields) + writer = csv.DictWriter(f, StructResult._by + + ['struct_'+k for k in StructResult._fields]) writer.writeheader() for r in results: - writer.writerow(r._asdict()) + writer.writerow( + {k: getattr(r, k) for k in StructResult._by} + | {'struct_'+k: getattr(r, k) + for k in StructResult._fields}) # find previous results? if args.get('diff'): @@ -442,22 +530,28 @@ def main(obj_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - diff_results.append(StructResult(**{ - k: v for k, v in r.items() - if k in StructResult._fields})) + diff_results.append(StructResult( + **{k: r[k] for k in StructResult._by + if k in r and r[k].strip()}, + **{k: r['struct_'+k] + for k in StructResult._fields + if 'struct_'+k in r + and r['struct_'+k].strip()})) except TypeError: pass except FileNotFoundError: pass - # fold to remove duplicates - diff_results = fold(diff_results) + # fold + diff_results = fold(StructResult, diff_results, by=by, defines=defines) # print table if not args.get('quiet'): - table( - results, + table(StructResult, results, diff_results if args.get('diff') else None, + by=by if by is not None else ['struct'], + fields=fields, + sort=sort, **args) @@ -498,22 +592,39 @@ if __name__ == "__main__": action='store_true', help="Only show percentage change, not a full diff.") parser.add_argument( - '-b', '--by-file', - action='store_true', - help="Group by file. Note this does not include padding " - "so sizes may differ from other tools.") + '-b', '--by', + action='append', + choices=StructResult._by, + help="Group by this field.") parser.add_argument( - '-s', '--size-sort', - action='store_true', - help="Sort by size.") + '-f', '--field', + dest='fields', + action='append', + choices=StructResult._fields, + help="Show this field.") parser.add_argument( - '-S', '--reverse-size-sort', - action='store_true', - help="Sort by size, but backwards.") + '-D', '--define', + dest='defines', + action='append', + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), + help="Only include results where this field is this value.") + class AppendSort(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.sort is None: + namespace.sort = [] + namespace.sort.append((value, True if option == '-S' else False)) + parser.add_argument( + '-s', '--sort', + action=AppendSort, + help="Sort by this field.") + parser.add_argument( + '-S', '--reverse-sort', + action=AppendSort, + help="Sort by this field, but backwards.") parser.add_argument( '-Y', '--summary', action='store_true', - help="Only show the total size.") + help="Only show the total.") parser.add_argument( '-A', '--everything', action='store_true', diff --git a/scripts/summary.py b/scripts/summary.py index 98c1a538..9a13703f 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -33,36 +33,25 @@ OPS = { 'prod': lambda xs: m.prod(xs[1:], start=xs[0]), 'min': min, 'max': max, - 'mean': lambda xs: FloatField(sum(float(x) for x in xs) / len(xs)), + 'mean': lambda xs: Float(sum(float(x) for x in xs) / len(xs)), 'stddev': lambda xs: ( - lambda mean: FloatField( + lambda mean: Float( m.sqrt(sum((float(x) - mean)**2 for x in xs) / len(xs))) )(sum(float(x) for x in xs) / len(xs)), - 'gmean': lambda xs: FloatField(m.prod(float(x) for x in xs)**(1/len(xs))), + 'gmean': lambda xs: Float(m.prod(float(x) for x in xs)**(1/len(xs))), 'gstddev': lambda xs: ( - lambda gmean: FloatField( + lambda gmean: Float( m.exp(m.sqrt(sum(m.log(float(x)/gmean)**2 for x in xs) / len(xs))) if gmean else m.inf) - )(m.prod(float(x) for x in xs)**(1/len(xs))) - + )(m.prod(float(x) for x in xs)**(1/len(xs))), } -def openio(path, mode='r'): - if path == '-': - if mode == 'r': - return os.fdopen(os.dup(sys.stdin.fileno()), 'r') - else: - return os.fdopen(os.dup(sys.stdout.fileno()), 'w') - else: - return open(path, mode) - - # integer fields -class IntField(co.namedtuple('IntField', 'x')): +class Int(co.namedtuple('Int', 'x')): __slots__ = () def __new__(cls, x=0): - if isinstance(x, IntField): + if isinstance(x, Int): return x if isinstance(x, str): try: @@ -128,31 +117,19 @@ class IntField(co.namedtuple('IntField', 'x')): return (new-old) / old def __add__(self, other): - return IntField(self.x + other.x) + return self.__class__(self.x + other.x) def __sub__(self, other): - return IntField(self.x - other.x) + return self.__class__(self.x - other.x) def __mul__(self, other): - return IntField(self.x * other.x) - - def __lt__(self, other): - return self.x < other.x - - def __gt__(self, other): - return self.__class__.__lt__(other, self) - - def __le__(self, other): - return not self.__gt__(other) - - def __ge__(self, other): - return not self.__lt__(other) + return self.__class__(self.x * other.x) # float fields -class FloatField(co.namedtuple('FloatField', 'x')): +class Float(co.namedtuple('Float', 'x')): __slots__ = () def __new__(cls, x=0.0): - if isinstance(x, FloatField): + if isinstance(x, Float): return x if isinstance(x, str): try: @@ -179,31 +156,27 @@ class FloatField(co.namedtuple('FloatField', 'x')): def __float__(self): return float(self.x) - none = IntField.none - table = IntField.table - diff_none = IntField.diff_none - diff_table = IntField.diff_table - diff_diff = IntField.diff_diff - ratio = IntField.ratio - __add__ = IntField.__add__ - __sub__ = IntField.__sub__ - __mul__ = IntField.__mul__ - __lt__ = IntField.__lt__ - __gt__ = IntField.__gt__ - __le__ = IntField.__le__ - __ge__ = IntField.__ge__ + none = Int.none + table = Int.table + diff_none = Int.diff_none + diff_table = Int.diff_table + diff_diff = Int.diff_diff + ratio = Int.ratio + __add__ = Int.__add__ + __sub__ = Int.__sub__ + __mul__ = Int.__mul__ # fractional fields, a/b -class FracField(co.namedtuple('FracField', 'a,b')): +class Frac(co.namedtuple('Frac', 'a,b')): __slots__ = () def __new__(cls, a=0, b=None): - if isinstance(a, FracField) and b is None: + if isinstance(a, Frac) and b is None: return a if isinstance(a, str) and b is None: a, b = a.split('/', 1) if b is None: b = a - return super().__new__(cls, IntField(a), IntField(b)) + return super().__new__(cls, Int(a), Int(b)) def __str__(self): return '%s/%s' % (self.a, self.b) @@ -213,10 +186,7 @@ class FracField(co.namedtuple('FracField', 'a,b')): none = '%11s %7s' % ('-', '-') def table(self): - if not self.b.x: - return self.none - - t = self.a.x/self.b.x + t = self.a.x/self.b.x if self.b.x else 1.0 return '%11s %7s' % ( self, '∞%' if t == +m.inf @@ -225,38 +195,35 @@ class FracField(co.namedtuple('FracField', 'a,b')): diff_none = '%11s' % '-' def diff_table(self): - if not self.b.x: - return self.diff_none - return '%11s' % (self,) def diff_diff(self, other): - new_a, new_b = self if self else (IntField(0), IntField(0)) - old_a, old_b = other if other else (IntField(0), IntField(0)) + new_a, new_b = self if self else (Int(0), Int(0)) + old_a, old_b = other if other else (Int(0), Int(0)) return '%11s' % ('%s/%s' % ( new_a.diff_diff(old_a).strip(), new_b.diff_diff(old_b).strip())) def ratio(self, other): - new_a, new_b = self if self else (IntField(0), IntField(0)) - old_a, old_b = other if other else (IntField(0), IntField(0)) + new_a, new_b = self if self else (Int(0), Int(0)) + old_a, old_b = other if other else (Int(0), Int(0)) new = new_a.x/new_b.x if new_b.x else 1.0 old = old_a.x/old_b.x if old_b.x else 1.0 return new - old def __add__(self, other): - return FracField(self.a + other.a, self.b + other.b) + return self.__class__(self.a + other.a, self.b + other.b) def __sub__(self, other): - return FracField(self.a - other.a, self.b - other.b) + return self.__class__(self.a - other.a, self.b - other.b) def __mul__(self, other): - return FracField(self.a * other.a, self.b + other.b) + return self.__class__(self.a * other.a, self.b + other.b) def __lt__(self, other): - self_r = self.a.x/self.b.x if self.b.x else -m.inf - other_r = other.a.x/other.b.x if other.b.x else -m.inf - return self_r < other_r + self_t = self.a.x/self.b.x if self.b.x else 1.0 + other_t = other.a.x/other.b.x if other.b.x else 1.0 + return (self_t, self.a.x) < (other_t, other.a.x) def __gt__(self, other): return self.__class__.__lt__(other, self) @@ -268,54 +235,41 @@ class FracField(co.namedtuple('FracField', 'a,b')): return not self.__lt__(other) # available types -TYPES = [IntField, FloatField, FracField] +TYPES = co.OrderedDict([ + ('int', Int), + ('float', Float), + ('frac', Frac) +]) -def homogenize(results, *, +def infer(results, *, by=None, fields=None, + types={}, + ops={}, renames=[], - define={}, - types=None, **_): - results = results.copy() - - # rename fields? - if renames: - for r in results: - # make a copy so renames can overlap - r_ = {} - for new_k, old_k in renames: - if old_k in r: - r_[new_k] = r[old_k] - r.update(r_) - - # filter by matching defines - if define: - results_ = [] - for r in results: - if all(k in r and r[k] in vs for k, vs in define): - results_.append(r) - results = results_ - # if fields not specified, try to guess from data if fields is None: fields = co.OrderedDict() for r in results: for k, v in r.items(): - if by is not None and k in by: - continue - types_ = [] - for type in fields.get(k, TYPES): - try: - type(v) - types_.append(type) - except ValueError: - pass - fields[k] = types_ - fields = list(k for k,v in fields.items() if v) + if (by is None or k not in by) and v.strip(): + types_ = [] + for t in fields.get(k, TYPES.values()): + try: + t(v) + types_.append(t) + except ValueError: + pass + fields[k] = types_ + fields = list(k for k, v in fields.items() if v) - # infer 'by' fields? + # deduplicate fields + fields = list(co.OrderedDict.fromkeys(fields).keys()) + + # if by not specified, guess it's anything not in fields and not a + # source of a rename if by is None: by = co.OrderedDict() for r in results: @@ -327,295 +281,355 @@ def homogenize(results, *, and not any(k == old_k for _, old_k in renames)) by = list(by.keys()) - # go ahead and clean up none values, these can have a few forms - results_ = [] - for r in results: - results_.append({ - k: r[k] for k in it.chain(by, fields) - if r.get(k) is not None and not ( - isinstance(r[k], str) - and re.match('^\s*[+-]?\s*$', r[k]))}) - results = results_ + # deduplicate fields + by = list(co.OrderedDict.fromkeys(by).keys()) # find best type for all fields - if types is None: - def is_type(x, type): - try: - type(x) - return True - except ValueError: - return False - - types = {} - for k in fields: - for type in TYPES: - if all(k not in r or is_type(r[k], type) for r in results_): - types[k] = type + types_ = {} + for k in fields: + if k in types: + types_[k] = types[k] + else: + for t in TYPES.values(): + for r in results: + if k in r and r[k].strip(): + try: + t(r[k]) + except ValueError: + break + else: + types_[k] = t break else: - print("no type matches field %r?" % k) + print("error: no type matches field %r?" % k) sys.exit(-1) + types = types_ - # homogenize types - for r in results: - for k in fields: - if k in r: - r[k] = types[k](r[k]) - - return by, fields, types, results + # does folding change the type? + types_ = {} + for k, t in types.items(): + types_[k] = ops.get(k, OPS['sum'])([t()]).__class__ -def fold(results, *, - by=[], - fields=[], - types=None, - ops={}, + # create result class + def __new__(cls, **r): + return cls.__mro__[1].__new__(cls, + **{k: r.get(k) for k in by}, + **{k: r[k] if k in r and isinstance(r[k], list) + else [types[k](r[k])] if k in r + else [] + for k in fields}) + + def __add__(self, other): + return self.__class__( + **{k: getattr(self, k) for k in by}, + **{k: object.__getattribute__(self, k) + + object.__getattribute__(other, k) + for k in fields}) + + def __getattribute__(self, k): + if k in fields: + if object.__getattribute__(self, k): + return ops.get(k, OPS['sum'])(object.__getattribute__(self, k)) + else: + return None + return object.__getattribute__(self, k) + + return type('Result', (co.namedtuple('Result', by + fields),), { + '__slots__': (), + '__new__': __new__, + '__add__': __add__, + '__getattribute__': __getattribute__, + '_by': by, + '_fields': fields, + '_types': types_, + }) + + +def fold(Result, results, *, + by=None, + defines=None, **_): + if by is None: + by = Result._by + + for k in it.chain(by or [], (k for k, _ in defines or [])): + if k not in Result._by and k not in Result._fields: + print("error: could not find field %r?" % k) + sys.exit(-1) + + # filter by matching defines + if defines is not None: + results_ = [] + for r in results: + if all(getattr(r, k) in vs for k, vs in defines): + results_.append(r) + results = results_ + + # organize results into conflicts folding = co.OrderedDict() for r in results: - name = tuple(r.get(k, '') for k in by) + name = tuple(getattr(r, k) for k in by) if name not in folding: - folding[name] = {k: [] for k in fields} - for k in fields: - if k in r: - folding[name][k].append(r[k]) + folding[name] = [] + folding[name].append(r) - # merge fields, we need the count at this point for averages + # merge conflicts folded = [] - for name, r in folding.items(): - r_ = {} - for k, vs in r.items(): - if vs: - # sum fields by default - op = OPS[ops.get(k, 'sum')] - r_[k] = op(vs) + for name, rs in folding.items(): + folded.append(sum(rs[1:], start=rs[0])) - # drop any rows without fields and any empty keys - if r_: - folded.append(dict( - {k: v for k, v in zip(by, name) if v}, - **r_)) + return folded - # what is the type of merged fields? - if types is not None: - types_ = {} - for k in fields: - op = OPS[ops.get(k, 'sum')] - types_[k] = op([types[k]()]).__class__ - - if types is None: - return folded - else: - return types_, folded - - -def table(results, total, diff_results=None, diff_total=None, *, - by=[], - fields=[], - types={}, - ops={}, +def table(Result, results, diff_results=None, *, + by=None, + fields=None, sort=None, - reverse_sort=None, summary=False, all=False, percent=False, **_): all_, all = all, __builtins__.all + if by is None: + by = Result._by + if fields is None: + fields = Result._fields + types = Result._types + + # fold again + results = fold(Result, results, by=by) + if diff_results is not None: + diff_results = fold(Result, diff_results, by=by) + + # organize by name table = { - ','.join(r.get(k,'') for k in by): r + ','.join(str(getattr(r, k) or '') for k in by): r for r in results} diff_table = { - ','.join(r.get(k,'') for k in by): r + ','.join(str(getattr(r, k) or '') for k in by): r for r in diff_results or []} - - # sort, note that python's sort is stable names = list(table.keys() | diff_table.keys()) + + # sort again, now with diff info, note that python's sort is stable names.sort() if diff_results is not None: names.sort(key=lambda n: tuple( - -types[k].ratio( - table.get(n,{}).get(k), - diff_table.get(n,{}).get(k)) - for k in fields)) - if sort: - names.sort(key=lambda n: tuple( - (table[n][k],) if k in table.get(n,{}) else () - for k in sort), + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), reverse=True) - elif reverse_sort: - names.sort(key=lambda n: tuple( - (table[n][k],) if k in table.get(n,{}) else () - for k in reverse_sort), - reverse=False) + if sort: + for k, reverse in reversed(sort): + names.sort(key=lambda n: (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None else (), + reverse=reverse ^ (not k or k in Result._fields)) - # print header - if not summary: - title = '%s%s' % ( - ','.join(by), - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - name_width = max(it.chain([23, len(title)], (len(n) for n in names))) - else: - title = '' - name_width = 23 - name_width = 4*((name_width+1+4-1)//4)-1 - print('%-*s ' % (name_width, title), end='') + # build up our lines + lines = [] + + # header + line = [] + line.append('%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else '') if diff_results is None: - widths = [ - 4*((max(len(types[k].none), len(k))+1+4-1)//4)-1 - for k in fields] - print(' %s' % ( - ' '.join(k.rjust(w) for w, k in zip(widths, fields)))) + for k in fields: + line.append(k) elif percent: - widths = [ - 4*((max(len(types[k].diff_none), len(k))+1+4-1)//4)-1 - for k in fields] - print(' %s' % ( - ' '.join(k.rjust(w) for w, k in zip(widths, fields)))) + for k in fields: + line.append(k) else: - widths = [ - 4*((max(len(types[k].diff_none), 1+len(k))+1+4-1)//4)-1 - for k in fields] - print(' %s %s %s' % ( - ' '.join(('o'+k).rjust(w) for w, k in zip(widths, fields)), - ' '.join(('n'+k).rjust(w) for w, k in zip(widths, fields)), - ' '.join(('d'+k).rjust(w) for w, k in zip(widths, fields)))) + for k in fields: + line.append('o'+k) + for k in fields: + line.append('n'+k) + for k in fields: + line.append('d'+k) + line.append('') + lines.append(line) - # print entries + # entries if not summary: for name in names: - r = table.get(name, {}) + r = table.get(name) if diff_results is not None: - diff_r = diff_table.get(name, {}) - ratios = [types[k].ratio(r.get(k), diff_r.get(k)) + diff_r = diff_table.get(name) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) for k in fields] if not any(ratios) and not all_: continue - print('%-*s ' % (name_width, name), end='') + line = [] + line.append(name) if diff_results is None: - print(' %s' % ( - ' '.join( - (r[k].table() - if k in r - else types[k].none).rjust(w) - for w, k in zip(widths, fields)))) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - ' '.join( - (r[k].diff_table().rjust(w) - if k in r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in ratios))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - ' '.join( - (diff_r[k].diff_table() - if k in diff_r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' '.join( - (r[k].diff_table() - if k in r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' '.join( - (types[k].diff_diff(r.get(k), diff_r.get(k)) - if k in r or k in diff_r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in ratios - if t) - if any(ratios) else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) - # print total - r = total - if diff_total is not None: - diff_r = diff_total - ratios = [types[k].ratio(r.get(k), diff_r.get(k)) + # total + r = next(iter(fold(Result, results, by=[])), None) + if diff_results is not None: + diff_r = next(iter(fold(Result, diff_results, by=[])), None) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) for k in fields] - print('%-*s ' % (name_width, 'TOTAL'), end='') + line = [] + line.append('TOTAL') if diff_results is None: - print(' %s' % ( - ' '.join( - (r[k].table() - if k in r - else types[k].none).rjust(w) - for w, k in zip(widths, fields)))) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - ' '.join( - (r[k].diff_table().rjust(w) - if k in r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in ratios))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - ' '.join( - (diff_r[k].diff_table() - if k in diff_r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' '.join( - (r[k].diff_table() - if k in r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' '.join( - (types[k].diff_diff(r.get(k), diff_r.get(k)) - if k in r or k in diff_r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in ratios - if t) - if any(ratios) else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) + # find the best widths, note that column 0 contains the names and column -1 + # the ratios, so those are handled a bit differently + widths = [ + ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1 + for w, i in zip( + it.chain([23], it.repeat(7)), + range(len(lines[0])-1))] + + # print our table + for line in lines: + print('%-*s %s%s' % ( + widths[0], line[0], + ' '.join('%*s' % (w, x) + for w, x in zip(widths[1:], line[1:-1])), + line[-1])) + + +def openio(path, mode='r'): + if path == '-': + if mode == 'r': + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) def main(csv_paths, *, by=None, fields=None, - define=[], + defines=None, + sort=None, **args): # separate out renames - renames = [k.split('=', 1) - for k in it.chain(by or [], fields or []) - if '=' in k] + renames = list(it.chain.from_iterable( + ((k, v) for v in vs) + for k, vs in it.chain(by or [], fields or []))) if by is not None: - by = [k.split('=', 1)[0] for k in by] + by = [k for k, _ in by] if fields is not None: - fields = [k.split('=', 1)[0] for k in fields] + fields = [k for k, _ in fields] + + # figure out types + types = {} + for t in TYPES.keys(): + for k in args.get(t, []): + if k in types: + print("error: conflicting type for field %r?" % k) + sys.exit(-1) + types[k] = TYPES[t] + # rename types? + if renames: + types_ = {} + for new_k, old_k in renames: + if old_k in types: + types_[new_k] = types[old_k] + types.update(types_) # figure out merge operations ops = {} - for m in OPS.keys(): - for k in args.get(m, []): + for o in OPS.keys(): + for k in args.get(o, []): if k in ops: - print("conflicting op for field %r?" % k) + print("error: conflicting op for field %r?" % k) sys.exit(-1) - ops[k] = m + ops[k] = OPS[o] # rename ops? if renames: ops_ = {} @@ -634,7 +648,7 @@ def main(csv_paths, *, paths.append(path) if not paths: - print('no .csv files found in %r?' % csv_paths) + print("error: no .csv files found in %r?" % csv_paths) sys.exit(-1) results = [] @@ -643,29 +657,56 @@ def main(csv_paths, *, with openio(path) as f: reader = csv.DictReader(f, restval='') for r in reader: + # rename fields? + if renames: + # make a copy so renames can overlap + r_ = {} + for new_k, old_k in renames: + if old_k in r: + r_[new_k] = r[old_k] + r.update(r_) + results.append(r) except FileNotFoundError: pass # homogenize - by, fields, types, results = homogenize(results, - by=by, fields=fields, renames=renames, define=define) + Result = infer(results, + by=by, + fields=fields, + types=types, + ops=ops, + renames=renames) + results_ = [] + for r in results: + try: + results_.append(Result(**{ + k: r[k] for k in Result._by + Result._fields + if k in r and r[k].strip()})) + except TypeError: + pass + results = results_ - # fold for total, note we do this with the raw data to avoid - # issues with lossy operations - total = fold(results, fields=fields, ops=ops) - total = total[0] if total else {} + # fold + results = fold(Result, results, by=by, defines=defines) - # fold to remove duplicates - types_, results = fold(results, by=by, fields=fields, types=types, ops=ops) + # sort, note that python's sort is stable + results.sort() + if sort: + for k, reverse in reversed(sort): + results.sort(key=lambda r: (getattr(r, k),) + if getattr(r, k) is not None else (), + reverse=reverse ^ (not k or k in Result._fields)) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, by + fields) + writer = csv.DictWriter(f, Result._by + Result._fields) writer.writeheader() for r in results: - writer.writerow(r) + # note we need to go through getattr to resolve lazy fields + writer.writerow({ + k: getattr(r, k) for k in Result._by + Result._fields}) # find previous results? if args.get('diff'): @@ -674,33 +715,34 @@ def main(csv_paths, *, with openio(args['diff']) as f: reader = csv.DictReader(f, restval='') for r in reader: - diff_results.append(r) + # rename fields? + if renames: + # make a copy so renames can overlap + r_ = {} + for new_k, old_k in renames: + if old_k in r: + r_[new_k] = r[old_k] + r.update(r_) + + try: + diff_results.append(Result(**{ + k: r[k] for k in Result._by + Result._fields + if k in r and r[k].strip()})) + except TypeError: + pass except FileNotFoundError: pass - # homogenize - _, _, _, diff_results = homogenize(diff_results, - by=by, fields=fields, renames=renames, define=define, types=types) - - # fold for total, note we do this with the raw data to avoid - # issues with lossy operations - diff_total = fold(diff_results, fields=fields, ops=ops) - diff_total = diff_total[0] if diff_total else {} - - # fold to remove duplicates - diff_results = fold(diff_results, by=by, fields=fields, ops=ops) + # fold + diff_results = fold(Result, diff_results, by=by, defines=defines) # print table if not args.get('quiet'): - table( - results, - total, + table(Result, results, diff_results if args.get('diff') else None, - diff_total if args.get('diff') else None, by=by, fields=fields, - types=types_, - ops=ops, + sort=sort, **args) @@ -736,18 +778,54 @@ if __name__ == "__main__": parser.add_argument( '-b', '--by', action='append', - help="Group by these fields. All other fields will be merged as " - "needed. Can rename fields with new_name=old_name.") + type=lambda x: ( + lambda k,v=None: (k, v.split(',') if v is not None else ()) + )(*x.split('=', 1)), + help="Group by this field. Can rename fields with new_name=old_name.") parser.add_argument( - '-f', '--fields', + '-f', '--field', + dest='fields', action='append', - help="Use these fields. Can rename fields with new_name=old_name.") + type=lambda x: ( + lambda k,v=None: (k, v.split(',') if v is not None else ()) + )(*x.split('=', 1)), + help="Show this field. Can rename fields with new_name=old_name.") parser.add_argument( '-D', '--define', + dest='defines', action='append', type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), - help="Only include rows where this field is this value. May include " + help="Only include results where this field is this value. May include " "comma-separated options.") + class AppendSort(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.sort is None: + namespace.sort = [] + namespace.sort.append((value, True if option == '-S' else False)) + parser.add_argument( + '-s', '--sort', + action=AppendSort, + help="Sort by this fields.") + parser.add_argument( + '-S', '--reverse-sort', + action=AppendSort, + help="Sort by this fields, but backwards.") + parser.add_argument( + '-Y', '--summary', + action='store_true', + help="Only show the total.") + parser.add_argument( + '--int', + action='append', + help="Treat these fields as ints.") + parser.add_argument( + '--float', + action='append', + help="Treat these fields as floats.") + parser.add_argument( + '--frac', + action='append', + help="Treat these fields as fractions.") parser.add_argument( '--sum', action='append', @@ -780,18 +858,6 @@ if __name__ == "__main__": '--gstddev', action='append', help="Find the geometric standard deviation of these fields.") - parser.add_argument( - '-s', '--sort', - action='append', - help="Sort by these fields.") - parser.add_argument( - '-S', '--reverse-sort', - action='append', - help="Sort by these fields, but backwards.") - parser.add_argument( - '-Y', '--summary', - action='store_true', - help="Only show the totals.") sys.exit(main(**{k: v for k, v in vars(parser.parse_intermixed_args()).items() if v is not None}))