From 8a35b9870b52e5bf944c72d8ea142e89d2adf521 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 2 Feb 2026 00:41:51 -0600 Subject: [PATCH] scripts: Tweaked table renderer to not hide conflicting results I think this is currently only possible with overlapping by/field fields, but hiding results with conflicting by fields is not ideal. Especially since this function is central to so many scripts: cat test.csv a,b,c x,2,1 x,1,2 x,1,3 Before: ./scripts/csv.py test.csv -ba -bb -fb -fc warning: by fields are unstable a,b b c x,2 2 1 TOTAL 4 6 After: ./scripts/csv.py test.csv -ba -bb -fb -fc a,b b c x,2 2 5 x,2 2 1 TOTAL 4 6 This solves the main issue with unstable by fields, so no more warning. Note that some features rely on by being unique to work (added/removed numbers, compare fields, etc). They shouldn't error, but may be incorrect/unintuitive with conflicting by fields, so avoiding conflicting by fields is still a good idea. --- scripts/code.py | 96 ++++++++++++++++++---------------------------- scripts/cov.py | 96 ++++++++++++++++++---------------------------- scripts/csv.py | 96 ++++++++++++++++++---------------------------- scripts/ctx.py | 96 ++++++++++++++++++---------------------------- scripts/data.py | 96 ++++++++++++++++++---------------------------- scripts/perf.py | 96 ++++++++++++++++++---------------------------- scripts/perfbd.py | 96 ++++++++++++++++++---------------------------- scripts/stack.py | 96 ++++++++++++++++++---------------------------- scripts/structs.py | 96 ++++++++++++++++++---------------------------- 9 files changed, 342 insertions(+), 522 deletions(-) diff --git a/scripts/code.py b/scripts/code.py index 8d68d1f0..58dcf5b8 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -640,30 +640,13 @@ def table(Result, results, diff_results=None, *, types = Result._types # organize by name - table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results} - diff_table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results or []} - - # lost results? note this can happen if a by field references the - # same field as a field field, and the field field changes during - # folding - # - # it's not an _error_, but can lead to really confusing results, so - # at least warn - if (len(table) != len(results) - or (diff_results is not None - and len(diff_table) != len(diff_results))): - print("warning: by fields are unstable", - file=sys.stderr) + def table_name(r): + return ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by) + table = {table_name(r): r for r in results} + diff_table = {table_name(r): r for r in diff_results or []} # find compare entry if there is one if compare: @@ -705,9 +688,9 @@ def table(Result, results, diff_results=None, *, del diff_table # entry helper - def table_entry(name, r, diff_r=None): + def table_entry(n, r, diff_r=None): # prepend name - entry = [name] + entry = [n] # normal entry? if ((compare is None or r == compare_r) @@ -782,30 +765,26 @@ def table(Result, results, diff_results=None, *, depth_, prefixes=('', '', '', '')): # build the children table at each layer - table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results_} - diff_table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results_ or []} - names_ = [n - for n in table_.keys() | diff_table_.keys() + table_ = {table_name(r): r for r in results_} + diff_table_ = {table_name(r): r for r in diff_results_ or []} + # this gets a bit tricky, we want to merge both result and diff + # result names, while preserving duplicates in the result list + results__ = [(n, r) + for n, r in it.chain( + ((table_name(r), r) for r in results_), + ((table_name(r), None) + for r in diff_results_ or [] + if table_name(r) not in table_)) if diff_results is None or all_ or any( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)] # sort again, now with diff info, note that python's sort is stable - names_.sort(key=lambda n: ( + results__.sort(key=lambda nr: (lambda n, r: ( # sort by explicit sort fields next( tuple((Rev @@ -816,35 +795,34 @@ def table(Result, results, diff_results=None, *, else () for k_ in ([k] if k else Result._sort))) for k, reverse in (sort or [])) - for r_ in [table_.get(n), diff_table_.get(n)] + for r_ in [r, diff_table_.get(n)] if r_ is not None), # sort by ratio if diffing Rev(tuple(types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)) if diff_results is not None else (), # move compare entry to the top, note this can be # overridden by explicitly sorting by fields - (table_.get(n) != compare_r, + (r != compare_r, # sort by ratio if comparing Rev(tuple( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(compare_r, k, None)) for k in fields))) if compare else (), # sort by result - (table_[n],) if n in table_ else (), + (r,) if r is not None else (), # and finally by name (diffs may be missing results) - n)) + n))(*nr)) - for i, name in enumerate(names_): + for i, (n, r) in enumerate(results__): # find comparable results - r = table_.get(name) - diff_r = diff_table_.get(name) + diff_r = diff_table_.get(n) # figure out a good label if labels is not None: @@ -856,26 +834,28 @@ def table(Result, results, diff_results=None, *, for r_ in [r, diff_r] if r_ is not None) else: - label = name + label = n # build line line = table_entry(label, r, diff_r) # add prefixes line = [x if isinstance(x, tuple) else (x, []) for x in line] - line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) + line[0] = ( + prefixes[0+(i==len(results__)-1)] + line[0][0], + line[0][1]) lines.append(line) # recurse? - if name in table_ and depth_ > 1: + if r is not None and depth_ > 1: table_recurse( getattr(r, Result._children), getattr(diff_r, Result._children, None), depth_-1, - (prefixes[2+(i==len(names_)-1)] + "|-> ", - prefixes[2+(i==len(names_)-1)] + "'-> ", - prefixes[2+(i==len(names_)-1)] + "| ", - prefixes[2+(i==len(names_)-1)] + " ")) + (prefixes[2+(i==len(results__)-1)] + "|-> ", + prefixes[2+(i==len(results__)-1)] + "'-> ", + prefixes[2+(i==len(results__)-1)] + "| ", + prefixes[2+(i==len(results__)-1)] + " ")) # build entries if not summary: diff --git a/scripts/cov.py b/scripts/cov.py index a409cb05..beff654e 100755 --- a/scripts/cov.py +++ b/scripts/cov.py @@ -503,30 +503,13 @@ def table(Result, results, diff_results=None, *, types = Result._types # organize by name - table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results} - diff_table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results or []} - - # lost results? note this can happen if a by field references the - # same field as a field field, and the field field changes during - # folding - # - # it's not an _error_, but can lead to really confusing results, so - # at least warn - if (len(table) != len(results) - or (diff_results is not None - and len(diff_table) != len(diff_results))): - print("warning: by fields are unstable", - file=sys.stderr) + def table_name(r): + return ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by) + table = {table_name(r): r for r in results} + diff_table = {table_name(r): r for r in diff_results or []} # find compare entry if there is one if compare: @@ -568,9 +551,9 @@ def table(Result, results, diff_results=None, *, del diff_table # entry helper - def table_entry(name, r, diff_r=None): + def table_entry(n, r, diff_r=None): # prepend name - entry = [name] + entry = [n] # normal entry? if ((compare is None or r == compare_r) @@ -645,30 +628,26 @@ def table(Result, results, diff_results=None, *, depth_, prefixes=('', '', '', '')): # build the children table at each layer - table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results_} - diff_table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results_ or []} - names_ = [n - for n in table_.keys() | diff_table_.keys() + table_ = {table_name(r): r for r in results_} + diff_table_ = {table_name(r): r for r in diff_results_ or []} + # this gets a bit tricky, we want to merge both result and diff + # result names, while preserving duplicates in the result list + results__ = [(n, r) + for n, r in it.chain( + ((table_name(r), r) for r in results_), + ((table_name(r), None) + for r in diff_results_ or [] + if table_name(r) not in table_)) if diff_results is None or all_ or any( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)] # sort again, now with diff info, note that python's sort is stable - names_.sort(key=lambda n: ( + results__.sort(key=lambda nr: (lambda n, r: ( # sort by explicit sort fields next( tuple((Rev @@ -679,35 +658,34 @@ def table(Result, results, diff_results=None, *, else () for k_ in ([k] if k else Result._sort))) for k, reverse in (sort or [])) - for r_ in [table_.get(n), diff_table_.get(n)] + for r_ in [r, diff_table_.get(n)] if r_ is not None), # sort by ratio if diffing Rev(tuple(types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)) if diff_results is not None else (), # move compare entry to the top, note this can be # overridden by explicitly sorting by fields - (table_.get(n) != compare_r, + (r != compare_r, # sort by ratio if comparing Rev(tuple( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(compare_r, k, None)) for k in fields))) if compare else (), # sort by result - (table_[n],) if n in table_ else (), + (r,) if r is not None else (), # and finally by name (diffs may be missing results) - n)) + n))(*nr)) - for i, name in enumerate(names_): + for i, (n, r) in enumerate(results__): # find comparable results - r = table_.get(name) - diff_r = diff_table_.get(name) + diff_r = diff_table_.get(n) # figure out a good label if labels is not None: @@ -719,26 +697,28 @@ def table(Result, results, diff_results=None, *, for r_ in [r, diff_r] if r_ is not None) else: - label = name + label = n # build line line = table_entry(label, r, diff_r) # add prefixes line = [x if isinstance(x, tuple) else (x, []) for x in line] - line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) + line[0] = ( + prefixes[0+(i==len(results__)-1)] + line[0][0], + line[0][1]) lines.append(line) # recurse? - if name in table_ and depth_ > 1: + if r is not None and depth_ > 1: table_recurse( getattr(r, Result._children), getattr(diff_r, Result._children, None), depth_-1, - (prefixes[2+(i==len(names_)-1)] + "|-> ", - prefixes[2+(i==len(names_)-1)] + "'-> ", - prefixes[2+(i==len(names_)-1)] + "| ", - prefixes[2+(i==len(names_)-1)] + " ")) + (prefixes[2+(i==len(results__)-1)] + "|-> ", + prefixes[2+(i==len(results__)-1)] + "'-> ", + prefixes[2+(i==len(results__)-1)] + "| ", + prefixes[2+(i==len(results__)-1)] + " ")) # build entries if not summary: diff --git a/scripts/csv.py b/scripts/csv.py index 96dc4009..5b4ce95c 100755 --- a/scripts/csv.py +++ b/scripts/csv.py @@ -1962,30 +1962,13 @@ def table(Result, results, diff_results=None, *, types = Result._types # organize by name - table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results} - diff_table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results or []} - - # lost results? note this can happen if a by field references the - # same field as a field field, and the field field changes during - # folding - # - # it's not an _error_, but can lead to really confusing results, so - # at least warn - if (len(table) != len(results) - or (diff_results is not None - and len(diff_table) != len(diff_results))): - print("warning: by fields are unstable", - file=sys.stderr) + def table_name(r): + return ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by) + table = {table_name(r): r for r in results} + diff_table = {table_name(r): r for r in diff_results or []} # find compare entry if there is one if compare: @@ -2027,9 +2010,9 @@ def table(Result, results, diff_results=None, *, del diff_table # entry helper - def table_entry(name, r, diff_r=None): + def table_entry(n, r, diff_r=None): # prepend name - entry = [name] + entry = [n] # normal entry? if ((compare is None or r == compare_r) @@ -2104,30 +2087,26 @@ def table(Result, results, diff_results=None, *, depth_, prefixes=('', '', '', '')): # build the children table at each layer - table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results_} - diff_table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results_ or []} - names_ = [n - for n in table_.keys() | diff_table_.keys() + table_ = {table_name(r): r for r in results_} + diff_table_ = {table_name(r): r for r in diff_results_ or []} + # this gets a bit tricky, we want to merge both result and diff + # result names, while preserving duplicates in the result list + results__ = [(n, r) + for n, r in it.chain( + ((table_name(r), r) for r in results_), + ((table_name(r), None) + for r in diff_results_ or [] + if table_name(r) not in table_)) if diff_results is None or all_ or any( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)] # sort again, now with diff info, note that python's sort is stable - names_.sort(key=lambda n: ( + results__.sort(key=lambda nr: (lambda n, r: ( # sort by explicit sort fields next( tuple((Rev @@ -2138,35 +2117,34 @@ def table(Result, results, diff_results=None, *, else () for k_ in ([k] if k else Result._sort))) for k, reverse in (sort or [])) - for r_ in [table_.get(n), diff_table_.get(n)] + for r_ in [r, diff_table_.get(n)] if r_ is not None), # sort by ratio if diffing Rev(tuple(types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)) if diff_results is not None else (), # move compare entry to the top, note this can be # overridden by explicitly sorting by fields - (table_.get(n) != compare_r, + (r != compare_r, # sort by ratio if comparing Rev(tuple( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(compare_r, k, None)) for k in fields))) if compare else (), # sort by result - (table_[n],) if n in table_ else (), + (r,) if r is not None else (), # and finally by name (diffs may be missing results) - n)) + n))(*nr)) - for i, name in enumerate(names_): + for i, (n, r) in enumerate(results__): # find comparable results - r = table_.get(name) - diff_r = diff_table_.get(name) + diff_r = diff_table_.get(n) # figure out a good label if labels is not None: @@ -2178,26 +2156,28 @@ def table(Result, results, diff_results=None, *, for r_ in [r, diff_r] if r_ is not None) else: - label = name + label = n # build line line = table_entry(label, r, diff_r) # add prefixes line = [x if isinstance(x, tuple) else (x, []) for x in line] - line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) + line[0] = ( + prefixes[0+(i==len(results__)-1)] + line[0][0], + line[0][1]) lines.append(line) # recurse? - if name in table_ and depth_ > 1: + if r is not None and depth_ > 1: table_recurse( getattr(r, Result._children), getattr(diff_r, Result._children, None), depth_-1, - (prefixes[2+(i==len(names_)-1)] + "|-> ", - prefixes[2+(i==len(names_)-1)] + "'-> ", - prefixes[2+(i==len(names_)-1)] + "| ", - prefixes[2+(i==len(names_)-1)] + " ")) + (prefixes[2+(i==len(results__)-1)] + "|-> ", + prefixes[2+(i==len(results__)-1)] + "'-> ", + prefixes[2+(i==len(results__)-1)] + "| ", + prefixes[2+(i==len(results__)-1)] + " ")) # build entries if not summary: diff --git a/scripts/ctx.py b/scripts/ctx.py index a4f41669..81ebc0ab 100755 --- a/scripts/ctx.py +++ b/scripts/ctx.py @@ -899,30 +899,13 @@ def table(Result, results, diff_results=None, *, types = Result._types # organize by name - table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results} - diff_table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results or []} - - # lost results? note this can happen if a by field references the - # same field as a field field, and the field field changes during - # folding - # - # it's not an _error_, but can lead to really confusing results, so - # at least warn - if (len(table) != len(results) - or (diff_results is not None - and len(diff_table) != len(diff_results))): - print("warning: by fields are unstable", - file=sys.stderr) + def table_name(r): + return ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by) + table = {table_name(r): r for r in results} + diff_table = {table_name(r): r for r in diff_results or []} # find compare entry if there is one if compare: @@ -964,9 +947,9 @@ def table(Result, results, diff_results=None, *, del diff_table # entry helper - def table_entry(name, r, diff_r=None): + def table_entry(n, r, diff_r=None): # prepend name - entry = [name] + entry = [n] # normal entry? if ((compare is None or r == compare_r) @@ -1041,30 +1024,26 @@ def table(Result, results, diff_results=None, *, depth_, prefixes=('', '', '', '')): # build the children table at each layer - table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results_} - diff_table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results_ or []} - names_ = [n - for n in table_.keys() | diff_table_.keys() + table_ = {table_name(r): r for r in results_} + diff_table_ = {table_name(r): r for r in diff_results_ or []} + # this gets a bit tricky, we want to merge both result and diff + # result names, while preserving duplicates in the result list + results__ = [(n, r) + for n, r in it.chain( + ((table_name(r), r) for r in results_), + ((table_name(r), None) + for r in diff_results_ or [] + if table_name(r) not in table_)) if diff_results is None or all_ or any( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)] # sort again, now with diff info, note that python's sort is stable - names_.sort(key=lambda n: ( + results__.sort(key=lambda nr: (lambda n, r: ( # sort by explicit sort fields next( tuple((Rev @@ -1075,35 +1054,34 @@ def table(Result, results, diff_results=None, *, else () for k_ in ([k] if k else Result._sort))) for k, reverse in (sort or [])) - for r_ in [table_.get(n), diff_table_.get(n)] + for r_ in [r, diff_table_.get(n)] if r_ is not None), # sort by ratio if diffing Rev(tuple(types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)) if diff_results is not None else (), # move compare entry to the top, note this can be # overridden by explicitly sorting by fields - (table_.get(n) != compare_r, + (r != compare_r, # sort by ratio if comparing Rev(tuple( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(compare_r, k, None)) for k in fields))) if compare else (), # sort by result - (table_[n],) if n in table_ else (), + (r,) if r is not None else (), # and finally by name (diffs may be missing results) - n)) + n))(*nr)) - for i, name in enumerate(names_): + for i, (n, r) in enumerate(results__): # find comparable results - r = table_.get(name) - diff_r = diff_table_.get(name) + diff_r = diff_table_.get(n) # figure out a good label if labels is not None: @@ -1115,26 +1093,28 @@ def table(Result, results, diff_results=None, *, for r_ in [r, diff_r] if r_ is not None) else: - label = name + label = n # build line line = table_entry(label, r, diff_r) # add prefixes line = [x if isinstance(x, tuple) else (x, []) for x in line] - line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) + line[0] = ( + prefixes[0+(i==len(results__)-1)] + line[0][0], + line[0][1]) lines.append(line) # recurse? - if name in table_ and depth_ > 1: + if r is not None and depth_ > 1: table_recurse( getattr(r, Result._children), getattr(diff_r, Result._children, None), depth_-1, - (prefixes[2+(i==len(names_)-1)] + "|-> ", - prefixes[2+(i==len(names_)-1)] + "'-> ", - prefixes[2+(i==len(names_)-1)] + "| ", - prefixes[2+(i==len(names_)-1)] + " ")) + (prefixes[2+(i==len(results__)-1)] + "|-> ", + prefixes[2+(i==len(results__)-1)] + "'-> ", + prefixes[2+(i==len(results__)-1)] + "| ", + prefixes[2+(i==len(results__)-1)] + " ")) # build entries if not summary: diff --git a/scripts/data.py b/scripts/data.py index 72a32198..ff3d395f 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -640,30 +640,13 @@ def table(Result, results, diff_results=None, *, types = Result._types # organize by name - table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results} - diff_table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results or []} - - # lost results? note this can happen if a by field references the - # same field as a field field, and the field field changes during - # folding - # - # it's not an _error_, but can lead to really confusing results, so - # at least warn - if (len(table) != len(results) - or (diff_results is not None - and len(diff_table) != len(diff_results))): - print("warning: by fields are unstable", - file=sys.stderr) + def table_name(r): + return ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by) + table = {table_name(r): r for r in results} + diff_table = {table_name(r): r for r in diff_results or []} # find compare entry if there is one if compare: @@ -705,9 +688,9 @@ def table(Result, results, diff_results=None, *, del diff_table # entry helper - def table_entry(name, r, diff_r=None): + def table_entry(n, r, diff_r=None): # prepend name - entry = [name] + entry = [n] # normal entry? if ((compare is None or r == compare_r) @@ -782,30 +765,26 @@ def table(Result, results, diff_results=None, *, depth_, prefixes=('', '', '', '')): # build the children table at each layer - table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results_} - diff_table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results_ or []} - names_ = [n - for n in table_.keys() | diff_table_.keys() + table_ = {table_name(r): r for r in results_} + diff_table_ = {table_name(r): r for r in diff_results_ or []} + # this gets a bit tricky, we want to merge both result and diff + # result names, while preserving duplicates in the result list + results__ = [(n, r) + for n, r in it.chain( + ((table_name(r), r) for r in results_), + ((table_name(r), None) + for r in diff_results_ or [] + if table_name(r) not in table_)) if diff_results is None or all_ or any( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)] # sort again, now with diff info, note that python's sort is stable - names_.sort(key=lambda n: ( + results__.sort(key=lambda nr: (lambda n, r: ( # sort by explicit sort fields next( tuple((Rev @@ -816,35 +795,34 @@ def table(Result, results, diff_results=None, *, else () for k_ in ([k] if k else Result._sort))) for k, reverse in (sort or [])) - for r_ in [table_.get(n), diff_table_.get(n)] + for r_ in [r, diff_table_.get(n)] if r_ is not None), # sort by ratio if diffing Rev(tuple(types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)) if diff_results is not None else (), # move compare entry to the top, note this can be # overridden by explicitly sorting by fields - (table_.get(n) != compare_r, + (r != compare_r, # sort by ratio if comparing Rev(tuple( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(compare_r, k, None)) for k in fields))) if compare else (), # sort by result - (table_[n],) if n in table_ else (), + (r,) if r is not None else (), # and finally by name (diffs may be missing results) - n)) + n))(*nr)) - for i, name in enumerate(names_): + for i, (n, r) in enumerate(results__): # find comparable results - r = table_.get(name) - diff_r = diff_table_.get(name) + diff_r = diff_table_.get(n) # figure out a good label if labels is not None: @@ -856,26 +834,28 @@ def table(Result, results, diff_results=None, *, for r_ in [r, diff_r] if r_ is not None) else: - label = name + label = n # build line line = table_entry(label, r, diff_r) # add prefixes line = [x if isinstance(x, tuple) else (x, []) for x in line] - line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) + line[0] = ( + prefixes[0+(i==len(results__)-1)] + line[0][0], + line[0][1]) lines.append(line) # recurse? - if name in table_ and depth_ > 1: + if r is not None and depth_ > 1: table_recurse( getattr(r, Result._children), getattr(diff_r, Result._children, None), depth_-1, - (prefixes[2+(i==len(names_)-1)] + "|-> ", - prefixes[2+(i==len(names_)-1)] + "'-> ", - prefixes[2+(i==len(names_)-1)] + "| ", - prefixes[2+(i==len(names_)-1)] + " ")) + (prefixes[2+(i==len(results__)-1)] + "|-> ", + prefixes[2+(i==len(results__)-1)] + "'-> ", + prefixes[2+(i==len(results__)-1)] + "| ", + prefixes[2+(i==len(results__)-1)] + " ")) # build entries if not summary: diff --git a/scripts/perf.py b/scripts/perf.py index 3d462af4..8c1a8d7c 100755 --- a/scripts/perf.py +++ b/scripts/perf.py @@ -998,30 +998,13 @@ def table(Result, results, diff_results=None, *, types = Result._types # organize by name - table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results} - diff_table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results or []} - - # lost results? note this can happen if a by field references the - # same field as a field field, and the field field changes during - # folding - # - # it's not an _error_, but can lead to really confusing results, so - # at least warn - if (len(table) != len(results) - or (diff_results is not None - and len(diff_table) != len(diff_results))): - print("warning: by fields are unstable", - file=sys.stderr) + def table_name(r): + return ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by) + table = {table_name(r): r for r in results} + diff_table = {table_name(r): r for r in diff_results or []} # find compare entry if there is one if compare: @@ -1063,9 +1046,9 @@ def table(Result, results, diff_results=None, *, del diff_table # entry helper - def table_entry(name, r, diff_r=None): + def table_entry(n, r, diff_r=None): # prepend name - entry = [name] + entry = [n] # normal entry? if ((compare is None or r == compare_r) @@ -1140,30 +1123,26 @@ def table(Result, results, diff_results=None, *, depth_, prefixes=('', '', '', '')): # build the children table at each layer - table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results_} - diff_table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results_ or []} - names_ = [n - for n in table_.keys() | diff_table_.keys() + table_ = {table_name(r): r for r in results_} + diff_table_ = {table_name(r): r for r in diff_results_ or []} + # this gets a bit tricky, we want to merge both result and diff + # result names, while preserving duplicates in the result list + results__ = [(n, r) + for n, r in it.chain( + ((table_name(r), r) for r in results_), + ((table_name(r), None) + for r in diff_results_ or [] + if table_name(r) not in table_)) if diff_results is None or all_ or any( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)] # sort again, now with diff info, note that python's sort is stable - names_.sort(key=lambda n: ( + results__.sort(key=lambda nr: (lambda n, r: ( # sort by explicit sort fields next( tuple((Rev @@ -1174,35 +1153,34 @@ def table(Result, results, diff_results=None, *, else () for k_ in ([k] if k else Result._sort))) for k, reverse in (sort or [])) - for r_ in [table_.get(n), diff_table_.get(n)] + for r_ in [r, diff_table_.get(n)] if r_ is not None), # sort by ratio if diffing Rev(tuple(types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)) if diff_results is not None else (), # move compare entry to the top, note this can be # overridden by explicitly sorting by fields - (table_.get(n) != compare_r, + (r != compare_r, # sort by ratio if comparing Rev(tuple( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(compare_r, k, None)) for k in fields))) if compare else (), # sort by result - (table_[n],) if n in table_ else (), + (r,) if r is not None else (), # and finally by name (diffs may be missing results) - n)) + n))(*nr)) - for i, name in enumerate(names_): + for i, (n, r) in enumerate(results__): # find comparable results - r = table_.get(name) - diff_r = diff_table_.get(name) + diff_r = diff_table_.get(n) # figure out a good label if labels is not None: @@ -1214,26 +1192,28 @@ def table(Result, results, diff_results=None, *, for r_ in [r, diff_r] if r_ is not None) else: - label = name + label = n # build line line = table_entry(label, r, diff_r) # add prefixes line = [x if isinstance(x, tuple) else (x, []) for x in line] - line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) + line[0] = ( + prefixes[0+(i==len(results__)-1)] + line[0][0], + line[0][1]) lines.append(line) # recurse? - if name in table_ and depth_ > 1: + if r is not None and depth_ > 1: table_recurse( getattr(r, Result._children), getattr(diff_r, Result._children, None), depth_-1, - (prefixes[2+(i==len(names_)-1)] + "|-> ", - prefixes[2+(i==len(names_)-1)] + "'-> ", - prefixes[2+(i==len(names_)-1)] + "| ", - prefixes[2+(i==len(names_)-1)] + " ")) + (prefixes[2+(i==len(results__)-1)] + "|-> ", + prefixes[2+(i==len(results__)-1)] + "'-> ", + prefixes[2+(i==len(results__)-1)] + "| ", + prefixes[2+(i==len(results__)-1)] + " ")) # build entries if not summary: diff --git a/scripts/perfbd.py b/scripts/perfbd.py index d5a2d615..70a1fb57 100755 --- a/scripts/perfbd.py +++ b/scripts/perfbd.py @@ -972,30 +972,13 @@ def table(Result, results, diff_results=None, *, types = Result._types # organize by name - table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results} - diff_table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results or []} - - # lost results? note this can happen if a by field references the - # same field as a field field, and the field field changes during - # folding - # - # it's not an _error_, but can lead to really confusing results, so - # at least warn - if (len(table) != len(results) - or (diff_results is not None - and len(diff_table) != len(diff_results))): - print("warning: by fields are unstable", - file=sys.stderr) + def table_name(r): + return ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by) + table = {table_name(r): r for r in results} + diff_table = {table_name(r): r for r in diff_results or []} # find compare entry if there is one if compare: @@ -1037,9 +1020,9 @@ def table(Result, results, diff_results=None, *, del diff_table # entry helper - def table_entry(name, r, diff_r=None): + def table_entry(n, r, diff_r=None): # prepend name - entry = [name] + entry = [n] # normal entry? if ((compare is None or r == compare_r) @@ -1114,30 +1097,26 @@ def table(Result, results, diff_results=None, *, depth_, prefixes=('', '', '', '')): # build the children table at each layer - table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results_} - diff_table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results_ or []} - names_ = [n - for n in table_.keys() | diff_table_.keys() + table_ = {table_name(r): r for r in results_} + diff_table_ = {table_name(r): r for r in diff_results_ or []} + # this gets a bit tricky, we want to merge both result and diff + # result names, while preserving duplicates in the result list + results__ = [(n, r) + for n, r in it.chain( + ((table_name(r), r) for r in results_), + ((table_name(r), None) + for r in diff_results_ or [] + if table_name(r) not in table_)) if diff_results is None or all_ or any( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)] # sort again, now with diff info, note that python's sort is stable - names_.sort(key=lambda n: ( + results__.sort(key=lambda nr: (lambda n, r: ( # sort by explicit sort fields next( tuple((Rev @@ -1148,35 +1127,34 @@ def table(Result, results, diff_results=None, *, else () for k_ in ([k] if k else Result._sort))) for k, reverse in (sort or [])) - for r_ in [table_.get(n), diff_table_.get(n)] + for r_ in [r, diff_table_.get(n)] if r_ is not None), # sort by ratio if diffing Rev(tuple(types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)) if diff_results is not None else (), # move compare entry to the top, note this can be # overridden by explicitly sorting by fields - (table_.get(n) != compare_r, + (r != compare_r, # sort by ratio if comparing Rev(tuple( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(compare_r, k, None)) for k in fields))) if compare else (), # sort by result - (table_[n],) if n in table_ else (), + (r,) if r is not None else (), # and finally by name (diffs may be missing results) - n)) + n))(*nr)) - for i, name in enumerate(names_): + for i, (n, r) in enumerate(results__): # find comparable results - r = table_.get(name) - diff_r = diff_table_.get(name) + diff_r = diff_table_.get(n) # figure out a good label if labels is not None: @@ -1188,26 +1166,28 @@ def table(Result, results, diff_results=None, *, for r_ in [r, diff_r] if r_ is not None) else: - label = name + label = n # build line line = table_entry(label, r, diff_r) # add prefixes line = [x if isinstance(x, tuple) else (x, []) for x in line] - line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) + line[0] = ( + prefixes[0+(i==len(results__)-1)] + line[0][0], + line[0][1]) lines.append(line) # recurse? - if name in table_ and depth_ > 1: + if r is not None and depth_ > 1: table_recurse( getattr(r, Result._children), getattr(diff_r, Result._children, None), depth_-1, - (prefixes[2+(i==len(names_)-1)] + "|-> ", - prefixes[2+(i==len(names_)-1)] + "'-> ", - prefixes[2+(i==len(names_)-1)] + "| ", - prefixes[2+(i==len(names_)-1)] + " ")) + (prefixes[2+(i==len(results__)-1)] + "|-> ", + prefixes[2+(i==len(results__)-1)] + "'-> ", + prefixes[2+(i==len(results__)-1)] + "| ", + prefixes[2+(i==len(results__)-1)] + " ")) # build entries if not summary: diff --git a/scripts/stack.py b/scripts/stack.py index bf283d17..42d4c49d 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -640,30 +640,13 @@ def table(Result, results, diff_results=None, *, types = Result._types # organize by name - table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results} - diff_table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results or []} - - # lost results? note this can happen if a by field references the - # same field as a field field, and the field field changes during - # folding - # - # it's not an _error_, but can lead to really confusing results, so - # at least warn - if (len(table) != len(results) - or (diff_results is not None - and len(diff_table) != len(diff_results))): - print("warning: by fields are unstable", - file=sys.stderr) + def table_name(r): + return ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by) + table = {table_name(r): r for r in results} + diff_table = {table_name(r): r for r in diff_results or []} # find compare entry if there is one if compare: @@ -705,9 +688,9 @@ def table(Result, results, diff_results=None, *, del diff_table # entry helper - def table_entry(name, r, diff_r=None): + def table_entry(n, r, diff_r=None): # prepend name - entry = [name] + entry = [n] # normal entry? if ((compare is None or r == compare_r) @@ -782,30 +765,26 @@ def table(Result, results, diff_results=None, *, depth_, prefixes=('', '', '', '')): # build the children table at each layer - table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results_} - diff_table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results_ or []} - names_ = [n - for n in table_.keys() | diff_table_.keys() + table_ = {table_name(r): r for r in results_} + diff_table_ = {table_name(r): r for r in diff_results_ or []} + # this gets a bit tricky, we want to merge both result and diff + # result names, while preserving duplicates in the result list + results__ = [(n, r) + for n, r in it.chain( + ((table_name(r), r) for r in results_), + ((table_name(r), None) + for r in diff_results_ or [] + if table_name(r) not in table_)) if diff_results is None or all_ or any( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)] # sort again, now with diff info, note that python's sort is stable - names_.sort(key=lambda n: ( + results__.sort(key=lambda nr: (lambda n, r: ( # sort by explicit sort fields next( tuple((Rev @@ -816,35 +795,34 @@ def table(Result, results, diff_results=None, *, else () for k_ in ([k] if k else Result._sort))) for k, reverse in (sort or [])) - for r_ in [table_.get(n), diff_table_.get(n)] + for r_ in [r, diff_table_.get(n)] if r_ is not None), # sort by ratio if diffing Rev(tuple(types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)) if diff_results is not None else (), # move compare entry to the top, note this can be # overridden by explicitly sorting by fields - (table_.get(n) != compare_r, + (r != compare_r, # sort by ratio if comparing Rev(tuple( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(compare_r, k, None)) for k in fields))) if compare else (), # sort by result - (table_[n],) if n in table_ else (), + (r,) if r is not None else (), # and finally by name (diffs may be missing results) - n)) + n))(*nr)) - for i, name in enumerate(names_): + for i, (n, r) in enumerate(results__): # find comparable results - r = table_.get(name) - diff_r = diff_table_.get(name) + diff_r = diff_table_.get(n) # figure out a good label if labels is not None: @@ -856,26 +834,28 @@ def table(Result, results, diff_results=None, *, for r_ in [r, diff_r] if r_ is not None) else: - label = name + label = n # build line line = table_entry(label, r, diff_r) # add prefixes line = [x if isinstance(x, tuple) else (x, []) for x in line] - line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) + line[0] = ( + prefixes[0+(i==len(results__)-1)] + line[0][0], + line[0][1]) lines.append(line) # recurse? - if name in table_ and depth_ > 1: + if r is not None and depth_ > 1: table_recurse( getattr(r, Result._children), getattr(diff_r, Result._children, None), depth_-1, - (prefixes[2+(i==len(names_)-1)] + "|-> ", - prefixes[2+(i==len(names_)-1)] + "'-> ", - prefixes[2+(i==len(names_)-1)] + "| ", - prefixes[2+(i==len(names_)-1)] + " ")) + (prefixes[2+(i==len(results__)-1)] + "|-> ", + prefixes[2+(i==len(results__)-1)] + "'-> ", + prefixes[2+(i==len(results__)-1)] + "| ", + prefixes[2+(i==len(results__)-1)] + " ")) # build entries if not summary: diff --git a/scripts/structs.py b/scripts/structs.py index d43ba6de..e2a5bc01 100755 --- a/scripts/structs.py +++ b/scripts/structs.py @@ -788,30 +788,13 @@ def table(Result, results, diff_results=None, *, types = Result._types # organize by name - table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results} - diff_table = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results or []} - - # lost results? note this can happen if a by field references the - # same field as a field field, and the field field changes during - # folding - # - # it's not an _error_, but can lead to really confusing results, so - # at least warn - if (len(table) != len(results) - or (diff_results is not None - and len(diff_table) != len(diff_results))): - print("warning: by fields are unstable", - file=sys.stderr) + def table_name(r): + return ','.join(str(getattr(r, k) + if getattr(r, k) is not None + else '') + for k in by) + table = {table_name(r): r for r in results} + diff_table = {table_name(r): r for r in diff_results or []} # find compare entry if there is one if compare: @@ -853,9 +836,9 @@ def table(Result, results, diff_results=None, *, del diff_table # entry helper - def table_entry(name, r, diff_r=None): + def table_entry(n, r, diff_r=None): # prepend name - entry = [name] + entry = [n] # normal entry? if ((compare is None or r == compare_r) @@ -930,30 +913,26 @@ def table(Result, results, diff_results=None, *, depth_, prefixes=('', '', '', '')): # build the children table at each layer - table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in results_} - diff_table_ = { - ','.join(str(getattr(r, k) - if getattr(r, k) is not None - else '') - for k in by): r - for r in diff_results_ or []} - names_ = [n - for n in table_.keys() | diff_table_.keys() + table_ = {table_name(r): r for r in results_} + diff_table_ = {table_name(r): r for r in diff_results_ or []} + # this gets a bit tricky, we want to merge both result and diff + # result names, while preserving duplicates in the result list + results__ = [(n, r) + for n, r in it.chain( + ((table_name(r), r) for r in results_), + ((table_name(r), None) + for r in diff_results_ or [] + if table_name(r) not in table_)) if diff_results is None or all_ or any( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)] # sort again, now with diff info, note that python's sort is stable - names_.sort(key=lambda n: ( + results__.sort(key=lambda nr: (lambda n, r: ( # sort by explicit sort fields next( tuple((Rev @@ -964,35 +943,34 @@ def table(Result, results, diff_results=None, *, else () for k_ in ([k] if k else Result._sort))) for k, reverse in (sort or [])) - for r_ in [table_.get(n), diff_table_.get(n)] + for r_ in [r, diff_table_.get(n)] if r_ is not None), # sort by ratio if diffing Rev(tuple(types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(diff_table_.get(n), k, None)) for k in fields)) if diff_results is not None else (), # move compare entry to the top, note this can be # overridden by explicitly sorting by fields - (table_.get(n) != compare_r, + (r != compare_r, # sort by ratio if comparing Rev(tuple( types[k].ratio( - getattr(table_.get(n), k, None), + getattr(r, k, None), getattr(compare_r, k, None)) for k in fields))) if compare else (), # sort by result - (table_[n],) if n in table_ else (), + (r,) if r is not None else (), # and finally by name (diffs may be missing results) - n)) + n))(*nr)) - for i, name in enumerate(names_): + for i, (n, r) in enumerate(results__): # find comparable results - r = table_.get(name) - diff_r = diff_table_.get(name) + diff_r = diff_table_.get(n) # figure out a good label if labels is not None: @@ -1004,26 +982,28 @@ def table(Result, results, diff_results=None, *, for r_ in [r, diff_r] if r_ is not None) else: - label = name + label = n # build line line = table_entry(label, r, diff_r) # add prefixes line = [x if isinstance(x, tuple) else (x, []) for x in line] - line[0] = (prefixes[0+(i==len(names_)-1)] + line[0][0], line[0][1]) + line[0] = ( + prefixes[0+(i==len(results__)-1)] + line[0][0], + line[0][1]) lines.append(line) # recurse? - if name in table_ and depth_ > 1: + if r is not None and depth_ > 1: table_recurse( getattr(r, Result._children), getattr(diff_r, Result._children, None), depth_-1, - (prefixes[2+(i==len(names_)-1)] + "|-> ", - prefixes[2+(i==len(names_)-1)] + "'-> ", - prefixes[2+(i==len(names_)-1)] + "| ", - prefixes[2+(i==len(names_)-1)] + " ")) + (prefixes[2+(i==len(results__)-1)] + "|-> ", + prefixes[2+(i==len(results__)-1)] + "'-> ", + prefixes[2+(i==len(results__)-1)] + "| ", + prefixes[2+(i==len(results__)-1)] + " ")) # build entries if not summary: