Github/littlefs
1
Fork 0
Code Actions Activity

scripts: Disentangled -r/--hot and -i/--enumerate

Browse Source 
This removes most of the special behavior around how -r/--hot and
-i/--enumerate interact. This does mean -r/--hot risks folding results
if -i/--enumerate is not specified, but this is _technically_ a valid
operation.

For most of the recursive result scripts, I've replaced the "i" field
with separate "z" and "i" fields for depth and field number, which I
think is a bit more informative/useful.

I've also added a default-hidden "off" field to structs.py/ctx.py, since
we have that info available. I considered replacing "i" with this, but
decided against it since non-zero offsets for union members would risk
being confusing/mistake prone.
This commit is contained in:
Christopher Haster
2025-02-27 14:55:46 -06:00
parent ac30a20d12
commit 748815bb46
9 changed files with 268 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/code.py
+7 -5
View File
@@ -922,11 +922,13 @@ def write_csv(path, Result, results, *,
with openio(path, 'w') as f: with openio(path, 'w') as f:
# write csv? # write csv?
if not json: if not json:
writer = csv.DictWriter(f, writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain(
(by if by is not None else Result._by) by
+ [k for k in (fields if by is not None
if fields is not None else Result._by,
else Result._fields)]) fields
if fields is not None
else Result._fields)).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
scripts/cov.py
+7 -5
View File
@@ -783,11 +783,13 @@ def write_csv(path, Result, results, *,
with openio(path, 'w') as f: with openio(path, 'w') as f:
# write csv? # write csv?
if not json: if not json:
writer = csv.DictWriter(f, writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain(
(by if by is not None else Result._by) by
+ [k for k in (fields if by is not None
if fields is not None else Result._by,
else Result._fields)]) fields
if fields is not None
else Result._fields)).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
scripts/csv.py
+51 -61
View File
@@ -1399,20 +1399,16 @@ def compile(fields_, results,
sort=None, sort=None,
enumerate=None, enumerate=None,
children=None, children=None,
notes=None, hot=None,
hot=None): notes=None):
import builtins import builtins
enumerate_, enumerate = enumerate, builtins.enumerate enumerate_, enumerate = enumerate, builtins.enumerate
by = by.copy() by = by.copy()
fields = fields.copy() fields = fields.copy()
# we need _something_ to order hot results by, so default to
# i if no enumerate field is specified
if hot is not None and enumerate_ is None:
enumerate_ = 'i'
# make sure enumerate fields are included # make sure enumerate fields are included
if enumerate_ is not None or hot is not None: if enumerate_ is not None:
if enumerate_ not in by: if enumerate_ not in by:
by.insert(0, enumerate_) by.insert(0, enumerate_)
# make sure define fields are included # make sure define fields are included
@@ -1543,15 +1539,12 @@ def compile(fields_, results,
_types={k: t for k, (_, t) in folds.items()}, _types={k: t for k, (_, t) in folds.items()},
_mods=mods, _mods=mods,
_exprs=exprs, _exprs=exprs,
**{'_i': enumerate_} if enumerate_ is not None else {},
**{'_children': children} if children is not None else {}, **{'_children': children} if children is not None else {},
**{'_notes': notes} if notes is not None else {})) **{'_notes': notes} if notes is not None else {}))
def homogenize(Result, results, *, def homogenize(Result, results, *,
depth=1,
hot=None,
enumerate=None, enumerate=None,
children=None): depth=1):
import builtins import builtins
enumerate_, enumerate = enumerate, builtins.enumerate enumerate_, enumerate = enumerate, builtins.enumerate
@@ -1562,19 +1555,17 @@ def homogenize(Result, results, *,
results_.append(Result(**( results_.append(Result(**(
r r
# enumerate? # enumerate?
| ({Result._i: RInt(i)} | ({enumerate_: RInt(i)}
if enumerate_ is not None if enumerate_ is not None
else {}) else {})
# recurse? # recurse?
| ({children: homogenize( | ({Result._children: homogenize(
Result, r[children], Result, r[Result._children],
depth=depth-1, enumerate=enumerate_,
# only enumerate top-level if hotifying depth=depth-1)}
enumerate=(enumerate_ if hot is None else None), if hasattr(Result, '_children')
children=children)} and Result._children in r
if children is not None and r[Result._children] is not None
and children in r
and r[children] is not None
and depth > 1 and depth > 1
else {})))) else {}))))
return results_ return results_
@@ -1663,17 +1654,17 @@ def fold(Result, results, *,
return folded return folded
def hotify(Result, results, *, def hotify(Result, results, *,
fields=None, enumerate=None,
sort=None,
depth=1, depth=1,
hot=None, hot=None,
**_): **_):
# hotify only makes sense for recursive results # note! hotifying risks confusion if you don't enumerate/have a z
assert hasattr(Result, '_i') # field, since it will allow folding across recursive boundaries
assert hasattr(Result, '_children') import builtins
enumerate_, enumerate = enumerate, builtins.enumerate
if fields is None: # hotify only makes sense for recursive results
fields = Result._fields assert hasattr(Result, '_children')
results_ = [] results_ = []
for r in results: for r in results:
@@ -1694,9 +1685,10 @@ def hotify(Result, results, *,
for k_ in ([k] if k else Result._sort))) for k_ in ([k] if k else Result._sort)))
for k, reverse in it.chain(hot, [(None, False)]))) for k, reverse in it.chain(hot, [(None, False)])))
hot_.append(r._replace(**{ hot_.append(r._replace(**(
Result._i: RInt(len(hot_)), ({enumerate_: len(hot_)}
Result._children: []})) if enumerate_ is not None else {})
| {Result._children: []})))
# recurse? # recurse?
if depth_ > 1: if depth_ > 1:
@@ -1704,8 +1696,7 @@ def hotify(Result, results, *,
depth_-1) depth_-1)
recurse(getattr(r, Result._children), depth-1) recurse(getattr(r, Result._children), depth-1)
results_.append(r._replace(**{ results_.append(r._replace(**{Result._children: hot_}))
Result._children: hot_}))
return results_ return results_
@@ -2052,11 +2043,13 @@ def write_csv(path, Result, results, *,
with openio(path, 'w') as f: with openio(path, 'w') as f:
# write csv? # write csv?
if not json: if not json:
writer = csv.DictWriter(f, writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain(
(by if by is not None else Result._by) by
+ [k for k in (fields if by is not None
if fields is not None else Result._by,
else Result._fields)]) fields
if fields is not None
else Result._fields)).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
@@ -2113,11 +2106,11 @@ def main(csv_paths, *,
fields=None, fields=None,
defines=[], defines=[],
sort=None, sort=None,
depth=None,
enumerate=None, enumerate=None,
depth=None,
children=None, children=None,
notes=None,
hot=None, hot=None,
notes=None,
**args): **args):
import builtins import builtins
enumerate_, enumerate = enumerate, builtins.enumerate enumerate_, enumerate = enumerate, builtins.enumerate
@@ -2194,8 +2187,8 @@ def main(csv_paths, *,
and not any(k == k_ for k_, _ in (sort or [])) and not any(k == k_ for k_, _ in (sort or []))
and k != enumerate_ and k != enumerate_
and k != children and k != children
and k != notes
and not any(k == k_ for k_, _ in (hot or [])) and not any(k == k_ for k_, _ in (hot or []))
and k != notes
and not any(k == k_ and not any(k == k_
for _, expr in exprs for _, expr in exprs
for k_ in expr.fields())] for k_ in expr.fields())]
@@ -2208,8 +2201,8 @@ def main(csv_paths, *,
and not any(k == k_ for k_, _ in (sort or [])) and not any(k == k_ for k_, _ in (sort or []))
and k != enumerate_ and k != enumerate_
and k != children and k != children
and k != notes
and not any(k == k_ for k_, _ in (hot or [])) and not any(k == k_ for k_, _ in (hot or []))
and k != notes
and not any(k == k_ and not any(k == k_
for _, expr in exprs for _, expr in exprs
for k_ in expr.fields())] for k_ in expr.fields())]
@@ -2224,14 +2217,13 @@ def main(csv_paths, *,
sort=sort, sort=sort,
enumerate=enumerate_, enumerate=enumerate_,
children=children, children=children,
notes=notes, hot=hot,
hot=hot) notes=notes)
# homogenize # homogenize
results = homogenize(Result, results, results = homogenize(Result, results,
depth=depth,
enumerate=enumerate_, enumerate=enumerate_,
children=children) depth=depth)
# fold # fold
results = fold(Result, results, results = fold(Result, results,
@@ -2242,10 +2234,9 @@ def main(csv_paths, *,
# hotify? # hotify?
if hot: if hot:
results = hotify(Result, results, results = hotify(Result, results,
fields=fields, enumerate=enumerate_,
depth=depth, depth=depth,
hot=hot, hot=hot)
**args)
# write results to CSV/JSON # write results to CSV/JSON
if args.get('output'): if args.get('output'):
@@ -2394,6 +2385,12 @@ if __name__ == "__main__":
const=(None, None), const=(None, None),
help="Sort by this field, but backwards. Can include an expression " help="Sort by this field, but backwards. Can include an expression "
"of the form field=expr.") "of the form field=expr.")
parser.add_argument(
'-i', '--enumerate',
nargs='?',
const='i',
help="Field to use for enumerating results. This will prevent "
"result folding.")
parser.add_argument( parser.add_argument(
'-z', '--depth', '-z', '--depth',
nargs='?', nargs='?',
@@ -2401,24 +2398,12 @@ if __name__ == "__main__":
const=0, const=0,
help="Depth of function calls to show. 0 shows all calls unless " help="Depth of function calls to show. 0 shows all calls unless "
"we find a cycle. Defaults to 0.") "we find a cycle. Defaults to 0.")
parser.add_argument(
'-i', '--enumerate',
nargs='?',
const='i',
help="Field to use for enumerating results. This will prevent "
"result folding. This can also be used to override which "
"field -r/--hot uses to order results.")
parser.add_argument( parser.add_argument(
'-Z', '--children', '-Z', '--children',
nargs='?', nargs='?',
const='children', const='children',
help="Field to use for recursive results. This expects a list " help="Field to use for recursive results. This expects a list "
"and really only works with JSON input.") "and really only works with JSON input.")
parser.add_argument(
'-N', '--notes',
nargs='?',
const='notes',
help="Field to use for notes.")
class AppendHot(argparse.Action): class AppendHot(argparse.Action):
def __call__(self, parser, namespace, value, option): def __call__(self, parser, namespace, value, option):
if namespace.hot is None: if namespace.hot is None:
@@ -2448,6 +2433,11 @@ if __name__ == "__main__":
)(*x.split('=', 1)), )(*x.split('=', 1)),
const=(None, None), const=(None, None),
help="Like -r/--hot, but backwards.") help="Like -r/--hot, but backwards.")
parser.add_argument(
'-N', '--notes',
nargs='?',
const='notes',
help="Field to use for notes.")
parser.add_argument( parser.add_argument(
'--no-header', '--no-header',
action='store_true', action='store_true',
scripts/ctx.py
+52 -35
View File
@@ -133,27 +133,27 @@ class RInt(co.namedtuple('RInt', 'x')):
# ctx size results # ctx size results
class CtxResult(co.namedtuple('CtxResult', [ class CtxResult(co.namedtuple('CtxResult', [
'i', 'file', 'function', 'z', 'i', 'file', 'function',
'size', 'off', 'size',
'children', 'notes'])): 'children', 'notes'])):
_by = ['i', 'file', 'function'] _by = ['z', 'i', 'file', 'function']
_fields = ['size'] _fields = ['off', 'size']
_sort = ['size'] _sort = ['size']
_types = {'size': RInt} _types = {'off': RInt, 'size': RInt}
_i = 'i'
_children = 'children' _children = 'children'
_notes = 'notes' _notes = 'notes'
__slots__ = () __slots__ = ()
def __new__(cls, i=None, file='', function='', size=0, def __new__(cls, z=0, i=0, file='', function='', off=0, size=0,
children=None, notes=None): children=None, notes=None):
return super().__new__(cls, i, file, function, return super().__new__(cls, z, i, file, function,
RInt(size), RInt(off), RInt(size),
children if children is not None else [], children if children is not None else [],
notes if notes is not None else set()) notes if notes is not None else set())
def __add__(self, other): def __add__(self, other):
return CtxResult(self.i, self.file, self.function, return CtxResult(self.z, self.i, self.file, self.function,
min(self.off, other.off),
max(self.size, other.size), max(self.size, other.size),
self.children + other.children, self.children + other.children,
self.notes | other.notes) self.notes | other.notes)
@@ -596,11 +596,13 @@ def collect_ctx(obj_paths, *,
type = info[int(type['DW_AT_type'].strip('<>'), 0)] type = info[int(type['DW_AT_type'].strip('<>'), 0)]
if (type.name is not None if (type.name is not None
and type.tag != 'DW_TAG_subroutine_type'): and type.tag != 'DW_TAG_subroutine_type'):
# find size, etc
name_ = type.name name_ = type.name
size_ = sizeof(type, seen | {entry.off}) size_ = sizeof(type, seen | {entry.off})
children_, notes_, dirty_ = childrenof( children_, notes_, dirty_ = childrenof(
type, depth-1, seen | {entry.off}) type, depth-1, seen | {entry.off})
children.append(CtxResult(0, file, name_, size_, children.append(CtxResult(
0, 0, file, name_, 0, size_,
children=children_, children=children_,
notes=notes_)) notes=notes_))
dirty = dirty or dirty_ dirty = dirty or dirty_
@@ -608,15 +610,22 @@ def collect_ctx(obj_paths, *,
elif entry.tag in { elif entry.tag in {
'DW_TAG_structure_type', 'DW_TAG_structure_type',
'DW_TAG_union_type'}: 'DW_TAG_union_type'}:
# iterate over children in struct/union
children, notes, dirty = [], set(), False children, notes, dirty = [], set(), False
for child in entry.children: for child in entry.children:
if child.tag != 'DW_TAG_member': if child.tag != 'DW_TAG_member':
continue continue
# find name
name_ = child.name name_ = child.name
# try to find offset for struct members, note this
# is _not_ the same as the dwarf entry offset
off_ = int(child.get('DW_AT_data_member_location', 0))
# find size, children, etc
size_ = sizeof(child, seen | {entry.off}) size_ = sizeof(child, seen | {entry.off})
children_, notes_, dirty_ = childrenof( children_, notes_, dirty_ = childrenof(
child, depth-1, seen | {entry.off}) child, depth-1, seen | {entry.off})
children.append(CtxResult(child.off, file, name_, size_, children.append(CtxResult(
0, len(children), file, name_, off_, size_,
children=children_, children=children_,
notes=notes_)) notes=notes_))
dirty = dirty or dirty_ dirty = dirty or dirty_
@@ -678,16 +687,24 @@ def collect_ctx(obj_paths, *,
# find children, recursing if necessary # find children, recursing if necessary
children_, notes_, _ = childrenof(param, depth-2) children_, notes_, _ = childrenof(param, depth-2)
params.append(CtxResult(param.off, file, name_, size_, params.append(CtxResult(
0, len(params), file, name_, 0, size_,
children=children_, children=children_,
notes=notes_)) notes=notes_))
# context = sum of params # context = sum of params
name = entry.name name = entry.name
size = sum((param.size for param in params), start=RInt(0)) size = sum((param.size for param in params), start=RInt(0))
results.append(CtxResult(None, file, name, size, results.append(CtxResult(
0, 0, file, name, 0, size,
children=params)) children=params))
# assign z at the end to avoid issues with caching
def zed(results, z):
return [r._replace(z=z, children=zed(r.children, z+1))
for r in results]
results = zed(results, 0)
return results return results
@@ -774,17 +791,17 @@ def fold(Result, results, *,
return folded return folded
def hotify(Result, results, *, def hotify(Result, results, *,
fields=None, enumerate=None,
sort=None,
depth=1, depth=1,
hot=None, hot=None,
**_): **_):
# hotify only makes sense for recursive results # note! hotifying risks confusion if you don't enumerate/have a z
assert hasattr(Result, '_i') # field, since it will allow folding across recursive boundaries
assert hasattr(Result, '_children') import builtins
enumerate_, enumerate = enumerate, builtins.enumerate
if fields is None: # hotify only makes sense for recursive results
fields = Result._fields assert hasattr(Result, '_children')
results_ = [] results_ = []
for r in results: for r in results:
@@ -805,9 +822,10 @@ def hotify(Result, results, *,
for k_ in ([k] if k else Result._sort))) for k_ in ([k] if k else Result._sort)))
for k, reverse in it.chain(hot, [(None, False)]))) for k, reverse in it.chain(hot, [(None, False)])))
hot_.append(r._replace(**{ hot_.append(r._replace(**(
Result._i: RInt(len(hot_)), ({enumerate_: len(hot_)}
Result._children: []})) if enumerate_ is not None else {})
| {Result._children: []})))
# recurse? # recurse?
if depth_ > 1: if depth_ > 1:
@@ -815,8 +833,7 @@ def hotify(Result, results, *,
depth_-1) depth_-1)
recurse(getattr(r, Result._children), depth-1) recurse(getattr(r, Result._children), depth-1)
results_.append(r._replace(**{ results_.append(r._replace(**{Result._children: hot_}))
Result._children: hot_}))
return results_ return results_
@@ -1163,11 +1180,13 @@ def write_csv(path, Result, results, *,
with openio(path, 'w') as f: with openio(path, 'w') as f:
# write csv? # write csv?
if not json: if not json:
writer = csv.DictWriter(f, writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain(
(by if by is not None else Result._by) by
+ [k for k in (fields if by is not None
if fields is not None else Result._by,
else Result._fields)]) fields
if fields is not None
else Result._fields)).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
@@ -1260,10 +1279,8 @@ def main(obj_paths, *,
# hotify? # hotify?
if hot: if hot:
results = hotify(CtxResult, results, results = hotify(CtxResult, results,
fields=fields,
depth=depth, depth=depth,
hot=hot, hot=hot)
**args)
# write results to CSV/JSON # write results to CSV/JSON
if args.get('output'): if args.get('output'):
@@ -1302,7 +1319,7 @@ def main(obj_paths, *,
if not args.get('quiet'): if not args.get('quiet'):
table(CtxResult, results, diff_results, table(CtxResult, results, diff_results,
by=by if by is not None else ['function'], by=by if by is not None else ['function'],
fields=fields, fields=fields if fields is not None else ['size'],
sort=sort, sort=sort,
depth=depth, depth=depth,
**args) **args)
scripts/data.py
+7 -5
View File
@@ -922,11 +922,13 @@ def write_csv(path, Result, results, *,
with openio(path, 'w') as f: with openio(path, 'w') as f:
# write csv? # write csv?
if not json: if not json:
writer = csv.DictWriter(f, writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain(
(by if by is not None else Result._by) by
+ [k for k in (fields if by is not None
if fields is not None else Result._by,
else Result._fields)]) fields
if fields is not None
else Result._fields)).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
scripts/perf.py
+29 -30
View File
@@ -147,31 +147,30 @@ class RInt(co.namedtuple('RInt', 'x')):
# perf results # perf results
class PerfResult(co.namedtuple('PerfResult', [ class PerfResult(co.namedtuple('PerfResult', [
'i', 'file', 'function', 'line', 'z', 'file', 'function', 'line',
'cycles', 'bmisses', 'branches', 'cmisses', 'caches', 'cycles', 'bmisses', 'branches', 'cmisses', 'caches',
'children'])): 'children'])):
_by = ['i', 'file', 'function', 'line'] _by = ['z', 'file', 'function', 'line']
_fields = ['cycles', 'bmisses', 'branches', 'cmisses', 'caches'] _fields = ['cycles', 'bmisses', 'branches', 'cmisses', 'caches']
_sort = ['cycles', 'bmisses', 'cmisses', 'branches', 'caches'] _sort = ['cycles', 'bmisses', 'cmisses', 'branches', 'caches']
_types = { _types = {
'cycles': RInt, 'cycles': RInt,
'bmisses': RInt, 'branches': RInt, 'bmisses': RInt, 'branches': RInt,
'cmisses': RInt, 'caches': RInt} 'cmisses': RInt, 'caches': RInt}
_i = 'i'
_children = 'children' _children = 'children'
__slots__ = () __slots__ = ()
def __new__(cls, i=None, file='', function='', line=0, def __new__(cls, z=0, file='', function='', line=0,
cycles=0, bmisses=0, branches=0, cmisses=0, caches=0, cycles=0, bmisses=0, branches=0, cmisses=0, caches=0,
children=None): children=None):
return super().__new__(cls, i, file, function, int(RInt(line)), return super().__new__(cls, z, file, function, int(RInt(line)),
RInt(cycles), RInt(cycles),
RInt(bmisses), RInt(branches), RInt(bmisses), RInt(branches),
RInt(cmisses), RInt(caches), RInt(cmisses), RInt(caches),
children if children is not None else []) children if children is not None else [])
def __add__(self, other): def __add__(self, other):
return PerfResult(self.i, self.file, self.function, self.line, return PerfResult(self.z, self.file, self.function, self.line,
self.cycles + other.cycles, self.cycles + other.cycles,
self.bmisses + other.bmisses, self.bmisses + other.bmisses,
self.branches + other.branches, self.branches + other.branches,
@@ -756,15 +755,15 @@ def collect_decompressed(path, *,
raise sp.CalledProcessError(proc.returncode, proc.args) raise sp.CalledProcessError(proc.returncode, proc.args)
# rearrange results into result type # rearrange results into result type
def to_results(results): def to_results(results, z):
results_ = [] results_ = []
for name, (r, children) in results.items(): for name, (r, children) in results.items():
results_.append(PerfResult(None, *name, results_.append(PerfResult(z, *name,
**{events[k]: v for k, v in r.items()}, **{events[k]: v for k, v in r.items()},
children=to_results(children))) children=to_results(children, z+1)))
return results_ return results_
return to_results(results) return to_results(results, 0)
def collect_job(path, i, **args): def collect_job(path, i, **args):
# decompress into a temporary file, this is to work around # decompress into a temporary file, this is to work around
@@ -896,17 +895,17 @@ def fold(Result, results, *,
return folded return folded
def hotify(Result, results, *, def hotify(Result, results, *,
fields=None, enumerate=None,
sort=None,
depth=1, depth=1,
hot=None, hot=None,
**_): **_):
# hotify only makes sense for recursive results # note! hotifying risks confusion if you don't enumerate/have a z
assert hasattr(Result, '_i') # field, since it will allow folding across recursive boundaries
assert hasattr(Result, '_children') import builtins
enumerate_, enumerate = enumerate, builtins.enumerate
if fields is None: # hotify only makes sense for recursive results
fields = Result._fields assert hasattr(Result, '_children')
results_ = [] results_ = []
for r in results: for r in results:
@@ -927,9 +926,10 @@ def hotify(Result, results, *,
for k_ in ([k] if k else Result._sort))) for k_ in ([k] if k else Result._sort)))
for k, reverse in it.chain(hot, [(None, False)]))) for k, reverse in it.chain(hot, [(None, False)])))
hot_.append(r._replace(**{ hot_.append(r._replace(**(
Result._i: RInt(len(hot_)), ({enumerate_: len(hot_)}
Result._children: []})) if enumerate_ is not None else {})
| {Result._children: []})))
# recurse? # recurse?
if depth_ > 1: if depth_ > 1:
@@ -937,8 +937,7 @@ def hotify(Result, results, *,
depth_-1) depth_-1)
recurse(getattr(r, Result._children), depth-1) recurse(getattr(r, Result._children), depth-1)
results_.append(r._replace(**{ results_.append(r._replace(**{Result._children: hot_}))
Result._children: hot_}))
return results_ return results_
@@ -1285,11 +1284,13 @@ def write_csv(path, Result, results, *,
with openio(path, 'w') as f: with openio(path, 'w') as f:
# write csv? # write csv?
if not json: if not json:
writer = csv.DictWriter(f, writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain(
(by if by is not None else Result._by) by
+ [k for k in (fields if by is not None
if fields is not None else Result._by,
else Result._fields)]) fields
if fields is not None
else Result._fields)).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
@@ -1491,10 +1492,8 @@ def report(perf_paths, *,
# hotify? # hotify?
if hot: if hot:
results = hotify(PerfResult, results, results = hotify(PerfResult, results,
fields=fields,
depth=depth, depth=depth,
hot=hot, hot=hot)
**args)
# write results to CSV/JSON # write results to CSV/JSON
if args.get('output'): if args.get('output'):
scripts/perfbd.py
+31 -31
View File
@@ -138,26 +138,25 @@ class RInt(co.namedtuple('RInt', 'x')):
# perf results # perf results
class PerfBdResult(co.namedtuple('PerfBdResult', [ class PerfBdResult(co.namedtuple('PerfBdResult', [
'i', 'file', 'function', 'line', 'z', 'file', 'function', 'line',
'readed', 'proged', 'erased', 'readed', 'proged', 'erased',
'children'])): 'children'])):
_by = ['i', 'file', 'function', 'line'] _by = ['z', 'file', 'function', 'line']
_fields = ['readed', 'proged', 'erased'] _fields = ['readed', 'proged', 'erased']
_sort = ['erased', 'proged', 'readed'] _sort = ['erased', 'proged', 'readed']
_types = {'readed': RInt, 'proged': RInt, 'erased': RInt} _types = {'readed': RInt, 'proged': RInt, 'erased': RInt}
_i = 'i'
_children = 'children' _children = 'children'
__slots__ = () __slots__ = ()
def __new__(cls, i=None, file='', function='', line=0, def __new__(cls, z=0, file='', function='', line=0,
readed=0, proged=0, erased=0, readed=0, proged=0, erased=0,
children=None): children=None):
return super().__new__(cls, i, file, function, int(RInt(line)), return super().__new__(cls, z, file, function, int(RInt(line)),
RInt(readed), RInt(proged), RInt(erased), RInt(readed), RInt(proged), RInt(erased),
children if children is not None else []) children if children is not None else [])
def __add__(self, other): def __add__(self, other):
return PerfBdResult(self.i, self.file, self.function, self.line, return PerfBdResult(self.z, self.file, self.function, self.line,
self.readed + other.readed, self.readed + other.readed,
self.proged + other.proged, self.proged + other.proged,
self.erased + other.erased, self.erased + other.erased,
@@ -715,15 +714,15 @@ def collect_job(path, start, stop, syms, lines, *,
commit() commit()
# rearrange results into result type # rearrange results into result type
def to_results(results): def to_results(results, z):
results_ = [] results_ = []
for name, (r, p, e, children) in results.items(): for name, (r, p, e, children) in results.items():
results_.append(PerfBdResult(None, *name, results_.append(PerfBdResult(z, *name,
r, p, e, r, p, e,
children=to_results(children))) children=to_results(children, z+1)))
return results_ return results_
return to_results(results) return to_results(results, 0)
def starapply(args): def starapply(args):
f, args, kwargs = args f, args, kwargs = args
@@ -866,17 +865,17 @@ def fold(Result, results, *,
return folded return folded
def hotify(Result, results, *, def hotify(Result, results, *,
fields=None, enumerate=None,
sort=None,
depth=1, depth=1,
hot=None, hot=None,
**_): **_):
# hotify only makes sense for recursive results # note! hotifying risks confusion if you don't enumerate/have a z
assert hasattr(Result, '_i') # field, since it will allow folding across recursive boundaries
assert hasattr(Result, '_children') import builtins
enumerate_, enumerate = enumerate, builtins.enumerate
if fields is None: # hotify only makes sense for recursive results
fields = Result._fields assert hasattr(Result, '_children')
results_ = [] results_ = []
for r in results: for r in results:
@@ -897,9 +896,10 @@ def hotify(Result, results, *,
for k_ in ([k] if k else Result._sort))) for k_ in ([k] if k else Result._sort)))
for k, reverse in it.chain(hot, [(None, False)]))) for k, reverse in it.chain(hot, [(None, False)])))
hot_.append(r._replace(**{ hot_.append(r._replace(**(
Result._i: RInt(len(hot_)), ({enumerate_: len(hot_)}
Result._children: []})) if enumerate_ is not None else {})
| {Result._children: []})))
# recurse? # recurse?
if depth_ > 1: if depth_ > 1:
@@ -907,8 +907,7 @@ def hotify(Result, results, *,
depth_-1) depth_-1)
recurse(getattr(r, Result._children), depth-1) recurse(getattr(r, Result._children), depth-1)
results_.append(r._replace(**{ results_.append(r._replace(**{Result._children: hot_}))
Result._children: hot_}))
return results_ return results_
@@ -1255,11 +1254,13 @@ def write_csv(path, Result, results, *,
with openio(path, 'w') as f: with openio(path, 'w') as f:
# write csv? # write csv?
if not json: if not json:
writer = csv.DictWriter(f, writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain(
(by if by is not None else Result._by) by
+ [k for k in (fields if by is not None
if fields is not None else Result._by,
else Result._fields)]) fields
if fields is not None
else Result._fields)).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
@@ -1490,10 +1491,8 @@ def report(paths, *,
# hotify? # hotify?
if hot: if hot:
results = hotify(PerfBdResult, results, results = hotify(PerfBdResult, results,
fields=fields,
depth=depth, depth=depth,
hot=hot, hot=hot)
**args)
# write results to CSV/JSON # write results to CSV/JSON
if args.get('output'): if args.get('output'):
@@ -1541,7 +1540,8 @@ def report(paths, *,
# print table # print table
table(PerfBdResult, results, diff_results, table(PerfBdResult, results, diff_results,
by=by if by is not None else ['function'], by=by if by is not None else ['function'],
fields=fields, fields=fields if fields is not None
else ['readed', 'proged', 'erased'],
sort=sort, sort=sort,
depth=depth, depth=depth,
**args) **args)
scripts/stack.py
+36 -29
View File
@@ -133,27 +133,26 @@ class RInt(co.namedtuple('RInt', 'x')):
# stack size results # stack size results
class StackResult(co.namedtuple('StackResult', [ class StackResult(co.namedtuple('StackResult', [
'i', 'file', 'function', 'z', 'file', 'function',
'frame', 'limit', 'frame', 'limit',
'children', 'notes'])): 'children', 'notes'])):
_by = ['i', 'file', 'function'] _by = ['z', 'file', 'function']
_fields = ['frame', 'limit'] _fields = ['frame', 'limit']
_sort = ['limit', 'frame'] _sort = ['limit', 'frame']
_types = {'frame': RInt, 'limit': RInt} _types = {'frame': RInt, 'limit': RInt}
_i = 'i'
_children = 'children' _children = 'children'
_notes = 'notes' _notes = 'notes'
__slots__ = () __slots__ = ()
def __new__(cls, i=None, file='', function='', frame=0, limit=0, def __new__(cls, z=0, file='', function='', frame=0, limit=0,
children=None, notes=None): children=None, notes=None):
return super().__new__(cls, i, file, function, return super().__new__(cls, z, file, function,
RInt(frame), RInt(limit), RInt(frame), RInt(limit),
children if children is not None else [], children if children is not None else [],
notes if notes is not None else set()) notes if notes is not None else set())
def __add__(self, other): def __add__(self, other):
return StackResult(self.i, self.file, self.function, return StackResult(self.z, self.file, self.function,
self.frame + other.frame, self.frame + other.frame,
max(self.limit, other.limit), max(self.limit, other.limit),
self.children + other.children, self.children + other.children,
@@ -417,7 +416,8 @@ def collect_stack(ci_paths, *,
limit_ = limitof(node_, seen | {node.name}) limit_ = limitof(node_, seen | {node.name})
children_, notes_, dirty_ = childrenof( children_, notes_, dirty_ = childrenof(
node_, depth-1, seen | {node.name}) node_, depth-1, seen | {node.name})
children.append(StackResult(None, file_, name_, frame_, limit_, children.append(StackResult(
file=file_, function=name_, frame=frame_, limit=limit_,
children=children_, children=children_,
notes=notes_)) notes=notes_))
dirty = dirty or dirty_ dirty = dirty or dirty_
@@ -440,10 +440,17 @@ def collect_stack(ci_paths, *,
frame = frameof(node) frame = frameof(node)
limit = limitof(node) limit = limitof(node)
children, notes, _ = childrenof(node, depth-1) children, notes, _ = childrenof(node, depth-1)
results.append(StackResult(None, file, name, frame, limit, results.append(StackResult(
file=file, function=name, frame=frame, limit=limit,
children=children, children=children,
notes=notes)) notes=notes))
# assign z at the end to avoid issues with caching
def zed(results, z):
return [r._replace(z=z, children=zed(r.children, z+1))
for r in results]
results = zed(results, 0)
return results return results
@@ -530,17 +537,17 @@ def fold(Result, results, *,
return folded return folded
def hotify(Result, results, *, def hotify(Result, results, *,
fields=None, enumerate=None,
sort=None,
depth=1, depth=1,
hot=None, hot=None,
**_): **_):
# hotify only makes sense for recursive results # note! hotifying risks confusion if you don't enumerate/have a z
assert hasattr(Result, '_i') # field, since it will allow folding across recursive boundaries
assert hasattr(Result, '_children') import builtins
enumerate_, enumerate = enumerate, builtins.enumerate
if fields is None: # hotify only makes sense for recursive results
fields = Result._fields assert hasattr(Result, '_children')
results_ = [] results_ = []
for r in results: for r in results:
@@ -561,9 +568,10 @@ def hotify(Result, results, *,
for k_ in ([k] if k else Result._sort))) for k_ in ([k] if k else Result._sort)))
for k, reverse in it.chain(hot, [(None, False)]))) for k, reverse in it.chain(hot, [(None, False)])))
hot_.append(r._replace(**{ hot_.append(r._replace(**(
Result._i: RInt(len(hot_)), ({enumerate_: len(hot_)}
Result._children: []})) if enumerate_ is not None else {})
| {Result._children: []})))
# recurse? # recurse?
if depth_ > 1: if depth_ > 1:
@@ -571,8 +579,7 @@ def hotify(Result, results, *,
depth_-1) depth_-1)
recurse(getattr(r, Result._children), depth-1) recurse(getattr(r, Result._children), depth-1)
results_.append(r._replace(**{ results_.append(r._replace(**{Result._children: hot_}))
Result._children: hot_}))
return results_ return results_
@@ -919,11 +926,13 @@ def write_csv(path, Result, results, *,
with openio(path, 'w') as f: with openio(path, 'w') as f:
# write csv? # write csv?
if not json: if not json:
writer = csv.DictWriter(f, writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain(
(by if by is not None else Result._by) by
+ [k for k in (fields if by is not None
if fields is not None else Result._by,
else Result._fields)]) fields
if fields is not None
else Result._fields)).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
@@ -1016,10 +1025,8 @@ def main(ci_paths,
# hotify? # hotify?
if hot: if hot:
results = hotify(StackResult, results, results = hotify(StackResult, results,
fields=fields,
depth=depth, depth=depth,
hot=hot, hot=hot)
**args)
# write results to CSV/JSON # write results to CSV/JSON
if args.get('output'): if args.get('output'):
@@ -1058,7 +1065,7 @@ def main(ci_paths,
if not args.get('quiet'): if not args.get('quiet'):
table(StackResult, results, diff_results, table(StackResult, results, diff_results,
by=by if by is not None else ['function'], by=by if by is not None else ['function'],
fields=fields, fields=fields if fields is not None else ['frame', 'limit'],
sort=sort, sort=sort,
depth=depth, depth=depth,
**args) **args)
scripts/structs.py
+48 -34
View File
@@ -133,25 +133,25 @@ class RInt(co.namedtuple('RInt', 'x')):
# struct size results # struct size results
class StructResult(co.namedtuple('StructResult', [ class StructResult(co.namedtuple('StructResult', [
'i', 'file', 'struct', 'z', 'i', 'file', 'struct',
'size', 'align', 'off', 'size', 'align',
'children'])): 'children'])):
_by = ['i', 'file', 'struct'] _by = ['z', 'i', 'file', 'struct']
_fields = ['size', 'align'] _fields = ['off', 'size', 'align']
_sort = ['size', 'align'] _sort = ['size', 'align']
_types = {'size': RInt, 'align': RInt} _types = {'off': RInt, 'size': RInt, 'align': RInt}
_i = 'i'
_children = 'children' _children = 'children'
__slots__ = () __slots__ = ()
def __new__(cls, i=None, file='', struct='', size=0, align=0, def __new__(cls, z=0, i=0, file='', struct='', off=0, size=0, align=0,
children=None): children=None):
return super().__new__(cls, i, file, struct, return super().__new__(cls, z, i, file, struct,
RInt(size), RInt(align), RInt(off), RInt(size), RInt(align),
children if children is not None else []) children if children is not None else [])
def __add__(self, other): def __add__(self, other):
return StructResult(self.i, self.file, self.struct, return StructResult(self.z, self.i, self.file, self.struct,
min(self.off, other.off),
self.size + other.size, self.size + other.size,
max(self.align, other.align), max(self.align, other.align),
self.children + other.children) self.children + other.children)
@@ -437,14 +437,22 @@ def collect_structs(obj_paths, *,
elif entry.tag in { elif entry.tag in {
'DW_TAG_structure_type', 'DW_TAG_structure_type',
'DW_TAG_union_type'}: 'DW_TAG_union_type'}:
# iterate over children in struct/union
children = [] children = []
for child in entry.children: for child in entry.children:
if child.tag != 'DW_TAG_member':
continue
# find name
name_ = child.name name_ = child.name
# try to find offset for struct members, note this
# is _not_ the same as the dwarf entry offset
off_ = int(child.get('DW_AT_data_member_location', 0))
# find size, align, children, etc
size_ = sizeof(child) size_ = sizeof(child)
align_ = alignof(child) align_ = alignof(child)
children_ = childrenof(child, depth-1) children_ = childrenof(child, depth-1)
children.append(StructResult( children.append(StructResult(
child.off, file, name_, size_, align_, 0, len(children), file, name_, off_, size_, align_,
children=children_)) children=children_))
# indirect type? # indirect type?
elif entry.tag in { elif entry.tag in {
@@ -497,12 +505,12 @@ def collect_structs(obj_paths, *,
# these separately # these separately
if entry.tag == 'DW_TAG_typedef': if entry.tag == 'DW_TAG_typedef':
typedefs[entry.off] = StructResult( typedefs[entry.off] = StructResult(
None, file, name, size, align, 0, 0, file, name, 0, size, align,
children=children) children=children)
typedefed.add(int(entry['DW_AT_type'].strip('<>'), 0)) typedefed.add(int(entry['DW_AT_type'].strip('<>'), 0))
else: else:
types[entry.off] = StructResult( types[entry.off] = StructResult(
None, file, name, size, align, 0, 0, file, name, 0, size, align,
children=children) children=children)
# let typedefs take priority # let typedefs take priority
@@ -511,6 +519,12 @@ def collect_structs(obj_paths, *,
for off, type in types.items() for off, type in types.items()
if off not in typedefed) if off not in typedefed)
# assign z at the end to avoid issues with caching
def zed(results, z):
return [r._replace(z=z, children=zed(r.children, z+1))
for r in results]
results = zed(results, 0)
return results return results
@@ -597,17 +611,17 @@ def fold(Result, results, *,
return folded return folded
def hotify(Result, results, *, def hotify(Result, results, *,
fields=None, enumerate=None,
sort=None,
depth=1, depth=1,
hot=None, hot=None,
**_): **_):
# hotify only makes sense for recursive results # note! hotifying risks confusion if you don't enumerate/have a z
assert hasattr(Result, '_i') # field, since it will allow folding across recursive boundaries
assert hasattr(Result, '_children') import builtins
enumerate_, enumerate = enumerate, builtins.enumerate
if fields is None: # hotify only makes sense for recursive results
fields = Result._fields assert hasattr(Result, '_children')
results_ = [] results_ = []
for r in results: for r in results:
@@ -628,9 +642,10 @@ def hotify(Result, results, *,
for k_ in ([k] if k else Result._sort))) for k_ in ([k] if k else Result._sort)))
for k, reverse in it.chain(hot, [(None, False)]))) for k, reverse in it.chain(hot, [(None, False)])))
hot_.append(r._replace(**{ hot_.append(r._replace(**(
Result._i: RInt(len(hot_)), ({enumerate_: len(hot_)}
Result._children: []})) if enumerate_ is not None else {})
| {Result._children: []})))
# recurse? # recurse?
if depth_ > 1: if depth_ > 1:
@@ -638,8 +653,7 @@ def hotify(Result, results, *,
depth_-1) depth_-1)
recurse(getattr(r, Result._children), depth-1) recurse(getattr(r, Result._children), depth-1)
results_.append(r._replace(**{ results_.append(r._replace(**{Result._children: hot_}))
Result._children: hot_}))
return results_ return results_
@@ -986,11 +1000,13 @@ def write_csv(path, Result, results, *,
with openio(path, 'w') as f: with openio(path, 'w') as f:
# write csv? # write csv?
if not json: if not json:
writer = csv.DictWriter(f, writer = csv.DictWriter(f, list(co.OrderedDict.fromkeys(it.chain(
(by if by is not None else Result._by) by
+ [k for k in (fields if by is not None
if fields is not None else Result._by,
else Result._fields)]) fields
if fields is not None
else Result._fields)).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
@@ -1083,10 +1099,8 @@ def main(obj_paths, *,
# hotify? # hotify?
if hot: if hot:
results = hotify(StructResult, results, results = hotify(StructResult, results,
fields=fields,
depth=depth, depth=depth,
hot=hot, hot=hot)
**args)
# write results to CSV/JSON # write results to CSV/JSON
if args.get('output'): if args.get('output'):
@@ -1125,7 +1139,7 @@ def main(obj_paths, *,
if not args.get('quiet'): if not args.get('quiet'):
table(StructResult, results, diff_results, table(StructResult, results, diff_results,
by=by if by is not None else ['struct'], by=by if by is not None else ['struct'],
fields=fields, fields=fields if fields is not None else ['size', 'align'],
sort=sort, sort=sort,
depth=depth, depth=depth,
**args) **args)