Github/littlefs
1
Fork 0
Code Actions Activity

scripts: Fully connected graph in stack.py, no more recursive folding

Browse Source 
This makes -D/--define more useful in stack.py/perf.py/perfbd.py by no
longer hiding undfined children entries.

For example:

  $ ./scripts/stack.py lfs.ci lfs_util.ci -Dfunction=lfsr_mount -t
  function                       frame    limit
  lfsr_mount                        96     2816
  |-> lfsr_fs_gc                    80     2720
  |-> lfsr_mtree_gc                176     2640
  |-> lfsr_mdir_commit             576     2464
  ... snip ...

Now shows all functions in the hot path of lfsr_mount, where before it
would only show functions in the hot path of lfsr_mount that were also
_named_ lfsr_mount.

The previous behavior was technically not wrong... but not very useful
(and confusing).

---

This was actually quite a bit annoying to get working because of the
possibility of function call cycles.

I ended up turning stack.py's result type into a fully connected graph,
which only works because Python has a cycle detector. (Actually this
script is so short-lived we probably wouldn't care if this leaked
memory.)

A nice side effect of this is now all the recursive scripts (stack.py,
perf.py, and perfbd.py) share the same internal result representation
and recursive printing logic, which is probably a good thing.
This commit is contained in:
Christopher Haster
2024-11-04 15:39:50 -06:00
parent 711cebfcf3
commit 0c3868f92c
3 changed files with 93 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/perf.py
+8 -16
View File
@@ -657,15 +657,6 @@ def fold(Result, results, by=None, defines=[]):
for name, rs in folding.items(): for name, rs in folding.items():
folded.append(sum(rs[1:], start=rs[0])) folded.append(sum(rs[1:], start=rs[0]))
# fold recursively
folded_ = []
for r in folded:
folded_.append(r._replace(children=fold(
Result, r.children,
by=by,
defines=defines)))
folded = folded_
return folded return folded
def table(Result, results, diff_results=None, *, def table(Result, results, diff_results=None, *,
@@ -838,7 +829,8 @@ def table(Result, results, diff_results=None, *,
depth_ = 2 depth_ = 2
elif m.isinf(depth_): elif m.isinf(depth_):
def rec_depth(results_, seen=set()): def rec_depth(results_, seen=set()):
# rebuild our tables at each layer # build the children table at each layer
results_ = fold(Result, results_, by=by)
table_ = { table_ = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results_} for r in results_}
@@ -871,7 +863,8 @@ def table(Result, results, diff_results=None, *,
if hot: if hot:
def recurse(results_, depth_, seen=set(), def recurse(results_, depth_, seen=set(),
prefixes=('', '', '', '')): prefixes=('', '', '', '')):
# rebuild our tables at each layer # build the children table at each layer
results_ = fold(Result, results_, by=by)
table_ = { table_ = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results_} for r in results_}
@@ -929,14 +922,14 @@ def table(Result, results, diff_results=None, *,
else: else:
def recurse(results_, depth_, seen=set(), def recurse(results_, depth_, seen=set(),
prefixes=('', '', '', '')): prefixes=('', '', '', '')):
# rebuild our tables at each layer # build the children table at each layer
results_ = fold(Result, results_, by=by)
table_ = { table_ = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results_} for r in results_}
names_ = list(table_.keys()) names_ = list(table_.keys())
# sort again at each layer, keep in mind the numbers are # sort the children layer
# changing as we descend
names_.sort() names_.sort()
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
@@ -984,8 +977,7 @@ def table(Result, results, diff_results=None, *,
prefixes[2+is_last] + "| ", prefixes[2+is_last] + "| ",
prefixes[2+is_last] + " ")) prefixes[2+is_last] + " "))
# we have enough going on with diffing to make the top layer # the top layer is a bit of a special case
# a special case
for name, line in zip(names, lines[1:-1]): for name, line in zip(names, lines[1:-1]):
print('%-*s %s' % ( print('%-*s %s' % (
widths[0], line[0][0], widths[0], line[0][0],
scripts/perfbd.py
+8 -16
View File
@@ -621,15 +621,6 @@ def fold(Result, results, by=None, defines=[]):
for name, rs in folding.items(): for name, rs in folding.items():
folded.append(sum(rs[1:], start=rs[0])) folded.append(sum(rs[1:], start=rs[0]))
# fold recursively
folded_ = []
for r in folded:
folded_.append(r._replace(children=fold(
Result, r.children,
by=by,
defines=defines)))
folded = folded_
return folded return folded
def table(Result, results, diff_results=None, *, def table(Result, results, diff_results=None, *,
@@ -802,7 +793,8 @@ def table(Result, results, diff_results=None, *,
depth_ = 2 depth_ = 2
elif m.isinf(depth_): elif m.isinf(depth_):
def rec_depth(results_, seen=set()): def rec_depth(results_, seen=set()):
# rebuild our tables at each layer # build the children table at each layer
results_ = fold(Result, results_, by=by)
table_ = { table_ = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results_} for r in results_}
@@ -835,7 +827,8 @@ def table(Result, results, diff_results=None, *,
if hot: if hot:
def recurse(results_, depth_, seen=set(), def recurse(results_, depth_, seen=set(),
prefixes=('', '', '', '')): prefixes=('', '', '', '')):
# rebuild our tables at each layer # build the children table at each layer
results_ = fold(Result, results_, by=by)
table_ = { table_ = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results_} for r in results_}
@@ -893,14 +886,14 @@ def table(Result, results, diff_results=None, *,
else: else:
def recurse(results_, depth_, seen=set(), def recurse(results_, depth_, seen=set(),
prefixes=('', '', '', '')): prefixes=('', '', '', '')):
# rebuild our tables at each layer # build the children table at each layer
results_ = fold(Result, results_, by=by)
table_ = { table_ = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results_} for r in results_}
names_ = list(table_.keys()) names_ = list(table_.keys())
# sort again at each layer, keep in mind the numbers are # sort the children layer
# changing as we descend
names_.sort() names_.sort()
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
@@ -948,8 +941,7 @@ def table(Result, results, diff_results=None, *,
prefixes[2+is_last] + "| ", prefixes[2+is_last] + "| ",
prefixes[2+is_last] + " ")) prefixes[2+is_last] + " "))
# we have enough going on with diffing to make the top layer # the top layer is a bit of a special case
# a special case
for name, line in zip(names, lines[1:-1]): for name, line in zip(names, lines[1:-1]):
print('%-*s %s' % ( print('%-*s %s' % (
widths[0], line[0][0], widths[0], line[0][0],
scripts/stack.py
+65 -44
View File
@@ -104,7 +104,8 @@ class StackResult(co.namedtuple('StackResult', [
__slots__ = () __slots__ = ()
def __new__(cls, file='', function='', def __new__(cls, file='', function='',
frame=0, limit=0, children=set()): frame=0, limit=0,
children=[]):
return super().__new__(cls, file, function, return super().__new__(cls, file, function,
RInt(frame), RInt(limit), RInt(frame), RInt(limit),
children) children)
@@ -113,7 +114,7 @@ class StackResult(co.namedtuple('StackResult', [
return StackResult(self.file, self.function, return StackResult(self.file, self.function,
self.frame + other.frame, self.frame + other.frame,
max(self.limit, other.limit), max(self.limit, other.limit),
self.children | other.children) self.children + other.children)
def openio(path, mode='r', buffering=-1): def openio(path, mode='r', buffering=-1):
@@ -257,22 +258,21 @@ def collect(ci_paths, *,
find_limit.cache[source] = frame + limit find_limit.cache[source] = frame + limit
return frame + limit return frame + limit
def find_children(targets):
children = set()
for target in targets:
if target in callgraph:
t_file, t_function, _, _ = callgraph[target]
children.add((t_file, t_function))
return children
# build results # build results
results = [] results = {}
for source, (s_file, s_function, frame, targets) in callgraph.items(): for source, (s_file, s_function, frame, _) in callgraph.items():
limit = find_limit(source) limit = find_limit(source)
children = find_children(targets) results[source] = StackResult(s_file, s_function, frame, limit, [])
results.append(StackResult(s_file, s_function, frame, limit, children))
return results # connect parents to their children, this may create a fully cyclic graph
# in the case of recursion
for source, (_, _, _, targets) in callgraph.items():
results[source].children.extend(
results[target]
for target in targets
if target in results)
return list(results.values())
def fold(Result, results, by=None, defines=[]): def fold(Result, results, by=None, defines=[]):
@@ -477,14 +477,16 @@ def table(Result, results, diff_results=None, *,
if hot: if hot:
depth_ = 2 depth_ = 2
elif m.isinf(depth_): elif m.isinf(depth_):
def rec_depth(children_, seen=set()): def rec_depth(results_, seen=set()):
names_ = { # build the children table at each layer
','.join(str(getattr(Result(*c), k) or '') results_ = fold(Result, results_, by=by)
for k in by) table_ = {
for c in children_} ','.join(str(getattr(r, k) or '') for k in by): r
for r in results_}
names_ = list(table_.keys())
return max( return max(
(rec_depth(table[name].children, seen | {name}) (rec_depth(table_[name].children, seen | {name})
for name in names_ for name in names_
if name not in seen), if name not in seen),
default=-1) + 1 default=-1) + 1
@@ -507,14 +509,15 @@ def table(Result, results, diff_results=None, *,
for i, x in enumerate(lines[0][1:], 1)))) for i, x in enumerate(lines[0][1:], 1))))
if not summary: if not summary:
line_table = {n: l for n, l in zip(names, lines[1:-1])}
if hot: if hot:
def recurse(children_, depth_, seen=set(), def recurse(results_, depth_, seen=set(),
prefixes=('', '', '', '')): prefixes=('', '', '', '')):
names_ = {','.join(str(getattr(Result(*c), k) or '') # build the children table at each layer
for k in by) results_ = fold(Result, results_, by=by)
for c in children_} table_ = {
','.join(str(getattr(r, k) or '') for k in by): r
for r in results_}
names_ = list(table_.keys())
if not names_: if not names_:
return return
@@ -523,8 +526,9 @@ def table(Result, results, diff_results=None, *,
name = max(names_, name = max(names_,
key=lambda n: tuple( key=lambda n: tuple(
tuple( tuple(
(getattr(table[n], k),) # make sure to use the rebuilt table
if getattr(table.get(n), k, None) is not None (getattr(table_[n], k),)
if getattr(table_.get(n), k, None) is not None
else () else ()
for k in ([k] if k else [ for k in ([k] if k else [
k for k in Result._sort if k in fields]) k for k in Result._sort if k in fields])
@@ -533,10 +537,12 @@ def table(Result, results, diff_results=None, *,
sort or [], sort or [],
[(None, False)]))) [(None, False)])))
if name in line_table: r = table_[name]
line = line_table[name] is_last = not r.children
is_last = not table[name].children
line = table_entry(name, r)
line = [x if isinstance(x, tuple) else (x, [])
for x in line]
print('%s%-*s %s' % ( print('%s%-*s %s' % (
prefixes[0+is_last], prefixes[0+is_last],
widths[0] - len(prefixes[0+is_last]), line[0][0], widths[0] - len(prefixes[0+is_last]), line[0][0],
@@ -557,26 +563,41 @@ def table(Result, results, diff_results=None, *,
# recurse? # recurse?
if depth_ > 1: if depth_ > 1:
recurse( recurse(
table[name].children, r.children,
depth_-1, depth_-1,
seen | {name}, seen | {name},
prefixes) prefixes)
else: else:
def recurse(children_, depth_, seen=set(), def recurse(results_, depth_, seen=set(),
prefixes=('', '', '', '')): prefixes=('', '', '', '')):
# note we're maintaining sort order # build the children table at each layer
names_ = {','.join(str(getattr(Result(*c), k) or '') results_ = fold(Result, results_, by=by)
for k in by) table_ = {
for c in children_} ','.join(str(getattr(r, k) or '') for k in by): r
names_ = [n for n in names if n in names_] for r in results_}
names_ = list(table_.keys())
# sort the children layer
names_.sort()
if sort:
for k, reverse in reversed(sort):
names_.sort(
key=lambda n: tuple(
(getattr(table_[n], k),)
if getattr(table_.get(n), k, None) is not None
else ()
for k in ([k] if k else [
k for k in Result._sort if k in fields])),
reverse=reverse ^ (not k or k in Result._fields))
for i, name in enumerate(names_): for i, name in enumerate(names_):
if name not in line_table: r = table_[name]
continue
line = line_table[name]
is_last = (i == len(names_)-1) is_last = (i == len(names_)-1)
line = table_entry(name, r)
line = [x if isinstance(x, tuple) else (x, [])
for x in line]
print('%s%-*s %s' % ( print('%s%-*s %s' % (
prefixes[0+is_last], prefixes[0+is_last],
widths[0] - len(prefixes[0+is_last]), line[0][0], widths[0] - len(prefixes[0+is_last]), line[0][0],
@@ -597,7 +618,7 @@ def table(Result, results, diff_results=None, *,
# recurse? # recurse?
if depth_ > 1: if depth_ > 1:
recurse( recurse(
table[name].children, r.children,
depth_-1, depth_-1,
seen | {name}, seen | {name},
(prefixes[2+is_last] + "|-> ", (prefixes[2+is_last] + "|-> ",
@@ -605,7 +626,7 @@ def table(Result, results, diff_results=None, *,
prefixes[2+is_last] + "| ", prefixes[2+is_last] + "| ",
prefixes[2+is_last] + " ")) prefixes[2+is_last] + " "))
# make the top layer a special case # the top layer is a bit of a special case
for name, line in zip(names, lines[1:-1]): for name, line in zip(names, lines[1:-1]):
print('%-*s %s' % ( print('%-*s %s' % (
widths[0], line[0][0], widths[0], line[0][0],