Github/littlefs
1
Fork 0
Code Actions Activity

scripts: bench.py: Fixed issue where cumul results were mixed together

Browse Source 
Whoops, looks like cumulative results were overlooked when multiple
bench measurements per bench were added. We were just adding all
cumulative results together!

This led to some very confusing bench results.

The solution here is to keep track of per-measurement cumulative results
via a Python dict. Which adds some memory usage, but definitely not
enough to be noticeable in the context of the bench-runner.
This commit is contained in:
Christopher Haster
2025-05-15 16:16:41 -05:00
parent 48daeed509
commit 275ca0e0ec
1 changed files with 21 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bench.py
+21 -21
View File
@@ -1042,9 +1042,9 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
last_defines = None # fetched on demand last_defines = None # fetched on demand
last_stdout = co.deque(maxlen=args.get('context', 5) + 1) last_stdout = co.deque(maxlen=args.get('context', 5) + 1)
last_assert = None last_assert = None
readed_ = None creaded = co.defaultdict(lambda: 0)
proged_ = None cproged = co.defaultdict(lambda: 0)
erased_ = None cerased = co.defaultdict(lambda: 0)
try: try:
while True: while True:
# parse a line for state changes # parse a line for state changes
@@ -1075,9 +1075,9 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
last_defines = None last_defines = None
last_stdout.clear() last_stdout.clear()
last_assert = None last_assert = None
readed_ = 0 creaded.clear()
proged_ = 0 cproged.clear()
erased_ = 0 cerased.clear()
elif op == 'finished': elif op == 'finished':
# force a failure # force a failure
if args.get('fail'): if args.get('fail'):
@@ -1110,13 +1110,13 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
return float(v) return float(v)
else: else:
return int(v) return int(v)
readed__ = dat(m.group('readed')) readed_ = dat(m.group('readed'))
proged__ = dat(m.group('proged')) proged_ = dat(m.group('proged'))
erased__ = dat(m.group('erased')) erased_ = dat(m.group('erased'))
# keep track of cumulative measurements # keep track of cumulative measurements
readed_ += readed__ creaded[m_] += readed_
proged_ += proged__ cproged[m_] += proged_
erased_ += erased__ cerased[m_] += erased_
if output_: if output_:
# fetch defines if needed, only do this at most # fetch defines if needed, only do this at most
# once per perm # once per perm
@@ -1131,16 +1131,16 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
**last_defines, **last_defines,
'm': m_, 'm': m_,
'n': n_, 'n': n_,
'bench_readed': readed__, 'bench_readed': readed_,
'bench_proged': proged__, 'bench_proged': proged_,
'bench_erased': erased__, 'bench_erased': erased_,
'bench_creaded': readed_, 'bench_creaded': creaded[m_],
'bench_cproged': proged_, 'bench_cproged': cproged[m_],
'bench_cerased': erased_}) 'bench_cerased': cerased[m_]})
# keep track of total for summary # keep track of total for summary
readed += readed__ readed += readed_
proged += proged__ proged += proged_
erased += erased__ erased += erased_
except KeyboardInterrupt: except KeyboardInterrupt:
proc.kill() proc.kill()
raise BenchFailure(last_id, 0, list(last_stdout)) raise BenchFailure(last_id, 0, list(last_stdout))