Github/littlefs
1
Fork 0
Code Actions Activity

scripts: test.py/bench.py: Added i field to test/bench marks

Browse Source 
For consistent ordering in later scripts. The previous
-F=min(enumerate()) trick mostly worked, but would get messed up by
running things in parallel (-j).

I've already confused myself a couple times looking at script output,
which is never a good sign.
This commit is contained in:
Christopher Haster
2026-02-10 15:06:54 -06:00
parent 1a98d8089b
commit 34026948ef
3 changed files with 48 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+12 -12
View File
@@ -531,8 +531,8 @@ test-list list-tests: test-runner
test-marks: SUMMARYFLAGS+=-Si test-marks: SUMMARYFLAGS+=-Si
test-marks: $(TEST_CSV) test-marks: $(TEST_CSV)
$(strip ./scripts/csv.py $^ \ $(strip ./scripts/csv.py $^ \
-Fi='min(i)' \
-bsuite \ -bsuite \
-Fi='min(enumerate())' \
-fpassed=test_passed \ -fpassed=test_passed \
-fruntime=test_runtime \ -fruntime=test_runtime \
$(SUMMARYFLAGS)) $(SUMMARYFLAGS))
@@ -545,8 +545,8 @@ test-marks-csv: $(BUILDDIR)/lfs3.test.csv
.PHONY: test-marks-diff .PHONY: test-marks-diff
test-marks-diff: $(TEST_CSV) test-marks-diff: $(TEST_CSV)
$(strip ./scripts/csv.py $^ \ $(strip ./scripts/csv.py $^ \
-Fi='min(i)' \
-bsuite \ -bsuite \
-Fi='min(enumerate())' \
-fpassed=test_passed \ -fpassed=test_passed \
-fruntime=test_runtime \ -fruntime=test_runtime \
$(SUMMARYFLAGS) -d $(BUILDDIR)/lfs3.test.csv) $(SUMMARYFLAGS) -d $(BUILDDIR)/lfs3.test.csv)
@@ -556,8 +556,8 @@ test-marks-diff: $(TEST_CSV)
test-bottlenecks: SUMMARYFLAGS+=-Sruntime test-bottlenecks: SUMMARYFLAGS+=-Sruntime
test-bottlenecks: $(TEST_CSV) test-bottlenecks: $(TEST_CSV)
$(strip ./scripts/csv.py $^ \ $(strip ./scripts/csv.py $^ \
-Fi='min(i)' \
-bcase \ -bcase \
-Fi='min(enumerate())' \
-fpassed=test_passed \ -fpassed=test_passed \
-fruntime=test_runtime \ -fruntime=test_runtime \
$(SUMMARYFLAGS)) $(SUMMARYFLAGS))
@@ -595,8 +595,8 @@ bench-marks: $(BENCH_CSV)
$(strip ./scripts/csv.py \ $(strip ./scripts/csv.py \
<(./scripts/csv.py $^ \ <(./scripts/csv.py $^ \
-Uprobe=stack,heap,usage \ -Uprobe=stack,heap,usage \
-Fi='min(i)' \
-bprobe='%(case)s+%(probe)s' \ -bprobe='%(case)s+%(probe)s' \
-Fi='min(enumerate())' \
-fn='max(n)' \ -fn='max(n)' \
-ft='max(float(bench_simtime)/1.0e9)' \ -ft='max(float(bench_simtime)/1.0e9)' \
-o-) \ -o-) \
@@ -617,15 +617,15 @@ bench-marks-diff: $(BENCH_CSV)
$(strip ./scripts/csv.py \ $(strip ./scripts/csv.py \
<(./scripts/csv.py $^ \ <(./scripts/csv.py $^ \
-Uprobe=stack,heap,usage \ -Uprobe=stack,heap,usage \
-Fi='min(i)' \
-bprobe='%(case)s+%(probe)s' \ -bprobe='%(case)s+%(probe)s' \
-Fi='min(enumerate())' \
-fn='max(n)' \ -fn='max(n)' \
-ft='max(float(bench_simtime)/1.0e9)' \ -ft='max(float(bench_simtime)/1.0e9)' \
-o-) \ -o-) \
-d <(./scripts/csv.py $(BUILDDIR)/lfs3.bench.csv \ -d <(./scripts/csv.py $(BUILDDIR)/lfs3.bench.csv \
-Uprobe=stack,heap,usage \ -Uprobe=stack,heap,usage \
-Fi='min(i)' \
-bprobe='%(case)s+%(probe)s' \ -bprobe='%(case)s+%(probe)s' \
-Fi='min(enumerate())' \
-fn='max(n)' \ -fn='max(n)' \
-ft='max(float(bench_simtime)/1.0e9)' \ -ft='max(float(bench_simtime)/1.0e9)' \
-o-) \ -o-) \
@@ -641,8 +641,8 @@ bench-bottlenecks: $(BENCH_CSV)
$(strip ./scripts/csv.py \ $(strip ./scripts/csv.py \
<(./scripts/csv.py $^ \ <(./scripts/csv.py $^ \
-Uprobe=stack,heap,usage \ -Uprobe=stack,heap,usage \
-Fi='min(i)' \
-bprobe='%(case)s+%(probe)s' \ -bprobe='%(case)s+%(probe)s' \
-Fi='min(enumerate())' \
-fn='max(n)' \ -fn='max(n)' \
-ft='max(float(bench_simtime)/1.0e9)' \ -ft='max(float(bench_simtime)/1.0e9)' \
-fruntime='max(bench_runtime)' \ -fruntime='max(bench_runtime)' \
@@ -660,9 +660,9 @@ bench-ops: SUMMARYFLAGS+=-Si
bench-ops: $(BENCH_CSV) bench-ops: $(BENCH_CSV)
$(strip ./scripts/csv.py $^ \ $(strip ./scripts/csv.py $^ \
-Uprobe=stack,heap,usage \ -Uprobe=stack,heap,usage \
-Fi='min(i)' \
-bprobe='%(case)s+%(probe)s' \ -bprobe='%(case)s+%(probe)s' \
-Hprobe=bench+probe \ -Hprobe=bench+probe \
-Fi='min(enumerate())' \
-freads='bench_reads' \ -freads='bench_reads' \
-fprogs='bench_progs' \ -fprogs='bench_progs' \
-ferases='bench_erases' \ -ferases='bench_erases' \
@@ -677,9 +677,9 @@ bench-widths: SUMMARYFLAGS+=-Si
bench-widths: $(BENCH_CSV) bench-widths: $(BENCH_CSV)
$(strip ./scripts/csv.py $^ \ $(strip ./scripts/csv.py $^ \
-Uprobe=stack,heap,usage \ -Uprobe=stack,heap,usage \
-Fi='min(i)' \
-bprobe='%(case)s+%(probe)s' \ -bprobe='%(case)s+%(probe)s' \
-Hprobe=bench+probe \ -Hprobe=bench+probe \
-Fi='min(enumerate())' \
-freaded="avg(saturate(ffrac( \ -freaded="avg(saturate(ffrac( \
float(bench_readed)/float(bench_reads), \ float(bench_readed)/float(bench_reads), \
max(1, $$( \ max(1, $$( \
@@ -707,20 +707,20 @@ bench-ram bench-usage: $(BENCH_CSV)
$(strip ./scripts/csv.py \ $(strip ./scripts/csv.py \
<(./scripts/csv.py $^ \ <(./scripts/csv.py $^ \
-Dprobe=stack \ -Dprobe=stack \
-Fi='min(i)' \
-bcase \ -bcase \
-Fi='min(enumerate())' \
-fstack='max(bench_simtime)' \ -fstack='max(bench_simtime)' \
-o-) \ -o-) \
<(./scripts/csv.py $^ \ <(./scripts/csv.py $^ \
-Dprobe=heap \ -Dprobe=heap \
-Fi='min(i)' \
-bcase \ -bcase \
-Fi='min(enumerate())' \
-fheap='max(bench_simtime)' \ -fheap='max(bench_simtime)' \
-o-) \ -o-) \
<(./scripts/csv.py $^ \ <(./scripts/csv.py $^ \
-Dprobe=usage \ -Dprobe=usage \
-Fi='min(i)' \
-bcase \ -bcase \
-Fi='min(enumerate())' \
-fdisk='max(bench_simtime)' \ -fdisk='max(bench_simtime)' \
-o-) \ -o-) \
-bcase \ -bcase \
scripts/bench.py
+19 -11
View File
@@ -1147,7 +1147,8 @@ class BenchFailure(Exception):
self.assert_ = assert_ self.assert_ = assert_
def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args): def run_stage(offset, name, runner, bench_ids,
stdout_, trace_, output_, **args):
# get expected suite/case/perm counts # get expected suite/case/perm counts
(case_suites, (case_suites,
expected_suite_perms, expected_suite_perms,
@@ -1301,6 +1302,9 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
# write measurements immediately, this allows # write measurements immediately, this allows
# analysis of partial results # analysis of partial results
output_.writerow({ output_.writerow({
'i': offset
+ locals.start
+ locals.seen_perms*locals.step,
'suite': last_suite, 'suite': last_suite,
'case': last_case, 'case': last_case,
**last_defines, **last_defines,
@@ -1338,14 +1342,16 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
nonlocal killed nonlocal killed
nonlocal locals nonlocal locals
start = start or 0 locals.start = start or 0
step = step or 1 locals.step = step or 1
while start < total_perms: while locals.start < total_perms:
runner_ = find_runner(runner, main=main, **args) runner_ = find_runner(runner, main=main, **args)
if args.get('isolate') or args.get('valgrind'): if args.get('isolate') or args.get('valgrind'):
runner_.append('--step=%s,%s,%s' % (start, start+step, step)) runner_.append('--step=%s,%s,%s' % (
elif start != 0 or step != 1: locals.start, locals.start+locals.step, locals.step))
runner_.append('--step=%s,,%s' % (start, step)) elif locals.start != 0 or locals.step != 1:
runner_.append('--step=%s,,%s' % (
locals.start, locals.step))
runner_.extend(bench_ids) runner_.extend(bench_ids)
@@ -1354,7 +1360,7 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
locals.seen_perms = 0 locals.seen_perms = 0
run_runner(runner_) run_runner(runner_)
assert locals.seen_perms > 0 assert locals.seen_perms > 0
start += locals.seen_perms*step locals.start += locals.seen_perms*locals.step
except BenchFailure as failure: except BenchFailure as failure:
# race condition for multiple failures? # race condition for multiple failures?
@@ -1371,7 +1377,7 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
if args.get('keep_going') and not killed: if args.get('keep_going') and not killed:
# resume after failed bench # resume after failed bench
assert locals.seen_perms > 0 assert locals.seen_perms > 0
start += locals.seen_perms*step locals.start += locals.seen_perms*locals.step
continue continue
else: else:
# stop other benches # stop other benches
@@ -1489,9 +1495,10 @@ def run(runner, bench_ids=[], **args):
output = None output = None
if args.get('output'): if args.get('output'):
output = BenchOutput(args['output'], output = BenchOutput(args['output'],
['suite', 'case'], ['i', 'suite', 'case'],
# defines go here # defines go here
['probe', 'n', ['probe',
'n',
'bench_reads', 'bench_reads',
'bench_progs', 'bench_progs',
'bench_erases', 'bench_erases',
@@ -1522,6 +1529,7 @@ def run(runner, bench_ids=[], **args):
erased_, erased_,
failures_, failures_,
killed) = run_stage( killed) = run_stage(
expected,
by or 'benches', by or 'benches',
runner, runner,
[by] if by is not None else [], [by] if by is not None else [],
scripts/test.py
+17 -10
View File
@@ -1134,7 +1134,8 @@ class TestFailure(Exception):
self.stdout = stdout self.stdout = stdout
self.assert_ = assert_ self.assert_ = assert_
def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args): def run_stage(offset, name, runner, test_ids,
stdout_, trace_, output_, **args):
# get expected suite/case/perm counts # get expected suite/case/perm counts
(case_suites, (case_suites,
expected_suite_perms, expected_suite_perms,
@@ -1228,6 +1229,9 @@ def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args):
defines = find_defines( defines = find_defines(
runner, m.group('id'), **args) runner, m.group('id'), **args)
output_.writerow({ output_.writerow({
'i': offset
+ locals.start
+ locals.seen_perms*locals.step,
'suite': suite, 'suite': suite,
'case': case, 'case': case,
**defines, **defines,
@@ -1265,14 +1269,16 @@ def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args):
nonlocal killed nonlocal killed
nonlocal locals nonlocal locals
start = start or 0 locals.start = start or 0
step = step or 1 locals.step = step or 1
while start < total_perms: while locals.start < total_perms:
runner_ = find_runner(runner, main=main, **args) runner_ = find_runner(runner, main=main, **args)
if args.get('isolate') or args.get('valgrind'): if args.get('isolate') or args.get('valgrind'):
runner_.append('--step=%s,%s,%s' % (start, start+step, step)) runner_.append('--step=%s,%s,%s' % (
elif start != 0 or step != 1: locals.start, locals.start+locals.step, locals.step))
runner_.append('--step=%s,,%s' % (start, step)) elif locals.start != 0 or locals.step != 1:
runner_.append('--step=%s,,%s' % (
locals.start, locals.step))
runner_.extend(test_ids) runner_.extend(test_ids)
@@ -1281,7 +1287,7 @@ def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args):
locals.seen_perms = 0 locals.seen_perms = 0
run_runner(runner_) run_runner(runner_)
assert locals.seen_perms > 0 assert locals.seen_perms > 0
start += locals.seen_perms*step locals.start += locals.seen_perms*locals.step
except TestFailure as failure: except TestFailure as failure:
# keep track of failures # keep track of failures
@@ -1310,7 +1316,7 @@ def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args):
if args.get('keep_going') and not killed: if args.get('keep_going') and not killed:
# resume after failed test # resume after failed test
assert locals.seen_perms > 0 assert locals.seen_perms > 0
start += locals.seen_perms*step locals.start += locals.seen_perms*locals.step
continue continue
else: else:
# stop other tests # stop other tests
@@ -1428,7 +1434,7 @@ def run(runner, test_ids=[], **args):
output = None output = None
if args.get('output'): if args.get('output'):
output = TestOutput(args['output'], output = TestOutput(args['output'],
['suite', 'case'], ['i', 'suite', 'case'],
# defines go here # defines go here
['test_passed', 'test_runtime']) ['test_passed', 'test_runtime'])
@@ -1449,6 +1455,7 @@ def run(runner, test_ids=[], **args):
powerlosses_, powerlosses_,
failures_, failures_,
killed) = run_stage( killed) = run_stage(
expected,
by or 'tests', by or 'tests',
runner, runner,
[by] if by is not None else [], [by] if by is not None else [],