scripts: test.py/bench.py: Added i field to test/bench marks
For consistent ordering in later scripts. The previous -F=min(enumerate()) trick mostly worked, but would get messed up by running things in parallel (-j). I've already confused myself a couple times looking at script output, which is never a good sign.
This commit is contained in:
Christopher Haster
@@ -531,8 +531,8 @@ test-list list-tests: test-runner
|
||||
test-marks: SUMMARYFLAGS+=-Si
|
||||
test-marks: $(TEST_CSV)
|
||||
$(strip ./scripts/csv.py $^ \
|
||||
-Fi='min(i)' \
|
||||
-bsuite \
|
||||
-Fi='min(enumerate())' \
|
||||
-fpassed=test_passed \
|
||||
-fruntime=test_runtime \
|
||||
$(SUMMARYFLAGS))
|
||||
@@ -545,8 +545,8 @@ test-marks-csv: $(BUILDDIR)/lfs3.test.csv
|
||||
.PHONY: test-marks-diff
|
||||
test-marks-diff: $(TEST_CSV)
|
||||
$(strip ./scripts/csv.py $^ \
|
||||
-Fi='min(i)' \
|
||||
-bsuite \
|
||||
-Fi='min(enumerate())' \
|
||||
-fpassed=test_passed \
|
||||
-fruntime=test_runtime \
|
||||
$(SUMMARYFLAGS) -d $(BUILDDIR)/lfs3.test.csv)
|
||||
@@ -556,8 +556,8 @@ test-marks-diff: $(TEST_CSV)
|
||||
test-bottlenecks: SUMMARYFLAGS+=-Sruntime
|
||||
test-bottlenecks: $(TEST_CSV)
|
||||
$(strip ./scripts/csv.py $^ \
|
||||
-Fi='min(i)' \
|
||||
-bcase \
|
||||
-Fi='min(enumerate())' \
|
||||
-fpassed=test_passed \
|
||||
-fruntime=test_runtime \
|
||||
$(SUMMARYFLAGS))
|
||||
@@ -595,8 +595,8 @@ bench-marks: $(BENCH_CSV)
|
||||
$(strip ./scripts/csv.py \
|
||||
<(./scripts/csv.py $^ \
|
||||
-Uprobe=stack,heap,usage \
|
||||
-Fi='min(i)' \
|
||||
-bprobe='%(case)s+%(probe)s' \
|
||||
-Fi='min(enumerate())' \
|
||||
-fn='max(n)' \
|
||||
-ft='max(float(bench_simtime)/1.0e9)' \
|
||||
-o-) \
|
||||
@@ -617,15 +617,15 @@ bench-marks-diff: $(BENCH_CSV)
|
||||
$(strip ./scripts/csv.py \
|
||||
<(./scripts/csv.py $^ \
|
||||
-Uprobe=stack,heap,usage \
|
||||
-Fi='min(i)' \
|
||||
-bprobe='%(case)s+%(probe)s' \
|
||||
-Fi='min(enumerate())' \
|
||||
-fn='max(n)' \
|
||||
-ft='max(float(bench_simtime)/1.0e9)' \
|
||||
-o-) \
|
||||
-d <(./scripts/csv.py $(BUILDDIR)/lfs3.bench.csv \
|
||||
-Uprobe=stack,heap,usage \
|
||||
-Fi='min(i)' \
|
||||
-bprobe='%(case)s+%(probe)s' \
|
||||
-Fi='min(enumerate())' \
|
||||
-fn='max(n)' \
|
||||
-ft='max(float(bench_simtime)/1.0e9)' \
|
||||
-o-) \
|
||||
@@ -641,8 +641,8 @@ bench-bottlenecks: $(BENCH_CSV)
|
||||
$(strip ./scripts/csv.py \
|
||||
<(./scripts/csv.py $^ \
|
||||
-Uprobe=stack,heap,usage \
|
||||
-Fi='min(i)' \
|
||||
-bprobe='%(case)s+%(probe)s' \
|
||||
-Fi='min(enumerate())' \
|
||||
-fn='max(n)' \
|
||||
-ft='max(float(bench_simtime)/1.0e9)' \
|
||||
-fruntime='max(bench_runtime)' \
|
||||
@@ -660,9 +660,9 @@ bench-ops: SUMMARYFLAGS+=-Si
|
||||
bench-ops: $(BENCH_CSV)
|
||||
$(strip ./scripts/csv.py $^ \
|
||||
-Uprobe=stack,heap,usage \
|
||||
-Fi='min(i)' \
|
||||
-bprobe='%(case)s+%(probe)s' \
|
||||
-Hprobe=bench+probe \
|
||||
-Fi='min(enumerate())' \
|
||||
-freads='bench_reads' \
|
||||
-fprogs='bench_progs' \
|
||||
-ferases='bench_erases' \
|
||||
@@ -677,9 +677,9 @@ bench-widths: SUMMARYFLAGS+=-Si
|
||||
bench-widths: $(BENCH_CSV)
|
||||
$(strip ./scripts/csv.py $^ \
|
||||
-Uprobe=stack,heap,usage \
|
||||
-Fi='min(i)' \
|
||||
-bprobe='%(case)s+%(probe)s' \
|
||||
-Hprobe=bench+probe \
|
||||
-Fi='min(enumerate())' \
|
||||
-freaded="avg(saturate(ffrac( \
|
||||
float(bench_readed)/float(bench_reads), \
|
||||
max(1, $$( \
|
||||
@@ -707,20 +707,20 @@ bench-ram bench-usage: $(BENCH_CSV)
|
||||
$(strip ./scripts/csv.py \
|
||||
<(./scripts/csv.py $^ \
|
||||
-Dprobe=stack \
|
||||
-Fi='min(i)' \
|
||||
-bcase \
|
||||
-Fi='min(enumerate())' \
|
||||
-fstack='max(bench_simtime)' \
|
||||
-o-) \
|
||||
<(./scripts/csv.py $^ \
|
||||
-Dprobe=heap \
|
||||
-Fi='min(i)' \
|
||||
-bcase \
|
||||
-Fi='min(enumerate())' \
|
||||
-fheap='max(bench_simtime)' \
|
||||
-o-) \
|
||||
<(./scripts/csv.py $^ \
|
||||
-Dprobe=usage \
|
||||
-Fi='min(i)' \
|
||||
-bcase \
|
||||
-Fi='min(enumerate())' \
|
||||
-fdisk='max(bench_simtime)' \
|
||||
-o-) \
|
||||
-bcase \
|
||||
|
||||
+19
-11
@@ -1147,7 +1147,8 @@ class BenchFailure(Exception):
|
||||
self.assert_ = assert_
|
||||
|
||||
|
||||
def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
|
||||
def run_stage(offset, name, runner, bench_ids,
|
||||
stdout_, trace_, output_, **args):
|
||||
# get expected suite/case/perm counts
|
||||
(case_suites,
|
||||
expected_suite_perms,
|
||||
@@ -1301,6 +1302,9 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
|
||||
# write measurements immediately, this allows
|
||||
# analysis of partial results
|
||||
output_.writerow({
|
||||
'i': offset
|
||||
+ locals.start
|
||||
+ locals.seen_perms*locals.step,
|
||||
'suite': last_suite,
|
||||
'case': last_case,
|
||||
**last_defines,
|
||||
@@ -1338,14 +1342,16 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
|
||||
nonlocal killed
|
||||
nonlocal locals
|
||||
|
||||
start = start or 0
|
||||
step = step or 1
|
||||
while start < total_perms:
|
||||
locals.start = start or 0
|
||||
locals.step = step or 1
|
||||
while locals.start < total_perms:
|
||||
runner_ = find_runner(runner, main=main, **args)
|
||||
if args.get('isolate') or args.get('valgrind'):
|
||||
runner_.append('--step=%s,%s,%s' % (start, start+step, step))
|
||||
elif start != 0 or step != 1:
|
||||
runner_.append('--step=%s,,%s' % (start, step))
|
||||
runner_.append('--step=%s,%s,%s' % (
|
||||
locals.start, locals.start+locals.step, locals.step))
|
||||
elif locals.start != 0 or locals.step != 1:
|
||||
runner_.append('--step=%s,,%s' % (
|
||||
locals.start, locals.step))
|
||||
|
||||
runner_.extend(bench_ids)
|
||||
|
||||
@@ -1354,7 +1360,7 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
|
||||
locals.seen_perms = 0
|
||||
run_runner(runner_)
|
||||
assert locals.seen_perms > 0
|
||||
start += locals.seen_perms*step
|
||||
locals.start += locals.seen_perms*locals.step
|
||||
|
||||
except BenchFailure as failure:
|
||||
# race condition for multiple failures?
|
||||
@@ -1371,7 +1377,7 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
|
||||
if args.get('keep_going') and not killed:
|
||||
# resume after failed bench
|
||||
assert locals.seen_perms > 0
|
||||
start += locals.seen_perms*step
|
||||
locals.start += locals.seen_perms*locals.step
|
||||
continue
|
||||
else:
|
||||
# stop other benches
|
||||
@@ -1489,9 +1495,10 @@ def run(runner, bench_ids=[], **args):
|
||||
output = None
|
||||
if args.get('output'):
|
||||
output = BenchOutput(args['output'],
|
||||
['suite', 'case'],
|
||||
['i', 'suite', 'case'],
|
||||
# defines go here
|
||||
['probe', 'n',
|
||||
['probe',
|
||||
'n',
|
||||
'bench_reads',
|
||||
'bench_progs',
|
||||
'bench_erases',
|
||||
@@ -1522,6 +1529,7 @@ def run(runner, bench_ids=[], **args):
|
||||
erased_,
|
||||
failures_,
|
||||
killed) = run_stage(
|
||||
expected,
|
||||
by or 'benches',
|
||||
runner,
|
||||
[by] if by is not None else [],
|
||||
|
||||
+17
-10
@@ -1134,7 +1134,8 @@ class TestFailure(Exception):
|
||||
self.stdout = stdout
|
||||
self.assert_ = assert_
|
||||
|
||||
def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args):
|
||||
def run_stage(offset, name, runner, test_ids,
|
||||
stdout_, trace_, output_, **args):
|
||||
# get expected suite/case/perm counts
|
||||
(case_suites,
|
||||
expected_suite_perms,
|
||||
@@ -1228,6 +1229,9 @@ def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args):
|
||||
defines = find_defines(
|
||||
runner, m.group('id'), **args)
|
||||
output_.writerow({
|
||||
'i': offset
|
||||
+ locals.start
|
||||
+ locals.seen_perms*locals.step,
|
||||
'suite': suite,
|
||||
'case': case,
|
||||
**defines,
|
||||
@@ -1265,14 +1269,16 @@ def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args):
|
||||
nonlocal killed
|
||||
nonlocal locals
|
||||
|
||||
start = start or 0
|
||||
step = step or 1
|
||||
while start < total_perms:
|
||||
locals.start = start or 0
|
||||
locals.step = step or 1
|
||||
while locals.start < total_perms:
|
||||
runner_ = find_runner(runner, main=main, **args)
|
||||
if args.get('isolate') or args.get('valgrind'):
|
||||
runner_.append('--step=%s,%s,%s' % (start, start+step, step))
|
||||
elif start != 0 or step != 1:
|
||||
runner_.append('--step=%s,,%s' % (start, step))
|
||||
runner_.append('--step=%s,%s,%s' % (
|
||||
locals.start, locals.start+locals.step, locals.step))
|
||||
elif locals.start != 0 or locals.step != 1:
|
||||
runner_.append('--step=%s,,%s' % (
|
||||
locals.start, locals.step))
|
||||
|
||||
runner_.extend(test_ids)
|
||||
|
||||
@@ -1281,7 +1287,7 @@ def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args):
|
||||
locals.seen_perms = 0
|
||||
run_runner(runner_)
|
||||
assert locals.seen_perms > 0
|
||||
start += locals.seen_perms*step
|
||||
locals.start += locals.seen_perms*locals.step
|
||||
|
||||
except TestFailure as failure:
|
||||
# keep track of failures
|
||||
@@ -1310,7 +1316,7 @@ def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args):
|
||||
if args.get('keep_going') and not killed:
|
||||
# resume after failed test
|
||||
assert locals.seen_perms > 0
|
||||
start += locals.seen_perms*step
|
||||
locals.start += locals.seen_perms*locals.step
|
||||
continue
|
||||
else:
|
||||
# stop other tests
|
||||
@@ -1428,7 +1434,7 @@ def run(runner, test_ids=[], **args):
|
||||
output = None
|
||||
if args.get('output'):
|
||||
output = TestOutput(args['output'],
|
||||
['suite', 'case'],
|
||||
['i', 'suite', 'case'],
|
||||
# defines go here
|
||||
['test_passed', 'test_runtime'])
|
||||
|
||||
@@ -1449,6 +1455,7 @@ def run(runner, test_ids=[], **args):
|
||||
powerlosses_,
|
||||
failures_,
|
||||
killed) = run_stage(
|
||||
expected,
|
||||
by or 'tests',
|
||||
runner,
|
||||
[by] if by is not None else [],
|
||||
|
||||
Reference in New Issue
Block a user