diff --git a/Makefile b/Makefile index 264e606a..c060bb63 100644 --- a/Makefile +++ b/Makefile @@ -531,8 +531,8 @@ test-list list-tests: test-runner test-marks: SUMMARYFLAGS+=-Si test-marks: $(TEST_CSV) $(strip ./scripts/csv.py $^ \ + -Fi='min(i)' \ -bsuite \ - -Fi='min(enumerate())' \ -fpassed=test_passed \ -fruntime=test_runtime \ $(SUMMARYFLAGS)) @@ -545,8 +545,8 @@ test-marks-csv: $(BUILDDIR)/lfs3.test.csv .PHONY: test-marks-diff test-marks-diff: $(TEST_CSV) $(strip ./scripts/csv.py $^ \ + -Fi='min(i)' \ -bsuite \ - -Fi='min(enumerate())' \ -fpassed=test_passed \ -fruntime=test_runtime \ $(SUMMARYFLAGS) -d $(BUILDDIR)/lfs3.test.csv) @@ -556,8 +556,8 @@ test-marks-diff: $(TEST_CSV) test-bottlenecks: SUMMARYFLAGS+=-Sruntime test-bottlenecks: $(TEST_CSV) $(strip ./scripts/csv.py $^ \ + -Fi='min(i)' \ -bcase \ - -Fi='min(enumerate())' \ -fpassed=test_passed \ -fruntime=test_runtime \ $(SUMMARYFLAGS)) @@ -595,8 +595,8 @@ bench-marks: $(BENCH_CSV) $(strip ./scripts/csv.py \ <(./scripts/csv.py $^ \ -Uprobe=stack,heap,usage \ + -Fi='min(i)' \ -bprobe='%(case)s+%(probe)s' \ - -Fi='min(enumerate())' \ -fn='max(n)' \ -ft='max(float(bench_simtime)/1.0e9)' \ -o-) \ @@ -617,15 +617,15 @@ bench-marks-diff: $(BENCH_CSV) $(strip ./scripts/csv.py \ <(./scripts/csv.py $^ \ -Uprobe=stack,heap,usage \ + -Fi='min(i)' \ -bprobe='%(case)s+%(probe)s' \ - -Fi='min(enumerate())' \ -fn='max(n)' \ -ft='max(float(bench_simtime)/1.0e9)' \ -o-) \ -d <(./scripts/csv.py $(BUILDDIR)/lfs3.bench.csv \ -Uprobe=stack,heap,usage \ + -Fi='min(i)' \ -bprobe='%(case)s+%(probe)s' \ - -Fi='min(enumerate())' \ -fn='max(n)' \ -ft='max(float(bench_simtime)/1.0e9)' \ -o-) \ @@ -641,8 +641,8 @@ bench-bottlenecks: $(BENCH_CSV) $(strip ./scripts/csv.py \ <(./scripts/csv.py $^ \ -Uprobe=stack,heap,usage \ + -Fi='min(i)' \ -bprobe='%(case)s+%(probe)s' \ - -Fi='min(enumerate())' \ -fn='max(n)' \ -ft='max(float(bench_simtime)/1.0e9)' \ -fruntime='max(bench_runtime)' \ @@ -660,9 +660,9 @@ bench-ops: SUMMARYFLAGS+=-Si bench-ops: $(BENCH_CSV) $(strip ./scripts/csv.py $^ \ -Uprobe=stack,heap,usage \ + -Fi='min(i)' \ -bprobe='%(case)s+%(probe)s' \ -Hprobe=bench+probe \ - -Fi='min(enumerate())' \ -freads='bench_reads' \ -fprogs='bench_progs' \ -ferases='bench_erases' \ @@ -677,9 +677,9 @@ bench-widths: SUMMARYFLAGS+=-Si bench-widths: $(BENCH_CSV) $(strip ./scripts/csv.py $^ \ -Uprobe=stack,heap,usage \ + -Fi='min(i)' \ -bprobe='%(case)s+%(probe)s' \ -Hprobe=bench+probe \ - -Fi='min(enumerate())' \ -freaded="avg(saturate(ffrac( \ float(bench_readed)/float(bench_reads), \ max(1, $$( \ @@ -707,20 +707,20 @@ bench-ram bench-usage: $(BENCH_CSV) $(strip ./scripts/csv.py \ <(./scripts/csv.py $^ \ -Dprobe=stack \ + -Fi='min(i)' \ -bcase \ - -Fi='min(enumerate())' \ -fstack='max(bench_simtime)' \ -o-) \ <(./scripts/csv.py $^ \ -Dprobe=heap \ + -Fi='min(i)' \ -bcase \ - -Fi='min(enumerate())' \ -fheap='max(bench_simtime)' \ -o-) \ <(./scripts/csv.py $^ \ -Dprobe=usage \ + -Fi='min(i)' \ -bcase \ - -Fi='min(enumerate())' \ -fdisk='max(bench_simtime)' \ -o-) \ -bcase \ diff --git a/scripts/bench.py b/scripts/bench.py index b9b89e5f..69b5a22c 100755 --- a/scripts/bench.py +++ b/scripts/bench.py @@ -1147,7 +1147,8 @@ class BenchFailure(Exception): self.assert_ = assert_ -def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args): +def run_stage(offset, name, runner, bench_ids, + stdout_, trace_, output_, **args): # get expected suite/case/perm counts (case_suites, expected_suite_perms, @@ -1301,6 +1302,9 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args): # write measurements immediately, this allows # analysis of partial results output_.writerow({ + 'i': offset + + locals.start + + locals.seen_perms*locals.step, 'suite': last_suite, 'case': last_case, **last_defines, @@ -1338,14 +1342,16 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args): nonlocal killed nonlocal locals - start = start or 0 - step = step or 1 - while start < total_perms: + locals.start = start or 0 + locals.step = step or 1 + while locals.start < total_perms: runner_ = find_runner(runner, main=main, **args) if args.get('isolate') or args.get('valgrind'): - runner_.append('--step=%s,%s,%s' % (start, start+step, step)) - elif start != 0 or step != 1: - runner_.append('--step=%s,,%s' % (start, step)) + runner_.append('--step=%s,%s,%s' % ( + locals.start, locals.start+locals.step, locals.step)) + elif locals.start != 0 or locals.step != 1: + runner_.append('--step=%s,,%s' % ( + locals.start, locals.step)) runner_.extend(bench_ids) @@ -1354,7 +1360,7 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args): locals.seen_perms = 0 run_runner(runner_) assert locals.seen_perms > 0 - start += locals.seen_perms*step + locals.start += locals.seen_perms*locals.step except BenchFailure as failure: # race condition for multiple failures? @@ -1371,7 +1377,7 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args): if args.get('keep_going') and not killed: # resume after failed bench assert locals.seen_perms > 0 - start += locals.seen_perms*step + locals.start += locals.seen_perms*locals.step continue else: # stop other benches @@ -1489,9 +1495,10 @@ def run(runner, bench_ids=[], **args): output = None if args.get('output'): output = BenchOutput(args['output'], - ['suite', 'case'], + ['i', 'suite', 'case'], # defines go here - ['probe', 'n', + ['probe', + 'n', 'bench_reads', 'bench_progs', 'bench_erases', @@ -1522,6 +1529,7 @@ def run(runner, bench_ids=[], **args): erased_, failures_, killed) = run_stage( + expected, by or 'benches', runner, [by] if by is not None else [], diff --git a/scripts/test.py b/scripts/test.py index e403d739..e2c0a7f8 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -1134,7 +1134,8 @@ class TestFailure(Exception): self.stdout = stdout self.assert_ = assert_ -def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args): +def run_stage(offset, name, runner, test_ids, + stdout_, trace_, output_, **args): # get expected suite/case/perm counts (case_suites, expected_suite_perms, @@ -1228,6 +1229,9 @@ def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args): defines = find_defines( runner, m.group('id'), **args) output_.writerow({ + 'i': offset + + locals.start + + locals.seen_perms*locals.step, 'suite': suite, 'case': case, **defines, @@ -1265,14 +1269,16 @@ def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args): nonlocal killed nonlocal locals - start = start or 0 - step = step or 1 - while start < total_perms: + locals.start = start or 0 + locals.step = step or 1 + while locals.start < total_perms: runner_ = find_runner(runner, main=main, **args) if args.get('isolate') or args.get('valgrind'): - runner_.append('--step=%s,%s,%s' % (start, start+step, step)) - elif start != 0 or step != 1: - runner_.append('--step=%s,,%s' % (start, step)) + runner_.append('--step=%s,%s,%s' % ( + locals.start, locals.start+locals.step, locals.step)) + elif locals.start != 0 or locals.step != 1: + runner_.append('--step=%s,,%s' % ( + locals.start, locals.step)) runner_.extend(test_ids) @@ -1281,7 +1287,7 @@ def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args): locals.seen_perms = 0 run_runner(runner_) assert locals.seen_perms > 0 - start += locals.seen_perms*step + locals.start += locals.seen_perms*locals.step except TestFailure as failure: # keep track of failures @@ -1310,7 +1316,7 @@ def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args): if args.get('keep_going') and not killed: # resume after failed test assert locals.seen_perms > 0 - start += locals.seen_perms*step + locals.start += locals.seen_perms*locals.step continue else: # stop other tests @@ -1428,7 +1434,7 @@ def run(runner, test_ids=[], **args): output = None if args.get('output'): output = TestOutput(args['output'], - ['suite', 'case'], + ['i', 'suite', 'case'], # defines go here ['test_passed', 'test_runtime']) @@ -1449,6 +1455,7 @@ def run(runner, test_ids=[], **args): powerlosses_, failures_, killed) = run_stage( + expected, by or 'tests', runner, [by] if by is not None else [],