make: Fixed throughput calculation for litmus benchmarks

So, avg seems like a poor way to merge throughput results, at least for
bench_rbyd (avg create/delete throughputs were wildly different).

To fix this, changed make benchmarks and friends to a two step
calculation:

1. find max results: -fn='max(n)' -ft='float(bench_simtime)/1.0e9'
2. calculate throughput: -fthroughput='avg(float(n) / max(t, 1.0e-9))'

As a plus, this is probably more robust toward accidentally introducing
new by fields.

Note this does require two csv.py calls. csv.py doesn't support exprs
after folding as an intentional simplification (in theory it's always
possible to chain multiple csv.py calls together). In make this can be a
bit annoying since we need SHELL=/bin/bash for subprocess substitution
(for benchmarks-diff specifically), but that's not the end of the world.

--

Also changed -Si='min(enumerate())' -> -Si -Fi='min(enumerate())', in
case users override SUMMARYFLAGS.
This commit is contained in:
Christopher Haster
2026-02-02 14:34:24 -06:00
parent 6c16cbd44f
commit b97c3b67c9
+47 -22
View File
@@ -325,7 +325,6 @@ funcs: $(OBJ) $(CI)
## Save function sizes
.PHONY: funcs-csv
funcs-csv: SHELL=/bin/bash
funcs-csv: \
$(BUILDDIR)/lfs3.code.csv \
$(BUILDDIR)/lfs3.stack.csv \
@@ -443,7 +442,6 @@ summary sizes: $(OBJ) $(CI)
## Save compile-time sizes
.PHONY: summary-csv sizes-csv
summary-csv sizes-csv: SHELL=/bin/bash
summary-csv sizes-csv: \
$(BUILDDIR)/lfs3.code.csv \
$(BUILDDIR)/lfs3.data.csv \
@@ -519,10 +517,11 @@ test-list list-tests: test-runner
## Summarize the test results
.PHONY: testmarks
testmarks: SUMMARYFLAGS+=-Si='min(enumerate())'
testmarks: SUMMARYFLAGS+=-Si
testmarks: $(TEST_CSV)
$(strip ./scripts/csv.py $^ \
-bsuite \
-Fi='min(enumerate())' \
-fpassed=test_passed \
-fruntime=test_runtime \
$(SUMMARYFLAGS))
@@ -536,6 +535,7 @@ testmarks-csv: $(BUILDDIR)/lfs3.test.csv
testmarks-diff: $(TEST_CSV)
$(strip ./scripts/csv.py $^ \
-bsuite \
-Fi='min(enumerate())' \
-fpassed=test_passed \
-fruntime=test_runtime \
$(SUMMARYFLAGS) -d $(BUILDDIR)/lfs3.test.csv)
@@ -546,6 +546,7 @@ testmarks-bottlenecks: SUMMARYFLAGS+=-Sruntime
testmarks-bottlenecks: $(TEST_CSV)
$(strip ./scripts/csv.py $^ \
-bcase \
-Fi='min(enumerate())' \
-fpassed=test_passed \
-fruntime=test_runtime \
$(SUMMARYFLAGS))
@@ -578,15 +579,20 @@ bench-list list-benches: bench-runner
## Summarize the bench results
.PHONY: benchmarks
benchmarks: SUMMARYFLAGS+=-Si='min(enumerate())'
benchmarks: SUMMARYFLAGS+=-Si
benchmarks: SHELL=/bin/bash
benchmarks: $(BENCH_CSV)
$(strip ./scripts/csv.py $^ \
-bprobe='%(case)s+%(probe)s' \
-fn='delta(n, case, probe)' \
-ft='float(bench_simtime)/1.0e9' \
-fthroughput='avg( \
float(delta(n, case, probe)) \
/ max(float(bench_simtime)/1.0e9, 1.0e-9))' \
$(strip ./scripts/csv.py \
<(./scripts/csv.py $^ \
-bprobe='%(case)s+%(probe)s' \
-Fi='min(enumerate())' \
-fn='max(n)' \
-ft='float(bench_simtime)/1.0e9' \
-o-) \
-bprobe \
-fn \
-ft \
-fthroughput='avg(float(n) / max(t, 1.0e-9))' \
$(SUMMARYFLAGS))
## Save the bench results
@@ -595,23 +601,42 @@ benchmarks-csv: $(BUILDDIR)/lfs3.bench.csv
## Compare bench results against a previous run
.PHONY: benchmarks-diff
benchmarks-diff: SHELL=/bin/bash
benchmarks-diff: $(BENCH_CSV)
$(strip ./scripts/csv.py $^ \
-bprobe='%(case)s+%(probe)s' \
-fthroughput='avg( \
float(delta(n, case, probe)) \
/ max(float(bench_simtime)/1.0e9, 1.0e-9))' \
$(SUMMARYFLAGS) -d $(BUILDDIR)/lfs3.bench.csv)
$(strip ./scripts/csv.py \
<(./scripts/csv.py $^ \
-bprobe='%(case)s+%(probe)s' \
-Fi='min(enumerate())' \
-fn='max(n)' \
-ft='float(bench_simtime)/1.0e9' \
-o-) \
-d <(./scripts/csv.py $(BUILDDIR)/lfs3.bench.csv \
-bprobe='%(case)s+%(probe)s' \
-Fi='min(enumerate())' \
-fn='max(n)' \
-ft='float(bench_simtime)/1.0e9' \
-o-) \
-bprobe \
-fthroughput='avg(float(n) / max(t, 1.0e-9))' \
$(SUMMARYFLAGS))
## Show which tests took the most time
.PHONY: benchmarks-bottlenecks
benchmarks-bottlenecks: SUMMARYFLAGS+=-Sruntime
benchmarks-bottlenecks: SHELL=/bin/bash
benchmarks-bottlenecks: $(BENCH_CSV)
$(strip ./scripts/csv.py $^ \
-bcase='%(case)s+%(probe)s' \
-fn='delta(n, case, probe)' \
-ft='float(bench_simtime)/1.0e9' \
-fruntime='delta(bench_runtime, case, probe)' \
$(strip ./scripts/csv.py \
<(./scripts/csv.py $^ \
-bprobe='%(case)s+%(probe)s' \
-Fi='min(enumerate())' \
-fn='max(n)' \
-ft='float(bench_simtime)/1.0e9' \
-fruntime='max(bench_runtime)' \
-o-) \
-bprobe \
-fn \
-ft \
-fruntime \
$(SUMMARYFLAGS))