Github/littlefs
1
Fork 0
Code Actions Activity

Reworked bench.py/bench_runner/how bench measurements are recorded

Browse Source 
This is based on how bench.py/bench_runners have actually been used in
practice. The main changes have been to make the output of bench.py more
readibly consumable by plot.py/plotmpl.py without needing a bunch of
hacky intermediary scripts.

Now instead of a single per-bench BENCH_START/BENCH_STOP, benches can
have multiple named BENCH_START/BENCH_STOP invocations to measure
multiple things in one run:

  BENCH_START("fetch", i, STEP);
  lfsr_rbyd_fetch(&lfs, &rbyd_, rbyd.block, CFG->block_size) => 0;
  BENCH_STOP("fetch");

Benches can also now report explicit results, for non-io measurements:

  BENCH_RESULT("usage", i, STEP, rbyd.eoff);

The extra iter/size parameters to BENCH_START/BENCH_RESULT also allow
some extra information to be calculated post-bench. This infomation gets
tagged with an extra bench_agg field to help organize results in
plot.py/plotmpl.py:

  - bench_meas=<meas>+amor, bench_agg=raw - amortized results
  - bench_meas=<meas>+div,  bench_agg=raw - per-byte results
  - bench_meas=<meas>+avg,  bench_agg=avg - average over BENCH_SEED
  - bench_meas=<meas>+min,  bench_agg=min - minimum over BENCH_SEED
  - bench_meas=<meas>+max,  bench_agg=max - maximum over BENCH_SEED

---

Also removed all bench.tomls for now. This may seem counterproductive in
a commit to improve benchmarking, but I'm not sure there's actual value
to keeping bench cases committed in tree.

These were alway quick to fall out of date (at the time of this commit
most of the low-level bench.tomls, rbyd, btree, etc, no longer
compiled), and most benchmarks were one-off collections of scripts/data
with results too large/cumbersome to commit and keep updated in tree.

I think the better way to approach benchmarking is a seperate repo
(multiple repos?) with all related scripts/state/code and results
committed into a hopefully reproducible snapshot. Keeping the
bench.tomls in that repo makes more sense in this model.

There may be some value to having benchmarks in CI in the future, but
for that to make sense they would need to actually fail on performance
regression. How to do that isn't so clear. Anyways we can always address
this in the future rather than now.
This commit is contained in:
Christopher Haster
2023-11-03 10:27:17 -05:00
parent 4069cf5701
commit e8bdd4d381
11 changed files with 272 additions and 1399 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runners/bench_runner.c
+78 -27
View File
@@ -637,24 +637,27 @@ void bench_permutation(size_t i, uint32_t *buffer, size_t size) {
// bench recording state
typedef struct bench_record {
const char *meas;
uintmax_t iter;
uintmax_t size;
lfs_emubd_io_t last_readed;
lfs_emubd_io_t last_proged;
lfs_emubd_io_t last_erased;
} bench_record_t;
static struct lfs_config *bench_cfg = NULL;
static lfs_emubd_io_t bench_last_readed = 0;
static lfs_emubd_io_t bench_last_proged = 0;
static lfs_emubd_io_t bench_last_erased = 0;
lfs_emubd_io_t bench_readed = 0;
lfs_emubd_io_t bench_proged = 0;
lfs_emubd_io_t bench_erased = 0;
static bench_record_t *bench_records;
size_t bench_record_count;
size_t bench_record_capacity;
void bench_reset(void) {
bench_readed = 0;
bench_proged = 0;
bench_erased = 0;
bench_last_readed = 0;
bench_last_proged = 0;
bench_last_erased = 0;
void bench_reset(struct lfs_config *cfg) {
bench_cfg = cfg;
bench_record_count = 0;
}
void bench_start(void) {
void bench_start(const char *meas, uintmax_t iter, uintmax_t size) {
// measure current read/prog/erase
assert(bench_cfg);
lfs_emubd_sio_t readed = lfs_emubd_readed(bench_cfg);
assert(readed >= 0);
@@ -663,12 +666,22 @@ void bench_start(void) {
lfs_emubd_sio_t erased = lfs_emubd_erased(bench_cfg);
assert(erased >= 0);
bench_last_readed = readed;
bench_last_proged = proged;
bench_last_erased = erased;
// allocate a new record
bench_record_t *record = mappend(
(void**)&bench_records,
sizeof(bench_record_t),
&bench_record_count,
&bench_record_capacity);
record->meas = meas;
record->iter = iter;
record->size = size;
record->last_readed = readed;
record->last_proged = proged;
record->last_erased = erased;
}
void bench_stop(void) {
void bench_stop(const char *meas) {
// measure current read/prog/erase
assert(bench_cfg);
lfs_emubd_sio_t readed = lfs_emubd_readed(bench_cfg);
assert(readed >= 0);
@@ -677,9 +690,52 @@ void bench_stop(void) {
lfs_emubd_sio_t erased = lfs_emubd_erased(bench_cfg);
assert(erased >= 0);
bench_readed += readed - bench_last_readed;
bench_proged += proged - bench_last_proged;
bench_erased += erased - bench_last_erased;
// find our record
for (size_t i = 0; i < bench_record_count; i++) {
if (strcmp(bench_records[i].meas, meas) == 0) {
// print results
printf("benched %s %zd %zd %"PRIu64" %"PRIu64" %"PRIu64"\n",
bench_records[i].meas,
bench_records[i].iter,
bench_records[i].size,
readed - bench_records[i].last_readed,
proged - bench_records[i].last_proged,
erased - bench_records[i].last_erased);
// remove our record
memmove(&bench_records[i],
&bench_records[i+1],
bench_record_count-(i+1));
bench_record_count -= 1;
return;
}
}
// not found?
fprintf(stderr, "error: bench stopped before it was started (%s)\n",
meas);
assert(false);
exit(-1);
}
void bench_result(const char *meas, uintmax_t iter, uintmax_t size,
uintmax_t result) {
// we just print these directly
printf("benched %s %zd %zd %"PRIu64"\n",
meas,
iter,
size,
result);
}
void bench_fresult(const char *meas, uintmax_t iter, uintmax_t size,
double result) {
// we just print these directly
printf("benched %s %zd %zd %.6f\n",
meas,
iter,
size,
result);
}
@@ -1404,8 +1460,7 @@ void perm_run(
}
// run the bench
bench_cfg = &cfg;
bench_reset();
bench_reset(&cfg);
printf("running ");
perm_printid(suite, case_);
printf("\n");
@@ -1414,10 +1469,6 @@ void perm_run(
printf("finished ");
perm_printid(suite, case_);
printf(" %"PRIu64" %"PRIu64" %"PRIu64,
bench_readed,
bench_proged,
bench_erased);
printf("\n");
// cleanup
runners/bench_runner.h
+19 -5
View File
@@ -19,12 +19,26 @@ void bench_trace(const char *fmt, ...);
#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")
#define LFS_EMUBD_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")
// provide BENCH_START/BENCH_STOP macros
void bench_start(void);
void bench_stop(void);
// BENCH_START/BENCH_STOP macros measure readed/proged/erased bytes
// through emubd
void bench_start(const char *meas, uintmax_t iter, uintmax_t size);
void bench_stop(const char *meas);
#define BENCH_START() bench_start()
#define BENCH_STOP() bench_stop()
#define BENCH_START(meas, iter, size) \
bench_start(meas, iter, size)
#define BENCH_STOP(meas) \
bench_stop(meas)
// BENCH_RESULT/BENCH_FRESULT allow for explicit non-io measurements
void bench_result(const char *meas, uintmax_t iter, uintmax_t size,
uintmax_t result);
void bench_fresult(const char *meas, uintmax_t iter, uintmax_t size,
double result);
#define BENCH_RESULT(meas, iter, size, result) \
bench_result(meas, iter, size, result)
#define BENCH_FRESULT(meas, iter, size, result) \
bench_fresult(meas, iter, size, result)
// note these are indirectly included in any generated files