Christopher Haster
1a98d8089b
This reworks bench_rt to write the target file gradually, mixing reads
and writes. This makes it so if the benchmark times out, the results are
still interesting, if less rigorous.
This is useful if you want to compare a change quickly, with more
SIM_TIME leading to a more accurate result.
---
The problem for bench_rt is that we first need to write a file. This can
take _quite_ a while for large SIM_TIME, completely failing in some
cases (bench_rt_many + BENCH_NAND).
The nice thing about bench_wt is that when it fails you still get
interesting numbers out of it. You don't find the relevant throughput
for the given SIZE, but you do find throughput for files _approaching_
SIZE. Unfortunately this didn't carry over to bench_rt.
The good news is the new bench probe system makes it easy to selectively
ignore parts of each bench, so we can rework bench_rt to start with a
CHUNK sized file and gradually increase it until it hits our target
size. This allows bench_rt to also fail gracefully.
Consider a 1 minute run:
$ make bench-runner -j \
&& BENCHFLAGS='bench_rt -DSIM_TIME=60000000000' make bench -j \
&& make bench-marks
bench+probe n t throughput
bench_rt_seq+read 302848 0.0 30690155.6
bench_rt_random+read 302592 0.0 59646605.2
bench_rt_logging+read 52992 5.9 8915.0
bench_rt_many+read 112896 0.2 482831.6
TOTAL 771328 6.2 22707126.8
Vs the default 1 hour run:
$ make bench-runner -j \
&& BENCHFLAGS='bench_rt' make bench -j \
&& make bench-marks
bench+probe n t throughput
bench_rt_seq+read 79600038272 3325.1 23939212.7
bench_rt_random+read 21482635264 3325.0 6461004.1
bench_rt_logging+read 3085056 740.9 4163.9
bench_rt_many+read 706571904 1800.7 392380.4
TOTAL 101792330496 9191.7 7699190.3
The main risk of doing this is cross-contaminating read results with
write operations. Fortunately the current benches appear to be isolated
well enough:
$ make bench-ops
bench+probe readed progged erased
bench_rt_logging+read 86886137 18936138 19767296
bench_rt_seq+read 83127251476 0 0
bench_rt_many+read 45018292401 0 0
bench_rt_random+read 83124213669 0 0
TOTAL 211356643683 18936138 19767296
---
Oh! This also lets us add bench_rt_logging, which needs a mixed writer
to make any sense.
Note bench_rt_logging also includes popping from the log (fifo?), so is
not a strictly read-only bench.
339 lines
9.9 KiB
TOML
339 lines
9.9 KiB
TOML
# High-level write-throughput benchmarks
|
|
after = ['bench_file', 'bench_dir']
|
|
|
|
# these are common and can be overridden suite-wide
|
|
#
|
|
# note for bench_*_many, file size defaults to CHUNK, and SIZE = sum of
|
|
# all files
|
|
#
|
|
defines.SIZE = '1024*1024' # 1 MiB
|
|
defines.CHUNK = 64
|
|
defines.SEED = 42
|
|
|
|
# simulated time, in nanoseconds, to run the bench
|
|
defines.SIM_TIME = '60ULL*60ULL*1000ULL*1000ULL*1000ULL' # 1 hour
|
|
# simulation size in bytes
|
|
defines.SIM_SIZE = 0
|
|
|
|
# set this to true to skip bench warmup
|
|
defines.SKIP_WARMUP = false
|
|
|
|
# include common bench helpers
|
|
code = '''
|
|
#include "benches/bench_helpers.h"
|
|
'''
|
|
|
|
|
|
# sequential write throughput
|
|
[cases.bench_wt_seq]
|
|
code = '''
|
|
lfs3_t lfs3;
|
|
lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
|
|
lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
|
|
if (!SKIP_WARMUP) {
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
int err = bench_helpers_warmup(&lfs3);
|
|
if (err) {
|
|
LFS3_ERROR("Bench warmup failed: %d", err);
|
|
return;
|
|
}
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
}
|
|
uint32_t prng = SEED;
|
|
|
|
// reset our timer
|
|
BENCH_SIMRESET();
|
|
|
|
// open a file
|
|
BENCH_START("write");
|
|
lfs3_file_t file;
|
|
lfs3_file_open(&lfs3, &file, "bench_seq",
|
|
LFS3_O_WRONLY | LFS3_O_CREAT | LFS3_O_EXCL) => 0;
|
|
lfs3_off_t size = 0;
|
|
uint64_t written = 0;
|
|
BENCH_STOP("write", written);
|
|
// ok, one of these needs to be non-zero
|
|
LFS3_ASSERT(SIM_TIME > 0 || SIM_SIZE > 0);
|
|
while (!(SIM_SIZE && written >= (uint64_t)SIM_SIZE)
|
|
&& !(SIM_TIME && BENCH_SIMTIME() >= SIM_TIME)) {
|
|
BENCH_START("write");
|
|
// arguably we should just rewind and continue writing to the
|
|
// front of the file when we hit the end, but this overly
|
|
// penalizes littlefs2, so instead we truncate
|
|
if (size >= SIZE) {
|
|
lfs3_file_rewind(&lfs3, &file) => 0;
|
|
lfs3_file_truncate(&lfs3, &file, 0) => 0;
|
|
size = 0;
|
|
}
|
|
|
|
// write to the file
|
|
uint8_t wbuf[CHUNK];
|
|
for (lfs3_size_t j = 0; j < CHUNK; j++) {
|
|
wbuf[j] = 'a' + (BENCH_PRNG(&prng) % 26);
|
|
}
|
|
lfs3_file_write(&lfs3, &file, wbuf, CHUNK) => CHUNK;
|
|
|
|
size += CHUNK;
|
|
written += CHUNK;
|
|
BENCH_STOP("write", written);
|
|
}
|
|
BENCH_START("write");
|
|
lfs3_file_close(&lfs3, &file) => 0;
|
|
// report the amount we managed to write
|
|
BENCH_STOP("write", written);
|
|
|
|
// report the total stack/heap usage after the benchmark
|
|
#ifdef BENCH_STACK
|
|
BENCH_RESULT("stack", written, BENCH_STACK_WATERMARK());
|
|
#endif
|
|
#ifdef BENCH_HEAP
|
|
BENCH_RESULT("heap", written, BENCH_HEAP_WATERMARK());
|
|
#endif
|
|
|
|
// find the total disk usage after the benchmark
|
|
//
|
|
// note this is garbage because of the above truncates!
|
|
uintmax_t usage = bench_helpers_usage(&lfs3);
|
|
BENCH_RESULT("usage", written, usage);
|
|
|
|
lfs3_unmount(&lfs3) => 0;
|
|
'''
|
|
|
|
# random write throughput
|
|
[cases.bench_wt_random]
|
|
code = '''
|
|
lfs3_t lfs3;
|
|
lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
|
|
lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
|
|
if (!SKIP_WARMUP) {
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
int err = bench_helpers_warmup(&lfs3);
|
|
if (err) {
|
|
return;
|
|
}
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
}
|
|
uint32_t prng = SEED;
|
|
|
|
// reset our timer
|
|
BENCH_SIMRESET();
|
|
|
|
// open a file
|
|
BENCH_START("write");
|
|
lfs3_file_t file;
|
|
lfs3_file_open(&lfs3, &file, "bench_random",
|
|
LFS3_O_WRONLY | LFS3_O_CREAT | LFS3_O_EXCL) => 0;
|
|
lfs3_off_t size = 0;
|
|
uint64_t written = 0;
|
|
BENCH_STOP("write", written);
|
|
// ok, one of these needs to be non-zero
|
|
LFS3_ASSERT(SIM_TIME > 0 || SIM_SIZE > 0);
|
|
while (!(SIM_SIZE && written >= (uint64_t)SIM_SIZE)
|
|
&& !(SIM_TIME && BENCH_SIMTIME() >= SIM_TIME)) {
|
|
BENCH_START("write");
|
|
// seek to a random location
|
|
lfs3_off_t pos = BENCH_PRNG(&prng) % SIZE;
|
|
lfs3_file_seek(&lfs3, &file, pos, LFS3_SEEK_SET) => pos;
|
|
|
|
// write to the file
|
|
uint8_t wbuf[CHUNK];
|
|
for (lfs3_size_t j = 0; j < CHUNK; j++) {
|
|
wbuf[j] = 'a' + (BENCH_PRNG(&prng) % 26);
|
|
}
|
|
lfs3_file_write(&lfs3, &file, wbuf, CHUNK) => CHUNK;
|
|
|
|
size = lfs3_max(size, pos + CHUNK);
|
|
written += CHUNK;
|
|
BENCH_STOP("write", written);
|
|
}
|
|
BENCH_START("write");
|
|
lfs3_file_close(&lfs3, &file) => 0;
|
|
// report the amount we managed to write
|
|
BENCH_STOP("write", written);
|
|
|
|
// report the total stack/heap usage after the benchmark
|
|
#ifdef BENCH_STACK
|
|
BENCH_RESULT("stack", written, BENCH_STACK_WATERMARK());
|
|
#endif
|
|
#ifdef BENCH_HEAP
|
|
BENCH_RESULT("heap", written, BENCH_HEAP_WATERMARK());
|
|
#endif
|
|
|
|
// find the total disk usage after the benchmark
|
|
uintmax_t usage = bench_helpers_usage(&lfs3);
|
|
BENCH_RESULT("usage", written, usage);
|
|
|
|
lfs3_unmount(&lfs3) => 0;
|
|
'''
|
|
|
|
# logging write throughput
|
|
#
|
|
# two big differences from seq:
|
|
# 1. fruncate/rotations instead of rewind+truncate
|
|
# 2. sync called on every write
|
|
[cases.bench_wt_logging]
|
|
defines.NO_FRUNCATE = false
|
|
code = '''
|
|
lfs3_t lfs3;
|
|
lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
|
|
lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
|
|
if (!SKIP_WARMUP) {
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
int err = bench_helpers_warmup(&lfs3);
|
|
if (err) {
|
|
return;
|
|
}
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
}
|
|
uint32_t prng = SEED;
|
|
|
|
// reset our timer
|
|
BENCH_SIMRESET();
|
|
|
|
// open a file
|
|
BENCH_START("write");
|
|
lfs3_file_t file;
|
|
lfs3_file_open(&lfs3, &file, "bench_log",
|
|
LFS3_O_WRONLY | LFS3_O_CREAT | LFS3_O_EXCL | LFS3_O_APPEND) => 0;
|
|
uint64_t written = 0;
|
|
BENCH_STOP("write", written);
|
|
// ok, one of these needs to be non-zero
|
|
LFS3_ASSERT(SIM_TIME > 0 || SIM_SIZE > 0);
|
|
while (!(SIM_SIZE && written >= (uint64_t)SIM_SIZE)
|
|
&& !(SIM_TIME && BENCH_SIMTIME() >= SIM_TIME)) {
|
|
BENCH_START("write");
|
|
// append to log
|
|
uint8_t wbuf[CHUNK];
|
|
for (lfs3_size_t j = 0; j < CHUNK; j++) {
|
|
wbuf[j] = 'a' + (BENCH_PRNG(&prng) % 26);
|
|
}
|
|
lfs3_file_write(&lfs3, &file, wbuf, CHUNK) => CHUNK;
|
|
|
|
// sync
|
|
lfs3_file_sync(&lfs3, &file) => 0;
|
|
|
|
// fruncate or rotate if full
|
|
lfs3_soff_t size = lfs3_file_size(&lfs3, &file);
|
|
assert(size >= 0);
|
|
if (size > SIZE) {
|
|
if (!NO_FRUNCATE) {
|
|
lfs3_file_fruncate(&lfs3, &file, SIZE) => 0;
|
|
} else {
|
|
lfs3_file_close(&lfs3, &file) => 0;
|
|
lfs3_rename(&lfs3, "bench_log", "bench_log.1") => 0;
|
|
lfs3_file_open(&lfs3, &file, "bench_log",
|
|
LFS3_O_WRONLY
|
|
| LFS3_O_CREAT
|
|
| LFS3_O_EXCL
|
|
| LFS3_O_APPEND) => 0;
|
|
}
|
|
}
|
|
|
|
written += CHUNK;
|
|
BENCH_STOP("write", written);
|
|
}
|
|
BENCH_START("write");
|
|
lfs3_file_close(&lfs3, &file) => 0;
|
|
// report the amount we managed to write
|
|
BENCH_STOP("write", written);
|
|
|
|
// report the total stack/heap usage after the benchmark
|
|
#ifdef BENCH_STACK
|
|
BENCH_RESULT("stack", written, BENCH_STACK_WATERMARK());
|
|
#endif
|
|
#ifdef BENCH_HEAP
|
|
BENCH_RESULT("heap", written, BENCH_HEAP_WATERMARK());
|
|
#endif
|
|
|
|
// find the total disk usage after the benchmark
|
|
uintmax_t usage = bench_helpers_usage(&lfs3);
|
|
BENCH_RESULT("usage", written, usage);
|
|
|
|
lfs3_unmount(&lfs3) => 0;
|
|
'''
|
|
|
|
# many small file write throughput
|
|
[cases.bench_wt_many]
|
|
defines.FILE_SIZE = 'CHUNK'
|
|
defines.FILE_COUNT = '(SIZE+(FILE_SIZE-1)) / lfs3_max(FILE_SIZE, 1)'
|
|
code = '''
|
|
lfs3_t lfs3;
|
|
lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
|
|
lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
|
|
if (!SKIP_WARMUP) {
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
int err = bench_helpers_warmup(&lfs3);
|
|
if (err) {
|
|
return;
|
|
}
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
}
|
|
uint32_t prng = SEED;
|
|
|
|
// reset our timer
|
|
BENCH_SIMRESET();
|
|
|
|
BENCH_START("write");
|
|
uint64_t written = 0;
|
|
BENCH_STOP("write", written);
|
|
// ok, one of these needs to be non-zero
|
|
LFS3_ASSERT(SIM_TIME > 0 || SIM_SIZE > 0);
|
|
while (!(SIM_SIZE && written >= (uint64_t)SIM_SIZE)
|
|
&& !(SIM_TIME && BENCH_SIMTIME() >= SIM_TIME)) {
|
|
BENCH_START("write");
|
|
// choose a random filename
|
|
lfs3_off_t pos = BENCH_PRNG(&prng) % FILE_COUNT;
|
|
char name[256];
|
|
sprintf(name, "bench_%08x", pos);
|
|
uint8_t wbuf[CHUNK];
|
|
|
|
// create the file
|
|
//
|
|
// note file == CHUNK here, the sum of all files should add up
|
|
// roughly to the benchmark SIZE
|
|
lfs3_file_t file;
|
|
lfs3_file_open(&lfs3, &file, name,
|
|
LFS3_O_WRONLY | LFS3_O_CREAT | LFS3_O_TRUNC) => 0;
|
|
for (lfs3_size_t i = 0; i < (FILE_SIZE+(CHUNK-1))/CHUNK; i++) {
|
|
lfs3_ssize_t d = lfs3_min(CHUNK, FILE_SIZE);
|
|
memset(wbuf, 'a'+(BENCH_PRNG(&prng) % 26), d);
|
|
lfs3_file_write(&lfs3, &file, wbuf, d) => d;
|
|
|
|
written += d;
|
|
|
|
// taking too long?
|
|
if (SIM_TIME && BENCH_SIMTIME() >= SIM_TIME) {
|
|
break;
|
|
}
|
|
}
|
|
lfs3_file_close(&lfs3, &file) => 0;
|
|
BENCH_STOP("write", written);
|
|
}
|
|
BENCH_START("write");
|
|
// report the amount we managed to write
|
|
BENCH_STOP("write", written);
|
|
|
|
// report the total stack/heap usage after the benchmark
|
|
#ifdef BENCH_STACK
|
|
BENCH_RESULT("stack", written, BENCH_STACK_WATERMARK());
|
|
#endif
|
|
#ifdef BENCH_HEAP
|
|
BENCH_RESULT("heap", written, BENCH_HEAP_WATERMARK());
|
|
#endif
|
|
|
|
// find the total disk usage after the benchmark
|
|
uintmax_t usage = bench_helpers_usage(&lfs3);
|
|
BENCH_RESULT("usage", written, usage);
|
|
|
|
lfs3_unmount(&lfs3) => 0;
|
|
'''
|
|
|