Christopher Haster
1a98d8089b
This reworks bench_rt to write the target file gradually, mixing reads
and writes. This makes it so if the benchmark times out, the results are
still interesting, if less rigorous.
This is useful if you want to compare a change quickly, with more
SIM_TIME leading to a more accurate result.
---
The problem for bench_rt is that we first need to write a file. This can
take _quite_ a while for large SIM_TIME, completely failing in some
cases (bench_rt_many + BENCH_NAND).
The nice thing about bench_wt is that when it fails you still get
interesting numbers out of it. You don't find the relevant throughput
for the given SIZE, but you do find throughput for files _approaching_
SIZE. Unfortunately this didn't carry over to bench_rt.
The good news is the new bench probe system makes it easy to selectively
ignore parts of each bench, so we can rework bench_rt to start with a
CHUNK sized file and gradually increase it until it hits our target
size. This allows bench_rt to also fail gracefully.
Consider a 1 minute run:
$ make bench-runner -j \
&& BENCHFLAGS='bench_rt -DSIM_TIME=60000000000' make bench -j \
&& make bench-marks
bench+probe n t throughput
bench_rt_seq+read 302848 0.0 30690155.6
bench_rt_random+read 302592 0.0 59646605.2
bench_rt_logging+read 52992 5.9 8915.0
bench_rt_many+read 112896 0.2 482831.6
TOTAL 771328 6.2 22707126.8
Vs the default 1 hour run:
$ make bench-runner -j \
&& BENCHFLAGS='bench_rt' make bench -j \
&& make bench-marks
bench+probe n t throughput
bench_rt_seq+read 79600038272 3325.1 23939212.7
bench_rt_random+read 21482635264 3325.0 6461004.1
bench_rt_logging+read 3085056 740.9 4163.9
bench_rt_many+read 706571904 1800.7 392380.4
TOTAL 101792330496 9191.7 7699190.3
The main risk of doing this is cross-contaminating read results with
write operations. Fortunately the current benches appear to be isolated
well enough:
$ make bench-ops
bench+probe readed progged erased
bench_rt_logging+read 86886137 18936138 19767296
bench_rt_seq+read 83127251476 0 0
bench_rt_many+read 45018292401 0 0
bench_rt_random+read 83124213669 0 0
TOTAL 211356643683 18936138 19767296
---
Oh! This also lets us add bench_rt_logging, which needs a mixed writer
to make any sense.
Note bench_rt_logging also includes popping from the log (fifo?), so is
not a strictly read-only bench.
394 lines
12 KiB
TOML
394 lines
12 KiB
TOML
# High-level read-throughput benchmarks
|
|
after = ['bench_file', 'bench_dir', 'bench_wt']
|
|
|
|
# these are common and can be overridden suite-wide
|
|
#
|
|
# note for bench_*_many, file size defaults to CHUNK, and SIZE = sum of
|
|
# all files
|
|
#
|
|
defines.SIZE = '1024*1024' # 1 MiB
|
|
defines.CHUNK = 64
|
|
defines.SEED = 42
|
|
|
|
# simulated time, in nanoseconds, to run the bench
|
|
defines.SIM_TIME = '60ULL*60ULL*1000ULL*1000ULL*1000ULL' # 1 hour
|
|
# simulation size in bytes
|
|
defines.SIM_SIZE = 0
|
|
|
|
# set this to true to skip bench warmup
|
|
defines.SKIP_WARMUP = false
|
|
|
|
# include common bench helpers
|
|
code = '''
|
|
#include "benches/bench_helpers.h"
|
|
'''
|
|
|
|
|
|
# sequential read throughput
|
|
[cases.bench_rt_seq]
|
|
code = '''
|
|
lfs3_t lfs3;
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
|
|
if (!SKIP_WARMUP) {
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
int err = bench_helpers_warmup(&lfs3);
|
|
if (err) {
|
|
LFS3_ERROR("Bench warmup failed: %d", err);
|
|
return;
|
|
}
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
}
|
|
uint32_t prng = SEED;
|
|
|
|
// reset our timer
|
|
BENCH_SIMRESET();
|
|
|
|
// open a file for both writing and reading
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
lfs3_file_t wfile;
|
|
lfs3_file_open(&lfs3, &wfile, "bench_seq",
|
|
LFS3_O_WRONLY | LFS3_O_CREAT | LFS3_O_EXCL) => 0;
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
BENCH_START("read");
|
|
lfs3_file_t rfile;
|
|
lfs3_file_open(&lfs3, &rfile, "bench_seq",
|
|
LFS3_O_RDONLY) => 0;
|
|
lfs3_off_t size = 0;
|
|
lfs3_off_t readed_ = 0;
|
|
uint64_t readed = 0;
|
|
BENCH_STOP("read", readed);
|
|
// ok, one of these needs to be non-zero
|
|
LFS3_ASSERT(SIM_TIME > 0 || SIM_SIZE > 0);
|
|
while (!(SIM_SIZE && readed >= (uint64_t)SIM_SIZE)
|
|
&& !(SIM_TIME && BENCH_SIMTIME() >= SIM_TIME)) {
|
|
// gradually increase the file size, so we at least have somewhat
|
|
// meaningful results if we timeout early
|
|
if (size < SIZE) {
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
uint8_t wbuf[CHUNK];
|
|
for (lfs3_size_t j = 0; j < CHUNK; j++) {
|
|
wbuf[j] = 'a' + (BENCH_PRNG(&prng) % 26);
|
|
}
|
|
lfs3_file_write(&lfs3, &wfile, wbuf, CHUNK) => CHUNK;
|
|
lfs3_file_sync(&lfs3, &wfile) => 0;
|
|
|
|
size += CHUNK;
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
}
|
|
|
|
BENCH_START("read");
|
|
// rewind if we reach the end
|
|
if (readed_ >= size) {
|
|
lfs3_file_rewind(&lfs3, &rfile) => 0;
|
|
readed_ = 0;
|
|
}
|
|
|
|
// read from the file
|
|
uint8_t rbuf[CHUNK];
|
|
lfs3_file_read(&lfs3, &rfile, rbuf, CHUNK) => CHUNK;
|
|
|
|
readed_ += CHUNK;
|
|
readed += CHUNK;
|
|
BENCH_STOP("read", readed);
|
|
}
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
lfs3_file_close(&lfs3, &wfile) => 0;
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
BENCH_START("read");
|
|
lfs3_file_close(&lfs3, &rfile) => 0;
|
|
// report the amount we managed to read
|
|
BENCH_STOP("read", readed);
|
|
|
|
lfs3_unmount(&lfs3) => 0;
|
|
'''
|
|
|
|
# random read throughput
|
|
[cases.bench_rt_random]
|
|
code = '''
|
|
lfs3_t lfs3;
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
|
|
if (!SKIP_WARMUP) {
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
int err = bench_helpers_warmup(&lfs3);
|
|
if (err) {
|
|
return;
|
|
}
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
}
|
|
uint32_t prng = SEED;
|
|
|
|
// reset our timer
|
|
BENCH_SIMRESET();
|
|
|
|
// open a file for both writing and reading
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
lfs3_file_t wfile;
|
|
lfs3_file_open(&lfs3, &wfile, "bench_random",
|
|
LFS3_O_WRONLY | LFS3_O_CREAT | LFS3_O_EXCL) => 0;
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
BENCH_START("read");
|
|
lfs3_file_t rfile;
|
|
lfs3_file_open(&lfs3, &rfile, "bench_random",
|
|
LFS3_O_RDONLY) => 0;
|
|
lfs3_off_t size = 0;
|
|
uint64_t readed = 0;
|
|
BENCH_STOP("read", readed);
|
|
// ok, one of these needs to be non-zero
|
|
LFS3_ASSERT(SIM_TIME > 0 || SIM_SIZE > 0);
|
|
while (!(SIM_SIZE && readed >= (uint64_t)SIM_SIZE)
|
|
&& !(SIM_TIME && BENCH_SIMTIME() >= SIM_TIME)) {
|
|
// gradually increase the file size, so we at least have somewhat
|
|
// meaningful results if we timeout early
|
|
if (size < SIZE) {
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
uint8_t wbuf[CHUNK];
|
|
for (lfs3_size_t j = 0; j < CHUNK; j++) {
|
|
wbuf[j] = 'a' + (BENCH_PRNG(&prng) % 26);
|
|
}
|
|
lfs3_file_write(&lfs3, &wfile, wbuf, CHUNK) => CHUNK;
|
|
lfs3_file_sync(&lfs3, &wfile) => 0;
|
|
|
|
size += CHUNK;
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
}
|
|
|
|
BENCH_START("read");
|
|
// seek to a random location
|
|
lfs3_off_t pos = BENCH_PRNG(&prng) % SIZE;
|
|
lfs3_file_seek(&lfs3, &rfile, pos, LFS3_SEEK_SET) => pos;
|
|
|
|
// read from the file
|
|
uint8_t rbuf[CHUNK];
|
|
lfs3_ssize_t d = lfs3_file_read(&lfs3, &rfile, rbuf, CHUNK);
|
|
assert(d <= CHUNK);
|
|
|
|
readed += CHUNK;
|
|
BENCH_STOP("read", readed);
|
|
}
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
lfs3_file_close(&lfs3, &wfile) => 0;
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
BENCH_START("read");
|
|
lfs3_file_close(&lfs3, &rfile) => 0;
|
|
// report the amount we managed to read
|
|
BENCH_STOP("read", readed);
|
|
|
|
lfs3_unmount(&lfs3) => 0;
|
|
'''
|
|
|
|
# logging read throughput
|
|
#
|
|
# note this will write as we need to pop the most recent read off the file!
|
|
[cases.bench_rt_logging]
|
|
code = '''
|
|
lfs3_t lfs3;
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
|
|
if (!SKIP_WARMUP) {
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
int err = bench_helpers_warmup(&lfs3);
|
|
if (err) {
|
|
return;
|
|
}
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
}
|
|
uint32_t prng = SEED;
|
|
|
|
// reset our timer
|
|
BENCH_SIMRESET();
|
|
|
|
// open a file for both writing and reading
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
lfs3_file_t wfile;
|
|
lfs3_file_open(&lfs3, &wfile, "bench_random",
|
|
LFS3_O_WRONLY | LFS3_O_CREAT | LFS3_O_EXCL | LFS3_O_APPEND) => 0;
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
BENCH_START("read");
|
|
lfs3_file_t rfile;
|
|
lfs3_file_open(&lfs3, &rfile, "bench_random",
|
|
LFS3_O_RDWR) => 0;
|
|
lfs3_off_t size = 0;
|
|
uint64_t readed = 0;
|
|
BENCH_STOP("read", readed);
|
|
// ok, one of these needs to be non-zero
|
|
LFS3_ASSERT(SIM_TIME > 0 || SIM_SIZE > 0);
|
|
while (!(SIM_SIZE && readed >= (uint64_t)SIM_SIZE)
|
|
&& !(SIM_TIME && BENCH_SIMTIME() >= SIM_TIME)) {
|
|
// gradually increase the file size, so we at least have somewhat
|
|
// meaningful results if we timeout early
|
|
//
|
|
// note we need to write ~2x to keep up with the reader!
|
|
if (size < SIZE) {
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
uint8_t wbuf[CHUNK];
|
|
for (lfs3_size_t j_ = 0; j_ < 2; j_++) {
|
|
for (lfs3_size_t j = 0; j < CHUNK; j++) {
|
|
wbuf[j] = 'a' + (BENCH_PRNG(&prng) % 26);
|
|
}
|
|
lfs3_file_write(&lfs3, &wfile, wbuf, CHUNK) => CHUNK;
|
|
}
|
|
lfs3_file_sync(&lfs3, &wfile) => 0;
|
|
|
|
size += 2*CHUNK;
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
}
|
|
|
|
BENCH_START("read");
|
|
// read from the file
|
|
uint8_t rbuf[CHUNK];
|
|
lfs3_file_read(&lfs3, &rfile, rbuf, CHUNK) => CHUNK;
|
|
// fruncate front of file
|
|
lfs3_file_fruncate(&lfs3, &rfile, size-CHUNK) => 0;
|
|
// broadcast to writer
|
|
lfs3_file_sync(&lfs3, &rfile) => 0;
|
|
|
|
size -= CHUNK;
|
|
readed += CHUNK;
|
|
BENCH_STOP("read", readed);
|
|
}
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
lfs3_file_close(&lfs3, &wfile) => 0;
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
BENCH_START("read");
|
|
lfs3_file_close(&lfs3, &rfile) => 0;
|
|
// report the amount we managed to read
|
|
BENCH_STOP("read", readed);
|
|
|
|
lfs3_unmount(&lfs3) => 0;
|
|
'''
|
|
|
|
# many small file read throughput
|
|
[cases.bench_rt_many]
|
|
defines.FILE_SIZE = 'CHUNK'
|
|
defines.FILE_COUNT = '(SIZE+(FILE_SIZE-1)) / lfs3_max(FILE_SIZE, 1)'
|
|
code = '''
|
|
lfs3_t lfs3;
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
|
|
if (!SKIP_WARMUP) {
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
int err = bench_helpers_warmup(&lfs3);
|
|
if (err) {
|
|
return;
|
|
}
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
}
|
|
uint32_t prng = SEED;
|
|
|
|
// reset our timer
|
|
BENCH_SIMRESET();
|
|
|
|
BENCH_START("read");
|
|
lfs3_off_t size = 0;
|
|
uint64_t readed = 0;
|
|
BENCH_STOP("read", readed);
|
|
// ok, one of these needs to be non-zero
|
|
LFS3_ASSERT(SIM_TIME > 0 || SIM_SIZE > 0);
|
|
while (!(SIM_SIZE && readed >= (uint64_t)SIM_SIZE)
|
|
&& !(SIM_TIME && BENCH_SIMTIME() >= SIM_TIME)) {
|
|
// gradually increase the filesystem size, so we at least have
|
|
// somewhat meaningful results if we timeout early
|
|
if (size < SIZE) {
|
|
BENCH_STACK_PAUSE();
|
|
BENCH_HEAP_PAUSE();
|
|
char name[256];
|
|
sprintf(name, "bench_%08x", (lfs3_off_t)(size/CHUNK));
|
|
lfs3_file_t wfile;
|
|
lfs3_file_open(&lfs3, &wfile, name,
|
|
LFS3_O_WRONLY | LFS3_O_CREAT | LFS3_O_EXCL) => 0;
|
|
for (lfs3_size_t i = 0; i < (FILE_SIZE+(CHUNK-1))/CHUNK; i++) {
|
|
lfs3_ssize_t d = lfs3_min(CHUNK, FILE_SIZE);
|
|
uint8_t wbuf[CHUNK];
|
|
memset(wbuf, 'a'+(BENCH_PRNG(&prng) % 26), d);
|
|
lfs3_file_write(&lfs3, &wfile, wbuf, d) => d;
|
|
|
|
// taking too long?
|
|
if (SIM_TIME && BENCH_SIMTIME() >= SIM_TIME) {
|
|
break;
|
|
}
|
|
}
|
|
lfs3_file_close(&lfs3, &wfile) => 0;
|
|
|
|
size += CHUNK;
|
|
BENCH_HEAP_RESUME();
|
|
BENCH_STACK_RESUME();
|
|
}
|
|
|
|
BENCH_START("read");
|
|
// choose a random filename
|
|
lfs3_off_t pos = BENCH_PRNG(&prng) % ((size+(CHUNK-1))/CHUNK);
|
|
char name[256];
|
|
sprintf(name, "bench_%08x", pos);
|
|
uint8_t rbuf[CHUNK];
|
|
|
|
// read the file
|
|
//
|
|
// note file == CHUNK here, the sum of all files should add up
|
|
// roughly to the benchmark SIZE
|
|
lfs3_file_t rfile;
|
|
lfs3_file_open(&lfs3, &rfile, name, LFS3_O_RDONLY) => 0;
|
|
for (lfs3_size_t i = 0; i < (FILE_SIZE+(CHUNK-1))/CHUNK; i++) {
|
|
lfs3_ssize_t d = lfs3_min(CHUNK, FILE_SIZE);
|
|
lfs3_file_read(&lfs3, &rfile, rbuf, d) => d;
|
|
readed += d;
|
|
|
|
// taking too long?
|
|
if (SIM_TIME && BENCH_SIMTIME() >= SIM_TIME) {
|
|
break;
|
|
}
|
|
}
|
|
lfs3_file_close(&lfs3, &rfile) => 0;
|
|
BENCH_STOP("read", readed);
|
|
}
|
|
BENCH_START("read");
|
|
// report the amount we managed to read
|
|
BENCH_STOP("read", readed);
|
|
|
|
lfs3_unmount(&lfs3) => 0;
|
|
'''
|
|
|