Github/littlefs
1
Fork 0
Code Actions Activity

Added bench.py and bench_runner.c for benchmarking

Browse Source 
These are really just different flavors of test.py and test_runner.c
without support for power-loss testing, but with support for measuring
the cumulative number of bytes read, programmed, and erased.

Note that the existing define parameterization should work perfectly
fine for running benchmarks across various dimensions:

./scripts/bench.py \
    runners/bench_runner \
    bench_file_read \
    -gnor \
    -DSIZE='range(0,131072,1024)'

Also added a couple basic benchmarks as a starting point.
This commit is contained in:
Christopher Haster
2022-09-20 02:01:59 -05:00
parent 20ec0be875
commit 4fe0738ff4
20 changed files with 4253 additions and 405 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_alloc.toml
+3 -3
View File
@@ -370,7 +370,7 @@ code = '''
[cases.test_alloc_bad_blocks]
in = "lfs.c"
defines.ERASE_CYCLES = 0xffffffff
defines.BADBLOCK_BEHAVIOR = 'LFS_TESTBD_BADBLOCK_READERROR'
defines.BADBLOCK_BEHAVIOR = 'LFS_EMUBD_BADBLOCK_READERROR'
code = '''
lfs_t lfs;
lfs_format(&lfs, cfg) => 0;
@@ -409,7 +409,7 @@ code = '''
// but mark the head of our file as a "bad block", this is force our
// scan to bail early
lfs_testbd_setwear(cfg, fileblock, 0xffffffff) => 0;
lfs_emubd_setwear(cfg, fileblock, 0xffffffff) => 0;
lfs_file_open(&lfs, &file, "ghost", LFS_O_WRONLY | LFS_O_CREAT) => 0;
strcpy((char*)buffer, "chomp");
size = strlen("chomp");
@@ -424,7 +424,7 @@ code = '''
// now reverse the "bad block" and try to write the file again until we
// run out of space
lfs_testbd_setwear(cfg, fileblock, 0) => 0;
lfs_emubd_setwear(cfg, fileblock, 0) => 0;
lfs_file_open(&lfs, &file, "ghost", LFS_O_WRONLY | LFS_O_CREAT) => 0;
strcpy((char*)buffer, "chomp");
size = strlen("chomp");
tests/test_badblocks.toml
+26 -26
View File
@@ -6,18 +6,18 @@ defines.BLOCK_COUNT = 256 # small bd so test runs faster
defines.ERASE_CYCLES = 0xffffffff
defines.ERASE_VALUE = [0x00, 0xff, -1]
defines.BADBLOCK_BEHAVIOR = [
'LFS_TESTBD_BADBLOCK_PROGERROR',
'LFS_TESTBD_BADBLOCK_ERASEERROR',
'LFS_TESTBD_BADBLOCK_READERROR',
'LFS_TESTBD_BADBLOCK_PROGNOOP',
'LFS_TESTBD_BADBLOCK_ERASENOOP',
'LFS_EMUBD_BADBLOCK_PROGERROR',
'LFS_EMUBD_BADBLOCK_ERASEERROR',
'LFS_EMUBD_BADBLOCK_READERROR',
'LFS_EMUBD_BADBLOCK_PROGNOOP',
'LFS_EMUBD_BADBLOCK_ERASENOOP',
]
defines.NAMEMULT = 64
defines.FILEMULT = 1
code = '''
for (lfs_block_t badblock = 2; badblock < BLOCK_COUNT; badblock++) {
lfs_testbd_setwear(cfg, badblock-1, 0) => 0;
lfs_testbd_setwear(cfg, badblock, 0xffffffff) => 0;
lfs_emubd_setwear(cfg, badblock-1, 0) => 0;
lfs_emubd_setwear(cfg, badblock, 0xffffffff) => 0;
lfs_t lfs;
lfs_format(&lfs, cfg) => 0;
@@ -86,17 +86,17 @@ defines.BLOCK_COUNT = 256 # small bd so test runs faster
defines.ERASE_CYCLES = 0xffffffff
defines.ERASE_VALUE = [0x00, 0xff, -1]
defines.BADBLOCK_BEHAVIOR = [
'LFS_TESTBD_BADBLOCK_PROGERROR',
'LFS_TESTBD_BADBLOCK_ERASEERROR',
'LFS_TESTBD_BADBLOCK_READERROR',
'LFS_TESTBD_BADBLOCK_PROGNOOP',
'LFS_TESTBD_BADBLOCK_ERASENOOP',
'LFS_EMUBD_BADBLOCK_PROGERROR',
'LFS_EMUBD_BADBLOCK_ERASEERROR',
'LFS_EMUBD_BADBLOCK_READERROR',
'LFS_EMUBD_BADBLOCK_PROGNOOP',
'LFS_EMUBD_BADBLOCK_ERASENOOP',
]
defines.NAMEMULT = 64
defines.FILEMULT = 1
code = '''
for (lfs_block_t i = 0; i < (BLOCK_COUNT-2)/2; i++) {
lfs_testbd_setwear(cfg, i+2, 0xffffffff) => 0;
lfs_emubd_setwear(cfg, i+2, 0xffffffff) => 0;
}
lfs_t lfs;
@@ -165,17 +165,17 @@ defines.BLOCK_COUNT = 256 # small bd so test runs faster
defines.ERASE_CYCLES = 0xffffffff
defines.ERASE_VALUE = [0x00, 0xff, -1]
defines.BADBLOCK_BEHAVIOR = [
'LFS_TESTBD_BADBLOCK_PROGERROR',
'LFS_TESTBD_BADBLOCK_ERASEERROR',
'LFS_TESTBD_BADBLOCK_READERROR',
'LFS_TESTBD_BADBLOCK_PROGNOOP',
'LFS_TESTBD_BADBLOCK_ERASENOOP',
'LFS_EMUBD_BADBLOCK_PROGERROR',
'LFS_EMUBD_BADBLOCK_ERASEERROR',
'LFS_EMUBD_BADBLOCK_READERROR',
'LFS_EMUBD_BADBLOCK_PROGNOOP',
'LFS_EMUBD_BADBLOCK_ERASENOOP',
]
defines.NAMEMULT = 64
defines.FILEMULT = 1
code = '''
for (lfs_block_t i = 0; i < (BLOCK_COUNT-2)/2; i++) {
lfs_testbd_setwear(cfg, (2*i) + 2, 0xffffffff) => 0;
lfs_emubd_setwear(cfg, (2*i) + 2, 0xffffffff) => 0;
}
lfs_t lfs;
@@ -244,15 +244,15 @@ code = '''
defines.ERASE_CYCLES = 0xffffffff
defines.ERASE_VALUE = [0x00, 0xff, -1]
defines.BADBLOCK_BEHAVIOR = [
'LFS_TESTBD_BADBLOCK_PROGERROR',
'LFS_TESTBD_BADBLOCK_ERASEERROR',
'LFS_TESTBD_BADBLOCK_READERROR',
'LFS_TESTBD_BADBLOCK_PROGNOOP',
'LFS_TESTBD_BADBLOCK_ERASENOOP',
'LFS_EMUBD_BADBLOCK_PROGERROR',
'LFS_EMUBD_BADBLOCK_ERASEERROR',
'LFS_EMUBD_BADBLOCK_READERROR',
'LFS_EMUBD_BADBLOCK_PROGNOOP',
'LFS_EMUBD_BADBLOCK_ERASENOOP',
]
code = '''
lfs_testbd_setwear(cfg, 0, 0xffffffff) => 0;
lfs_testbd_setwear(cfg, 1, 0xffffffff) => 0;
lfs_emubd_setwear(cfg, 0, 0xffffffff) => 0;
lfs_emubd_setwear(cfg, 1, 0xffffffff) => 0;
lfs_t lfs;
lfs_format(&lfs, cfg) => LFS_ERR_NOSPC;
tests/test_exhaustion.toml
+20 -20
View File
@@ -4,11 +4,11 @@ defines.ERASE_CYCLES = 10
defines.BLOCK_COUNT = 256 # small bd so test runs faster
defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2'
defines.BADBLOCK_BEHAVIOR = [
'LFS_TESTBD_BADBLOCK_PROGERROR',
'LFS_TESTBD_BADBLOCK_ERASEERROR',
'LFS_TESTBD_BADBLOCK_READERROR',
'LFS_TESTBD_BADBLOCK_PROGNOOP',
'LFS_TESTBD_BADBLOCK_ERASENOOP',
'LFS_EMUBD_BADBLOCK_PROGERROR',
'LFS_EMUBD_BADBLOCK_ERASEERROR',
'LFS_EMUBD_BADBLOCK_READERROR',
'LFS_EMUBD_BADBLOCK_PROGNOOP',
'LFS_EMUBD_BADBLOCK_ERASENOOP',
]
defines.FILES = 10
code = '''
@@ -97,11 +97,11 @@ defines.ERASE_CYCLES = 10
defines.BLOCK_COUNT = 256 # small bd so test runs faster
defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2'
defines.BADBLOCK_BEHAVIOR = [
'LFS_TESTBD_BADBLOCK_PROGERROR',
'LFS_TESTBD_BADBLOCK_ERASEERROR',
'LFS_TESTBD_BADBLOCK_READERROR',
'LFS_TESTBD_BADBLOCK_PROGNOOP',
'LFS_TESTBD_BADBLOCK_ERASENOOP',
'LFS_EMUBD_BADBLOCK_PROGERROR',
'LFS_EMUBD_BADBLOCK_ERASEERROR',
'LFS_EMUBD_BADBLOCK_READERROR',
'LFS_EMUBD_BADBLOCK_PROGNOOP',
'LFS_EMUBD_BADBLOCK_ERASENOOP',
]
defines.FILES = 10
code = '''
@@ -197,7 +197,7 @@ code = '''
for (int run = 0; run < 2; run++) {
for (lfs_block_t b = 0; b < BLOCK_COUNT; b++) {
lfs_testbd_setwear(cfg, b,
lfs_emubd_setwear(cfg, b,
(b < run_block_count[run]) ? 0 : ERASE_CYCLES) => 0;
}
@@ -297,7 +297,7 @@ code = '''
for (int run = 0; run < 2; run++) {
for (lfs_block_t b = 0; b < BLOCK_COUNT; b++) {
lfs_testbd_setwear(cfg, b,
lfs_emubd_setwear(cfg, b,
(b < run_block_count[run]) ? 0 : ERASE_CYCLES) => 0;
}
@@ -469,12 +469,12 @@ exhausted:
LFS_WARN("completed %d cycles", cycle);
// check the wear on our block device
lfs_testbd_wear_t minwear = -1;
lfs_testbd_wear_t totalwear = 0;
lfs_testbd_wear_t maxwear = 0;
lfs_emubd_wear_t minwear = -1;
lfs_emubd_wear_t totalwear = 0;
lfs_emubd_wear_t maxwear = 0;
// skip 0 and 1 as superblock movement is intentionally avoided
for (lfs_block_t b = 2; b < BLOCK_COUNT; b++) {
lfs_testbd_wear_t wear = lfs_testbd_getwear(cfg, b);
lfs_emubd_wear_t wear = lfs_emubd_getwear(cfg, b);
printf("%08x: wear %d\n", b, wear);
assert(wear >= 0);
if (wear < minwear) {
@@ -485,17 +485,17 @@ exhausted:
}
totalwear += wear;
}
lfs_testbd_wear_t avgwear = totalwear / BLOCK_COUNT;
lfs_emubd_wear_t avgwear = totalwear / BLOCK_COUNT;
LFS_WARN("max wear: %d cycles", maxwear);
LFS_WARN("avg wear: %d cycles", totalwear / (int)BLOCK_COUNT);
LFS_WARN("min wear: %d cycles", minwear);
// find standard deviation^2
lfs_testbd_wear_t dev2 = 0;
lfs_emubd_wear_t dev2 = 0;
for (lfs_block_t b = 2; b < BLOCK_COUNT; b++) {
lfs_testbd_wear_t wear = lfs_testbd_getwear(cfg, b);
lfs_emubd_wear_t wear = lfs_emubd_getwear(cfg, b);
assert(wear >= 0);
lfs_testbd_swear_t diff = wear - avgwear;
lfs_emubd_swear_t diff = wear - avgwear;
dev2 += diff*diff;
}
dev2 /= totalwear;
tests/test_move.toml
+10 -10
View File
@@ -1638,15 +1638,15 @@ code = '''
if (RELOCATIONS & 0x1) {
lfs_dir_t dir;
lfs_dir_open(&lfs, &dir, "/parent");
lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
lfs_dir_close(&lfs, &dir) => 0;
}
if (RELOCATIONS & 0x2) {
lfs_dir_t dir;
lfs_dir_open(&lfs, &dir, "/parent/child");
lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
lfs_dir_close(&lfs, &dir) => 0;
}
@@ -1784,22 +1784,22 @@ code = '''
if (RELOCATIONS & 0x1) {
lfs_dir_t dir;
lfs_dir_open(&lfs, &dir, "/parent");
lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
lfs_dir_close(&lfs, &dir) => 0;
}
if (RELOCATIONS & 0x2) {
lfs_dir_t dir;
lfs_dir_open(&lfs, &dir, "/parent/sibling");
lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
lfs_dir_close(&lfs, &dir) => 0;
}
if (RELOCATIONS & 0x4) {
lfs_dir_t dir;
lfs_dir_open(&lfs, &dir, "/parent/child");
lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
lfs_dir_close(&lfs, &dir) => 0;
}