diff --git a/.gitignore b/.gitignore index 58ff85d3..8f1b90e6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ *.ci *.csv *.t.c +*.b.c *.a.c *.gcno *.gcda @@ -17,3 +18,4 @@ tests/*.toml.* scripts/__pycache__ .gdb_history runners/test_runner +runners/bench_runner diff --git a/Makefile b/Makefile index ce9a5539..3eaa92d6 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,8 @@ $(if $(findstring n,$(MAKEFLAGS)),, $(shell mkdir -p \ $(BUILDDIR) \ $(BUILDDIR)bd \ $(BUILDDIR)runners \ - $(BUILDDIR)tests)) + $(BUILDDIR)tests \ + $(BUILDDIR)benches)) endif # overridable target/src/tools/flags/etc @@ -45,6 +46,18 @@ TEST_CI := $(TEST_TAC:%.t.a.c=%.t.a.ci) TEST_GCNO := $(TEST_TAC:%.t.a.c=%.t.a.gcno) TEST_GCDA := $(TEST_TAC:%.t.a.c=%.t.a.gcda) +BENCHES ?= $(wildcard benches/*.toml) +BENCH_SRC ?= $(SRC) \ + $(filter-out $(wildcard bd/*.*.c),$(wildcard bd/*.c)) \ + runners/bench_runner.c +BENCH_BC := $(BENCHES:%.toml=$(BUILDDIR)%.b.c) $(BENCH_SRC:%.c=$(BUILDDIR)%.b.c) +BENCH_BAC := $(BENCH_BC:%.b.c=%.b.a.c) +BENCH_OBJ := $(BENCH_BAC:%.b.a.c=%.b.a.o) +BENCH_DEP := $(BENCH_BAC:%.b.a.c=%.b.a.d) +BENCH_CI := $(BENCH_BAC:%.b.a.c=%.b.a.ci) +BENCH_GCNO := $(BENCH_BAC:%.b.a.c=%.b.a.gcno) +BENCH_GCDA := $(BENCH_BAC:%.b.a.c=%.b.a.gcda) + ifdef DEBUG override CFLAGS += -O0 else @@ -60,27 +73,31 @@ override CFLAGS += -Wextra -Wshadow -Wjump-misses-init -Wundef override CFLAGS += -ftrack-macro-expansion=0 override TESTFLAGS += -b +override BENCHFLAGS += -b # forward -j flag override TESTFLAGS += $(filter -j%,$(MAKEFLAGS)) +override BENCHFLAGS += $(filter -j%,$(MAKEFLAGS)) ifdef VERBOSE -override TESTFLAGS += -v -override CODEFLAGS += -v -override DATAFLAGS += -v -override STACKFLAGS += -v -override STRUCTFLAGS += -v -override COVERAGEFLAGS += -v -override TESTFLAGS += -v -override TESTCFLAGS += -v +override CODEFLAGS += -v +override DATAFLAGS += -v +override STACKFLAGS += -v +override STRUCTFLAGS += -v +override COVERAGEFLAGS += -v +override TESTFLAGS += -v +override TESTCFLAGS += -v +override BENCHFLAGS += -v +override BENCHCFLAGS += -v endif ifdef EXEC -override TESTFLAGS += --exec="$(EXEC)" +override TESTFLAGS += --exec="$(EXEC)" +override BENCHFLAGS += --exec="$(EXEC)" endif ifdef BUILDDIR -override CODEFLAGS += --build-dir="$(BUILDDIR:/=)" -override DATAFLAGS += --build-dir="$(BUILDDIR:/=)" -override STACKFLAGS += --build-dir="$(BUILDDIR:/=)" -override STRUCTFLAGS += --build-dir="$(BUILDDIR:/=)" -override COVERAGEFLAGS += --build-dir="$(BUILDDIR:/=)" +override CODEFLAGS += --build-dir="$(BUILDDIR:/=)" +override DATAFLAGS += --build-dir="$(BUILDDIR:/=)" +override STACKFLAGS += --build-dir="$(BUILDDIR:/=)" +override STRUCTFLAGS += --build-dir="$(BUILDDIR:/=)" +override COVERAGEFLAGS += --build-dir="$(BUILDDIR:/=)" endif ifneq ($(NM),nm) override CODEFLAGS += --nm-tool="$(NM)" @@ -119,6 +136,17 @@ test: test-runner test-list: test-runner ./scripts/test.py $(BUILDDIR)runners/test_runner $(TESTFLAGS) -l +.PHONY: bench-runner build-bench +bench-runner build-bench: $(BUILDDIR)runners/bench_runner + +.PHONY: bench +bench: bench-runner + ./scripts/bench.py $(BUILDDIR)runners/bench_runner $(BENCHFLAGS) + +.PHONY: bench-list +bench-list: bench-runner + ./scripts/bench.py $(BUILDDIR)runners/bench_runner $(BENCHFLAGS) -l + .PHONY: code code: $(OBJ) ./scripts/code.py $^ -S $(CODEFLAGS) @@ -186,6 +214,9 @@ $(BUILDDIR)lfs.csv: \ $(BUILDDIR)runners/test_runner: $(TEST_OBJ) $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ +$(BUILDDIR)runners/bench_runner: $(BENCH_OBJ) + $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ + # our main build rule generates .o, .d, and .ci files, the latter # used for stack analysis $(BUILDDIR)%.o $(BUILDDIR)%.ci: %.c @@ -206,6 +237,12 @@ $(BUILDDIR)%.t.c: %.toml $(BUILDDIR)%.t.c: %.c $(TESTS) ./scripts/test.py -c $(TESTS) -s $< $(TESTCFLAGS) -o $@ +$(BUILDDIR)%.b.c: %.toml + ./scripts/bench.py -c $< $(BENCHCFLAGS) -o $@ + +$(BUILDDIR)%.b.c: %.c $(BENCHES) + ./scripts/bench.py -c $(BENCHES) -s $< $(BENCHCFLAGS) -o $@ + # clean everything .PHONY: clean clean: @@ -219,6 +256,7 @@ clean: $(BUILDDIR)lfs.struct.csv \ $(BUILDDIR)lfs.coverage.csv) rm -f $(BUILDDIR)runners/test_runner + rm -f $(BUILDDIR)runners/bench_runner rm -f $(OBJ) rm -f $(DEP) rm -f $(ASM) @@ -230,3 +268,10 @@ clean: rm -f $(TEST_CI) rm -f $(TEST_GCNO) rm -f $(TEST_GCDA) + rm -f $(BENCH_BC) + rm -f $(BENCH_BAC) + rm -f $(BENCH_OBJ) + rm -f $(BENCH_DEP) + rm -f $(BENCH_CI) + rm -f $(BENCH_GCNO) + rm -f $(BENCH_GCDA) diff --git a/bd/lfs_testbd.c b/bd/lfs_emubd.c similarity index 57% rename from bd/lfs_testbd.c rename to bd/lfs_emubd.c index 61063af8..8a9da7a7 100644 --- a/bd/lfs_testbd.c +++ b/bd/lfs_emubd.c @@ -1,5 +1,5 @@ /* - * Testing block device, wraps filebd and rambd while providing a bunch + * Emulating block device, wraps filebd and rambd while providing a bunch * of hooks for testing littlefs in various conditions. * * Copyright (c) 2022, The littlefs authors. @@ -11,7 +11,7 @@ #define _POSIX_C_SOURCE 199309L #endif -#include "bd/lfs_testbd.h" +#include "bd/lfs_emubd.h" #include #include @@ -29,14 +29,14 @@ // Note we can only modify a block if we have exclusive access to it (rc == 1) // -static lfs_testbd_block_t *lfs_testbd_incblock(lfs_testbd_block_t *block) { +static lfs_emubd_block_t *lfs_emubd_incblock(lfs_emubd_block_t *block) { if (block) { block->rc += 1; } return block; } -static void lfs_testbd_decblock(lfs_testbd_block_t *block) { +static void lfs_emubd_decblock(lfs_emubd_block_t *block) { if (block) { block->rc -= 1; if (block->rc == 0) { @@ -45,34 +45,34 @@ static void lfs_testbd_decblock(lfs_testbd_block_t *block) { } } -static lfs_testbd_block_t *lfs_testbd_mutblock( +static lfs_emubd_block_t *lfs_emubd_mutblock( const struct lfs_config *cfg, - lfs_testbd_block_t **block) { - lfs_testbd_block_t *block_ = *block; + lfs_emubd_block_t **block) { + lfs_emubd_block_t *block_ = *block; if (block_ && block_->rc == 1) { // rc == 1? can modify return block_; } else if (block_) { // rc > 1? need to create a copy - lfs_testbd_block_t *nblock = malloc( - sizeof(lfs_testbd_block_t) + cfg->block_size); + lfs_emubd_block_t *nblock = malloc( + sizeof(lfs_emubd_block_t) + cfg->block_size); if (!nblock) { return NULL; } memcpy(nblock, block_, - sizeof(lfs_testbd_block_t) + cfg->block_size); + sizeof(lfs_emubd_block_t) + cfg->block_size); nblock->rc = 1; - lfs_testbd_decblock(block_); + lfs_emubd_decblock(block_); *block = nblock; return nblock; } else { // no block? need to allocate - lfs_testbd_block_t *nblock = malloc( - sizeof(lfs_testbd_block_t) + cfg->block_size); + lfs_emubd_block_t *nblock = malloc( + sizeof(lfs_emubd_block_t) + cfg->block_size); if (!nblock) { return NULL; } @@ -81,7 +81,7 @@ static lfs_testbd_block_t *lfs_testbd_mutblock( nblock->wear = 0; // zero for consistency - lfs_testbd_t *bd = cfg->context; + lfs_emubd_t *bd = cfg->context; memset(nblock->data, (bd->cfg->erase_value != -1) ? bd->cfg->erase_value : 0, cfg->block_size); @@ -92,11 +92,11 @@ static lfs_testbd_block_t *lfs_testbd_mutblock( } -// testbd create/destroy +// emubd create/destroy -int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, - const struct lfs_testbd_config *bdcfg) { - LFS_TESTBD_TRACE("lfs_testbd_createcfg(%p {.context=%p, " +int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, + const struct lfs_emubd_config *bdcfg) { + LFS_EMUBD_TRACE("lfs_emubd_createcfg(%p {.context=%p, " ".read=%p, .prog=%p, .erase=%p, .sync=%p, " ".read_size=%"PRIu32", .prog_size=%"PRIu32", " ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " @@ -113,25 +113,28 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, bdcfg->badblock_behavior, bdcfg->power_cycles, bdcfg->powerloss_behavior, (void*)(uintptr_t)bdcfg->powerloss_cb, bdcfg->powerloss_data, bdcfg->track_branches); - lfs_testbd_t *bd = cfg->context; + lfs_emubd_t *bd = cfg->context; bd->cfg = bdcfg; // allocate our block array, all blocks start as uninitialized - bd->blocks = malloc(cfg->block_count * sizeof(lfs_testbd_block_t*)); + bd->blocks = malloc(cfg->block_count * sizeof(lfs_emubd_block_t*)); if (!bd->blocks) { - LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } - memset(bd->blocks, 0, cfg->block_count * sizeof(lfs_testbd_block_t*)); + memset(bd->blocks, 0, cfg->block_count * sizeof(lfs_emubd_block_t*)); // setup testing things + bd->read = 0; + bd->prog = 0; + bd->erased = 0; bd->power_cycles = bd->cfg->power_cycles; bd->disk = NULL; if (bd->cfg->disk_path) { - bd->disk = malloc(sizeof(lfs_testbd_disk_t)); + bd->disk = malloc(sizeof(lfs_emubd_disk_t)); if (!bd->disk) { - LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } bd->disk->rc = 1; @@ -146,7 +149,7 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, #endif if (bd->disk->fd < 0) { int err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_create -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", err); return err; } @@ -155,7 +158,7 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, if (bd->cfg->erase_value != -1) { bd->disk->scratch = malloc(cfg->block_size); if (!bd->disk->scratch) { - LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } memset(bd->disk->scratch, @@ -164,12 +167,12 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, } } - LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", 0); return 0; } -int lfs_testbd_create(const struct lfs_config *cfg, const char *path) { - LFS_TESTBD_TRACE("lfs_testbd_create(%p {.context=%p, " +int lfs_emubd_create(const struct lfs_config *cfg, const char *path) { + LFS_EMUBD_TRACE("lfs_emubd_create(%p {.context=%p, " ".read=%p, .prog=%p, .erase=%p, .sync=%p, " ".read_size=%"PRIu32", .prog_size=%"PRIu32", " ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " @@ -179,19 +182,19 @@ int lfs_testbd_create(const struct lfs_config *cfg, const char *path) { (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, path); - static const struct lfs_testbd_config defaults = {.erase_value=-1}; - int err = lfs_testbd_createcfg(cfg, path, &defaults); - LFS_TESTBD_TRACE("lfs_testbd_create -> %d", err); + static const struct lfs_emubd_config defaults = {.erase_value=-1}; + int err = lfs_emubd_createcfg(cfg, path, &defaults); + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", err); return err; } -int lfs_testbd_destroy(const struct lfs_config *cfg) { - LFS_TESTBD_TRACE("lfs_testbd_destroy(%p)", (void*)cfg); - lfs_testbd_t *bd = cfg->context; +int lfs_emubd_destroy(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_destroy(%p)", (void*)cfg); + lfs_emubd_t *bd = cfg->context; // decrement reference counts for (lfs_block_t i = 0; i < cfg->block_count; i++) { - lfs_testbd_decblock(bd->blocks[i]); + lfs_emubd_decblock(bd->blocks[i]); } free(bd->blocks); @@ -205,7 +208,7 @@ int lfs_testbd_destroy(const struct lfs_config *cfg) { } } - LFS_TESTBD_TRACE("lfs_testbd_destroy -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_destroy -> %d", 0); return 0; } @@ -213,12 +216,12 @@ int lfs_testbd_destroy(const struct lfs_config *cfg) { // block device API -int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, +int lfs_emubd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { - LFS_TESTBD_TRACE("lfs_testbd_read(%p, " + LFS_EMUBD_TRACE("lfs_emubd_read(%p, " "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")", (void*)cfg, block, off, buffer, size); - lfs_testbd_t *bd = cfg->context; + lfs_emubd_t *bd = cfg->context; // check if read is valid LFS_ASSERT(block < cfg->block_count); @@ -227,12 +230,12 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, LFS_ASSERT(off+size <= cfg->block_size); // get the block - const lfs_testbd_block_t *b = bd->blocks[block]; + const lfs_emubd_block_t *b = bd->blocks[block]; if (b) { // block bad? if (bd->cfg->erase_cycles && b->wear >= bd->cfg->erase_cycles && - bd->cfg->badblock_behavior == LFS_TESTBD_BADBLOCK_READERROR) { - LFS_TESTBD_TRACE("lfs_testbd_read -> %d", LFS_ERR_CORRUPT); + bd->cfg->badblock_behavior == LFS_EMUBD_BADBLOCK_READERROR) { + LFS_EMUBD_TRACE("lfs_emubd_read -> %d", LFS_ERR_CORRUPT); return LFS_ERR_CORRUPT; } @@ -243,8 +246,10 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, memset(buffer, (bd->cfg->erase_value != -1) ? bd->cfg->erase_value : 0, size); - } + } + // track reads + bd->read += size; if (bd->cfg->read_sleep) { int err = nanosleep(&(struct timespec){ .tv_sec=bd->cfg->read_sleep/1000000000, @@ -252,21 +257,21 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, NULL); if (err) { err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_read -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_read -> %d", err); return err; } } - LFS_TESTBD_TRACE("lfs_testbd_read -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_read -> %d", 0); return 0; } -int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, +int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size) { - LFS_TESTBD_TRACE("lfs_testbd_prog(%p, " + LFS_EMUBD_TRACE("lfs_emubd_prog(%p, " "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")", (void*)cfg, block, off, buffer, size); - lfs_testbd_t *bd = cfg->context; + lfs_emubd_t *bd = cfg->context; // check if write is valid LFS_ASSERT(block < cfg->block_count); @@ -275,23 +280,23 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, LFS_ASSERT(off+size <= cfg->block_size); // get the block - lfs_testbd_block_t *b = lfs_testbd_mutblock(cfg, &bd->blocks[block]); + lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); if (!b) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } // block bad? if (bd->cfg->erase_cycles && b->wear >= bd->cfg->erase_cycles) { if (bd->cfg->badblock_behavior == - LFS_TESTBD_BADBLOCK_PROGERROR) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_CORRUPT); + LFS_EMUBD_BADBLOCK_PROGERROR) { + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", LFS_ERR_CORRUPT); return LFS_ERR_CORRUPT; } else if (bd->cfg->badblock_behavior == - LFS_TESTBD_BADBLOCK_PROGNOOP || + LFS_EMUBD_BADBLOCK_PROGNOOP || bd->cfg->badblock_behavior == - LFS_TESTBD_BADBLOCK_ERASENOOP) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0); + LFS_EMUBD_BADBLOCK_ERASENOOP) { + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", 0); return 0; } } @@ -313,18 +318,20 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, SEEK_SET); if (res1 < 0) { int err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", err); return err; } ssize_t res2 = write(bd->disk->fd, buffer, size); if (res2 < 0) { int err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", err); return err; } } + // track progs + bd->prog += size; if (bd->cfg->prog_sleep) { int err = nanosleep(&(struct timespec){ .tv_sec=bd->cfg->prog_sleep/1000000000, @@ -332,7 +339,7 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, NULL); if (err) { err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", err); return err; } } @@ -346,21 +353,21 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, } } - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", 0); return 0; } -int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { - LFS_TESTBD_TRACE("lfs_testbd_erase(%p, 0x%"PRIx32")", (void*)cfg, block); - lfs_testbd_t *bd = cfg->context; +int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block) { + LFS_EMUBD_TRACE("lfs_emubd_erase(%p, 0x%"PRIx32")", (void*)cfg, block); + lfs_emubd_t *bd = cfg->context; // check if erase is valid LFS_ASSERT(block < cfg->block_count); // get the block - lfs_testbd_block_t *b = lfs_testbd_mutblock(cfg, &bd->blocks[block]); + lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); if (!b) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } @@ -368,12 +375,12 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { if (bd->cfg->erase_cycles) { if (b->wear >= bd->cfg->erase_cycles) { if (bd->cfg->badblock_behavior == - LFS_TESTBD_BADBLOCK_ERASEERROR) { - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", LFS_ERR_CORRUPT); + LFS_EMUBD_BADBLOCK_ERASEERROR) { + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", LFS_ERR_CORRUPT); return LFS_ERR_CORRUPT; } else if (bd->cfg->badblock_behavior == - LFS_TESTBD_BADBLOCK_ERASENOOP) { - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", 0); + LFS_EMUBD_BADBLOCK_ERASENOOP) { + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", 0); return 0; } } else { @@ -393,7 +400,7 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { SEEK_SET); if (res1 < 0) { int err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", err); return err; } @@ -402,12 +409,14 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { cfg->block_size); if (res2 < 0) { int err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", err); return err; } } } + // track erases + bd->erased += cfg->block_size; if (bd->cfg->erase_sleep) { int err = nanosleep(&(struct timespec){ .tv_sec=bd->cfg->erase_sleep/1000000000, @@ -415,7 +424,7 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { NULL); if (err) { err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", err); return err; } } @@ -429,98 +438,142 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { } } - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", 0); return 0; } -int lfs_testbd_sync(const struct lfs_config *cfg) { - LFS_TESTBD_TRACE("lfs_testbd_sync(%p)", (void*)cfg); +int lfs_emubd_sync(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_sync(%p)", (void*)cfg); // do nothing (void)cfg; - LFS_TESTBD_TRACE("lfs_testbd_sync -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_sync -> %d", 0); return 0; } +/// Additional extended API for driving test features /// -// simulated wear operations +lfs_emubd_sio_t lfs_emubd_getread(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_getread(%p)", (void*)cfg); + lfs_emubd_t *bd = cfg->context; + LFS_EMUBD_TRACE("lfs_emubd_getread -> %"PRIu64, bd->read); + return bd->read; +} -lfs_testbd_swear_t lfs_testbd_getwear(const struct lfs_config *cfg, +lfs_emubd_sio_t lfs_emubd_getprog(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_getprog(%p)", (void*)cfg); + lfs_emubd_t *bd = cfg->context; + LFS_EMUBD_TRACE("lfs_emubd_getprog -> %"PRIu64, bd->prog); + return bd->prog; +} + +lfs_emubd_sio_t lfs_emubd_geterased(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_geterased(%p)", (void*)cfg); + lfs_emubd_t *bd = cfg->context; + LFS_EMUBD_TRACE("lfs_emubd_geterased -> %"PRIu64, bd->erased); + return bd->erased; +} + +int lfs_emubd_setread(const struct lfs_config *cfg, lfs_emubd_io_t read) { + LFS_EMUBD_TRACE("lfs_emubd_setread(%p, %"PRIu64")", (void*)cfg, read); + lfs_emubd_t *bd = cfg->context; + bd->read = read; + LFS_EMUBD_TRACE("lfs_emubd_setread -> %d", 0); + return 0; +} + +int lfs_emubd_setprog(const struct lfs_config *cfg, lfs_emubd_io_t prog) { + LFS_EMUBD_TRACE("lfs_emubd_setprog(%p, %"PRIu64")", (void*)cfg, prog); + lfs_emubd_t *bd = cfg->context; + bd->prog = prog; + LFS_EMUBD_TRACE("lfs_emubd_setprog -> %d", 0); + return 0; +} + +int lfs_emubd_seterased(const struct lfs_config *cfg, lfs_emubd_io_t erased) { + LFS_EMUBD_TRACE("lfs_emubd_seterased(%p, %"PRIu64")", (void*)cfg, erased); + lfs_emubd_t *bd = cfg->context; + bd->erased = erased; + LFS_EMUBD_TRACE("lfs_emubd_seterased -> %d", 0); + return 0; +} + +lfs_emubd_swear_t lfs_emubd_getwear(const struct lfs_config *cfg, lfs_block_t block) { - LFS_TESTBD_TRACE("lfs_testbd_getwear(%p, %"PRIu32")", (void*)cfg, block); - lfs_testbd_t *bd = cfg->context; + LFS_EMUBD_TRACE("lfs_emubd_getwear(%p, %"PRIu32")", (void*)cfg, block); + lfs_emubd_t *bd = cfg->context; // check if block is valid LFS_ASSERT(block < cfg->block_count); // get the wear - lfs_testbd_wear_t wear; - const lfs_testbd_block_t *b = bd->blocks[block]; + lfs_emubd_wear_t wear; + const lfs_emubd_block_t *b = bd->blocks[block]; if (b) { wear = b->wear; } else { wear = 0; } - LFS_TESTBD_TRACE("lfs_testbd_getwear -> %"PRIu32, wear); + LFS_EMUBD_TRACE("lfs_emubd_getwear -> %"PRIu32, wear); return wear; } -int lfs_testbd_setwear(const struct lfs_config *cfg, - lfs_block_t block, lfs_testbd_wear_t wear) { - LFS_TESTBD_TRACE("lfs_testbd_setwear(%p, %"PRIu32")", (void*)cfg, block); - lfs_testbd_t *bd = cfg->context; +int lfs_emubd_setwear(const struct lfs_config *cfg, + lfs_block_t block, lfs_emubd_wear_t wear) { + LFS_EMUBD_TRACE("lfs_emubd_setwear(%p, %"PRIu32")", (void*)cfg, block); + lfs_emubd_t *bd = cfg->context; // check if block is valid LFS_ASSERT(block < cfg->block_count); // set the wear - lfs_testbd_block_t *b = lfs_testbd_mutblock(cfg, &bd->blocks[block]); + lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); if (!b) { - LFS_TESTBD_TRACE("lfs_testbd_setwear -> %"PRIu32, LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_setwear -> %"PRIu32, LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } b->wear = wear; - LFS_TESTBD_TRACE("lfs_testbd_setwear -> %"PRIu32, 0); + LFS_EMUBD_TRACE("lfs_emubd_setwear -> %"PRIu32, 0); return 0; } -lfs_testbd_spowercycles_t lfs_testbd_getpowercycles( +lfs_emubd_spowercycles_t lfs_emubd_getpowercycles( const struct lfs_config *cfg) { - LFS_TESTBD_TRACE("lfs_testbd_getpowercycles(%p)", (void*)cfg); - lfs_testbd_t *bd = cfg->context; + LFS_EMUBD_TRACE("lfs_emubd_getpowercycles(%p)", (void*)cfg); + lfs_emubd_t *bd = cfg->context; - LFS_TESTBD_TRACE("lfs_testbd_getpowercycles -> %"PRIi32, bd->power_cycles); + LFS_EMUBD_TRACE("lfs_emubd_getpowercycles -> %"PRIi32, bd->power_cycles); return bd->power_cycles; } -int lfs_testbd_setpowercycles(const struct lfs_config *cfg, - lfs_testbd_powercycles_t power_cycles) { - LFS_TESTBD_TRACE("lfs_testbd_setpowercycles(%p, %"PRIi32")", +int lfs_emubd_setpowercycles(const struct lfs_config *cfg, + lfs_emubd_powercycles_t power_cycles) { + LFS_EMUBD_TRACE("lfs_emubd_setpowercycles(%p, %"PRIi32")", (void*)cfg, power_cycles); - lfs_testbd_t *bd = cfg->context; + lfs_emubd_t *bd = cfg->context; bd->power_cycles = power_cycles; - LFS_TESTBD_TRACE("lfs_testbd_getpowercycles -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_getpowercycles -> %d", 0); return 0; } -int lfs_testbd_copy(const struct lfs_config *cfg, lfs_testbd_t *copy) { - LFS_TESTBD_TRACE("lfs_testbd_copy(%p, %p)", (void*)cfg, (void*)copy); - lfs_testbd_t *bd = cfg->context; +int lfs_emubd_copy(const struct lfs_config *cfg, lfs_emubd_t *copy) { + LFS_EMUBD_TRACE("lfs_emubd_copy(%p, %p)", (void*)cfg, (void*)copy); + lfs_emubd_t *bd = cfg->context; // lazily copy over our block array - copy->blocks = malloc(cfg->block_count * sizeof(lfs_testbd_block_t*)); + copy->blocks = malloc(cfg->block_count * sizeof(lfs_emubd_block_t*)); if (!copy->blocks) { - LFS_TESTBD_TRACE("lfs_testbd_copy -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_copy -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } for (size_t i = 0; i < cfg->block_count; i++) { - copy->blocks[i] = lfs_testbd_incblock(bd->blocks[i]); + copy->blocks[i] = lfs_emubd_incblock(bd->blocks[i]); } // other state @@ -531,7 +584,7 @@ int lfs_testbd_copy(const struct lfs_config *cfg, lfs_testbd_t *copy) { } copy->cfg = bd->cfg; - LFS_TESTBD_TRACE("lfs_testbd_copy -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_copy -> %d", 0); return 0; } diff --git a/bd/lfs_testbd.h b/bd/lfs_emubd.h similarity index 52% rename from bd/lfs_testbd.h rename to bd/lfs_emubd.h index b0c9d005..8aff161f 100644 --- a/bd/lfs_testbd.h +++ b/bd/lfs_emubd.h @@ -1,13 +1,13 @@ /* - * Testing block device, wraps filebd and rambd while providing a bunch + * Emulating block device, wraps filebd and rambd while providing a bunch * of hooks for testing littlefs in various conditions. * * Copyright (c) 2022, The littlefs authors. * Copyright (c) 2017, Arm Limited. All rights reserved. * SPDX-License-Identifier: BSD-3-Clause */ -#ifndef LFS_TESTBD_H -#define LFS_TESTBD_H +#ifndef LFS_EMUBD_H +#define LFS_EMUBD_H #include "lfs.h" #include "lfs_util.h" @@ -21,11 +21,11 @@ extern "C" // Block device specific tracing -#ifndef LFS_TESTBD_TRACE -#ifdef LFS_TESTBD_YES_TRACE -#define LFS_TESTBD_TRACE(...) LFS_TRACE(__VA_ARGS__) +#ifndef LFS_EMUBD_TRACE +#ifdef LFS_EMUBD_YES_TRACE +#define LFS_EMUBD_TRACE(...) LFS_TRACE(__VA_ARGS__) #else -#define LFS_TESTBD_TRACE(...) +#define LFS_EMUBD_TRACE(...) #endif #endif @@ -35,34 +35,38 @@ extern "C" // // Not that read-noop is not allowed. Read _must_ return a consistent (but // may be arbitrary) value on every read. -typedef enum lfs_testbd_badblock_behavior { - LFS_TESTBD_BADBLOCK_PROGERROR, - LFS_TESTBD_BADBLOCK_ERASEERROR, - LFS_TESTBD_BADBLOCK_READERROR, - LFS_TESTBD_BADBLOCK_PROGNOOP, - LFS_TESTBD_BADBLOCK_ERASENOOP, -} lfs_testbd_badblock_behavior_t; +typedef enum lfs_emubd_badblock_behavior { + LFS_EMUBD_BADBLOCK_PROGERROR, + LFS_EMUBD_BADBLOCK_ERASEERROR, + LFS_EMUBD_BADBLOCK_READERROR, + LFS_EMUBD_BADBLOCK_PROGNOOP, + LFS_EMUBD_BADBLOCK_ERASENOOP, +} lfs_emubd_badblock_behavior_t; // Mode determining how power-loss behaves during testing. For now this // only supports a noop behavior, leaving the data on-disk untouched. -typedef enum lfs_testbd_powerloss_behavior { - LFS_TESTBD_POWERLOSS_NOOP, -} lfs_testbd_powerloss_behavior_t; +typedef enum lfs_emubd_powerloss_behavior { + LFS_EMUBD_POWERLOSS_NOOP, +} lfs_emubd_powerloss_behavior_t; + +// Type for measuring read/program/erase operations +typedef uint64_t lfs_emubd_io_t; +typedef int64_t lfs_emubd_sio_t; // Type for measuring wear -typedef uint32_t lfs_testbd_wear_t; -typedef int32_t lfs_testbd_swear_t; +typedef uint32_t lfs_emubd_wear_t; +typedef int32_t lfs_emubd_swear_t; // Type for tracking power-cycles -typedef uint32_t lfs_testbd_powercycles_t; -typedef int32_t lfs_testbd_spowercycles_t; +typedef uint32_t lfs_emubd_powercycles_t; +typedef int32_t lfs_emubd_spowercycles_t; // Type for delays in nanoseconds -typedef uint64_t lfs_testbd_sleep_t; -typedef int64_t lfs_testbd_ssleep_t; +typedef uint64_t lfs_emubd_sleep_t; +typedef int64_t lfs_emubd_ssleep_t; -// testbd config, this is required for testing -struct lfs_testbd_config { +// emubd config, this is required for testing +struct lfs_emubd_config { // 8-bit erase value to use for simulating erases. -1 does not simulate // erases, which can speed up testing by avoiding the extra block-device // operations to store the erase value. @@ -73,15 +77,15 @@ struct lfs_testbd_config { uint32_t erase_cycles; // The mode determining how bad-blocks fail - lfs_testbd_badblock_behavior_t badblock_behavior; + lfs_emubd_badblock_behavior_t badblock_behavior; // Number of write operations (erase/prog) before triggering a power-loss. // power_cycles=0 disables this. The exact behavior of power-loss is // controlled by a combination of powerloss_behavior and powerloss_cb. - lfs_testbd_powercycles_t power_cycles; + lfs_emubd_powercycles_t power_cycles; // The mode determining how power-loss affects disk - lfs_testbd_powerloss_behavior_t powerloss_behavior; + lfs_emubd_powerloss_behavior_t powerloss_behavior; // Function to call to emulate power-loss. The exact behavior of power-loss // is up to the runner to provide. @@ -100,98 +104,119 @@ struct lfs_testbd_config { // Artificial delay in nanoseconds, there is no purpose for this other // than slowing down the simulation. - lfs_testbd_sleep_t read_sleep; + lfs_emubd_sleep_t read_sleep; // Artificial delay in nanoseconds, there is no purpose for this other // than slowing down the simulation. - lfs_testbd_sleep_t prog_sleep; + lfs_emubd_sleep_t prog_sleep; // Artificial delay in nanoseconds, there is no purpose for this other // than slowing down the simulation. - lfs_testbd_sleep_t erase_sleep; + lfs_emubd_sleep_t erase_sleep; }; // A reference counted block -typedef struct lfs_testbd_block { +typedef struct lfs_emubd_block { uint32_t rc; - lfs_testbd_wear_t wear; + lfs_emubd_wear_t wear; uint8_t data[]; -} lfs_testbd_block_t; +} lfs_emubd_block_t; // Disk mirror -typedef struct lfs_testbd_disk { +typedef struct lfs_emubd_disk { uint32_t rc; int fd; uint8_t *scratch; -} lfs_testbd_disk_t; +} lfs_emubd_disk_t; -// testbd state -typedef struct lfs_testbd { +// emubd state +typedef struct lfs_emubd { // array of copy-on-write blocks - lfs_testbd_block_t **blocks; + lfs_emubd_block_t **blocks; // some other test state - uint32_t power_cycles; - lfs_testbd_disk_t *disk; + lfs_emubd_io_t read; + lfs_emubd_io_t prog; + lfs_emubd_io_t erased; + lfs_emubd_powercycles_t power_cycles; + lfs_emubd_disk_t *disk; - const struct lfs_testbd_config *cfg; -} lfs_testbd_t; + const struct lfs_emubd_config *cfg; +} lfs_emubd_t; /// Block device API /// -// Create a test block device using the geometry in lfs_config +// Create an emulating block device using the geometry in lfs_config // // Note that filebd is used if a path is provided, if path is NULL -// testbd will use rambd which can be much faster. -int lfs_testbd_create(const struct lfs_config *cfg, const char *path); -int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, - const struct lfs_testbd_config *bdcfg); +// emubd will use rambd which can be much faster. +int lfs_emubd_create(const struct lfs_config *cfg, const char *path); +int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, + const struct lfs_emubd_config *bdcfg); // Clean up memory associated with block device -int lfs_testbd_destroy(const struct lfs_config *cfg); +int lfs_emubd_destroy(const struct lfs_config *cfg); // Read a block -int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, +int lfs_emubd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size); // Program a block // // The block must have previously been erased. -int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, +int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size); // Erase a block // // A block must be erased before being programmed. The // state of an erased block is undefined. -int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block); +int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block); // Sync the block device -int lfs_testbd_sync(const struct lfs_config *cfg); +int lfs_emubd_sync(const struct lfs_config *cfg); /// Additional extended API for driving test features /// +// Get total amount of bytes read +lfs_emubd_sio_t lfs_emubd_getread(const struct lfs_config *cfg); + +// Get total amount of bytes programmed +lfs_emubd_sio_t lfs_emubd_getprog(const struct lfs_config *cfg); + +// Get total amount of bytes erased +lfs_emubd_sio_t lfs_emubd_geterased(const struct lfs_config *cfg); + +// Manually set amount of bytes read +int lfs_emubd_setread(const struct lfs_config *cfg, lfs_emubd_io_t read); + +// Manually set amount of bytes programmed +int lfs_emubd_setprog(const struct lfs_config *cfg, lfs_emubd_io_t prog); + +// Manually set amount of bytes erased +int lfs_emubd_seterased(const struct lfs_config *cfg, lfs_emubd_io_t erased); + // Get simulated wear on a given block -lfs_testbd_swear_t lfs_testbd_getwear(const struct lfs_config *cfg, +lfs_emubd_swear_t lfs_emubd_getwear(const struct lfs_config *cfg, lfs_block_t block); // Manually set simulated wear on a given block -int lfs_testbd_setwear(const struct lfs_config *cfg, - lfs_block_t block, lfs_testbd_wear_t wear); +int lfs_emubd_setwear(const struct lfs_config *cfg, + lfs_block_t block, lfs_emubd_wear_t wear); // Get the remaining power-cycles -lfs_testbd_spowercycles_t lfs_testbd_getpowercycles( +lfs_emubd_spowercycles_t lfs_emubd_getpowercycles( const struct lfs_config *cfg); // Manually set the remaining power-cycles -int lfs_testbd_setpowercycles(const struct lfs_config *cfg, - lfs_testbd_powercycles_t power_cycles); +int lfs_emubd_setpowercycles(const struct lfs_config *cfg, + lfs_emubd_powercycles_t power_cycles); // Create a copy-on-write copy of the state of this block device -int lfs_testbd_copy(const struct lfs_config *cfg, lfs_testbd_t *copy); +int lfs_emubd_copy(const struct lfs_config *cfg, lfs_emubd_t *copy); #ifdef __cplusplus diff --git a/benches/bench_dir.toml b/benches/bench_dir.toml new file mode 100644 index 00000000..937f70d2 --- /dev/null +++ b/benches/bench_dir.toml @@ -0,0 +1,284 @@ + + +# deterministic prng +code = ''' +static uint32_t xorshift32(uint32_t *state) { + uint32_t x = *state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *state = x; + return x; +} +''' + +[cases.bench_dir_open] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.N = 1024 +defines.FILE_SIZE = 8 +defines.CHUNK_SIZE = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + // first create the files + char name[256]; + uint8_t buffer[CHUNK_SIZE]; + for (lfs_size_t i = 0; i < N; i++) { + sprintf(name, "file%08x", i); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint32_t file_prng = i; + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + buffer[k] = xorshift32(&file_prng); + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + } + + // then read the files + BENCH_START(); + uint32_t prng = 42; + for (lfs_size_t i = 0; i < N; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (N-1-i) + : xorshift32(&prng) % N; + sprintf(name, "file%08x", i_); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, LFS_O_RDONLY) => 0; + + uint32_t file_prng = i_; + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + lfs_file_read(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + assert(buffer[k] == xorshift32(&file_prng)); + } + } + + lfs_file_close(&lfs, &file) => 0; + } + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_dir_creat] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.N = 1024 +defines.FILE_SIZE = 8 +defines.CHUNK_SIZE = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + BENCH_START(); + uint32_t prng = 42; + char name[256]; + uint8_t buffer[CHUNK_SIZE]; + for (lfs_size_t i = 0; i < N; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (N-1-i) + : xorshift32(&prng) % N; + sprintf(name, "file%08x", i_); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + + uint32_t file_prng = i_; + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + buffer[k] = xorshift32(&file_prng); + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + } + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_dir_remove] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.N = 1024 +defines.FILE_SIZE = 8 +defines.CHUNK_SIZE = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + // first create the files + char name[256]; + uint8_t buffer[CHUNK_SIZE]; + for (lfs_size_t i = 0; i < N; i++) { + sprintf(name, "file%08x", i); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint32_t file_prng = i; + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + buffer[k] = xorshift32(&file_prng); + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + } + + // then remove the files + BENCH_START(); + uint32_t prng = 42; + for (lfs_size_t i = 0; i < N; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (N-1-i) + : xorshift32(&prng) % N; + sprintf(name, "file%08x", i_); + int err = lfs_remove(&lfs, name); + assert(!err || err == LFS_ERR_NOENT); + } + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_dir_read] +defines.N = 1024 +defines.FILE_SIZE = 8 +defines.CHUNK_SIZE = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + // first create the files + char name[256]; + uint8_t buffer[CHUNK_SIZE]; + for (lfs_size_t i = 0; i < N; i++) { + sprintf(name, "file%08x", i); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint32_t file_prng = i; + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + buffer[k] = xorshift32(&file_prng); + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + } + + // then read the directory + BENCH_START(); + lfs_dir_t dir; + lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; + lfs_dir_read(&lfs, &dir, &info) => 1; + assert(info.type == LFS_TYPE_DIR); + assert(strcmp(info.name, ".") == 0); + lfs_dir_read(&lfs, &dir, &info) => 1; + assert(info.type == LFS_TYPE_DIR); + assert(strcmp(info.name, "..") == 0); + for (int i = 0; i < N; i++) { + sprintf(name, "file%08x", i); + lfs_dir_read(&lfs, &dir, &info) => 1; + assert(info.type == LFS_TYPE_REG); + assert(strcmp(info.name, name) == 0); + } + lfs_dir_read(&lfs, &dir, &info) => 0; + lfs_dir_close(&lfs, &dir) => 0; + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_dir_mkdir] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.N = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + BENCH_START(); + uint32_t prng = 42; + char name[256]; + for (lfs_size_t i = 0; i < N; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (N-1-i) + : xorshift32(&prng) % N; + printf("hm %d\n", i); + sprintf(name, "dir%08x", i_); + int err = lfs_mkdir(&lfs, name); + assert(!err || err == LFS_ERR_EXIST); + } + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_dir_rmdir] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.N = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + // first create the dirs + char name[256]; + for (lfs_size_t i = 0; i < N; i++) { + sprintf(name, "dir%08x", i); + lfs_mkdir(&lfs, name) => 0; + } + + // then remove the dirs + BENCH_START(); + uint32_t prng = 42; + for (lfs_size_t i = 0; i < N; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (N-1-i) + : xorshift32(&prng) % N; + sprintf(name, "dir%08x", i_); + int err = lfs_remove(&lfs, name); + assert(!err || err == LFS_ERR_NOENT); + } + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + + diff --git a/benches/bench_file.toml b/benches/bench_file.toml new file mode 100644 index 00000000..7e556428 --- /dev/null +++ b/benches/bench_file.toml @@ -0,0 +1,109 @@ + + +# deterministic prng +code = ''' +static uint32_t xorshift32(uint32_t *state) { + uint32_t x = *state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *state = x; + return x; +} +''' + +[cases.bench_file_read] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.SIZE = '128*1024' +defines.CHUNK_SIZE = 64 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_size_t chunks = (SIZE+CHUNK_SIZE-1)/CHUNK_SIZE; + + // first write the file + lfs_file_t file; + uint8_t buffer[CHUNK_SIZE]; + lfs_file_open(&lfs, &file, "file", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + for (lfs_size_t i = 0; i < chunks; i++) { + uint32_t chunk_prng = i; + for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) { + buffer[j] = xorshift32(&chunk_prng); + } + + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + lfs_file_close(&lfs, &file) => 0; + + // then read the file + BENCH_START(); + lfs_file_open(&lfs, &file, "file", LFS_O_RDONLY) => 0; + + uint32_t prng = 42; + for (lfs_size_t i = 0; i < chunks; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (chunks-1-i) + : xorshift32(&prng) % chunks; + lfs_file_seek(&lfs, &file, i_*CHUNK_SIZE, LFS_SEEK_SET) + => i_*CHUNK_SIZE; + lfs_file_read(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + + uint32_t chunk_prng = i_; + for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) { + assert(buffer[j] == xorshift32(&chunk_prng)); + } + } + + lfs_file_close(&lfs, &file) => 0; + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_file_write] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.SIZE = '128*1024' +defines.CHUNK_SIZE = 64 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_size_t chunks = (SIZE+CHUNK_SIZE-1)/CHUNK_SIZE; + + BENCH_START(); + lfs_file_t file; + lfs_file_open(&lfs, &file, "file", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint8_t buffer[CHUNK_SIZE]; + uint32_t prng = 42; + for (lfs_size_t i = 0; i < chunks; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (chunks-1-i) + : xorshift32(&prng) % chunks; + uint32_t chunk_prng = i_; + for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) { + buffer[j] = xorshift32(&chunk_prng); + } + + lfs_file_seek(&lfs, &file, i_*CHUNK_SIZE, LFS_SEEK_SET) + => i_*CHUNK_SIZE; + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' diff --git a/benches/bench_superblock.toml b/benches/bench_superblock.toml new file mode 100644 index 00000000..37659d47 --- /dev/null +++ b/benches/bench_superblock.toml @@ -0,0 +1,56 @@ +[cases.bench_superblocks_found] +# support benchmarking with files +defines.N = [0, 1024] +defines.FILE_SIZE = 8 +defines.CHUNK_SIZE = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + // create files? + lfs_mount(&lfs, cfg) => 0; + char name[256]; + uint8_t buffer[CHUNK_SIZE]; + for (lfs_size_t i = 0; i < N; i++) { + sprintf(name, "file%08x", i); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + buffer[k] = i+j+k; + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + } + lfs_unmount(&lfs) => 0; + + BENCH_START(); + lfs_mount(&lfs, cfg) => 0; + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_superblocks_missing] +code = ''' + lfs_t lfs; + + BENCH_START(); + int err = lfs_mount(&lfs, cfg); + assert(err != 0); + BENCH_STOP(); +''' + +[cases.bench_superblocks_format] +code = ''' + lfs_t lfs; + + BENCH_START(); + lfs_format(&lfs, cfg) => 0; + BENCH_STOP(); +''' + diff --git a/runners/bench_runner.c b/runners/bench_runner.c new file mode 100644 index 00000000..39a38f38 --- /dev/null +++ b/runners/bench_runner.c @@ -0,0 +1,1795 @@ + +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 199309L +#endif + +#include "runners/bench_runner.h" +#include "bd/lfs_emubd.h" + +#include +#include +#include +#include +#include +#include +#include +#include + + +// some helpers + +// append to an array with amortized doubling +void *mappend(void **p, + size_t size, + size_t *count, + size_t *capacity) { + uint8_t *p_ = *p; + size_t count_ = *count; + size_t capacity_ = *capacity; + + count_ += 1; + if (count_ > capacity_) { + capacity_ = (2*capacity_ < 4) ? 4 : 2*capacity_; + + p_ = realloc(p_, capacity_*size); + if (!p_) { + return NULL; + } + } + + *p = p_; + *count = count_; + *capacity = capacity_; + return &p_[(count_-1)*size]; +} + +// a quick self-terminating text-safe varint scheme +static void leb16_print(uintmax_t x) { + while (true) { + char nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0); + printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10); + if (x <= 0xf) { + break; + } + x >>= 4; + } +} + +static uintmax_t leb16_parse(const char *s, char **tail) { + uintmax_t x = 0; + size_t i = 0; + while (true) { + uintmax_t nibble = s[i]; + if (nibble >= '0' && nibble <= '9') { + nibble = nibble - '0'; + } else if (nibble >= 'a' && nibble <= 'v') { + nibble = nibble - 'a' + 10; + } else { + // invalid? + if (tail) { + *tail = (char*)s; + } + return 0; + } + + x |= (nibble & 0xf) << (4*i); + i += 1; + if (!(nibble & 0x10)) { + break; + } + } + + if (tail) { + *tail = (char*)s + i; + } + return x; +} + + + +// bench_runner types + +typedef struct bench_geometry { + char short_name; + const char *long_name; + + bench_define_t defines[BENCH_GEOMETRY_DEFINE_COUNT]; +} bench_geometry_t; + +typedef struct bench_id { + const char *name; + const bench_define_t *defines; + size_t define_count; +} bench_id_t; + + +// bench suites are linked into a custom ld section +extern struct bench_suite __start__bench_suites; +extern struct bench_suite __stop__bench_suites; + +const struct bench_suite *bench_suites = &__start__bench_suites; +#define BENCH_SUITE_COUNT \ + ((size_t)(&__stop__bench_suites - &__start__bench_suites)) + + +// bench define management +typedef struct bench_define_map { + const bench_define_t *defines; + size_t count; +} bench_define_map_t; + +typedef struct bench_define_names { + const char *const *names; + size_t count; +} bench_define_names_t; + +intmax_t bench_define_lit(void *data) { + return (intmax_t)data; +} + +#define BENCH_CONST(x) {bench_define_lit, (void*)(uintptr_t)(x)} +#define BENCH_LIT(x) ((bench_define_t)BENCH_CONST(x)) + + +#define BENCH_DEF(k, v) \ + intmax_t bench_define_##k(void *data) { \ + (void)data; \ + return v; \ + } + + BENCH_IMPLICIT_DEFINES +#undef BENCH_DEF + +#define BENCH_DEFINE_MAP_EXPLICIT 0 +#define BENCH_DEFINE_MAP_OVERRIDE 1 +#define BENCH_DEFINE_MAP_PERMUTATION 2 +#define BENCH_DEFINE_MAP_GEOMETRY 3 +#define BENCH_DEFINE_MAP_IMPLICIT 4 +#define BENCH_DEFINE_MAP_COUNT 5 + +bench_define_map_t bench_define_maps[BENCH_DEFINE_MAP_COUNT] = { + [BENCH_DEFINE_MAP_IMPLICIT] = { + (const bench_define_t[BENCH_IMPLICIT_DEFINE_COUNT]) { + #define BENCH_DEF(k, v) \ + [k##_i] = {bench_define_##k, NULL}, + + BENCH_IMPLICIT_DEFINES + #undef BENCH_DEF + }, + BENCH_IMPLICIT_DEFINE_COUNT, + }, +}; + +#define BENCH_DEFINE_NAMES_SUITE 0 +#define BENCH_DEFINE_NAMES_IMPLICIT 1 +#define BENCH_DEFINE_NAMES_COUNT 2 + +bench_define_names_t bench_define_names[BENCH_DEFINE_NAMES_COUNT] = { + [BENCH_DEFINE_NAMES_IMPLICIT] = { + (const char *const[BENCH_IMPLICIT_DEFINE_COUNT]){ + #define BENCH_DEF(k, v) \ + [k##_i] = #k, + + BENCH_IMPLICIT_DEFINES + #undef BENCH_DEF + }, + BENCH_IMPLICIT_DEFINE_COUNT, + }, +}; + +intmax_t *bench_define_cache; +size_t bench_define_cache_count; +unsigned *bench_define_cache_mask; + +const char *bench_define_name(size_t define) { + // lookup in our bench names + for (size_t i = 0; i < BENCH_DEFINE_NAMES_COUNT; i++) { + if (define < bench_define_names[i].count + && bench_define_names[i].names + && bench_define_names[i].names[define]) { + return bench_define_names[i].names[define]; + } + } + + return NULL; +} + +bool bench_define_ispermutation(size_t define) { + // is this define specific to the permutation? + for (size_t i = 0; i < BENCH_DEFINE_MAP_IMPLICIT; i++) { + if (define < bench_define_maps[i].count + && bench_define_maps[i].defines[define].cb) { + return true; + } + } + + return false; +} + +intmax_t bench_define(size_t define) { + // is the define in our cache? + if (define < bench_define_cache_count + && (bench_define_cache_mask[define/(8*sizeof(unsigned))] + & (1 << (define%(8*sizeof(unsigned)))))) { + return bench_define_cache[define]; + } + + // lookup in our bench defines + for (size_t i = 0; i < BENCH_DEFINE_MAP_COUNT; i++) { + if (define < bench_define_maps[i].count + && bench_define_maps[i].defines[define].cb) { + intmax_t v = bench_define_maps[i].defines[define].cb( + bench_define_maps[i].defines[define].data); + + // insert into cache! + bench_define_cache[define] = v; + bench_define_cache_mask[define / (8*sizeof(unsigned))] + |= 1 << (define%(8*sizeof(unsigned))); + + return v; + } + } + + return 0; + + // not found? + const char *name = bench_define_name(define); + fprintf(stderr, "error: undefined define %s (%zd)\n", + name ? name : "(unknown)", + define); + assert(false); + exit(-1); +} + +void bench_define_flush(void) { + // clear cache between permutations + memset(bench_define_cache_mask, 0, + sizeof(unsigned)*( + (bench_define_cache_count+(8*sizeof(unsigned))-1) + / (8*sizeof(unsigned)))); +} + +// geometry updates +const bench_geometry_t *bench_geometry = NULL; + +void bench_define_geometry(const bench_geometry_t *geometry) { + bench_define_maps[BENCH_DEFINE_MAP_GEOMETRY] = (bench_define_map_t){ + geometry->defines, BENCH_GEOMETRY_DEFINE_COUNT}; +} + +// override updates +typedef struct bench_override { + const char *name; + const intmax_t *defines; + size_t permutations; +} bench_override_t; + +const bench_override_t *bench_overrides = NULL; +size_t bench_override_count = 0; + +bench_define_t *bench_override_defines = NULL; +size_t bench_override_define_count = 0; +size_t bench_override_define_permutations = 1; +size_t bench_override_define_capacity = 0; + +// suite/perm updates +void bench_define_suite(const struct bench_suite *suite) { + bench_define_names[BENCH_DEFINE_NAMES_SUITE] = (bench_define_names_t){ + suite->define_names, suite->define_count}; + + // make sure our cache is large enough + if (lfs_max(suite->define_count, BENCH_IMPLICIT_DEFINE_COUNT) + > bench_define_cache_count) { + // align to power of two to avoid any superlinear growth + size_t ncount = 1 << lfs_npw2( + lfs_max(suite->define_count, BENCH_IMPLICIT_DEFINE_COUNT)); + bench_define_cache = realloc(bench_define_cache, ncount*sizeof(intmax_t)); + bench_define_cache_mask = realloc(bench_define_cache_mask, + sizeof(unsigned)*( + (ncount+(8*sizeof(unsigned))-1) + / (8*sizeof(unsigned)))); + bench_define_cache_count = ncount; + } + + // map any overrides + if (bench_override_count > 0) { + // first figure out the total size of override permutations + size_t count = 0; + size_t permutations = 1; + for (size_t i = 0; i < bench_override_count; i++) { + for (size_t d = 0; + d < lfs_max( + suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + // define name match? + const char *name = bench_define_name(d); + if (name && strcmp(name, bench_overrides[i].name) == 0) { + count = lfs_max(count, d+1); + permutations *= bench_overrides[i].permutations; + break; + } + } + } + bench_override_define_count = count; + bench_override_define_permutations = permutations; + + // make sure our override arrays are big enough + if (count * permutations > bench_override_define_capacity) { + // align to power of two to avoid any superlinear growth + size_t ncapacity = 1 << lfs_npw2(count * permutations); + bench_override_defines = realloc( + bench_override_defines, + sizeof(bench_define_t)*ncapacity); + bench_override_define_capacity = ncapacity; + } + + // zero unoverridden defines + memset(bench_override_defines, 0, + sizeof(bench_define_t) * count * permutations); + + // compute permutations + size_t p = 1; + for (size_t i = 0; i < bench_override_count; i++) { + for (size_t d = 0; + d < lfs_max( + suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + // define name match? + const char *name = bench_define_name(d); + if (name && strcmp(name, bench_overrides[i].name) == 0) { + // scatter the define permutations based on already + // seen permutations + for (size_t j = 0; j < permutations; j++) { + bench_override_defines[j*count + d] = BENCH_LIT( + bench_overrides[i].defines[(j/p) + % bench_overrides[i].permutations]); + } + + // keep track of how many permutations we've seen so far + p *= bench_overrides[i].permutations; + break; + } + } + } + } +} + +void bench_define_perm( + const struct bench_suite *suite, + const struct bench_case *case_, + size_t perm) { + if (case_->defines) { + bench_define_maps[BENCH_DEFINE_MAP_PERMUTATION] = (bench_define_map_t){ + case_->defines + perm*suite->define_count, + suite->define_count}; + } else { + bench_define_maps[BENCH_DEFINE_MAP_PERMUTATION] = (bench_define_map_t){ + NULL, 0}; + } +} + +void bench_define_override(size_t perm) { + bench_define_maps[BENCH_DEFINE_MAP_OVERRIDE] = (bench_define_map_t){ + bench_override_defines + perm*bench_override_define_count, + bench_override_define_count}; +} + +void bench_define_explicit( + const bench_define_t *defines, + size_t define_count) { + bench_define_maps[BENCH_DEFINE_MAP_EXPLICIT] = (bench_define_map_t){ + defines, define_count}; +} + +void bench_define_cleanup(void) { + // bench define management can allocate a few things + free(bench_define_cache); + free(bench_define_cache_mask); + free(bench_override_defines); +} + + + +// bench state +extern const bench_geometry_t *bench_geometries; +extern size_t bench_geometry_count; + +const bench_id_t *bench_ids = (const bench_id_t[]) { + {NULL, NULL, 0}, +}; +size_t bench_id_count = 1; + +size_t bench_step_start = 0; +size_t bench_step_stop = -1; +size_t bench_step_step = 1; + +const char *bench_disk_path = NULL; +const char *bench_trace_path = NULL; +FILE *bench_trace_file = NULL; +uint32_t bench_trace_cycles = 0; +lfs_emubd_sleep_t bench_read_sleep = 0.0; +lfs_emubd_sleep_t bench_prog_sleep = 0.0; +lfs_emubd_sleep_t bench_erase_sleep = 0.0; + + +// trace printing +void bench_trace(const char *fmt, ...) { + if (bench_trace_path) { + if (!bench_trace_file) { + // Tracing output is heavy and trying to open every trace + // call is slow, so we only try to open the trace file every + // so often. Note this doesn't affect successfully opened files + if (bench_trace_cycles % 128 != 0) { + bench_trace_cycles += 1; + return; + } + bench_trace_cycles += 1; + + int fd; + if (strcmp(bench_trace_path, "-") == 0) { + fd = dup(1); + if (fd < 0) { + return; + } + } else { + fd = open( + bench_trace_path, + O_WRONLY | O_CREAT | O_APPEND | O_NONBLOCK, + 0666); + if (fd < 0) { + return; + } + int err = fcntl(fd, F_SETFL, O_WRONLY | O_CREAT | O_APPEND); + assert(!err); + } + + FILE *f = fdopen(fd, "a"); + assert(f); + int err = setvbuf(f, NULL, _IOLBF, BUFSIZ); + assert(!err); + bench_trace_file = f; + } + + va_list va; + va_start(va, fmt); + int res = vfprintf(bench_trace_file, fmt, va); + if (res < 0) { + fclose(bench_trace_file); + bench_trace_file = NULL; + } + va_end(va); + } +} + + +// bench recording state +static struct lfs_config *bench_cfg = NULL; +static lfs_emubd_io_t bench_last_read = 0; +static lfs_emubd_io_t bench_last_prog = 0; +static lfs_emubd_io_t bench_last_erased = 0; +lfs_emubd_io_t bench_read = 0; +lfs_emubd_io_t bench_prog = 0; +lfs_emubd_io_t bench_erased = 0; + +void bench_reset(void) { + bench_read = 0; + bench_prog = 0; + bench_erased = 0; + bench_last_read = 0; + bench_last_prog = 0; + bench_last_erased = 0; +} + +void bench_start(void) { + assert(bench_cfg); + lfs_emubd_sio_t read = lfs_emubd_getread(bench_cfg); + assert(read >= 0); + lfs_emubd_sio_t prog = lfs_emubd_getprog(bench_cfg); + assert(prog >= 0); + lfs_emubd_sio_t erased = lfs_emubd_geterased(bench_cfg); + assert(erased >= 0); + + bench_last_read = read; + bench_last_prog = prog; + bench_last_erased = erased; +} + +void bench_stop(void) { + assert(bench_cfg); + lfs_emubd_sio_t read = lfs_emubd_getread(bench_cfg); + assert(read >= 0); + lfs_emubd_sio_t prog = lfs_emubd_getprog(bench_cfg); + assert(prog >= 0); + lfs_emubd_sio_t erased = lfs_emubd_geterased(bench_cfg); + assert(erased >= 0); + + bench_read += read - bench_last_read; + bench_prog += prog - bench_last_prog; + bench_erased += erased - bench_last_erased; +} + + +// encode our permutation into a reusable id +static void perm_printid( + const struct bench_suite *suite, + const struct bench_case *case_) { + (void)suite; + // case[:permutation] + printf("%s:", case_->name); + for (size_t d = 0; + d < lfs_max( + suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + if (bench_define_ispermutation(d)) { + leb16_print(d); + leb16_print(BENCH_DEFINE(d)); + } + } +} + +// iterate through permutations in a bench case +static void case_forperm( + const struct bench_suite *suite, + const struct bench_case *case_, + const bench_define_t *defines, + size_t define_count, + void (*cb)( + void *data, + const struct bench_suite *suite, + const struct bench_case *case_), + void *data) { + if (defines) { + bench_define_explicit(defines, define_count); + bench_define_flush(); + + cb(data, suite, case_); + } else { + for (size_t k = 0; k < case_->permutations; k++) { + // define permutation + bench_define_perm(suite, case_, k); + + for (size_t v = 0; v < bench_override_define_permutations; v++) { + // define override permutation + bench_define_override(v); + + for (size_t g = 0; g < bench_geometry_count; g++) { + // define geometry + bench_define_geometry(&bench_geometries[g]); + bench_define_flush(); + + cb(data, suite, case_); + } + } + } + } +} + + +// how many permutations are there actually in a bench case +struct perm_count_state { + size_t total; + size_t filtered; +}; + +void perm_count( + void *data, + const struct bench_suite *suite, + const struct bench_case *case_) { + struct perm_count_state *state = data; + (void)suite; + (void)case_; + + state->total += 1; + + if (case_->filter && !case_->filter()) { + return; + } + + state->filtered += 1; +} + + +// operations we can do +static void summary(void) { + printf("%-36s %7s %7s %7s %11s\n", + "", "flags", "suites", "cases", "perms"); + size_t suites = 0; + size_t cases = 0; + bench_flags_t flags = 0; + struct perm_count_state perms = {0, 0}; + + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + cases += 1; + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_count, + &perms); + } + + suites += 1; + flags |= bench_suites[i].flags; + } + } + + char perm_buf[64]; + sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); + char flag_buf[64]; + sprintf(flag_buf, "%s%s", + (flags & BENCH_REENTRANT) ? "r" : "", + (!flags) ? "-" : ""); + printf("%-36s %7s %7zu %7zu %11s\n", + "TOTAL", + flag_buf, + suites, + cases, + perm_buf); +} + +static void list_suites(void) { + printf("%-36s %7s %7s %11s\n", "suite", "flags", "cases", "perms"); + + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + size_t cases = 0; + struct perm_count_state perms = {0, 0}; + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + cases += 1; + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_count, + &perms); + } + + // no benches found? + if (!cases) { + continue; + } + + char perm_buf[64]; + sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); + char flag_buf[64]; + sprintf(flag_buf, "%s%s", + (bench_suites[i].flags & BENCH_REENTRANT) ? "r" : "", + (!bench_suites[i].flags) ? "-" : ""); + printf("%-36s %7s %7zu %11s\n", + bench_suites[i].name, + flag_buf, + cases, + perm_buf); + } + } +} + +static void list_cases(void) { + printf("%-36s %7s %11s\n", "case", "flags", "perms"); + + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + struct perm_count_state perms = {0, 0}; + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_count, + &perms); + + char perm_buf[64]; + sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); + char flag_buf[64]; + sprintf(flag_buf, "%s%s", + (bench_suites[i].cases[j].flags & BENCH_REENTRANT) + ? "r" : "", + (!bench_suites[i].cases[j].flags) + ? "-" : ""); + printf("%-36s %7s %11s\n", + bench_suites[i].cases[j].name, + flag_buf, + perm_buf); + } + } + } +} + +static void list_suite_paths(void) { + printf("%-36s %s\n", "suite", "path"); + + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + size_t cases = 0; + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + } + + // no benches found? + if (!cases) { + continue; + } + + printf("%-36s %s\n", + bench_suites[i].name, + bench_suites[i].path); + } + } +} + +static void list_case_paths(void) { + printf("%-36s %s\n", "case", "path"); + + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + printf("%-36s %s\n", + bench_suites[i].cases[j].name, + bench_suites[i].cases[j].path); + } + } + } +} + +struct list_defines_define { + const char *name; + intmax_t *values; + size_t value_count; + size_t value_capacity; +}; + +struct list_defines_defines { + struct list_defines_define *defines; + size_t define_count; + size_t define_capacity; +}; + +static void list_defines_add( + struct list_defines_defines *defines, + size_t d) { + const char *name = bench_define_name(d); + intmax_t value = BENCH_DEFINE(d); + + // define already in defines? + for (size_t i = 0; i < defines->define_count; i++) { + if (strcmp(defines->defines[i].name, name) == 0) { + // value already in values? + for (size_t j = 0; j < defines->defines[i].value_count; j++) { + if (defines->defines[i].values[j] == value) { + return; + } + } + + *(intmax_t*)mappend( + (void**)&defines->defines[i].values, + sizeof(intmax_t), + &defines->defines[i].value_count, + &defines->defines[i].value_capacity) = value; + + return; + } + } + + // new define? + struct list_defines_define *define = mappend( + (void**)&defines->defines, + sizeof(struct list_defines_define), + &defines->define_count, + &defines->define_capacity); + define->name = name; + define->values = malloc(sizeof(intmax_t)); + define->values[0] = value; + define->value_count = 1; + define->value_capacity = 1; +} + +void perm_list_defines( + void *data, + const struct bench_suite *suite, + const struct bench_case *case_) { + struct list_defines_defines *defines = data; + (void)suite; + (void)case_; + + // collect defines + for (size_t d = 0; + d < lfs_max(suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + if (d < BENCH_IMPLICIT_DEFINE_COUNT + || bench_define_ispermutation(d)) { + list_defines_add(defines, d); + } + } +} + +void perm_list_permutation_defines( + void *data, + const struct bench_suite *suite, + const struct bench_case *case_) { + struct list_defines_defines *defines = data; + (void)suite; + (void)case_; + + // collect permutation_defines + for (size_t d = 0; + d < lfs_max(suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + if (bench_define_ispermutation(d)) { + list_defines_add(defines, d); + } + } +} + +extern const bench_geometry_t builtin_geometries[]; + +static void list_defines(void) { + struct list_defines_defines defines = {NULL, 0, 0}; + + // add defines + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_list_defines, + &defines); + } + } + } + + for (size_t i = 0; i < defines.define_count; i++) { + printf("%s=", defines.defines[i].name); + for (size_t j = 0; j < defines.defines[i].value_count; j++) { + printf("%jd", defines.defines[i].values[j]); + if (j != defines.defines[i].value_count-1) { + printf(","); + } + } + printf("\n"); + } + + for (size_t i = 0; i < defines.define_count; i++) { + free(defines.defines[i].values); + } + free(defines.defines); +} + +static void list_permutation_defines(void) { + struct list_defines_defines defines = {NULL, 0, 0}; + + // add permutation defines + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_list_permutation_defines, + &defines); + } + } + } + + for (size_t i = 0; i < defines.define_count; i++) { + printf("%s=", defines.defines[i].name); + for (size_t j = 0; j < defines.defines[i].value_count; j++) { + printf("%jd", defines.defines[i].values[j]); + if (j != defines.defines[i].value_count-1) { + printf(","); + } + } + printf("\n"); + } + + for (size_t i = 0; i < defines.define_count; i++) { + free(defines.defines[i].values); + } + free(defines.defines); +} + +static void list_implicit_defines(void) { + struct list_defines_defines defines = {NULL, 0, 0}; + + // yes we do need to define a suite, this does a bit of bookeeping + // such as setting up the define cache + bench_define_suite(&(const struct bench_suite){0}); + + // make sure to include builtin geometries here + extern const bench_geometry_t builtin_geometries[]; + for (size_t g = 0; builtin_geometries[g].long_name; g++) { + bench_define_geometry(&builtin_geometries[g]); + bench_define_flush(); + + // add implicit defines + for (size_t d = 0; d < BENCH_IMPLICIT_DEFINE_COUNT; d++) { + list_defines_add(&defines, d); + } + } + + for (size_t i = 0; i < defines.define_count; i++) { + printf("%s=", defines.defines[i].name); + for (size_t j = 0; j < defines.defines[i].value_count; j++) { + printf("%jd", defines.defines[i].values[j]); + if (j != defines.defines[i].value_count-1) { + printf(","); + } + } + printf("\n"); + } + + for (size_t i = 0; i < defines.define_count; i++) { + free(defines.defines[i].values); + } + free(defines.defines); +} + + + +// geometries to bench + +const bench_geometry_t builtin_geometries[] = { + {'d', "default", {{NULL}, BENCH_CONST(16), BENCH_CONST(512), {NULL}}}, + {'e', "eeprom", {{NULL}, BENCH_CONST(1), BENCH_CONST(512), {NULL}}}, + {'E', "emmc", {{NULL}, {NULL}, BENCH_CONST(512), {NULL}}}, + {'n', "nor", {{NULL}, BENCH_CONST(1), BENCH_CONST(4096), {NULL}}}, + {'N', "nand", {{NULL}, BENCH_CONST(4096), BENCH_CONST(32768), {NULL}}}, + {0, NULL, {{NULL}, {NULL}, {NULL}, {NULL}}}, +}; + +const bench_geometry_t *bench_geometries = builtin_geometries; +size_t bench_geometry_count = 5; + +static void list_geometries(void) { + // yes we do need to define a suite, this does a bit of bookeeping + // such as setting up the define cache + bench_define_suite(&(const struct bench_suite){0}); + + printf("%-24s %7s %7s %7s %7s %11s\n", + "geometry", "read", "prog", "erase", "count", "size"); + for (size_t g = 0; builtin_geometries[g].long_name; g++) { + bench_define_geometry(&builtin_geometries[g]); + bench_define_flush(); + printf("%c,%-22s %7ju %7ju %7ju %7ju %11ju\n", + builtin_geometries[g].short_name, + builtin_geometries[g].long_name, + READ_SIZE, + PROG_SIZE, + BLOCK_SIZE, + BLOCK_COUNT, + BLOCK_SIZE*BLOCK_COUNT); + } +} + + + +// global bench step count +size_t bench_step = 0; + +void perm_run( + void *data, + const struct bench_suite *suite, + const struct bench_case *case_) { + (void)data; + + // skip this step? + if (!(bench_step >= bench_step_start + && bench_step < bench_step_stop + && (bench_step-bench_step_start) % bench_step_step == 0)) { + bench_step += 1; + return; + } + bench_step += 1; + + // filter? + if (case_->filter && !case_->filter()) { + printf("skipped "); + perm_printid(suite, case_); + printf("\n"); + return; + } + + // create block device and configuration + lfs_emubd_t bd; + + struct lfs_config cfg = { + .context = &bd, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .block_size = BLOCK_SIZE, + .block_count = BLOCK_COUNT, + .block_cycles = BLOCK_CYCLES, + .cache_size = CACHE_SIZE, + .lookahead_size = LOOKAHEAD_SIZE, + }; + + struct lfs_emubd_config bdcfg = { + .erase_value = ERASE_VALUE, + .erase_cycles = ERASE_CYCLES, + .badblock_behavior = BADBLOCK_BEHAVIOR, + .disk_path = bench_disk_path, + .read_sleep = bench_read_sleep, + .prog_sleep = bench_prog_sleep, + .erase_sleep = bench_erase_sleep, + }; + + int err = lfs_emubd_createcfg(&cfg, bench_disk_path, &bdcfg); + if (err) { + fprintf(stderr, "error: could not create block device: %d\n", err); + exit(-1); + } + + // run the bench + bench_cfg = &cfg; + bench_reset(); + printf("running "); + perm_printid(suite, case_); + printf("\n"); + + case_->run(&cfg); + + printf("finished "); + perm_printid(suite, case_); + printf(" %"PRIu64" %"PRIu64" %"PRIu64, + bench_read, + bench_prog, + bench_erased); + printf("\n"); + + // cleanup + err = lfs_emubd_destroy(&cfg); + if (err) { + fprintf(stderr, "error: could not destroy block device: %d\n", err); + exit(-1); + } +} + +static void run(void) { + // ignore disconnected pipes + signal(SIGPIPE, SIG_IGN); + + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_run, + NULL); + } + } + } +} + + + +// option handling +enum opt_flags { + OPT_HELP = 'h', + OPT_SUMMARY = 'Y', + OPT_LIST_SUITES = 'l', + OPT_LIST_CASES = 'L', + OPT_LIST_SUITE_PATHS = 1, + OPT_LIST_CASE_PATHS = 2, + OPT_LIST_DEFINES = 3, + OPT_LIST_PERMUTATION_DEFINES = 4, + OPT_LIST_IMPLICIT_DEFINES = 5, + OPT_LIST_GEOMETRIES = 6, + OPT_DEFINE = 'D', + OPT_GEOMETRY = 'g', + OPT_STEP = 's', + OPT_DISK = 'd', + OPT_TRACE = 't', + OPT_READ_SLEEP = 7, + OPT_PROG_SLEEP = 8, + OPT_ERASE_SLEEP = 9, +}; + +const char *short_opts = "hYlLD:g:s:d:t:"; + +const struct option long_opts[] = { + {"help", no_argument, NULL, OPT_HELP}, + {"summary", no_argument, NULL, OPT_SUMMARY}, + {"list-suites", no_argument, NULL, OPT_LIST_SUITES}, + {"list-cases", no_argument, NULL, OPT_LIST_CASES}, + {"list-suite-paths", no_argument, NULL, OPT_LIST_SUITE_PATHS}, + {"list-case-paths", no_argument, NULL, OPT_LIST_CASE_PATHS}, + {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, + {"list-permutation-defines", + no_argument, NULL, OPT_LIST_PERMUTATION_DEFINES}, + {"list-implicit-defines", + no_argument, NULL, OPT_LIST_IMPLICIT_DEFINES}, + {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, + {"define", required_argument, NULL, OPT_DEFINE}, + {"geometry", required_argument, NULL, OPT_GEOMETRY}, + {"step", required_argument, NULL, OPT_STEP}, + {"disk", required_argument, NULL, OPT_DISK}, + {"trace", required_argument, NULL, OPT_TRACE}, + {"read-sleep", required_argument, NULL, OPT_READ_SLEEP}, + {"prog-sleep", required_argument, NULL, OPT_PROG_SLEEP}, + {"erase-sleep", required_argument, NULL, OPT_ERASE_SLEEP}, + {NULL, 0, NULL, 0}, +}; + +const char *const help_text[] = { + "Show this help message.", + "Show quick summary.", + "List bench suites.", + "List bench cases.", + "List the path for each bench suite.", + "List the path and line number for each bench case.", + "List all defines in this bench-runner.", + "List explicit defines in this bench-runner.", + "List implicit defines in this bench-runner.", + "List the available disk geometries.", + "Override a bench define.", + "Comma-separated list of disk geometries to bench. Defaults to d,e,E,n,N.", + "Comma-separated range of bench permutations to run (start,stop,step).", + "Redirect block device operations to this file.", + "Redirect trace output to this file.", + "Artificial read delay in seconds.", + "Artificial prog delay in seconds.", + "Artificial erase delay in seconds.", +}; + +int main(int argc, char **argv) { + void (*op)(void) = run; + + size_t bench_override_capacity = 0; + size_t bench_geometry_capacity = 0; + size_t bench_id_capacity = 0; + + // parse options + while (true) { + int c = getopt_long(argc, argv, short_opts, long_opts, NULL); + switch (c) { + // generate help message + case OPT_HELP: { + printf("usage: %s [options] [bench_id]\n", argv[0]); + printf("\n"); + + printf("options:\n"); + size_t i = 0; + while (long_opts[i].name) { + size_t indent; + if (long_opts[i].has_arg == no_argument) { + if (long_opts[i].val >= '0' && long_opts[i].val < 'z') { + indent = printf(" -%c, --%s ", + long_opts[i].val, + long_opts[i].name); + } else { + indent = printf(" --%s ", + long_opts[i].name); + } + } else { + if (long_opts[i].val >= '0' && long_opts[i].val < 'z') { + indent = printf(" -%c %s, --%s %s ", + long_opts[i].val, + long_opts[i].name, + long_opts[i].name, + long_opts[i].name); + } else { + indent = printf(" --%s %s ", + long_opts[i].name, + long_opts[i].name); + } + } + + // a quick, hacky, byte-level method for text wrapping + size_t len = strlen(help_text[i]); + size_t j = 0; + if (indent < 24) { + printf("%*s %.80s\n", + (int)(24-1-indent), + "", + &help_text[i][j]); + j += 80; + } else { + printf("\n"); + } + + while (j < len) { + printf("%24s%.80s\n", "", &help_text[i][j]); + j += 80; + } + + i += 1; + } + + printf("\n"); + exit(0); + } + // summary/list flags + case OPT_SUMMARY: + op = summary; + break; + case OPT_LIST_SUITES: + op = list_suites; + break; + case OPT_LIST_CASES: + op = list_cases; + break; + case OPT_LIST_SUITE_PATHS: + op = list_suite_paths; + break; + case OPT_LIST_CASE_PATHS: + op = list_case_paths; + break; + case OPT_LIST_DEFINES: + op = list_defines; + break; + case OPT_LIST_PERMUTATION_DEFINES: + op = list_permutation_defines; + break; + case OPT_LIST_IMPLICIT_DEFINES: + op = list_implicit_defines; + break; + case OPT_LIST_GEOMETRIES: + op = list_geometries; + break; + // configuration + case OPT_DEFINE: { + // allocate space + bench_override_t *override = mappend( + (void**)&bench_overrides, + sizeof(bench_override_t), + &bench_override_count, + &bench_override_capacity); + + // parse into string key/intmax_t value, cannibalizing the + // arg in the process + char *sep = strchr(optarg, '='); + char *parsed = NULL; + if (!sep) { + goto invalid_define; + } + *sep = '\0'; + override->name = optarg; + optarg = sep+1; + + // parse comma-separated permutations + { + override->defines = NULL; + override->permutations = 0; + size_t override_capacity = 0; + while (true) { + optarg += strspn(optarg, " "); + + if (strncmp(optarg, "range", strlen("range")) == 0) { + // range of values + optarg += strlen("range"); + optarg += strspn(optarg, " "); + if (*optarg != '(') { + goto invalid_define; + } + optarg += 1; + + intmax_t start = strtoumax(optarg, &parsed, 0); + intmax_t stop = -1; + intmax_t step = 1; + // allow empty string for start=0 + if (parsed == optarg) { + start = 0; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != ')') { + goto invalid_define; + } + + if (*optarg == ',') { + optarg += 1; + stop = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=end + if (parsed == optarg) { + stop = -1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != ')') { + goto invalid_define; + } + + if (*optarg == ',') { + optarg += 1; + step = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=1 + if (parsed == optarg) { + step = 1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ')') { + goto invalid_define; + } + } + } else { + // single value = stop only + stop = start; + start = 0; + } + + if (*optarg != ')') { + goto invalid_define; + } + optarg += 1; + + // calculate the range of values + assert(step != 0); + for (intmax_t i = start; + (step < 0) + ? i > stop + : (uintmax_t)i < (uintmax_t)stop; + i += step) { + *(intmax_t*)mappend( + (void**)&override->defines, + sizeof(intmax_t), + &override->permutations, + &override_capacity) = i; + } + } else if (*optarg != '\0') { + // single value + intmax_t define = strtoimax(optarg, &parsed, 0); + if (parsed == optarg) { + goto invalid_define; + } + optarg = parsed + strspn(parsed, " "); + *(intmax_t*)mappend( + (void**)&override->defines, + sizeof(intmax_t), + &override->permutations, + &override_capacity) = define; + } else { + break; + } + + if (*optarg == ',') { + optarg += 1; + } + } + } + assert(override->permutations > 0); + break; + +invalid_define: + fprintf(stderr, "error: invalid define: %s\n", optarg); + exit(-1); + } + case OPT_GEOMETRY: { + // reset our geometry scenarios + if (bench_geometry_capacity > 0) { + free((bench_geometry_t*)bench_geometries); + } + bench_geometries = NULL; + bench_geometry_count = 0; + bench_geometry_capacity = 0; + + // parse the comma separated list of disk geometries + while (*optarg) { + // allocate space + bench_geometry_t *geometry = mappend( + (void**)&bench_geometries, + sizeof(bench_geometry_t), + &bench_geometry_count, + &bench_geometry_capacity); + + // parse the disk geometry + optarg += strspn(optarg, " "); + + // named disk geometry + size_t len = strcspn(optarg, " ,"); + for (size_t i = 0; builtin_geometries[i].long_name; i++) { + if ((len == 1 + && *optarg == builtin_geometries[i].short_name) + || (len == strlen( + builtin_geometries[i].long_name) + && memcmp(optarg, + builtin_geometries[i].long_name, + len) == 0)) { + *geometry = builtin_geometries[i]; + optarg += len; + goto geometry_next; + } + } + + // comma-separated read/prog/erase/count + if (*optarg == '{') { + lfs_size_t sizes[4]; + size_t count = 0; + + char *s = optarg + 1; + while (count < 4) { + char *parsed = NULL; + sizes[count] = strtoumax(s, &parsed, 0); + count += 1; + + s = parsed + strspn(parsed, " "); + if (*s == ',') { + s += 1; + continue; + } else if (*s == '}') { + s += 1; + break; + } else { + goto geometry_unknown; + } + } + + // allow implicit r=p and p=e for common geometries + memset(geometry, 0, sizeof(bench_geometry_t)); + if (count >= 3) { + geometry->defines[READ_SIZE_i] + = BENCH_LIT(sizes[0]); + geometry->defines[PROG_SIZE_i] + = BENCH_LIT(sizes[1]); + geometry->defines[BLOCK_SIZE_i] + = BENCH_LIT(sizes[2]); + } else if (count >= 2) { + geometry->defines[PROG_SIZE_i] + = BENCH_LIT(sizes[0]); + geometry->defines[BLOCK_SIZE_i] + = BENCH_LIT(sizes[1]); + } else { + geometry->defines[BLOCK_SIZE_i] + = BENCH_LIT(sizes[0]); + } + if (count >= 4) { + geometry->defines[BLOCK_COUNT_i] + = BENCH_LIT(sizes[3]); + } + optarg = s; + goto geometry_next; + } + + // leb16-encoded read/prog/erase/count + if (*optarg == ':') { + lfs_size_t sizes[4]; + size_t count = 0; + + char *s = optarg + 1; + while (true) { + char *parsed = NULL; + uintmax_t x = leb16_parse(s, &parsed); + if (parsed == s || count >= 4) { + break; + } + + sizes[count] = x; + count += 1; + s = parsed; + } + + // allow implicit r=p and p=e for common geometries + memset(geometry, 0, sizeof(bench_geometry_t)); + if (count >= 3) { + geometry->defines[READ_SIZE_i] + = BENCH_LIT(sizes[0]); + geometry->defines[PROG_SIZE_i] + = BENCH_LIT(sizes[1]); + geometry->defines[BLOCK_SIZE_i] + = BENCH_LIT(sizes[2]); + } else if (count >= 2) { + geometry->defines[PROG_SIZE_i] + = BENCH_LIT(sizes[0]); + geometry->defines[BLOCK_SIZE_i] + = BENCH_LIT(sizes[1]); + } else { + geometry->defines[BLOCK_SIZE_i] + = BENCH_LIT(sizes[0]); + } + if (count >= 4) { + geometry->defines[BLOCK_COUNT_i] + = BENCH_LIT(sizes[3]); + } + optarg = s; + goto geometry_next; + } + +geometry_unknown: + // unknown scenario? + fprintf(stderr, "error: unknown disk geometry: %s\n", + optarg); + exit(-1); + +geometry_next: + optarg += strspn(optarg, " "); + if (*optarg == ',') { + optarg += 1; + } else if (*optarg == '\0') { + break; + } else { + goto geometry_unknown; + } + } + break; + } + case OPT_STEP: { + char *parsed = NULL; + bench_step_start = strtoumax(optarg, &parsed, 0); + bench_step_stop = -1; + bench_step_step = 1; + // allow empty string for start=0 + if (parsed == optarg) { + bench_step_start = 0; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != '\0') { + goto step_unknown; + } + + if (*optarg == ',') { + optarg += 1; + bench_step_stop = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=end + if (parsed == optarg) { + bench_step_stop = -1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != '\0') { + goto step_unknown; + } + + if (*optarg == ',') { + optarg += 1; + bench_step_step = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=1 + if (parsed == optarg) { + bench_step_step = 1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != '\0') { + goto step_unknown; + } + } + } else { + // single value = stop only + bench_step_stop = bench_step_start; + bench_step_start = 0; + } + + break; +step_unknown: + fprintf(stderr, "error: invalid step: %s\n", optarg); + exit(-1); + } + case OPT_DISK: + bench_disk_path = optarg; + break; + case OPT_TRACE: + bench_trace_path = optarg; + break; + case OPT_READ_SLEEP: { + char *parsed = NULL; + double read_sleep = strtod(optarg, &parsed); + if (parsed == optarg) { + fprintf(stderr, "error: invalid read-sleep: %s\n", optarg); + exit(-1); + } + bench_read_sleep = read_sleep*1.0e9; + break; + } + case OPT_PROG_SLEEP: { + char *parsed = NULL; + double prog_sleep = strtod(optarg, &parsed); + if (parsed == optarg) { + fprintf(stderr, "error: invalid prog-sleep: %s\n", optarg); + exit(-1); + } + bench_prog_sleep = prog_sleep*1.0e9; + break; + } + case OPT_ERASE_SLEEP: { + char *parsed = NULL; + double erase_sleep = strtod(optarg, &parsed); + if (parsed == optarg) { + fprintf(stderr, "error: invalid erase-sleep: %s\n", optarg); + exit(-1); + } + bench_erase_sleep = erase_sleep*1.0e9; + break; + } + // done parsing + case -1: + goto getopt_done; + // unknown arg, getopt prints a message for us + default: + exit(-1); + } + } +getopt_done: ; + + if (argc > optind) { + // reset our bench identifier list + bench_ids = NULL; + bench_id_count = 0; + bench_id_capacity = 0; + } + + // parse bench identifier, if any, cannibalizing the arg in the process + for (; argc > optind; optind++) { + bench_define_t *defines = NULL; + size_t define_count = 0; + + // parse name, can be suite or case + char *name = argv[optind]; + char *defines_ = strchr(name, ':'); + if (defines_) { + *defines_ = '\0'; + defines_ += 1; + } + + // remove optional path and .toml suffix + char *slash = strrchr(name, '/'); + if (slash) { + name = slash+1; + } + + size_t name_len = strlen(name); + if (name_len > 5 && strcmp(&name[name_len-5], ".toml") == 0) { + name[name_len-5] = '\0'; + } + + if (defines_) { + // parse defines + while (true) { + char *parsed; + size_t d = leb16_parse(defines_, &parsed); + intmax_t v = leb16_parse(parsed, &parsed); + if (parsed == defines_) { + break; + } + defines_ = parsed; + + if (d >= define_count) { + // align to power of two to avoid any superlinear growth + size_t ncount = 1 << lfs_npw2(d+1); + defines = realloc(defines, + ncount*sizeof(bench_define_t)); + memset(defines+define_count, 0, + (ncount-define_count)*sizeof(bench_define_t)); + define_count = ncount; + } + defines[d] = BENCH_LIT(v); + } + } + + // append to identifier list + *(bench_id_t*)mappend( + (void**)&bench_ids, + sizeof(bench_id_t), + &bench_id_count, + &bench_id_capacity) = (bench_id_t){ + .name = name, + .defines = defines, + .define_count = define_count, + }; + } + + // do the thing + op(); + + // cleanup (need to be done for valgrind benching) + bench_define_cleanup(); + if (bench_overrides) { + for (size_t i = 0; i < bench_override_count; i++) { + free((void*)bench_overrides[i].defines); + } + free((void*)bench_overrides); + } + if (bench_geometry_capacity) { + free((void*)bench_geometries); + } + if (bench_id_capacity) { + for (size_t i = 0; i < bench_id_count; i++) { + free((void*)bench_ids[i].defines); + } + free((void*)bench_ids); + } +} diff --git a/runners/bench_runner.h b/runners/bench_runner.h new file mode 100644 index 00000000..a33c31d8 --- /dev/null +++ b/runners/bench_runner.h @@ -0,0 +1,119 @@ +#ifndef BENCH_RUNNER_H +#define BENCH_RUNNER_H + + +// override LFS_TRACE +void bench_trace(const char *fmt, ...); + +#define LFS_TRACE_(fmt, ...) \ + bench_trace("%s:%d:trace: " fmt "%s\n", \ + __FILE__, \ + __LINE__, \ + __VA_ARGS__) +#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") +#define LFS_EMUBD_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") + +// provide BENCH_START/BENCH_STOP macros +void bench_start(void); +void bench_stop(void); + +#define BENCH_START() bench_start() +#define BENCH_STOP() bench_stop() + + +// note these are indirectly included in any generated files +#include "bd/lfs_emubd.h" +#include + +// give source a chance to define feature macros +#undef _FEATURES_H +#undef _STDIO_H + + +// generated bench configurations +struct lfs_config; + +enum bench_flags { + BENCH_REENTRANT = 0x1, +}; +typedef uint8_t bench_flags_t; + +typedef struct bench_define { + intmax_t (*cb)(void *data); + void *data; +} bench_define_t; + +struct bench_case { + const char *name; + const char *path; + bench_flags_t flags; + size_t permutations; + + const bench_define_t *defines; + + bool (*filter)(void); + void (*run)(struct lfs_config *cfg); +}; + +struct bench_suite { + const char *name; + const char *path; + bench_flags_t flags; + + const char *const *define_names; + size_t define_count; + + const struct bench_case *cases; + size_t case_count; +}; + + +// access generated bench defines +intmax_t bench_define(size_t define); + +#define BENCH_DEFINE(i) bench_define(i) + +// a few preconfigured defines that control how benches run + +#define READ_SIZE_i 0 +#define PROG_SIZE_i 1 +#define BLOCK_SIZE_i 2 +#define BLOCK_COUNT_i 3 +#define CACHE_SIZE_i 4 +#define LOOKAHEAD_SIZE_i 5 +#define BLOCK_CYCLES_i 6 +#define ERASE_VALUE_i 7 +#define ERASE_CYCLES_i 8 +#define BADBLOCK_BEHAVIOR_i 9 +#define POWERLOSS_BEHAVIOR_i 10 + +#define READ_SIZE bench_define(READ_SIZE_i) +#define PROG_SIZE bench_define(PROG_SIZE_i) +#define BLOCK_SIZE bench_define(BLOCK_SIZE_i) +#define BLOCK_COUNT bench_define(BLOCK_COUNT_i) +#define CACHE_SIZE bench_define(CACHE_SIZE_i) +#define LOOKAHEAD_SIZE bench_define(LOOKAHEAD_SIZE_i) +#define BLOCK_CYCLES bench_define(BLOCK_CYCLES_i) +#define ERASE_VALUE bench_define(ERASE_VALUE_i) +#define ERASE_CYCLES bench_define(ERASE_CYCLES_i) +#define BADBLOCK_BEHAVIOR bench_define(BADBLOCK_BEHAVIOR_i) +#define POWERLOSS_BEHAVIOR bench_define(POWERLOSS_BEHAVIOR_i) + +#define BENCH_IMPLICIT_DEFINES \ + BENCH_DEF(READ_SIZE, PROG_SIZE) \ + BENCH_DEF(PROG_SIZE, BLOCK_SIZE) \ + BENCH_DEF(BLOCK_SIZE, 0) \ + BENCH_DEF(BLOCK_COUNT, (1024*1024)/BLOCK_SIZE) \ + BENCH_DEF(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ + BENCH_DEF(LOOKAHEAD_SIZE, 16) \ + BENCH_DEF(BLOCK_CYCLES, -1) \ + BENCH_DEF(ERASE_VALUE, 0xff) \ + BENCH_DEF(ERASE_CYCLES, 0) \ + BENCH_DEF(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR) \ + BENCH_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP) + +#define BENCH_GEOMETRY_DEFINE_COUNT 4 +#define BENCH_IMPLICIT_DEFINE_COUNT 11 + + +#endif diff --git a/runners/test_runner.c b/runners/test_runner.c index 55f237dc..8333e388 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -4,7 +4,7 @@ #endif #include "runners/test_runner.h" -#include "bd/lfs_testbd.h" +#include "bd/lfs_emubd.h" #include #include @@ -46,7 +46,7 @@ void *mappend(void **p, // a quick self-terminating text-safe varint scheme static void leb16_print(uintmax_t x) { while (true) { - lfs_testbd_powercycles_t nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0); + char nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0); printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10); if (x <= 0xf) { break; @@ -101,11 +101,11 @@ typedef struct test_powerloss { const char *long_name; void (*run)( - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, const struct test_suite *suite, const struct test_case *case_); - const lfs_testbd_powercycles_t *cycles; + const lfs_emubd_powercycles_t *cycles; size_t cycle_count; } test_powerloss_t; @@ -113,7 +113,7 @@ typedef struct test_id { const char *name; const test_define_t *defines; size_t define_count; - const lfs_testbd_powercycles_t *cycles; + const lfs_emubd_powercycles_t *cycles; size_t cycle_count; } test_id_t; @@ -141,17 +141,19 @@ typedef struct test_define_names { intmax_t test_define_lit(void *data) { return (intmax_t)data; } -#define TEST_LIT(x) {test_define_lit, (void*)(uintptr_t)(x)} + +#define TEST_CONST(x) {test_define_lit, (void*)(uintptr_t)(x)} +#define TEST_LIT(x) ((test_define_t)TEST_CONST(x)) -#define TEST_DEFINE(k, v) \ +#define TEST_DEF(k, v) \ intmax_t test_define_##k(void *data) { \ (void)data; \ return v; \ } TEST_IMPLICIT_DEFINES -#undef TEST_DEFINE +#undef TEST_DEF #define TEST_DEFINE_MAP_EXPLICIT 0 #define TEST_DEFINE_MAP_OVERRIDE 1 @@ -163,11 +165,11 @@ intmax_t test_define_lit(void *data) { test_define_map_t test_define_maps[TEST_DEFINE_MAP_COUNT] = { [TEST_DEFINE_MAP_IMPLICIT] = { (const test_define_t[TEST_IMPLICIT_DEFINE_COUNT]) { - #define TEST_DEFINE(k, v) \ + #define TEST_DEF(k, v) \ [k##_i] = {test_define_##k, NULL}, TEST_IMPLICIT_DEFINES - #undef TEST_DEFINE + #undef TEST_DEF }, TEST_IMPLICIT_DEFINE_COUNT, }, @@ -180,11 +182,11 @@ test_define_map_t test_define_maps[TEST_DEFINE_MAP_COUNT] = { test_define_names_t test_define_names[TEST_DEFINE_NAMES_COUNT] = { [TEST_DEFINE_NAMES_IMPLICIT] = { (const char *const[TEST_IMPLICIT_DEFINE_COUNT]){ - #define TEST_DEFINE(k, v) \ + #define TEST_DEF(k, v) \ [k##_i] = #k, TEST_IMPLICIT_DEFINES - #undef TEST_DEFINE + #undef TEST_DEF }, TEST_IMPLICIT_DEFINE_COUNT, }, @@ -318,7 +320,7 @@ void test_define_suite(const struct test_suite *suite) { // define name match? const char *name = test_define_name(d); if (name && strcmp(name, test_overrides[i].name) == 0) { - count = d+1; + count = lfs_max(count, d+1); permutations *= test_overrides[i].permutations; break; } @@ -355,10 +357,9 @@ void test_define_suite(const struct test_suite *suite) { // scatter the define permutations based on already // seen permutations for (size_t j = 0; j < permutations; j++) { - test_override_defines[j*count + d] - = (test_define_t)TEST_LIT( - test_overrides[i].defines[(j/p) - % test_overrides[i].permutations]); + test_override_defines[j*count + d] = TEST_LIT( + test_overrides[i].defines[(j/p) + % test_overrides[i].permutations]); } // keep track of how many permutations we've seen so far @@ -426,9 +427,9 @@ const char *test_disk_path = NULL; const char *test_trace_path = NULL; FILE *test_trace_file = NULL; uint32_t test_trace_cycles = 0; -lfs_testbd_sleep_t test_read_sleep = 0.0; -lfs_testbd_sleep_t test_prog_sleep = 0.0; -lfs_testbd_sleep_t test_erase_sleep = 0.0; +lfs_emubd_sleep_t test_read_sleep = 0.0; +lfs_emubd_sleep_t test_prog_sleep = 0.0; +lfs_emubd_sleep_t test_erase_sleep = 0.0; // trace printing @@ -485,10 +486,10 @@ void test_trace(const char *fmt, ...) { static void perm_printid( const struct test_suite *suite, const struct test_case *case_, - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count) { (void)suite; - // case[:permutation[:powercycles]]] + // case[:permutation[:powercycles]] printf("%s:", case_->name); for (size_t d = 0; d < lfs_max( @@ -497,7 +498,7 @@ static void perm_printid( d++) { if (test_define_ispermutation(d)) { leb16_print(d); - leb16_print(test_define(d)); + leb16_print(TEST_DEFINE(d)); } } @@ -511,7 +512,7 @@ static void perm_printid( } static void run_powerloss_cycles( - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, const struct test_suite *suite, const struct test_case *case_); @@ -522,7 +523,7 @@ static void case_forperm( const struct test_case *case_, const test_define_t *defines, size_t define_count, - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, void (*cb)( void *data, @@ -834,7 +835,7 @@ static void list_defines_add( struct list_defines_defines *defines, size_t d) { const char *name = test_define_name(d); - intmax_t value = test_define(d); + intmax_t value = TEST_DEFINE(d); // define already in defines? for (size_t i = 0; i < defines->define_count; i++) { @@ -1051,11 +1052,11 @@ static void list_implicit_defines(void) { // geometries to test const test_geometry_t builtin_geometries[] = { - {'d', "default", {{NULL}, TEST_LIT(16), TEST_LIT(512), {NULL}}}, - {'e', "eeprom", {{NULL}, TEST_LIT(1), TEST_LIT(512), {NULL}}}, - {'E', "emmc", {{NULL}, {NULL}, TEST_LIT(512), {NULL}}}, - {'n', "nor", {{NULL}, TEST_LIT(1), TEST_LIT(4096), {NULL}}}, - {'N', "nand", {{NULL}, TEST_LIT(4096), TEST_LIT(32768), {NULL}}}, + {'d', "default", {{NULL}, TEST_CONST(16), TEST_CONST(512), {NULL}}}, + {'e', "eeprom", {{NULL}, TEST_CONST(1), TEST_CONST(512), {NULL}}}, + {'E', "emmc", {{NULL}, {NULL}, TEST_CONST(512), {NULL}}}, + {'n', "nor", {{NULL}, TEST_CONST(1), TEST_CONST(4096), {NULL}}}, + {'N', "nand", {{NULL}, TEST_CONST(4096), TEST_CONST(32768), {NULL}}}, {0, NULL, {{NULL}, {NULL}, {NULL}, {NULL}}}, }; @@ -1087,7 +1088,7 @@ static void list_geometries(void) { // scenarios to run tests under power-loss static void run_powerloss_none( - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, const struct test_suite *suite, const struct test_case *case_) { @@ -1096,14 +1097,14 @@ static void run_powerloss_none( (void)suite; // create block device and configuration - lfs_testbd_t bd; + lfs_emubd_t bd; struct lfs_config cfg = { .context = &bd, - .read = lfs_testbd_read, - .prog = lfs_testbd_prog, - .erase = lfs_testbd_erase, - .sync = lfs_testbd_sync, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, .block_size = BLOCK_SIZE, @@ -1113,7 +1114,7 @@ static void run_powerloss_none( .lookahead_size = LOOKAHEAD_SIZE, }; - struct lfs_testbd_config bdcfg = { + struct lfs_emubd_config bdcfg = { .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1123,7 +1124,7 @@ static void run_powerloss_none( .erase_sleep = test_erase_sleep, }; - int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1141,7 +1142,7 @@ static void run_powerloss_none( printf("\n"); // cleanup - err = lfs_testbd_destroy(&cfg); + err = lfs_emubd_destroy(&cfg); if (err) { fprintf(stderr, "error: could not destroy block device: %d\n", err); exit(-1); @@ -1154,7 +1155,7 @@ static void powerloss_longjmp(void *c) { } static void run_powerloss_linear( - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, const struct test_suite *suite, const struct test_case *case_) { @@ -1163,16 +1164,16 @@ static void run_powerloss_linear( (void)suite; // create block device and configuration - lfs_testbd_t bd; + lfs_emubd_t bd; jmp_buf powerloss_jmp; - volatile lfs_testbd_powercycles_t i = 1; + volatile lfs_emubd_powercycles_t i = 1; struct lfs_config cfg = { .context = &bd, - .read = lfs_testbd_read, - .prog = lfs_testbd_prog, - .erase = lfs_testbd_erase, - .sync = lfs_testbd_sync, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, .block_size = BLOCK_SIZE, @@ -1182,7 +1183,7 @@ static void run_powerloss_linear( .lookahead_size = LOOKAHEAD_SIZE, }; - struct lfs_testbd_config bdcfg = { + struct lfs_emubd_config bdcfg = { .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1196,7 +1197,7 @@ static void run_powerloss_linear( .powerloss_data = &powerloss_jmp, }; - int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1218,13 +1219,13 @@ static void run_powerloss_linear( printf("powerloss "); perm_printid(suite, case_, NULL, 0); printf(":"); - for (lfs_testbd_powercycles_t j = 1; j <= i; j++) { + for (lfs_emubd_powercycles_t j = 1; j <= i; j++) { leb16_print(j); } printf("\n"); i += 1; - lfs_testbd_setpowercycles(&cfg, i); + lfs_emubd_setpowercycles(&cfg, i); } printf("finished "); @@ -1232,7 +1233,7 @@ static void run_powerloss_linear( printf("\n"); // cleanup - err = lfs_testbd_destroy(&cfg); + err = lfs_emubd_destroy(&cfg); if (err) { fprintf(stderr, "error: could not destroy block device: %d\n", err); exit(-1); @@ -1240,7 +1241,7 @@ static void run_powerloss_linear( } static void run_powerloss_exponential( - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, const struct test_suite *suite, const struct test_case *case_) { @@ -1249,16 +1250,16 @@ static void run_powerloss_exponential( (void)suite; // create block device and configuration - lfs_testbd_t bd; + lfs_emubd_t bd; jmp_buf powerloss_jmp; - volatile lfs_testbd_powercycles_t i = 1; + volatile lfs_emubd_powercycles_t i = 1; struct lfs_config cfg = { .context = &bd, - .read = lfs_testbd_read, - .prog = lfs_testbd_prog, - .erase = lfs_testbd_erase, - .sync = lfs_testbd_sync, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, .block_size = BLOCK_SIZE, @@ -1268,7 +1269,7 @@ static void run_powerloss_exponential( .lookahead_size = LOOKAHEAD_SIZE, }; - struct lfs_testbd_config bdcfg = { + struct lfs_emubd_config bdcfg = { .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1282,7 +1283,7 @@ static void run_powerloss_exponential( .powerloss_data = &powerloss_jmp, }; - int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1304,13 +1305,13 @@ static void run_powerloss_exponential( printf("powerloss "); perm_printid(suite, case_, NULL, 0); printf(":"); - for (lfs_testbd_powercycles_t j = 1; j <= i; j *= 2) { + for (lfs_emubd_powercycles_t j = 1; j <= i; j *= 2) { leb16_print(j); } printf("\n"); i *= 2; - lfs_testbd_setpowercycles(&cfg, i); + lfs_emubd_setpowercycles(&cfg, i); } printf("finished "); @@ -1318,7 +1319,7 @@ static void run_powerloss_exponential( printf("\n"); // cleanup - err = lfs_testbd_destroy(&cfg); + err = lfs_emubd_destroy(&cfg); if (err) { fprintf(stderr, "error: could not destroy block device: %d\n", err); exit(-1); @@ -1326,23 +1327,23 @@ static void run_powerloss_exponential( } static void run_powerloss_cycles( - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, const struct test_suite *suite, const struct test_case *case_) { (void)suite; // create block device and configuration - lfs_testbd_t bd; + lfs_emubd_t bd; jmp_buf powerloss_jmp; volatile size_t i = 0; struct lfs_config cfg = { .context = &bd, - .read = lfs_testbd_read, - .prog = lfs_testbd_prog, - .erase = lfs_testbd_erase, - .sync = lfs_testbd_sync, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, .block_size = BLOCK_SIZE, @@ -1352,7 +1353,7 @@ static void run_powerloss_cycles( .lookahead_size = LOOKAHEAD_SIZE, }; - struct lfs_testbd_config bdcfg = { + struct lfs_emubd_config bdcfg = { .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1366,7 +1367,7 @@ static void run_powerloss_cycles( .powerloss_data = &powerloss_jmp, }; - int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1391,7 +1392,7 @@ static void run_powerloss_cycles( printf("\n"); i += 1; - lfs_testbd_setpowercycles(&cfg, + lfs_emubd_setpowercycles(&cfg, (i < cycle_count) ? cycles[i] : 0); } @@ -1400,7 +1401,7 @@ static void run_powerloss_cycles( printf("\n"); // cleanup - err = lfs_testbd_destroy(&cfg); + err = lfs_emubd_destroy(&cfg); if (err) { fprintf(stderr, "error: could not destroy block device: %d\n", err); exit(-1); @@ -1410,13 +1411,13 @@ static void run_powerloss_cycles( struct powerloss_exhaustive_state { struct lfs_config *cfg; - lfs_testbd_t *branches; + lfs_emubd_t *branches; size_t branch_count; size_t branch_capacity; }; struct powerloss_exhaustive_cycles { - lfs_testbd_powercycles_t *cycles; + lfs_emubd_powercycles_t *cycles; size_t cycle_count; size_t cycle_capacity; }; @@ -1424,9 +1425,9 @@ struct powerloss_exhaustive_cycles { static void powerloss_exhaustive_branch(void *c) { struct powerloss_exhaustive_state *state = c; // append to branches - lfs_testbd_t *branch = mappend( + lfs_emubd_t *branch = mappend( (void**)&state->branches, - sizeof(lfs_testbd_t), + sizeof(lfs_emubd_t), &state->branch_count, &state->branch_capacity); if (!branch) { @@ -1435,14 +1436,14 @@ static void powerloss_exhaustive_branch(void *c) { } // create copy-on-write copy - int err = lfs_testbd_copy(state->cfg, branch); + int err = lfs_emubd_copy(state->cfg, branch); if (err) { fprintf(stderr, "error: exhaustive: could not create bd copy\n"); exit(-1); } // also trigger on next power cycle - lfs_testbd_setpowercycles(state->cfg, 1); + lfs_emubd_setpowercycles(state->cfg, 1); } static void run_powerloss_exhaustive_layer( @@ -1450,7 +1451,7 @@ static void run_powerloss_exhaustive_layer( const struct test_suite *suite, const struct test_case *case_, struct lfs_config *cfg, - struct lfs_testbd_config *bdcfg, + struct lfs_emubd_config *bdcfg, size_t depth) { (void)suite; @@ -1463,14 +1464,14 @@ static void run_powerloss_exhaustive_layer( // run through the test without additional powerlosses, collecting possible // branches as we do so - lfs_testbd_setpowercycles(state.cfg, depth > 0 ? 1 : 0); + lfs_emubd_setpowercycles(state.cfg, depth > 0 ? 1 : 0); bdcfg->powerloss_data = &state; // run the tests case_->run(cfg); // aggressively clean up memory here to try to keep our memory usage low - int err = lfs_testbd_destroy(cfg); + int err = lfs_emubd_destroy(cfg); if (err) { fprintf(stderr, "error: could not destroy block device: %d\n", err); exit(-1); @@ -1479,9 +1480,9 @@ static void run_powerloss_exhaustive_layer( // recurse into each branch for (size_t i = 0; i < state.branch_count; i++) { // first push and print the branch - lfs_testbd_powercycles_t *cycle = mappend( + lfs_emubd_powercycles_t *cycle = mappend( (void**)&cycles->cycles, - sizeof(lfs_testbd_powercycles_t), + sizeof(lfs_emubd_powercycles_t), &cycles->cycle_count, &cycles->cycle_capacity); if (!cycle) { @@ -1509,7 +1510,7 @@ static void run_powerloss_exhaustive_layer( } static void run_powerloss_exhaustive( - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, const struct test_suite *suite, const struct test_case *case_) { @@ -1517,14 +1518,14 @@ static void run_powerloss_exhaustive( (void)suite; // create block device and configuration - lfs_testbd_t bd; + lfs_emubd_t bd; struct lfs_config cfg = { .context = &bd, - .read = lfs_testbd_read, - .prog = lfs_testbd_prog, - .erase = lfs_testbd_erase, - .sync = lfs_testbd_sync, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, .block_size = BLOCK_SIZE, @@ -1534,7 +1535,7 @@ static void run_powerloss_exhaustive( .lookahead_size = LOOKAHEAD_SIZE, }; - struct lfs_testbd_config bdcfg = { + struct lfs_emubd_config bdcfg = { .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1547,7 +1548,7 @@ static void run_powerloss_exhaustive( .powerloss_data = NULL, }; - int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1956,6 +1957,7 @@ int main(int argc, char **argv) { if (parsed == optarg) { goto invalid_define; } + optarg = parsed + strspn(parsed, " "); *(intmax_t*)mappend( (void**)&override->defines, sizeof(intmax_t), @@ -1965,7 +1967,6 @@ int main(int argc, char **argv) { break; } - optarg = parsed + strspn(parsed, " "); if (*optarg == ',') { optarg += 1; } @@ -2041,24 +2042,24 @@ invalid_define: // allow implicit r=p and p=e for common geometries memset(geometry, 0, sizeof(test_geometry_t)); if (count >= 3) { - geometry->defines[0] - = (test_define_t)TEST_LIT(sizes[0]); - geometry->defines[1] - = (test_define_t)TEST_LIT(sizes[1]); - geometry->defines[2] - = (test_define_t)TEST_LIT(sizes[2]); + geometry->defines[READ_SIZE_i] + = TEST_LIT(sizes[0]); + geometry->defines[PROG_SIZE_i] + = TEST_LIT(sizes[1]); + geometry->defines[BLOCK_SIZE_i] + = TEST_LIT(sizes[2]); } else if (count >= 2) { - geometry->defines[1] - = (test_define_t)TEST_LIT(sizes[0]); - geometry->defines[2] - = (test_define_t)TEST_LIT(sizes[1]); + geometry->defines[PROG_SIZE_i] + = TEST_LIT(sizes[0]); + geometry->defines[BLOCK_SIZE_i] + = TEST_LIT(sizes[1]); } else { - geometry->defines[2] - = (test_define_t)TEST_LIT(sizes[0]); + geometry->defines[BLOCK_SIZE_i] + = TEST_LIT(sizes[0]); } if (count >= 4) { - geometry->defines[3] - = (test_define_t)TEST_LIT(sizes[3]); + geometry->defines[BLOCK_COUNT_i] + = TEST_LIT(sizes[3]); } optarg = s; goto geometry_next; @@ -2085,24 +2086,24 @@ invalid_define: // allow implicit r=p and p=e for common geometries memset(geometry, 0, sizeof(test_geometry_t)); if (count >= 3) { - geometry->defines[0] - = (test_define_t)TEST_LIT(sizes[0]); - geometry->defines[1] - = (test_define_t)TEST_LIT(sizes[1]); - geometry->defines[2] - = (test_define_t)TEST_LIT(sizes[2]); + geometry->defines[READ_SIZE_i] + = TEST_LIT(sizes[0]); + geometry->defines[PROG_SIZE_i] + = TEST_LIT(sizes[1]); + geometry->defines[BLOCK_SIZE_i] + = TEST_LIT(sizes[2]); } else if (count >= 2) { - geometry->defines[1] - = (test_define_t)TEST_LIT(sizes[0]); - geometry->defines[2] - = (test_define_t)TEST_LIT(sizes[1]); + geometry->defines[PROG_SIZE_i] + = TEST_LIT(sizes[0]); + geometry->defines[BLOCK_SIZE_i] + = TEST_LIT(sizes[1]); } else { - geometry->defines[2] - = (test_define_t)TEST_LIT(sizes[0]); + geometry->defines[BLOCK_SIZE_i] + = TEST_LIT(sizes[0]); } if (count >= 4) { - geometry->defines[3] - = (test_define_t)TEST_LIT(sizes[3]); + geometry->defines[BLOCK_COUNT_i] + = TEST_LIT(sizes[3]); } optarg = s; goto geometry_next; @@ -2165,16 +2166,16 @@ geometry_next: // comma-separated permutation if (*optarg == '{') { - lfs_testbd_powercycles_t *cycles = NULL; + lfs_emubd_powercycles_t *cycles = NULL; size_t cycle_count = 0; size_t cycle_capacity = 0; char *s = optarg + 1; while (true) { char *parsed = NULL; - *(lfs_testbd_powercycles_t*)mappend( + *(lfs_emubd_powercycles_t*)mappend( (void**)&cycles, - sizeof(lfs_testbd_powercycles_t), + sizeof(lfs_emubd_powercycles_t), &cycle_count, &cycle_capacity) = strtoumax(s, &parsed, 0); @@ -2202,7 +2203,7 @@ geometry_next: // leb16-encoded permutation if (*optarg == ':') { - lfs_testbd_powercycles_t *cycles = NULL; + lfs_emubd_powercycles_t *cycles = NULL; size_t cycle_count = 0; size_t cycle_capacity = 0; @@ -2214,9 +2215,9 @@ geometry_next: break; } - *(lfs_testbd_powercycles_t*)mappend( + *(lfs_emubd_powercycles_t*)mappend( (void**)&cycles, - sizeof(lfs_testbd_powercycles_t), + sizeof(lfs_emubd_powercycles_t), &cycle_count, &cycle_capacity) = x; s = parsed; @@ -2374,7 +2375,7 @@ getopt_done: ; for (; argc > optind; optind++) { test_define_t *defines = NULL; size_t define_count = 0; - lfs_testbd_powercycles_t *cycles = NULL; + lfs_emubd_powercycles_t *cycles = NULL; size_t cycle_count = 0; // parse name, can be suite or case @@ -2422,7 +2423,7 @@ getopt_done: ; (ncount-define_count)*sizeof(test_define_t)); define_count = ncount; } - defines[d] = (test_define_t)TEST_LIT(v); + defines[d] = TEST_LIT(v); } if (cycles_) { @@ -2430,9 +2431,9 @@ getopt_done: ; size_t cycle_capacity = 0; while (*cycles_ != '\0') { char *parsed = NULL; - *(lfs_testbd_powercycles_t*)mappend( + *(lfs_emubd_powercycles_t*)mappend( (void**)&cycles, - sizeof(lfs_testbd_powercycles_t), + sizeof(lfs_emubd_powercycles_t), &cycle_count, &cycle_capacity) = leb16_parse(cycles_, &parsed); diff --git a/runners/test_runner.h b/runners/test_runner.h index e5986acc..f9561509 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -11,11 +11,11 @@ void test_trace(const char *fmt, ...); __LINE__, \ __VA_ARGS__) #define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") -#define LFS_TESTBD_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") +#define LFS_EMUBD_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") // note these are indirectly included in any generated files -#include "bd/lfs_testbd.h" +#include "bd/lfs_emubd.h" #include // give source a chance to define feature macros @@ -62,9 +62,10 @@ struct test_suite { // access generated test defines -//intmax_t test_predefine(size_t define); intmax_t test_define(size_t define); +#define TEST_DEFINE(i) test_define(i) + // a few preconfigured defines that control how tests run #define READ_SIZE_i 0 @@ -79,33 +80,33 @@ intmax_t test_define(size_t define); #define BADBLOCK_BEHAVIOR_i 9 #define POWERLOSS_BEHAVIOR_i 10 -#define READ_SIZE test_define(READ_SIZE_i) -#define PROG_SIZE test_define(PROG_SIZE_i) -#define BLOCK_SIZE test_define(BLOCK_SIZE_i) -#define BLOCK_COUNT test_define(BLOCK_COUNT_i) -#define CACHE_SIZE test_define(CACHE_SIZE_i) -#define LOOKAHEAD_SIZE test_define(LOOKAHEAD_SIZE_i) -#define BLOCK_CYCLES test_define(BLOCK_CYCLES_i) -#define ERASE_VALUE test_define(ERASE_VALUE_i) -#define ERASE_CYCLES test_define(ERASE_CYCLES_i) -#define BADBLOCK_BEHAVIOR test_define(BADBLOCK_BEHAVIOR_i) -#define POWERLOSS_BEHAVIOR test_define(POWERLOSS_BEHAVIOR_i) +#define READ_SIZE TEST_DEFINE(READ_SIZE_i) +#define PROG_SIZE TEST_DEFINE(PROG_SIZE_i) +#define BLOCK_SIZE TEST_DEFINE(BLOCK_SIZE_i) +#define BLOCK_COUNT TEST_DEFINE(BLOCK_COUNT_i) +#define CACHE_SIZE TEST_DEFINE(CACHE_SIZE_i) +#define LOOKAHEAD_SIZE TEST_DEFINE(LOOKAHEAD_SIZE_i) +#define BLOCK_CYCLES TEST_DEFINE(BLOCK_CYCLES_i) +#define ERASE_VALUE TEST_DEFINE(ERASE_VALUE_i) +#define ERASE_CYCLES TEST_DEFINE(ERASE_CYCLES_i) +#define BADBLOCK_BEHAVIOR TEST_DEFINE(BADBLOCK_BEHAVIOR_i) +#define POWERLOSS_BEHAVIOR TEST_DEFINE(POWERLOSS_BEHAVIOR_i) #define TEST_IMPLICIT_DEFINES \ - TEST_DEFINE(READ_SIZE, PROG_SIZE) \ - TEST_DEFINE(PROG_SIZE, BLOCK_SIZE) \ - TEST_DEFINE(BLOCK_SIZE, 0) \ - TEST_DEFINE(BLOCK_COUNT, (1024*1024)/BLOCK_SIZE) \ - TEST_DEFINE(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ - TEST_DEFINE(LOOKAHEAD_SIZE, 16) \ - TEST_DEFINE(BLOCK_CYCLES, -1) \ - TEST_DEFINE(ERASE_VALUE, 0xff) \ - TEST_DEFINE(ERASE_CYCLES, 0) \ - TEST_DEFINE(BADBLOCK_BEHAVIOR, LFS_TESTBD_BADBLOCK_PROGERROR) \ - TEST_DEFINE(POWERLOSS_BEHAVIOR, LFS_TESTBD_POWERLOSS_NOOP) + TEST_DEF(READ_SIZE, PROG_SIZE) \ + TEST_DEF(PROG_SIZE, BLOCK_SIZE) \ + TEST_DEF(BLOCK_SIZE, 0) \ + TEST_DEF(BLOCK_COUNT, (1024*1024)/BLOCK_SIZE) \ + TEST_DEF(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ + TEST_DEF(LOOKAHEAD_SIZE, 16) \ + TEST_DEF(BLOCK_CYCLES, -1) \ + TEST_DEF(ERASE_VALUE, 0xff) \ + TEST_DEF(ERASE_CYCLES, 0) \ + TEST_DEF(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR) \ + TEST_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP) -#define TEST_GEOMETRY_DEFINE_COUNT 4 #define TEST_IMPLICIT_DEFINE_COUNT 11 +#define TEST_GEOMETRY_DEFINE_COUNT 4 #endif diff --git a/scripts/bench.py b/scripts/bench.py new file mode 100755 index 00000000..93c18a2d --- /dev/null +++ b/scripts/bench.py @@ -0,0 +1,1355 @@ +#!/usr/bin/env python3 +# +# Script to compile and runs benches. +# +# Example: +# ./scripts/bench.py runners/bench_runner -b +# +# Copyright (c) 2022, The littlefs authors. +# SPDX-License-Identifier: BSD-3-Clause +# + +import collections as co +import csv +import errno +import glob +import itertools as it +import math as m +import os +import pty +import re +import shlex +import shutil +import signal +import subprocess as sp +import threading as th +import time +import toml + + +RUNNER_PATH = 'runners/bench_runner' +HEADER_PATH = 'runners/bench_runner.h' + + +def openio(path, mode='r', buffering=-1, nb=False): + if path == '-': + if mode == 'r': + return os.fdopen(os.dup(sys.stdin.fileno()), 'r', buffering) + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w', buffering) + elif nb and 'a' in mode: + return os.fdopen(os.open( + path, + os.O_WRONLY | os.O_CREAT | os.O_APPEND | os.O_NONBLOCK, + 0o666), + mode, + buffering) + else: + return open(path, mode, buffering) + +class BenchCase: + # create a BenchCase object from a config + def __init__(self, config, args={}): + self.name = config.pop('name') + self.path = config.pop('path') + self.suite = config.pop('suite') + self.lineno = config.pop('lineno', None) + self.if_ = config.pop('if', None) + if isinstance(self.if_, bool): + self.if_ = 'true' if self.if_ else 'false' + self.code = config.pop('code') + self.code_lineno = config.pop('code_lineno', None) + self.in_ = config.pop('in', + config.pop('suite_in', None)) + + # figure out defines and build possible permutations + self.defines = set() + self.permutations = [] + + # defines can be a dict or a list or dicts + suite_defines = config.pop('suite_defines', {}) + if not isinstance(suite_defines, list): + suite_defines = [suite_defines] + defines = config.pop('defines', {}) + if not isinstance(defines, list): + defines = [defines] + + def csplit(v): + # split commas but only outside of parens + parens = 0 + i_ = 0 + for i in range(len(v)): + if v[i] == ',' and parens == 0: + yield v[i_:i] + i_ = i+1 + elif v[i] in '([{': + parens += 1 + elif v[i] in '}])': + parens -= 1 + if v[i_:].strip(): + yield v[i_:] + + def parse_define(v): + # a define entry can be a list + if isinstance(v, list): + for v_ in v: + yield from parse_define(v_) + # or a string + elif isinstance(v, str): + # which can be comma-separated values, with optional + # range statements. This matches the runtime define parser in + # the runner itself. + for v_ in csplit(v): + m = re.search(r'\brange\b\s*\(' + '(?P[^,\s]*)' + '\s*(?:,\s*(?P[^,\s]*)' + '\s*(?:,\s*(?P[^,\s]*)\s*)?)?\)', + v_) + if m: + start = (int(m.group('start'), 0) + if m.group('start') else 0) + stop = (int(m.group('stop'), 0) + if m.group('stop') else None) + step = (int(m.group('step'), 0) + if m.group('step') else 1) + if m.lastindex <= 1: + start, stop = 0, start + for x in range(start, stop, step): + yield from parse_define('%s(%d)%s' % ( + v_[:m.start()], x, v_[m.end():])) + else: + yield v_ + # or a literal value + else: + yield v + + # build possible permutations + for suite_defines_ in suite_defines: + self.defines |= suite_defines_.keys() + for defines_ in defines: + self.defines |= defines_.keys() + self.permutations.extend(dict(perm) for perm in it.product(*( + [(k, v) for v in parse_define(vs)] + for k, vs in sorted((suite_defines_ | defines_).items())))) + + for k in config.keys(): + print('%swarning:%s in %s, found unused key %r' % ( + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + self.name, + k), + file=sys.stderr) + + +class BenchSuite: + # create a BenchSuite object from a toml file + def __init__(self, path, args={}): + self.path = path + self.name = os.path.basename(path) + if self.name.endswith('.toml'): + self.name = self.name[:-len('.toml')] + + # load toml file and parse bench cases + with open(self.path) as f: + # load benches + config = toml.load(f) + + # find line numbers + f.seek(0) + case_linenos = [] + code_linenos = [] + for i, line in enumerate(f): + match = re.match( + '(?P\[\s*cases\s*\.\s*(?P\w+)\s*\])' + '|' '(?Pcode\s*=)', + line) + if match and match.group('case'): + case_linenos.append((i+1, match.group('name'))) + elif match and match.group('code'): + code_linenos.append(i+2) + + # sort in case toml parsing did not retain order + case_linenos.sort() + + cases = config.pop('cases') + for (lineno, name), (nlineno, _) in it.zip_longest( + case_linenos, case_linenos[1:], + fillvalue=(float('inf'), None)): + code_lineno = min( + (l for l in code_linenos if l >= lineno and l < nlineno), + default=None) + cases[name]['lineno'] = lineno + cases[name]['code_lineno'] = code_lineno + + self.if_ = config.pop('if', None) + if isinstance(self.if_, bool): + self.if_ = 'true' if self.if_ else 'false' + + self.code = config.pop('code', None) + self.code_lineno = min( + (l for l in code_linenos + if not case_linenos or l < case_linenos[0][0]), + default=None) + + # a couple of these we just forward to all cases + defines = config.pop('defines', {}) + in_ = config.pop('in', None) + + self.cases = [] + for name, case in sorted(cases.items(), + key=lambda c: c[1].get('lineno')): + self.cases.append(BenchCase(config={ + 'name': name, + 'path': path + (':%d' % case['lineno'] + if 'lineno' in case else ''), + 'suite': self.name, + 'suite_defines': defines, + 'suite_in': in_, + **case}, + args=args)) + + # combine per-case defines + self.defines = set.union(*( + set(case.defines) for case in self.cases)) + + for k in config.keys(): + print('%swarning:%s in %s, found unused key %r' % ( + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + self.name, + k), + file=sys.stderr) + + + +def compile(bench_paths, **args): + # find .toml files + paths = [] + for path in bench_paths: + if os.path.isdir(path): + path = path + '/*.toml' + + for path in glob.glob(path): + paths.append(path) + + if not paths: + print('no bench suites found in %r?' % bench_paths) + sys.exit(-1) + + # load the suites + suites = [BenchSuite(path, args) for path in paths] + suites.sort(key=lambda s: s.name) + + # check for name conflicts, these will cause ambiguity problems later + # when running benches + seen = {} + for suite in suites: + if suite.name in seen: + print('%swarning:%s conflicting suite %r, %s and %s' % ( + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + suite.name, + suite.path, + seen[suite.name].path), + file=sys.stderr) + seen[suite.name] = suite + + for case in suite.cases: + # only allow conflicts if a case and its suite share a name + if case.name in seen and not ( + isinstance(seen[case.name], BenchSuite) + and seen[case.name].cases == [case]): + print('%swarning:%s conflicting case %r, %s and %s' % ( + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + case.name, + case.path, + seen[case.name].path), + file=sys.stderr) + seen[case.name] = case + + # we can only compile one bench suite at a time + if not args.get('source'): + if len(suites) > 1: + print('more than one bench suite for compilation? (%r)' % bench_paths) + sys.exit(-1) + + suite = suites[0] + + # write generated bench source + if 'output' in args: + with openio(args['output'], 'w') as f: + _write = f.write + def write(s): + f.lineno += s.count('\n') + _write(s) + def writeln(s=''): + f.lineno += s.count('\n') + 1 + _write(s) + _write('\n') + f.lineno = 1 + f.write = write + f.writeln = writeln + + f.writeln("// Generated by %s:" % sys.argv[0]) + f.writeln("//") + f.writeln("// %s" % ' '.join(sys.argv)) + f.writeln("//") + f.writeln() + + # include bench_runner.h in every generated file + f.writeln("#include \"%s\"" % args['include']) + f.writeln() + + # write out generated functions, this can end up in different + # files depending on the "in" attribute + # + # note it's up to the specific generated file to declare + # the bench defines + def write_case_functions(f, suite, case): + # create case define functions + if case.defines: + # deduplicate defines by value to try to reduce the + # number of functions we generate + define_cbs = {} + for i, defines in enumerate(case.permutations): + for k, v in sorted(defines.items()): + if v not in define_cbs: + name = ('__bench__%s__%s__%s__%d' + % (suite.name, case.name, k, i)) + define_cbs[v] = name + f.writeln('intmax_t %s(' + '__attribute__((unused)) ' + 'void *data) {' % name) + f.writeln(4*' '+'return %s;' % v) + f.writeln('}') + f.writeln() + f.writeln('const bench_define_t ' + '__bench__%s__%s__defines[][' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d] = {' + % (suite.name, case.name, len(suite.defines))) + for defines in case.permutations: + f.writeln(4*' '+'{') + for k, v in sorted(defines.items()): + f.writeln(8*' '+'[%-24s] = {%s, NULL},' % ( + k+'_i', define_cbs[v])) + f.writeln(4*' '+'},') + f.writeln('};') + f.writeln() + + # create case filter function + if suite.if_ is not None or case.if_ is not None: + f.writeln('bool __bench__%s__%s__filter(void) {' + % (suite.name, case.name)) + f.writeln(4*' '+'return %s;' + % ' && '.join('(%s)' % if_ + for if_ in [suite.if_, case.if_] + if if_ is not None)) + f.writeln('}') + f.writeln() + + # create case run function + f.writeln('void __bench__%s__%s__run(' + '__attribute__((unused)) struct lfs_config *cfg) {' + % (suite.name, case.name)) + f.writeln(4*' '+'// bench case %s' % case.name) + if case.code_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (case.code_lineno, suite.path)) + f.write(case.code) + if case.code_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (f.lineno+1, args['output'])) + f.writeln('}') + f.writeln() + + if not args.get('source'): + if suite.code is not None: + if suite.code_lineno is not None: + f.writeln('#line %d "%s"' + % (suite.code_lineno, suite.path)) + f.write(suite.code) + if suite.code_lineno is not None: + f.writeln('#line %d "%s"' + % (f.lineno+1, args['output'])) + f.writeln() + + if suite.defines: + for i, define in enumerate(sorted(suite.defines)): + f.writeln('#ifndef %s' % define) + f.writeln('#define %-24s ' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d' % (define+'_i', i)) + f.writeln('#define %-24s ' + 'BENCH_DEFINE(%s)' % (define, define+'_i')) + f.writeln('#endif') + f.writeln() + + # create case functions + for case in suite.cases: + if case.in_ is None: + write_case_functions(f, suite, case) + else: + if case.defines: + f.writeln('extern const bench_define_t ' + '__bench__%s__%s__defines[][' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d];' + % (suite.name, case.name, len(suite.defines))) + if suite.if_ is not None or case.if_ is not None: + f.writeln('extern bool __bench__%s__%s__filter(' + 'void);' + % (suite.name, case.name)) + f.writeln('extern void __bench__%s__%s__run(' + 'struct lfs_config *cfg);' + % (suite.name, case.name)) + f.writeln() + + # create suite struct + # + # note we place this in the custom bench_suites section with + # minimum alignment, otherwise GCC ups the alignment to + # 32-bytes for some reason + f.writeln('__attribute__((section("_bench_suites"), ' + 'aligned(1)))') + f.writeln('const struct bench_suite __bench__%s__suite = {' + % suite.name) + f.writeln(4*' '+'.name = "%s",' % suite.name) + f.writeln(4*' '+'.path = "%s",' % suite.path) + f.writeln(4*' '+'.flags = 0,') + if suite.defines: + # create suite define names + f.writeln(4*' '+'.define_names = (const char *const[' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d]){' % ( + len(suite.defines))) + for k in sorted(suite.defines): + f.writeln(8*' '+'[%-24s] = "%s",' % (k+'_i', k)) + f.writeln(4*' '+'},') + f.writeln(4*' '+'.define_count = ' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d,' % len(suite.defines)) + f.writeln(4*' '+'.cases = (const struct bench_case[]){') + for case in suite.cases: + # create case structs + f.writeln(8*' '+'{') + f.writeln(12*' '+'.name = "%s",' % case.name) + f.writeln(12*' '+'.path = "%s",' % case.path) + f.writeln(12*' '+'.flags = 0,') + f.writeln(12*' '+'.permutations = %d,' + % len(case.permutations)) + if case.defines: + f.writeln(12*' '+'.defines ' + '= (const bench_define_t*)__bench__%s__%s__defines,' + % (suite.name, case.name)) + if suite.if_ is not None or case.if_ is not None: + f.writeln(12*' '+'.filter = __bench__%s__%s__filter,' + % (suite.name, case.name)) + f.writeln(12*' '+'.run = __bench__%s__%s__run,' + % (suite.name, case.name)) + f.writeln(8*' '+'},') + f.writeln(4*' '+'},') + f.writeln(4*' '+'.case_count = %d,' % len(suite.cases)) + f.writeln('};') + f.writeln() + + else: + # copy source + f.writeln('#line 1 "%s"' % args['source']) + with open(args['source']) as sf: + shutil.copyfileobj(sf, f) + f.writeln() + + # write any internal benches + for suite in suites: + for case in suite.cases: + if (case.in_ is not None + and os.path.normpath(case.in_) + == os.path.normpath(args['source'])): + # write defines, but note we need to undef any + # new defines since we're in someone else's file + if suite.defines: + for i, define in enumerate( + sorted(suite.defines)): + f.writeln('#ifndef %s' % define) + f.writeln('#define %-24s ' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d' % ( + define+'_i', i)) + f.writeln('#define %-24s ' + 'BENCH_DEFINE(%s)' % ( + define, define+'_i')) + f.writeln('#define ' + '__BENCH__%s__NEEDS_UNDEF' % ( + define)) + f.writeln('#endif') + f.writeln() + + write_case_functions(f, suite, case) + + if suite.defines: + for define in sorted(suite.defines): + f.writeln('#ifdef __BENCH__%s__NEEDS_UNDEF' + % define) + f.writeln('#undef __BENCH__%s__NEEDS_UNDEF' + % define) + f.writeln('#undef %s' % define) + f.writeln('#undef %s' % (define+'_i')) + f.writeln('#endif') + f.writeln() + +def find_runner(runner, **args): + cmd = runner.copy() + + # run under some external command? + if args.get('exec'): + cmd[:0] = args['exec'] + + # run under valgrind? + if args.get('valgrind'): + cmd[:0] = filter(None, [ + 'valgrind', + '--leak-check=full', + '--track-origins=yes', + '--error-exitcode=4', + '-q']) + + # other context + if args.get('geometry'): + cmd.append('-g%s' % args['geometry']) + if args.get('disk'): + cmd.append('-d%s' % args['disk']) + if args.get('trace'): + cmd.append('-t%s' % args['trace']) + if args.get('read_sleep'): + cmd.append('--read-sleep=%s' % args['read_sleep']) + if args.get('prog_sleep'): + cmd.append('--prog-sleep=%s' % args['prog_sleep']) + if args.get('erase_sleep'): + cmd.append('--erase-sleep=%s' % args['erase_sleep']) + + # defines? + if args.get('define'): + for define in args.get('define'): + cmd.append('-D%s' % define) + + return cmd + +def list_(runner, bench_ids=[], **args): + cmd = find_runner(runner, **args) + bench_ids + if args.get('summary'): cmd.append('--summary') + if args.get('list_suites'): cmd.append('--list-suites') + if args.get('list_cases'): cmd.append('--list-cases') + if args.get('list_suite_paths'): cmd.append('--list-suite-paths') + if args.get('list_case_paths'): cmd.append('--list-case-paths') + if args.get('list_defines'): cmd.append('--list-defines') + if args.get('list_permutation_defines'): + cmd.append('--list-permutation-defines') + if args.get('list_implicit_defines'): + cmd.append('--list-implicit-defines') + if args.get('list_geometries'): cmd.append('--list-geometries') + + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + return sp.call(cmd) + + +def find_perms(runner_, ids=[], **args): + case_suites = {} + expected_case_perms = co.defaultdict(lambda: 0) + expected_perms = 0 + total_perms = 0 + + # query cases from the runner + cmd = runner_ + ['--list-cases'] + ids + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + pattern = re.compile( + '^(?P[^\s]+)' + '\s+(?P[^\s]+)' + '\s+(?P\d+)/(?P\d+)') + # skip the first line + for line in it.islice(proc.stdout, 1, None): + m = pattern.match(line) + if m: + filtered = int(m.group('filtered')) + perms = int(m.group('perms')) + expected_case_perms[m.group('case')] += filtered + expected_perms += filtered + total_perms += perms + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + # get which suite each case belongs to via paths + cmd = runner_ + ['--list-case-paths'] + ids + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + pattern = re.compile( + '^(?P[^\s]+)' + '\s+(?P[^:]+):(?P\d+)') + # skip the first line + for line in it.islice(proc.stdout, 1, None): + m = pattern.match(line) + if m: + path = m.group('path') + # strip path/suffix here + suite = os.path.basename(path) + if suite.endswith('.toml'): + suite = suite[:-len('.toml')] + case_suites[m.group('case')] = suite + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + # figure out expected suite perms + expected_suite_perms = co.defaultdict(lambda: 0) + for case, suite in case_suites.items(): + expected_suite_perms[suite] += expected_case_perms[case] + + return ( + case_suites, + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) + +def find_path(runner_, id, **args): + path = None + # query from runner + cmd = runner_ + ['--list-case-paths', id] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + pattern = re.compile( + '^(?P[^\s]+)' + '\s+(?P[^:]+):(?P\d+)') + # skip the first line + for line in it.islice(proc.stdout, 1, None): + m = pattern.match(line) + if m and path is None: + path_ = m.group('path') + lineno = int(m.group('lineno')) + path = (path_, lineno) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return path + +def find_defines(runner_, id, **args): + # query permutation defines from runner + cmd = runner_ + ['--list-permutation-defines', id] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + defines = co.OrderedDict() + pattern = re.compile('^(?P\w+)=(?P.+)') + for line in proc.stdout: + m = pattern.match(line) + if m: + define = m.group('define') + value = m.group('value') + defines[define] = value + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return defines + + +# Thread-safe CSV writer +class BenchOutput: + def __init__(self, path, head=None, tail=None): + self.f = openio(path, 'w+', 1) + self.lock = th.Lock() + self.head = head or [] + self.tail = tail or [] + self.writer = csv.DictWriter(self.f, self.head + self.tail) + self.rows = [] + + def close(self): + self.f.close() + + def __enter__(self): + return self + + def __exit__(self, *_): + self.f.close() + + def writerow(self, row): + with self.lock: + self.rows.append(row) + if all(k in self.head or k in self.tail for k in row.keys()): + # can simply append + self.writer.writerow(row) + else: + # need to rewrite the file + self.head.extend(row.keys() - (self.head + self.tail)) + self.f.seek(0) + self.f.truncate() + self.writer = csv.DictWriter(self.f, self.head + self.tail) + self.writer.writeheader() + for row in self.rows: + self.writer.writerow(row) + +# A bench failure +class BenchFailure(Exception): + def __init__(self, id, returncode, stdout, assert_=None): + self.id = id + self.returncode = returncode + self.stdout = stdout + self.assert_ = assert_ + +def run_stage(name, runner_, ids, output_, **args): + # get expected suite/case/perm counts + (case_suites, + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) = find_perms(runner_, ids, **args) + + passed_suite_perms = co.defaultdict(lambda: 0) + passed_case_perms = co.defaultdict(lambda: 0) + passed_perms = 0 + read = 0 + prog = 0 + erased = 0 + failures = [] + killed = False + + pattern = re.compile('^(?:' + '(?Prunning|finished|skipped|powerloss)' + ' (?P(?P[^:]+)[^\s]*)' + '(?: (?P\d+))?' + '(?: (?P\d+))?' + '(?: (?P\d+))?' + '|' '(?P[^:]+):(?P\d+):(?Passert):' + ' *(?P.*)' + ')$') + locals = th.local() + children = set() + + def run_runner(runner_, ids=[]): + nonlocal passed_suite_perms + nonlocal passed_case_perms + nonlocal passed_perms + nonlocal read + nonlocal prog + nonlocal erased + nonlocal locals + + # run the benches! + cmd = runner_ + ids + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + + mpty, spty = pty.openpty() + proc = sp.Popen(cmd, stdout=spty, stderr=spty, close_fds=False) + os.close(spty) + children.add(proc) + mpty = os.fdopen(mpty, 'r', 1) + stdout = None + + last_id = None + last_stdout = [] + last_assert = None + try: + while True: + # parse a line for state changes + try: + line = mpty.readline() + except OSError as e: + if e.errno == errno.EIO: + break + raise + if not line: + break + last_stdout.append(line) + if args.get('stdout'): + try: + if not stdout: + stdout = openio(args['stdout'], 'a', 1, nb=True) + stdout.write(line) + except OSError as e: + if e.errno not in [ + errno.ENXIO, + errno.EPIPE, + errno.EAGAIN]: + raise + stdout = None + if args.get('verbose'): + sys.stdout.write(line) + + m = pattern.match(line) + if m: + op = m.group('op') or m.group('op_') + if op == 'running': + locals.seen_perms += 1 + last_id = m.group('id') + last_stdout = [] + last_assert = None + elif op == 'finished': + case = m.group('case') + suite = case_suites[case] + read_ = int(m.group('read')) + prog_ = int(m.group('prog')) + erased_ = int(m.group('erased')) + passed_suite_perms[suite] += 1 + passed_case_perms[case] += 1 + passed_perms += 1 + read += read_ + prog += prog_ + erased += erased_ + if output_: + # get defines and write to csv + defines = find_defines( + runner_, m.group('id'), **args) + output_.writerow({ + 'suite': suite, + 'case': case, + 'bench_read': read_, + 'bench_prog': prog_, + 'bench_erased': erased_, + **defines}) + elif op == 'skipped': + locals.seen_perms += 1 + elif op == 'assert': + last_assert = ( + m.group('path'), + int(m.group('lineno')), + m.group('message')) + # go ahead and kill the process, aborting takes a while + if args.get('keep_going'): + proc.kill() + except KeyboardInterrupt: + raise BenchFailure(last_id, 1, last_stdout) + finally: + children.remove(proc) + mpty.close() + + proc.wait() + if proc.returncode != 0: + raise BenchFailure( + last_id, + proc.returncode, + last_stdout, + last_assert) + + def run_job(runner_, ids=[], start=None, step=None): + nonlocal failures + nonlocal killed + nonlocal locals + + start = start or 0 + step = step or 1 + while start < total_perms: + job_runner = runner_.copy() + if args.get('isolate') or args.get('valgrind'): + job_runner.append('-s%s,%s,%s' % (start, start+step, step)) + else: + job_runner.append('-s%s,,%s' % (start, step)) + + try: + # run the benches + locals.seen_perms = 0 + run_runner(job_runner, ids) + assert locals.seen_perms > 0 + start += locals.seen_perms*step + + except BenchFailure as failure: + # keep track of failures + if output_: + case, _ = failure.id.split(':', 1) + suite = case_suites[case] + # get defines and write to csv + defines = find_defines(runner_, failure.id, **args) + output_.writerow({ + 'suite': suite, + 'case': case, + **defines}) + + # race condition for multiple failures? + if failures and not args.get('keep_going'): + break + + failures.append(failure) + + if args.get('keep_going') and not killed: + # resume after failed bench + assert locals.seen_perms > 0 + start += locals.seen_perms*step + continue + else: + # stop other benches + killed = True + for child in children.copy(): + child.kill() + break + + + # parallel jobs? + runners = [] + if 'jobs' in args: + for job in range(args['jobs']): + runners.append(th.Thread( + target=run_job, args=(runner_, ids, job, args['jobs']), + daemon=True)) + else: + runners.append(th.Thread( + target=run_job, args=(runner_, ids, None, None), + daemon=True)) + + def print_update(done): + if not args.get('verbose') and (args['color'] or done): + sys.stdout.write('%s%srunning %s%s:%s %s%s' % ( + '\r\x1b[K' if args['color'] else '', + '\x1b[?7l' if not done else '', + ('\x1b[34m' if not failures else '\x1b[31m') + if args['color'] else '', + name, + '\x1b[m' if args['color'] else '', + ', '.join(filter(None, [ + '%d/%d suites' % ( + sum(passed_suite_perms[k] == v + for k, v in expected_suite_perms.items()), + len(expected_suite_perms)) + if (not args.get('by_suites') + and not args.get('by_cases')) else None, + '%d/%d cases' % ( + sum(passed_case_perms[k] == v + for k, v in expected_case_perms.items()), + len(expected_case_perms)) + if not args.get('by_cases') else None, + '%d/%d perms' % (passed_perms, expected_perms), + '%s%d/%d failures%s' % ( + '\x1b[31m' if args['color'] else '', + len(failures), + expected_perms, + '\x1b[m' if args['color'] else '') + if failures else None])), + '\x1b[?7h' if not done else '\n')) + sys.stdout.flush() + + for r in runners: + r.start() + + try: + while any(r.is_alive() for r in runners): + time.sleep(0.01) + print_update(False) + except KeyboardInterrupt: + # this is handled by the runner threads, we just + # need to not abort here + killed = True + finally: + print_update(True) + + for r in runners: + r.join() + + return ( + expected_perms, + passed_perms, + read, + prog, + erased, + failures, + killed) + + +def run(runner, bench_ids=[], **args): + # query runner for benches + runner_ = find_runner(runner, **args) + print('using runner: %s' % ' '.join(shlex.quote(c) for c in runner_)) + (_, + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) = find_perms(runner_, bench_ids, **args) + print('found %d suites, %d cases, %d/%d permutations' % ( + len(expected_suite_perms), + len(expected_case_perms), + expected_perms, + total_perms)) + print() + + # truncate and open logs here so they aren't disconnected between benches + stdout = None + if args.get('stdout'): + stdout = openio(args['stdout'], 'w', 1) + trace = None + if args.get('trace'): + trace = openio(args['trace'], 'w', 1) + output = None + if args.get('output'): + output = BenchOutput(args['output'], + ['suite', 'case'], + ['bench_read', 'bench_prog', 'bench_erased']) + + # measure runtime + start = time.time() + + # spawn runners + expected = 0 + passed = 0 + read = 0 + prog = 0 + erased = 0 + failures = [] + for by in (expected_case_perms.keys() if args.get('by_cases') + else expected_suite_perms.keys() if args.get('by_suites') + else [None]): + # spawn jobs for stage + (expected_, + passed_, + read_, + prog_, + erased_, + failures_, + killed) = run_stage( + by or 'benches', + runner_, + [by] if by is not None else bench_ids, + output, + **args) + # collect passes/failures + expected += expected_ + passed += passed_ + read += read_ + prog += prog_ + erased += erased_ + failures.extend(failures_) + if (failures and not args.get('keep_going')) or killed: + break + + stop = time.time() + + if stdout: + stdout.close() + if trace: + trace.close() + if output: + output.close() + + # show summary + print() + print('%sdone:%s %s' % ( + ('\x1b[34m' if not failures else '\x1b[31m') + if args['color'] else '', + '\x1b[m' if args['color'] else '', + ', '.join(filter(None, [ + '%d read' % read, + '%d prog' % prog, + '%d erased' % erased, + 'in %.2fs' % (stop-start)])))) + print() + + # print each failure + for failure in failures: + assert failure.id is not None, '%s broken? %r' % ( + ' '.join(shlex.quote(c) for c in runner_), + failure) + + # get some extra info from runner + path, lineno = find_path(runner_, failure.id, **args) + defines = find_defines(runner_, failure.id, **args) + + # show summary of failure + print('%s%s:%d:%sfailure:%s %s%s failed' % ( + '\x1b[01m' if args['color'] else '', + path, lineno, + '\x1b[01;31m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + failure.id, + ' (%s)' % ', '.join('%s=%s' % (k,v) for k,v in defines.items()) + if defines else '')) + + if failure.stdout: + stdout = failure.stdout + if failure.assert_ is not None: + stdout = stdout[:-1] + for line in stdout[-args.get('context', 5):]: + sys.stdout.write(line) + + if failure.assert_ is not None: + path, lineno, message = failure.assert_ + print('%s%s:%d:%sassert:%s %s' % ( + '\x1b[01m' if args['color'] else '', + path, lineno, + '\x1b[01;31m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + message)) + with open(path) as f: + line = next(it.islice(f, lineno-1, None)).strip('\n') + print(line) + print() + + # drop into gdb? + if failures and (args.get('gdb') + or args.get('gdb_case') + or args.get('gdb_main')): + failure = failures[0] + cmd = runner_ + [failure.id] + + if args.get('gdb_main'): + cmd[:0] = ['gdb', + '-ex', 'break main', + '-ex', 'run', + '--args'] + elif args.get('gdb_case'): + path, lineno = find_path(runner_, failure.id, **args) + cmd[:0] = ['gdb', + '-ex', 'break %s:%d' % (path, lineno), + '-ex', 'run', + '--args'] + elif failure.assert_ is not None: + cmd[:0] = ['gdb', + '-ex', 'run', + '-ex', 'frame function raise', + '-ex', 'up 2', + '--args'] + else: + cmd[:0] = ['gdb', + '-ex', 'run', + '--args'] + + # exec gdb interactively + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + os.execvp(cmd[0], cmd) + + return 1 if failures else 0 + + +def main(**args): + # figure out what color should be + if args.get('color') == 'auto': + args['color'] = sys.stdout.isatty() + elif args.get('color') == 'always': + args['color'] = True + else: + args['color'] = False + + if args.get('compile'): + return compile(**args) + elif (args.get('summary') + or args.get('list_suites') + or args.get('list_cases') + or args.get('list_suite_paths') + or args.get('list_case_paths') + or args.get('list_defines') + or args.get('list_permutation_defines') + or args.get('list_implicit_defines') + or args.get('list_geometries')): + return list_(**args) + else: + return run(**args) + + +if __name__ == "__main__": + import argparse + import sys + argparse.ArgumentParser._handle_conflict_ignore = lambda *_: None + argparse._ArgumentGroup._handle_conflict_ignore = lambda *_: None + parser = argparse.ArgumentParser( + description="Build and run benches.", + conflict_handler='ignore') + parser.add_argument( + '-v', '--verbose', + action='store_true', + help="Output commands that run behind the scenes.") + parser.add_argument( + '--color', + choices=['never', 'always', 'auto'], + default='auto', + help="When to use terminal colors. Defaults to 'auto'.") + + # bench flags + bench_parser = parser.add_argument_group('bench options') + bench_parser.add_argument( + 'runner', + nargs='?', + type=lambda x: x.split(), + help="Bench runner to use for benching. Defaults to %r." % RUNNER_PATH) + bench_parser.add_argument( + 'bench_ids', + nargs='*', + help="Description of benches to run.") + bench_parser.add_argument( + '-Y', '--summary', + action='store_true', + help="Show quick summary.") + bench_parser.add_argument( + '-l', '--list-suites', + action='store_true', + help="List bench suites.") + bench_parser.add_argument( + '-L', '--list-cases', + action='store_true', + help="List bench cases.") + bench_parser.add_argument( + '--list-suite-paths', + action='store_true', + help="List the path for each bench suite.") + bench_parser.add_argument( + '--list-case-paths', + action='store_true', + help="List the path and line number for each bench case.") + bench_parser.add_argument( + '--list-defines', + action='store_true', + help="List all defines in this bench-runner.") + bench_parser.add_argument( + '--list-permutation-defines', + action='store_true', + help="List explicit defines in this bench-runner.") + bench_parser.add_argument( + '--list-implicit-defines', + action='store_true', + help="List implicit defines in this bench-runner.") + bench_parser.add_argument( + '--list-geometries', + action='store_true', + help="List the available disk geometries.") + bench_parser.add_argument( + '-D', '--define', + action='append', + help="Override a bench define.") + bench_parser.add_argument( + '-g', '--geometry', + help="Comma-separated list of disk geometries to bench. " + "Defaults to d,e,E,n,N.") + bench_parser.add_argument( + '-d', '--disk', + help="Direct block device operations to this file.") + bench_parser.add_argument( + '-t', '--trace', + help="Direct trace output to this file.") + bench_parser.add_argument( + '-O', '--stdout', + help="Direct stdout to this file. Note stderr is already merged here.") + bench_parser.add_argument( + '-o', '--output', + help="CSV file to store results.") + bench_parser.add_argument( + '--read-sleep', + help="Artificial read delay in seconds.") + bench_parser.add_argument( + '--prog-sleep', + help="Artificial prog delay in seconds.") + bench_parser.add_argument( + '--erase-sleep', + help="Artificial erase delay in seconds.") + bench_parser.add_argument( + '-j', '--jobs', + nargs='?', + type=lambda x: int(x, 0), + const=len(os.sched_getaffinity(0)), + help="Number of parallel runners to run.") + bench_parser.add_argument( + '-k', '--keep-going', + action='store_true', + help="Don't stop on first error.") + bench_parser.add_argument( + '-i', '--isolate', + action='store_true', + help="Run each bench permutation in a separate process.") + bench_parser.add_argument( + '-b', '--by-suites', + action='store_true', + help="Step through benches by suite.") + bench_parser.add_argument( + '-B', '--by-cases', + action='store_true', + help="Step through benches by case.") + bench_parser.add_argument( + '--context', + type=lambda x: int(x, 0), + default=5, + help="Show this many lines of stdout on bench failure. " + "Defaults to 5.") + bench_parser.add_argument( + '--gdb', + action='store_true', + help="Drop into gdb on bench failure.") + bench_parser.add_argument( + '--gdb-case', + action='store_true', + help="Drop into gdb on bench failure but stop at the beginning " + "of the failing bench case.") + bench_parser.add_argument( + '--gdb-main', + action='store_true', + help="Drop into gdb on bench failure but stop at the beginning " + "of main.") + bench_parser.add_argument( + '--exec', + type=lambda e: e.split(), + help="Run under another executable.") + bench_parser.add_argument( + '--valgrind', + action='store_true', + help="Run under Valgrind to find memory errors. Implicitly sets " + "--isolate.") + + # compilation flags + comp_parser = parser.add_argument_group('compilation options') + comp_parser.add_argument( + 'bench_paths', + nargs='*', + help="Description of *.toml files to compile. May be a directory " + "or a list of paths.") + comp_parser.add_argument( + '-c', '--compile', + action='store_true', + help="Compile a bench suite or source file.") + comp_parser.add_argument( + '-s', '--source', + help="Source file to compile, possibly injecting internal benches.") + comp_parser.add_argument( + '--include', + default=HEADER_PATH, + help="Inject this header file into every compiled bench file. " + "Defaults to %r." % HEADER_PATH) + comp_parser.add_argument( + '-o', '--output', + help="Output file.") + + # runner + bench_ids overlaps bench_paths, so we need to do some munging here + args = parser.parse_intermixed_args() + args.bench_paths = [' '.join(args.runner or [])] + args.bench_ids + args.runner = args.runner or [RUNNER_PATH] + + sys.exit(main(**{k: v + for k, v in vars(args).items() + if v is not None})) diff --git a/scripts/summary.py b/scripts/summary.py index 680a7150..0855ffb2 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -28,7 +28,8 @@ MERGES = { 'add': ( ['code_size', 'data_size', 'stack_frame', 'struct_size', 'coverage_lines', 'coverage_branches', - 'test_passed'], + 'test_passed', + 'bench_read', 'bench_prog', 'bench_erased'], lambda xs: sum(xs[1:], start=xs[0]) ), 'mul': ( diff --git a/scripts/tailpipe.py b/scripts/tailpipe.py index c9e742ca..08213cf6 100755 --- a/scripts/tailpipe.py +++ b/scripts/tailpipe.py @@ -48,7 +48,7 @@ def main(path='-', *, lines=1, sleep=0.01, keep_open=False): if not keep_open: break # don't just flood open calls - time.sleep(sleep) + time.sleep(sleep or 0.1) done = True th.Thread(target=read, daemon=True).start() diff --git a/scripts/test.py b/scripts/test.py index 42bc308a..71688547 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -388,7 +388,7 @@ def compile(test_paths, **args): f.writeln('#define %-24s ' 'TEST_IMPLICIT_DEFINE_COUNT+%d' % (define+'_i', i)) f.writeln('#define %-24s ' - 'test_define(%s)' % (define, define+'_i')) + 'TEST_DEFINE(%s)' % (define, define+'_i')) f.writeln('#endif') f.writeln() @@ -486,7 +486,7 @@ def compile(test_paths, **args): 'TEST_IMPLICIT_DEFINE_COUNT+%d' % ( define+'_i', i)) f.writeln('#define %-24s ' - 'test_define(%s)' % ( + 'TEST_DEFINE(%s)' % ( define, define+'_i')) f.writeln('#define ' '__TEST__%s__NEEDS_UNDEF' % ( @@ -1018,7 +1018,9 @@ def run(runner, test_ids=[], **args): trace = openio(args['trace'], 'w', 1) output = None if args.get('output'): - output = TestOutput(args['output'], ['suite', 'case'], ['test_passed']) + output = TestOutput(args['output'], + ['suite', 'case'], + ['test_passed']) # measure runtime start = time.time() diff --git a/scripts/tracebd.py b/scripts/tracebd.py index ff0bbbea..a8bdf8a4 100755 --- a/scripts/tracebd.py +++ b/scripts/tracebd.py @@ -607,7 +607,7 @@ def main(path='-', *, if not keep_open: break # don't just flood open calls - time.sleep(sleep) + time.sleep(sleep or 0.1) except KeyboardInterrupt: pass else: @@ -627,7 +627,7 @@ def main(path='-', *, if not keep_open: break # don't just flood open calls - time.sleep(sleep) + time.sleep(sleep or 0.1) done = True th.Thread(target=parse, daemon=True).start() diff --git a/tests/test_alloc.toml b/tests/test_alloc.toml index 64b805fa..205efbb1 100644 --- a/tests/test_alloc.toml +++ b/tests/test_alloc.toml @@ -370,7 +370,7 @@ code = ''' [cases.test_alloc_bad_blocks] in = "lfs.c" defines.ERASE_CYCLES = 0xffffffff -defines.BADBLOCK_BEHAVIOR = 'LFS_TESTBD_BADBLOCK_READERROR' +defines.BADBLOCK_BEHAVIOR = 'LFS_EMUBD_BADBLOCK_READERROR' code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -409,7 +409,7 @@ code = ''' // but mark the head of our file as a "bad block", this is force our // scan to bail early - lfs_testbd_setwear(cfg, fileblock, 0xffffffff) => 0; + lfs_emubd_setwear(cfg, fileblock, 0xffffffff) => 0; lfs_file_open(&lfs, &file, "ghost", LFS_O_WRONLY | LFS_O_CREAT) => 0; strcpy((char*)buffer, "chomp"); size = strlen("chomp"); @@ -424,7 +424,7 @@ code = ''' // now reverse the "bad block" and try to write the file again until we // run out of space - lfs_testbd_setwear(cfg, fileblock, 0) => 0; + lfs_emubd_setwear(cfg, fileblock, 0) => 0; lfs_file_open(&lfs, &file, "ghost", LFS_O_WRONLY | LFS_O_CREAT) => 0; strcpy((char*)buffer, "chomp"); size = strlen("chomp"); diff --git a/tests/test_badblocks.toml b/tests/test_badblocks.toml index 012d8765..b50b3933 100644 --- a/tests/test_badblocks.toml +++ b/tests/test_badblocks.toml @@ -6,18 +6,18 @@ defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] defines.BADBLOCK_BEHAVIOR = [ - 'LFS_TESTBD_BADBLOCK_PROGERROR', - 'LFS_TESTBD_BADBLOCK_ERASEERROR', - 'LFS_TESTBD_BADBLOCK_READERROR', - 'LFS_TESTBD_BADBLOCK_PROGNOOP', - 'LFS_TESTBD_BADBLOCK_ERASENOOP', + 'LFS_EMUBD_BADBLOCK_PROGERROR', + 'LFS_EMUBD_BADBLOCK_ERASEERROR', + 'LFS_EMUBD_BADBLOCK_READERROR', + 'LFS_EMUBD_BADBLOCK_PROGNOOP', + 'LFS_EMUBD_BADBLOCK_ERASENOOP', ] defines.NAMEMULT = 64 defines.FILEMULT = 1 code = ''' for (lfs_block_t badblock = 2; badblock < BLOCK_COUNT; badblock++) { - lfs_testbd_setwear(cfg, badblock-1, 0) => 0; - lfs_testbd_setwear(cfg, badblock, 0xffffffff) => 0; + lfs_emubd_setwear(cfg, badblock-1, 0) => 0; + lfs_emubd_setwear(cfg, badblock, 0xffffffff) => 0; lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -86,17 +86,17 @@ defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] defines.BADBLOCK_BEHAVIOR = [ - 'LFS_TESTBD_BADBLOCK_PROGERROR', - 'LFS_TESTBD_BADBLOCK_ERASEERROR', - 'LFS_TESTBD_BADBLOCK_READERROR', - 'LFS_TESTBD_BADBLOCK_PROGNOOP', - 'LFS_TESTBD_BADBLOCK_ERASENOOP', + 'LFS_EMUBD_BADBLOCK_PROGERROR', + 'LFS_EMUBD_BADBLOCK_ERASEERROR', + 'LFS_EMUBD_BADBLOCK_READERROR', + 'LFS_EMUBD_BADBLOCK_PROGNOOP', + 'LFS_EMUBD_BADBLOCK_ERASENOOP', ] defines.NAMEMULT = 64 defines.FILEMULT = 1 code = ''' for (lfs_block_t i = 0; i < (BLOCK_COUNT-2)/2; i++) { - lfs_testbd_setwear(cfg, i+2, 0xffffffff) => 0; + lfs_emubd_setwear(cfg, i+2, 0xffffffff) => 0; } lfs_t lfs; @@ -165,17 +165,17 @@ defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] defines.BADBLOCK_BEHAVIOR = [ - 'LFS_TESTBD_BADBLOCK_PROGERROR', - 'LFS_TESTBD_BADBLOCK_ERASEERROR', - 'LFS_TESTBD_BADBLOCK_READERROR', - 'LFS_TESTBD_BADBLOCK_PROGNOOP', - 'LFS_TESTBD_BADBLOCK_ERASENOOP', + 'LFS_EMUBD_BADBLOCK_PROGERROR', + 'LFS_EMUBD_BADBLOCK_ERASEERROR', + 'LFS_EMUBD_BADBLOCK_READERROR', + 'LFS_EMUBD_BADBLOCK_PROGNOOP', + 'LFS_EMUBD_BADBLOCK_ERASENOOP', ] defines.NAMEMULT = 64 defines.FILEMULT = 1 code = ''' for (lfs_block_t i = 0; i < (BLOCK_COUNT-2)/2; i++) { - lfs_testbd_setwear(cfg, (2*i) + 2, 0xffffffff) => 0; + lfs_emubd_setwear(cfg, (2*i) + 2, 0xffffffff) => 0; } lfs_t lfs; @@ -244,15 +244,15 @@ code = ''' defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] defines.BADBLOCK_BEHAVIOR = [ - 'LFS_TESTBD_BADBLOCK_PROGERROR', - 'LFS_TESTBD_BADBLOCK_ERASEERROR', - 'LFS_TESTBD_BADBLOCK_READERROR', - 'LFS_TESTBD_BADBLOCK_PROGNOOP', - 'LFS_TESTBD_BADBLOCK_ERASENOOP', + 'LFS_EMUBD_BADBLOCK_PROGERROR', + 'LFS_EMUBD_BADBLOCK_ERASEERROR', + 'LFS_EMUBD_BADBLOCK_READERROR', + 'LFS_EMUBD_BADBLOCK_PROGNOOP', + 'LFS_EMUBD_BADBLOCK_ERASENOOP', ] code = ''' - lfs_testbd_setwear(cfg, 0, 0xffffffff) => 0; - lfs_testbd_setwear(cfg, 1, 0xffffffff) => 0; + lfs_emubd_setwear(cfg, 0, 0xffffffff) => 0; + lfs_emubd_setwear(cfg, 1, 0xffffffff) => 0; lfs_t lfs; lfs_format(&lfs, cfg) => LFS_ERR_NOSPC; diff --git a/tests/test_exhaustion.toml b/tests/test_exhaustion.toml index fdcef24c..6003af47 100644 --- a/tests/test_exhaustion.toml +++ b/tests/test_exhaustion.toml @@ -4,11 +4,11 @@ defines.ERASE_CYCLES = 10 defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' defines.BADBLOCK_BEHAVIOR = [ - 'LFS_TESTBD_BADBLOCK_PROGERROR', - 'LFS_TESTBD_BADBLOCK_ERASEERROR', - 'LFS_TESTBD_BADBLOCK_READERROR', - 'LFS_TESTBD_BADBLOCK_PROGNOOP', - 'LFS_TESTBD_BADBLOCK_ERASENOOP', + 'LFS_EMUBD_BADBLOCK_PROGERROR', + 'LFS_EMUBD_BADBLOCK_ERASEERROR', + 'LFS_EMUBD_BADBLOCK_READERROR', + 'LFS_EMUBD_BADBLOCK_PROGNOOP', + 'LFS_EMUBD_BADBLOCK_ERASENOOP', ] defines.FILES = 10 code = ''' @@ -97,11 +97,11 @@ defines.ERASE_CYCLES = 10 defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' defines.BADBLOCK_BEHAVIOR = [ - 'LFS_TESTBD_BADBLOCK_PROGERROR', - 'LFS_TESTBD_BADBLOCK_ERASEERROR', - 'LFS_TESTBD_BADBLOCK_READERROR', - 'LFS_TESTBD_BADBLOCK_PROGNOOP', - 'LFS_TESTBD_BADBLOCK_ERASENOOP', + 'LFS_EMUBD_BADBLOCK_PROGERROR', + 'LFS_EMUBD_BADBLOCK_ERASEERROR', + 'LFS_EMUBD_BADBLOCK_READERROR', + 'LFS_EMUBD_BADBLOCK_PROGNOOP', + 'LFS_EMUBD_BADBLOCK_ERASENOOP', ] defines.FILES = 10 code = ''' @@ -197,7 +197,7 @@ code = ''' for (int run = 0; run < 2; run++) { for (lfs_block_t b = 0; b < BLOCK_COUNT; b++) { - lfs_testbd_setwear(cfg, b, + lfs_emubd_setwear(cfg, b, (b < run_block_count[run]) ? 0 : ERASE_CYCLES) => 0; } @@ -297,7 +297,7 @@ code = ''' for (int run = 0; run < 2; run++) { for (lfs_block_t b = 0; b < BLOCK_COUNT; b++) { - lfs_testbd_setwear(cfg, b, + lfs_emubd_setwear(cfg, b, (b < run_block_count[run]) ? 0 : ERASE_CYCLES) => 0; } @@ -469,12 +469,12 @@ exhausted: LFS_WARN("completed %d cycles", cycle); // check the wear on our block device - lfs_testbd_wear_t minwear = -1; - lfs_testbd_wear_t totalwear = 0; - lfs_testbd_wear_t maxwear = 0; + lfs_emubd_wear_t minwear = -1; + lfs_emubd_wear_t totalwear = 0; + lfs_emubd_wear_t maxwear = 0; // skip 0 and 1 as superblock movement is intentionally avoided for (lfs_block_t b = 2; b < BLOCK_COUNT; b++) { - lfs_testbd_wear_t wear = lfs_testbd_getwear(cfg, b); + lfs_emubd_wear_t wear = lfs_emubd_getwear(cfg, b); printf("%08x: wear %d\n", b, wear); assert(wear >= 0); if (wear < minwear) { @@ -485,17 +485,17 @@ exhausted: } totalwear += wear; } - lfs_testbd_wear_t avgwear = totalwear / BLOCK_COUNT; + lfs_emubd_wear_t avgwear = totalwear / BLOCK_COUNT; LFS_WARN("max wear: %d cycles", maxwear); LFS_WARN("avg wear: %d cycles", totalwear / (int)BLOCK_COUNT); LFS_WARN("min wear: %d cycles", minwear); // find standard deviation^2 - lfs_testbd_wear_t dev2 = 0; + lfs_emubd_wear_t dev2 = 0; for (lfs_block_t b = 2; b < BLOCK_COUNT; b++) { - lfs_testbd_wear_t wear = lfs_testbd_getwear(cfg, b); + lfs_emubd_wear_t wear = lfs_emubd_getwear(cfg, b); assert(wear >= 0); - lfs_testbd_swear_t diff = wear - avgwear; + lfs_emubd_swear_t diff = wear - avgwear; dev2 += diff*diff; } dev2 /= totalwear; diff --git a/tests/test_move.toml b/tests/test_move.toml index 6c89766c..0537f486 100644 --- a/tests/test_move.toml +++ b/tests/test_move.toml @@ -1638,15 +1638,15 @@ code = ''' if (RELOCATIONS & 0x1) { lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/parent"); - lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; - lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; lfs_dir_close(&lfs, &dir) => 0; } if (RELOCATIONS & 0x2) { lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/parent/child"); - lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; - lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; lfs_dir_close(&lfs, &dir) => 0; } @@ -1784,22 +1784,22 @@ code = ''' if (RELOCATIONS & 0x1) { lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/parent"); - lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; - lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; lfs_dir_close(&lfs, &dir) => 0; } if (RELOCATIONS & 0x2) { lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/parent/sibling"); - lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; - lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; lfs_dir_close(&lfs, &dir) => 0; } if (RELOCATIONS & 0x4) { lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/parent/child"); - lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; - lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; lfs_dir_close(&lfs, &dir) => 0; }