runners: bench: Added BENCH_SIMTIME/SIMRESET/PAUSE/RESUME/etc

- BENCH_SIMTIME()   => lfs3_kiwibd_simtime()
- BENCH_SIMRESET()  => lfs3_kiwibd_simreset()
- BENCH_SIMPAUSE()  => lfs3_kiwibd_simpause()
- BENCH_SIMRESUME() => lfs3_kiwibd_simresume()
- BENCH_RESET()     => lfs3_kiwibd_simreset() + BENCH_STACK/HEAP_RESET()
- BENCH_PAUSE()     => lfs3_kiwibd_simpause() + BENCH_STACK/HEAP_PAUSE()
- BENCH_RESUME()    => lfs3_kiwibd_simresume() + BENCH_STACK/HEAP_RESUME()

This does two things:

1. Adds pause/resume counters to bd counters to make it easier to
   exclude operations from the current bench (potentially useful for
   seq+disk usage).

2. Exposes bd simtime operations as BENCH_* macros, to make it a bit
   easier to interact with simtime without tying all the benches to
   kiwibd. (Not that we'll ever probably not use kiwibd, but still).

Also adopted 32-bit counters for stack/heap pause state instead of a
32-bit stack. Not that either are at a risk of overflowing, but better
safe than sorry.

Makefile

@@ -701,9 +701,9 @@ bench-widths: $(BENCH_CSV)
 		$(SUMMARYFLAGS))
 
 ## Show heap/stack/disk usage
-.PHONY: bench-usage
+.PHONY: bench-ram bench-usage
-bench-usage: SUMMARYFLAGS+=-Si
+bench-ram bench-usage: SUMMARYFLAGS+=-Si
-bench-usage: $(BENCH_CSV)
+bench-ram bench-usage: $(BENCH_CSV)
 	$(strip ./scripts/csv.py \
 		<(./scripts/csv.py $^ \
 			-Dprobe=stack \

bd/lfs3_emubd.c

@@ -217,6 +217,7 @@ int lfs3_emubd_createcfg(const struct lfs3_cfg *cfg, const char *path,
 
     // setup testing things
     bd->blocks = NULL;
+    bd->paused = false;
     bd->reads = 0;
     bd->progs = 0;
     bd->erases = 0;
@@ -437,10 +438,14 @@ int lfs3_emubd_read(const struct lfs3_cfg *cfg, lfs3_block_t block,
     }   
 
     // track reads
-    bd->reads += (lfs3_alignup(off + size, lfs3_max(bd->cfg->read_width, 1))
+    if (!bd->paused) {
-                - lfs3_aligndown(off, lfs3_max(bd->cfg->read_width, 1)))
+        bd->reads += (lfs3_alignup(off + size,
-            / lfs3_max(bd->cfg->read_width, 1);
+                        lfs3_max(bd->cfg->read_width, 1))
-    bd->readed += size;
+                    - lfs3_aligndown(off,
+                        lfs3_max(bd->cfg->read_width, 1)))
+                / lfs3_max(bd->cfg->read_width, 1);
+        bd->readed += size;
+    }
     if (bd->cfg->read_sleep) {
         int err = nanosleep(&(struct timespec){
                 .tv_sec=bd->cfg->read_sleep/1000000000,
@@ -751,10 +756,14 @@ progged:;
     }
 
     // track progs
-    bd->progs += (lfs3_alignup(off + size, lfs3_max(bd->cfg->prog_width, 1))
+    if (!bd->paused) {
-                - lfs3_aligndown(off, lfs3_max(bd->cfg->prog_width, 1)))
+        bd->progs += (lfs3_alignup(off + size,
-            / lfs3_max(bd->cfg->prog_width, 1);
+                        lfs3_max(bd->cfg->prog_width, 1))
-    bd->progged += size;
+                    - lfs3_aligndown(off,
+                        lfs3_max(bd->cfg->prog_width, 1)))
+                / lfs3_max(bd->cfg->prog_width, 1);
+        bd->progged += size;
+    }
     if (bd->cfg->prog_sleep) {
         int err = nanosleep(&(struct timespec){
                 .tv_sec=bd->cfg->prog_sleep/1000000000,
@@ -1049,10 +1058,12 @@ int lfs3_emubd_erase(const struct lfs3_cfg *cfg, lfs3_block_t block) {
 
 erased:;
     // track erases
-    bd->erases += lfs3_alignup(cfg->block_size,
+    if (!bd->paused) {
-                lfs3_max(bd->cfg->erase_width, 1))
+        bd->erases += lfs3_alignup(cfg->block_size,
-            / lfs3_max(bd->cfg->erase_width, 1);
+                    lfs3_max(bd->cfg->erase_width, 1))
-    bd->erased += cfg->block_size;
+                / lfs3_max(bd->cfg->erase_width, 1);
+        bd->erased += cfg->block_size;
+    }
     if (bd->cfg->erase_sleep) {
         int err = nanosleep(&(struct timespec){
                 .tv_sec=bd->cfg->erase_sleep/1000000000,
@@ -1128,6 +1139,23 @@ int lfs3_emubd_simreset(const struct lfs3_cfg *cfg) {
     return 0;
 }
 
+int lfs3_emubd_simpause(const struct lfs3_cfg *cfg) {
+    LFS3_EMUBD_TRACE("lfs3_emubd_simpause(%p)", (void*)cfg);
+    lfs3_emubd_t *bd = cfg->context;
+    bd->paused += 1;
+    LFS3_EMUBD_TRACE("lfs3_emubd_simpause -> %d", 0);
+    return 0;
+}
+
+int lfs3_emubd_simresume(const struct lfs3_cfg *cfg) {
+    LFS3_EMUBD_TRACE("lfs3_emubd_simresume(%p)", (void*)cfg);
+    lfs3_emubd_t *bd = cfg->context;
+    LFS3_ASSERT(bd->paused);
+    bd->paused -= 1;
+    LFS3_EMUBD_TRACE("lfs3_emubd_simresume -> %d", 0);
+    return 0;
+}
+
 lfs3_emubd_sio_t lfs3_emubd_reads(const struct lfs3_cfg *cfg) {
     LFS3_EMUBD_TRACE("lfs3_emubd_reads(%p)", (void*)cfg);
     lfs3_emubd_t *bd = cfg->context;
@@ -1519,6 +1547,7 @@ int lfs3_emubd_cpy(const struct lfs3_cfg *cfg, lfs3_emubd_t *copy) {
     }
 
     // other state
+    copy->paused = bd->paused;
     copy->reads = bd->reads;
     copy->progs = bd->progs;
     copy->erases = bd->erases;

bd/lfs3_emubd.h

@@ -179,13 +179,17 @@ typedef struct lfs3_emubd {
     // array of copy-on-write blocks
     lfs3_emubd_block_t **blocks;
 
-    // some other test state
+    // sim state
+    uint32_t paused;
+    // amount read/progged/erased
     lfs3_emubd_io_t reads;
     lfs3_emubd_io_t progs;
     lfs3_emubd_io_t erases;
     lfs3_emubd_io_t readed;
     lfs3_emubd_io_t progged;
     lfs3_emubd_io_t erased;
+
+    // some other test state
     uint32_t prng;
     lfs3_emubd_powercycles_t power_cycles;
     lfs3_emubd_block_t **ooo_before;
@@ -239,6 +243,12 @@ lfs3_emubd_sns_t lfs3_emubd_simtime(const struct lfs3_cfg *cfg);
 // Reset simulation counters
 int lfs3_emubd_simreset(const struct lfs3_cfg *cfg);
 
+// Pause simulation counters
+int lfs3_emubd_simpause(const struct lfs3_cfg *cfg);
+
+// Resume simulation counters
+int lfs3_emubd_simresume(const struct lfs3_cfg *cfg);
+
 // Get total number of read transactions
 lfs3_emubd_sio_t lfs3_emubd_reads(const struct lfs3_cfg *cfg);
 

bd/lfs3_kiwibd.c

@@ -117,6 +117,7 @@ int lfs3_kiwibd_createcfg(const struct lfs3_cfg *cfg, const char *path,
     bd->cfg = bdcfg;
 
     // setup some initial state
+    bd->paused = false;
     bd->reads = 0;
     bd->progs = 0;
     bd->erases = 0;
@@ -279,10 +280,14 @@ int lfs3_kiwibd_read(const struct lfs3_cfg *cfg, lfs3_block_t block,
     }
 
     // track reads
-    bd->reads += (lfs3_alignup(off + size, lfs3_max(bd->cfg->read_width, 1))
+    if (!bd->paused) {
-                - lfs3_aligndown(off, lfs3_max(bd->cfg->read_width, 1)))
+        bd->reads += (lfs3_alignup(off + size,
-            / lfs3_max(bd->cfg->read_width, 1);
+                        lfs3_max(bd->cfg->read_width, 1))
-    bd->readed += size;
+                    - lfs3_aligndown(off,
+                        lfs3_max(bd->cfg->read_width, 1)))
+                / lfs3_max(bd->cfg->read_width, 1);
+        bd->readed += size;
+    }
     if (bd->cfg->read_sleep) {
         int err = nanosleep(&(struct timespec){
                 .tv_sec=bd->cfg->read_sleep/1000000000,
@@ -410,10 +415,14 @@ int lfs3_kiwibd_prog(const struct lfs3_cfg *cfg, lfs3_block_t block,
     }
 
     // track progs
-    bd->progs += (lfs3_alignup(off + size, lfs3_max(bd->cfg->prog_width, 1))
+    if (!bd->paused) {
-                - lfs3_aligndown(off, lfs3_max(bd->cfg->prog_width, 1)))
+        bd->progs += (lfs3_alignup(off + size,
-            / lfs3_max(bd->cfg->prog_width, 1);
+                        lfs3_max(bd->cfg->prog_width, 1))
-    bd->progged += size;
+                    - lfs3_aligndown(off,
+                        lfs3_max(bd->cfg->prog_width, 1)))
+                / lfs3_max(bd->cfg->prog_width, 1);
+        bd->progged += size;
+    }
     if (bd->cfg->prog_sleep) {
         int err = nanosleep(&(struct timespec){
                 .tv_sec=bd->cfg->prog_sleep/1000000000,
@@ -474,10 +483,12 @@ int lfs3_kiwibd_erase(const struct lfs3_cfg *cfg, lfs3_block_t block) {
 
 erased:;
     // track erases
-    bd->erases += lfs3_alignup(cfg->block_size,
+    if (!bd->paused) {
-                lfs3_max(bd->cfg->erase_width, 1))
+        bd->erases += lfs3_alignup(cfg->block_size,
-            / lfs3_max(bd->cfg->erase_width, 1);
+                    lfs3_max(bd->cfg->erase_width, 1))
-    bd->erased += cfg->block_size;
+                / lfs3_max(bd->cfg->erase_width, 1);
+        bd->erased += cfg->block_size;
+    }
     if (bd->cfg->erase_sleep) {
         int err = nanosleep(&(struct timespec){
                 .tv_sec=bd->cfg->erase_sleep/1000000000,
@@ -553,6 +564,23 @@ int lfs3_kiwibd_simreset(const struct lfs3_cfg *cfg) {
     return 0;
 }
 
+int lfs3_kiwibd_simpause(const struct lfs3_cfg *cfg) {
+    LFS3_KIWIBD_TRACE("lfs3_kiwibd_simpause(%p)", (void*)cfg);
+    lfs3_kiwibd_t *bd = cfg->context;
+    bd->paused += 1;
+    LFS3_KIWIBD_TRACE("lfs3_kiwibd_simpause -> %d", 0);
+    return 0;
+}
+
+int lfs3_kiwibd_simresume(const struct lfs3_cfg *cfg) {
+    LFS3_KIWIBD_TRACE("lfs3_kiwibd_simresume(%p)", (void*)cfg);
+    lfs3_kiwibd_t *bd = cfg->context;
+    LFS3_ASSERT(bd->paused);
+    bd->paused -= 1;
+    LFS3_KIWIBD_TRACE("lfs3_kiwibd_simresume -> %d", 0);
+    return 0;
+}
+
 lfs3_kiwibd_sio_t lfs3_kiwibd_reads(const struct lfs3_cfg *cfg) {
     LFS3_KIWIBD_TRACE("lfs3_kiwibd_reads(%p)", (void*)cfg);
     lfs3_kiwibd_t *bd = cfg->context;

bd/lfs3_kiwibd.h

@@ -107,6 +107,8 @@ typedef struct lfs3_kiwibd {
         uint8_t *mem;
     } u;
 
+    // sim state
+    uint32_t paused;
     // amount read/progged/erased
     lfs3_kiwibd_io_t reads;
     lfs3_kiwibd_io_t progs;
@@ -161,6 +163,12 @@ lfs3_kiwibd_sns_t lfs3_kiwibd_simtime(const struct lfs3_cfg *cfg);
 // Reset simulation counters
 int lfs3_kiwibd_simreset(const struct lfs3_cfg *cfg);
 
+// Pause simulation counters
+int lfs3_kiwibd_simpause(const struct lfs3_cfg *cfg);
+
+// Resume simulation counters
+int lfs3_kiwibd_simresume(const struct lfs3_cfg *cfg);
+
 // Get total number of read transactions
 lfs3_kiwibd_sio_t lfs3_kiwibd_reads(const struct lfs3_cfg *cfg);
 

benches/bench_helpers.c

@@ -10,9 +10,6 @@
 // this writes a 1 block file 2*block_count times to get it into a good
 // state for benchmarking
 int bench_helpers_warmup(lfs3_t *lfs3) {
-    BENCH_STACK_PAUSE();
-    BENCH_HEAP_PAUSE();
-
     uint8_t *wbuf = malloc(BLOCK_SIZE);
     memset(wbuf, '1', BLOCK_SIZE);
 
@@ -29,18 +26,12 @@ int bench_helpers_warmup(lfs3_t *lfs3) {
     lfs3_remove(lfs3, "warmup") => 0;
 
     free(wbuf);
-
-    BENCH_HEAP_RESUME();
-    BENCH_STACK_RESUME();
     return 0;
 }
 
 
 // find tight disk usage
 uintmax_t bench_helpers_usage(lfs3_t *lfs3) {
-    BENCH_STACK_PAUSE();
-    BENCH_HEAP_PAUSE();
-
     // measure disk usage
     //
     // littlefs can be a dag, so build a bitmap to find the exact
@@ -70,9 +61,6 @@ uintmax_t bench_helpers_usage(lfs3_t *lfs3) {
     }
 
     free(usage_bmap);
-
-    BENCH_HEAP_RESUME();
-    BENCH_STACK_RESUME();
     return (uintmax_t)usage * (uintmax_t)BLOCK_SIZE;
 }
 

benches/bench_rt.toml

@@ -31,16 +31,18 @@ code = '''
     lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
     lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
     if (!SKIP_WARMUP) {
+        BENCH_PAUSE();
         int err = bench_helpers_warmup(&lfs3);
         if (err) {
             LFS3_ERROR("Bench warmup failed: %d", err);
             return;
         }
+        BENCH_RESUME();
     }
     uint32_t prng = SEED;
 
     // reset our timer
-    lfs3_kiwibd_simreset(CFG);
+    BENCH_SIMRESET();
 
     // create a file to read
     lfs3_file_t file;
@@ -54,14 +56,14 @@ code = '''
         lfs3_file_write(&lfs3, &file, wbuf, CHUNK) => CHUNK;
 
         // taking too long?
-        if (SIM_TIME && lfs3_kiwibd_simtime(CFG) >= SIM_TIME) {
+        if (SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME) {
             return;
         }
     }
     lfs3_file_close(&lfs3, &file) => 0;
 
     // reset our timer
-    lfs3_kiwibd_simreset(CFG);
+    BENCH_SIMRESET();
 
     // open the file
     BENCH_START("read");
@@ -69,7 +71,7 @@ code = '''
     lfs3_off_t size = 0;
     uint64_t readed = 0;
     while (!(SIM_SIZE && readed >= (uint64_t)SIM_SIZE)
-            && !(SIM_TIME && lfs3_kiwibd_simtime(CFG) >= SIM_TIME)) {
+            && !(SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME)) {
         // read from the file
         uint8_t rbuf[CHUNK];
         lfs3_file_read(&lfs3, &file, rbuf, CHUNK) => CHUNK;
@@ -97,15 +99,17 @@ code = '''
     lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
     lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
     if (!SKIP_WARMUP) {
+        BENCH_PAUSE();
         int err = bench_helpers_warmup(&lfs3);
         if (err) {
             return;
         }
+        BENCH_RESUME();
     }
     uint32_t prng = SEED;
 
     // reset our timer
-    lfs3_kiwibd_simreset(CFG);
+    BENCH_SIMRESET();
 
     // create a file to read
     lfs3_file_t file;
@@ -119,21 +123,21 @@ code = '''
         lfs3_file_write(&lfs3, &file, wbuf, CHUNK) => CHUNK;
 
         // taking too long?
-        if (SIM_TIME && lfs3_kiwibd_simtime(CFG) >= SIM_TIME) {
+        if (SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME) {
             return;
         }
     }
     lfs3_file_close(&lfs3, &file) => 0;
 
     // reset our timer
-    lfs3_kiwibd_simreset(CFG);
+    BENCH_SIMRESET();
 
     // open the file
     BENCH_START("read");
     lfs3_file_open(&lfs3, &file, "bench_random", LFS3_O_RDONLY) => 0;
     uint64_t readed = 0;
     while (!(SIM_SIZE && readed >= (uint64_t)SIM_SIZE)
-            && !(SIM_TIME && lfs3_kiwibd_simtime(CFG) >= SIM_TIME)) {
+            && !(SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME)) {
         // seek to a random location
         lfs3_off_t pos = BENCH_PRNG(&prng) % SIZE;
         lfs3_file_seek(&lfs3, &file, pos, LFS3_SEEK_SET) => pos;
@@ -161,15 +165,17 @@ code = '''
     lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
     lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
     if (!SKIP_WARMUP) {
+        BENCH_PAUSE();
         int err = bench_helpers_warmup(&lfs3);
         if (err) {
             return;
         }
+        BENCH_RESUME();
     }
     uint32_t prng = SEED;
 
     // reset our timer
-    lfs3_kiwibd_simreset(CFG);
+    BENCH_SIMRESET();
 
     // create files to read
     for (lfs3_size_t i = 0; i < (SIZE+(CHUNK-1))/CHUNK; i++) {
@@ -185,7 +191,7 @@ code = '''
             lfs3_file_write(&lfs3, &file, wbuf, d) => d;
 
             // taking too long?
-            if (SIM_TIME && lfs3_kiwibd_simtime(CFG) >= SIM_TIME) {
+            if (SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME) {
                 return;
             }
         }
@@ -193,13 +199,13 @@ code = '''
     }
 
     // reset our timer
-    lfs3_kiwibd_simreset(CFG);
+    BENCH_SIMRESET();
 
     // open the files
     BENCH_START("read");
     uint64_t readed = 0;
     while (!(SIM_SIZE && readed >= (uint64_t)SIM_SIZE)
-            && !(SIM_TIME && lfs3_kiwibd_simtime(CFG) >= SIM_TIME)) {
+            && !(SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME)) {
         // choose a random filename
         lfs3_off_t pos = BENCH_PRNG(&prng) % ((SIZE+(CHUNK-1))/CHUNK);
         char name[256];
@@ -218,7 +224,7 @@ code = '''
             readed += d;
 
             // taking too long?
-            if (SIM_TIME && lfs3_kiwibd_simtime(CFG) >= SIM_TIME) {
+            if (SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME) {
                 break;
             }
         }

benches/bench_wt.toml

@@ -31,16 +31,18 @@ code = '''
     lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
     lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
     if (!SKIP_WARMUP) {
+        BENCH_PAUSE();
         int err = bench_helpers_warmup(&lfs3);
         if (err) {
             LFS3_ERROR("Bench warmup failed: %d", err);
             return;
         }
+        BENCH_RESUME();
     }
     uint32_t prng = SEED;
 
     // reset our timer
-    lfs3_kiwibd_simreset(CFG);
+    BENCH_SIMRESET();
 
     // open a file
     BENCH_START("write");
@@ -52,7 +54,7 @@ code = '''
     // ok, one of these needs to be non-zero
     LFS3_ASSERT(SIM_TIME > 0 || SIM_SIZE > 0);
     while (!(SIM_SIZE && written >= (uint64_t)SIM_SIZE)
-            && !(SIM_TIME && lfs3_kiwibd_simtime(CFG) >= SIM_TIME)) {
+            && !(SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME)) {
         // arguably we should just rewind and continue writing to the
         // front of the file when we hit the end, but this overly
         // penalizes littlefs2, so instead we truncate
@@ -100,15 +102,17 @@ code = '''
     lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
     lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
     if (!SKIP_WARMUP) {
+        BENCH_PAUSE();
         int err = bench_helpers_warmup(&lfs3);
         if (err) {
             return;
         }
+        BENCH_RESUME();
     }
     uint32_t prng = SEED;
 
     // reset our timer
-    lfs3_kiwibd_simreset(CFG);
+    BENCH_SIMRESET();
 
     // open a file
     BENCH_START("write");
@@ -118,7 +122,7 @@ code = '''
     lfs3_off_t size = 0;
     uint64_t written = 0;
     while (!(SIM_SIZE && written >= (uint64_t)SIM_SIZE)
-            && !(SIM_TIME && lfs3_kiwibd_simtime(CFG) >= SIM_TIME)) {
+            && !(SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME)) {
         // seek to a random location
         lfs3_off_t pos = BENCH_PRNG(&prng) % SIZE;
         lfs3_file_seek(&lfs3, &file, pos, LFS3_SEEK_SET) => pos;
@@ -164,15 +168,17 @@ code = '''
     lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
     lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
     if (!SKIP_WARMUP) {
+        BENCH_PAUSE();
         int err = bench_helpers_warmup(&lfs3);
         if (err) {
             return;
         }
+        BENCH_RESUME();
     }
     uint32_t prng = SEED;
 
     // reset our timer
-    lfs3_kiwibd_simreset(CFG);
+    BENCH_SIMRESET();
 
     // open a file
     BENCH_START("write");
@@ -183,7 +189,7 @@ code = '''
     // ok, one of these needs to be non-zero
     LFS3_ASSERT(SIM_TIME > 0 || SIM_SIZE > 0);
     while (!(SIM_SIZE && written >= (uint64_t)SIM_SIZE)
-            && !(SIM_TIME && lfs3_kiwibd_simtime(CFG) >= SIM_TIME)) {
+            && !(SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME)) {
         // append to log
         uint8_t wbuf[CHUNK];
         for (lfs3_size_t j = 0; j < CHUNK; j++) {
@@ -241,15 +247,17 @@ code = '''
     lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
     lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
     if (!SKIP_WARMUP) {
+        BENCH_PAUSE();
         int err = bench_helpers_warmup(&lfs3);
         if (err) {
             return;
         }
+        BENCH_RESUME();
     }
     uint32_t prng = SEED;
 
     // reset our timer
-    lfs3_kiwibd_simreset(CFG);
+    BENCH_SIMRESET();
 
     // open a file
     BENCH_START("write");
@@ -257,7 +265,7 @@ code = '''
     // ok, one of these needs to be non-zero
     LFS3_ASSERT(SIM_TIME > 0 || SIM_SIZE > 0);
     while (!(SIM_SIZE && written >= (uint64_t)SIM_SIZE)
-            && !(SIM_TIME && lfs3_kiwibd_simtime(CFG) >= SIM_TIME)) {
+            && !(SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME)) {
         // choose a random filename
         lfs3_off_t pos = BENCH_PRNG(&prng) % FILE_COUNT;
         char name[256];
@@ -279,7 +287,7 @@ code = '''
             written += d;
 
             // taking too long?
-            if (SIM_TIME && lfs3_kiwibd_simtime(CFG) >= SIM_TIME) {
+            if (SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME) {
                 break;
             }
         }

runners/bench_runner.c

@@ -24,20 +24,6 @@
 #include <stddef.h>
 
 
-// some common types
-#ifndef BENCH_KIWIBD
-typedef lfs3_emubd_io_t   bench_io_t;
-typedef lfs3_emubd_sio_t  bench_sio_t;
-typedef lfs3_emubd_ns_t   bench_ns_t;
-typedef lfs3_emubd_sns_t  bench_sns_t;
-#else
-typedef lfs3_kiwibd_io_t  bench_io_t;
-typedef lfs3_kiwibd_sio_t bench_sio_t;
-typedef lfs3_kiwibd_ns_t  bench_ns_t;
-typedef lfs3_kiwibd_sns_t bench_sns_t;
-#endif
-
-
 // some helpers
 
 // append to an array with amortized doubling
@@ -625,7 +611,8 @@ void bench_permutation(size_t i, uint32_t *buffer, size_t size) {
 
 // stack hooks
 #ifdef BENCH_STACK
-uint32_t bench_stack_entered = 0;
+uint32_t bench_stack_entered = false;
+uint32_t bench_stack_paused = false;
 uint8_t *bench_stack_entrance = NULL;
 size_t bench_stack_watermark = 0;
 #endif
@@ -634,7 +621,8 @@ size_t bench_stack_watermark = 0;
 #ifdef BENCH_STACK
 __attribute__((noinline))
 void bench_stack_enter(void) {
-    bench_stack_entered = 1;
+    bench_stack_entered = true;
+    bench_stack_paused = false;
     bench_stack_entrance = __builtin_frame_address(0);
     bench_stack_watermark = 0;
 }
@@ -642,7 +630,14 @@ void bench_stack_enter(void) {
 
 #ifdef BENCH_STACK
 void bench_stack_exit(void) {
-    bench_stack_entered = 0;
+    assert(bench_stack_entered);
+    bench_stack_entered = false;
+}
+#endif
+
+#ifdef BENCH_STACK
+void bench_stack_reset(void) {
+    bench_stack_watermark = 0;
 }
 #endif
 
@@ -650,7 +645,7 @@ void bench_stack_exit(void) {
 #ifdef BENCH_STACK
 __attribute__((noinline))
 void bench_stack_pause(void) {
-    if (bench_stack_entered & 1) {
+    if (bench_stack_entered && !bench_stack_paused) {
         uint8_t *current = __builtin_frame_address(0);
 
         // keep track of the deepest stack
@@ -664,14 +659,14 @@ void bench_stack_pause(void) {
         }
     }
 
-    // haha, a little 32-bit stack
+    bench_stack_paused += 1;
-    bench_stack_entered <<= 1;
 }
 #endif
 
 #ifdef BENCH_STACK
 void bench_stack_resume(void) {
-    bench_stack_entered >>= 1;
+    assert(bench_stack_paused);
+    bench_stack_paused -= 1;
 }
 #endif
 
@@ -695,7 +690,8 @@ size_t bench_stack_current(void) {
 
 // heap hooks
 #ifdef BENCH_HEAP
-uint32_t bench_heap_entered = 0;
+uint32_t bench_heap_entered = false;
+uint32_t bench_heap_paused = false;
 size_t bench_heap_current = 0;
 size_t bench_heap_watermark = 0;
 #endif
@@ -703,7 +699,8 @@ size_t bench_heap_watermark = 0;
 // call me when entering/exiting a bench!
 #ifdef BENCH_HEAP
 void bench_heap_enter(void) {
-    bench_heap_entered = 1;
+    bench_heap_entered = true;
+    bench_heap_paused = false;
     bench_heap_current = 0;
     bench_heap_watermark = 0;
 }
@@ -711,31 +708,34 @@ void bench_heap_enter(void) {
 
 #ifdef BENCH_HEAP
 void bench_heap_exit(void) {
-    bench_heap_entered = 0;
+    assert(bench_heap_entered);
-    if (bench_heap_watermark != 0) {
+    bench_heap_entered = false;
-        fprintf(stderr, "warning: memory leak detected (%zd > 0)\n",
+}
-                bench_heap_watermark);
+#endif
-    }
+
+#ifdef BENCH_HEAP
+void bench_heap_reset(void) {
+    bench_heap_watermark = 0;
 }
 #endif
 
 // call me when entering/exiting a bd op!
 #ifdef BENCH_HEAP
 void bench_heap_pause(void) {
-    // haha, a little 32-bit stack
+    bench_heap_paused += 1;
-    bench_heap_entered <<= 1;
 }
 #endif
 
 #ifdef BENCH_HEAP
 void bench_heap_resume(void) {
-    bench_heap_entered >>= 1;
+    assert(bench_heap_paused);
+    bench_heap_paused -= 1;
 }
 #endif
 
 #ifdef BENCH_HEAP
 void bench_heap_inc(size_t size) {
-    if (bench_heap_entered & 1) {
+    if (bench_heap_entered && !bench_heap_paused) {
         bench_heap_current += size;
         // keep track of the deepest heap
         if (bench_heap_current > bench_heap_watermark) {
@@ -747,9 +747,8 @@ void bench_heap_inc(size_t size) {
 
 #ifdef BENCH_HEAP
 void bench_heap_dec(size_t size) {
-    if (bench_heap_entered & 1) {
+    if (bench_heap_entered && !bench_heap_paused) {
-        assert(bench_heap_current >= size);
+        bench_heap_current -= lfs3_min(size, bench_heap_current);
-        bench_heap_current -= size;
     }
 }
 #endif
@@ -885,11 +884,16 @@ static bench_record_t *bench_records;
 size_t bench_record_count;
 size_t bench_record_capacity;
 
-void bench_reset(const struct lfs3_cfg *cfg) {
+void bench_init(const struct lfs3_cfg *cfg) {
     bench_cfg = cfg;
     bench_record_count = 0;
 }
 
+void bench_deinit(const struct lfs3_cfg *cfg) {
+    (void)cfg;
+    // do nothing
+}
+
 void bench_start(const char *probe) {
     BENCH_STACK_PAUSE();
     BENCH_HEAP_PAUSE();
@@ -1091,6 +1095,95 @@ void bench_fresult(const char *probe, uintmax_t n, double result) {
     BENCH_STACK_RESUME();
 }
 
+bench_ns_t bench_simtime(void) {
+    // get the current simtime
+    assert(bench_cfg);
+    #ifndef BENCH_KIWIBD
+    // note this can error if no timings provided
+    bench_sns_t simtime = lfs3_emubd_simtime(bench_cfg);
+    assert(simtime >= 0);
+    #else
+    // note this can error if no timings provided
+    bench_sns_t simtime = lfs3_kiwibd_simtime(bench_cfg);
+    assert(simtime >= 0);
+    #endif
+    return simtime;
+}
+
+void bench_simreset(void) {
+    // reset bd
+    #ifndef BENCH_KIWIBD
+    int err = lfs3_emubd_simreset(bench_cfg);
+    assert(!err);
+    #else
+    int err = lfs3_kiwibd_simreset(bench_cfg);
+    assert(!err);
+    #endif
+}
+
+void bench_simpause(void) {
+    // pause bd simulation
+    #ifndef BENCH_KIWIBD
+    int err = lfs3_emubd_simpause(bench_cfg);
+    assert(!err);
+    #else
+    int err = lfs3_kiwibd_simpause(bench_cfg);
+    assert(!err);
+    #endif
+}
+
+void bench_simresume(void) {
+    // resume bd simulation
+    #ifndef BENCH_KIWIBD
+    int err = lfs3_emubd_simresume(bench_cfg);
+    assert(!err);
+    #else
+    int err = lfs3_kiwibd_simresume(bench_cfg);
+    assert(!err);
+    #endif
+}
+
+void bench_reset(void) {
+    // reset bd
+    bench_simreset();
+
+    // reset stack/heap measurements
+    #ifdef BENCH_HEAP
+    bench_heap_reset();
+    #endif
+    #ifdef BENCH_STACK
+    bench_stack_reset();
+    #endif
+}
+
+void bench_pause(void) {
+    // pause stack/heap measurements
+    #ifdef BENCH_STACK
+    bench_stack_pause();
+    #endif
+    #ifdef BENCH_HEAP
+    bench_heap_pause();
+    #endif
+
+    // pause bd simulation
+    bench_simpause();
+}
+
+void bench_resume(void) {
+    // resume bd simulation
+    bench_simresume();
+
+    // resume stack/heap measurements
+    #ifdef BENCH_HEAP
+    bench_heap_resume();
+    #endif
+    #ifdef BENCH_STACK
+    bench_stack_resume();
+    #endif
+}
+
+
+
 
 // encode our permutation into a reusable id
 static void perm_printid(
@@ -1868,22 +1961,23 @@ void perm_run(
     printf("running ");
     perm_printid(suite, case_);
     printf("\n");
-    bench_reset(CFG);
+    bench_init(CFG);
-    #ifdef BENCH_STACK
-    bench_stack_enter();
-    #endif
     #ifdef BENCH_HEAP
     bench_heap_enter();
     #endif
+    #ifdef BENCH_STACK
+    bench_stack_enter();
+    #endif
 
     case_->run(CFG);
 
-    #ifdef BENCH_HEAP
-    bench_heap_exit();
-    #endif
     #ifdef BENCH_STACK
     bench_stack_exit();
     #endif
+    #ifdef BENCH_HEAP
+    bench_heap_exit();
+    #endif
+    bench_deinit(CFG);
     printf("finished ");
     perm_printid(suite, case_);
     printf("\n");

runners/bench_runner.h

@@ -78,6 +78,19 @@ void bench_trace(const char *fmt, ...);
 #undef _STDIO_H
 
 
+// some common types
+#ifndef BENCH_KIWIBD
+typedef lfs3_emubd_io_t   bench_io_t;
+typedef lfs3_emubd_sio_t  bench_sio_t;
+typedef lfs3_emubd_ns_t   bench_ns_t;
+typedef lfs3_emubd_sns_t  bench_sns_t;
+#else
+typedef lfs3_kiwibd_io_t  bench_io_t;
+typedef lfs3_kiwibd_sio_t bench_sio_t;
+typedef lfs3_kiwibd_ns_t  bench_ns_t;
+typedef lfs3_kiwibd_sns_t bench_sns_t;
+#endif
+
 // generated bench configurations
 struct lfs3_cfg;
 
@@ -138,6 +151,24 @@ void bench_fresult(const char *probe, uintmax_t n, double result);
 #define BENCH_RESULT(probe, n, result) bench_result(probe, n, result)
 #define BENCH_FRESULT(probe, n, result) bench_fresult(probe, n, result)
 
+// extra hooks to get the current simtime, pause readed/progged/erased
+// counters, etc
+bench_ns_t bench_simtime(void);
+void bench_simreset(void);
+void bench_simpause(void);
+void bench_simresume(void);
+void bench_reset(void);
+void bench_pause(void);
+void bench_resume(void);
+
+#define BENCH_SIMTIME() bench_simtime()
+#define BENCH_SIMRESET() bench_simreset()
+#define BENCH_SIMPAUSE() bench_simpause()
+#define BENCH_SIMRESUME() bench_simresume()
+#define BENCH_RESET() bench_reset()
+#define BENCH_PAUSE() bench_pause()
+#define BENCH_RESUME() bench_resume()
+
 
 // deterministic prng for pseudo-randomness in benches
 uint32_t bench_prng(uint32_t *state);
@@ -155,15 +186,18 @@ void bench_permutation(size_t i, uint32_t *buffer, size_t size);
 // get the maximum/current stack usage for this run
 extern size_t bench_stack_watermark;
 __attribute__((noinline)) size_t bench_stack_current(void);
+void bench_stack_reset(void);
 __attribute__((noinline)) void bench_stack_pause(void);
 void bench_stack_resume(void);
 
 #define BENCH_STACK_WATERMARK() bench_stack_watermark
 #define BENCH_STACK_CURRENT() bench_stack_current()
+#define BENCH_STACK_RESET() bench_stack_reset()
 #define BENCH_STACK_PAUSE() bench_stack_pause()
 #define BENCH_STACK_RESUME() bench_stack_resume()
 #else
 // stubs if not measuring stack
+#define BENCH_STACK_RESET()
 #define BENCH_STACK_PAUSE()
 #define BENCH_STACK_RESUME()
 #endif
@@ -179,12 +213,14 @@ void bench_heap_dec(size_t size);
 
 #define BENCH_HEAP_WATERMARK() bench_heap_watermark
 #define BENCH_HEAP_CURRENT() bench_heap_current
+#define BENCH_HEAP_RESET() bench_heap_reset()
 #define BENCH_HEAP_PAUSE() bench_heap_pause()
 #define BENCH_HEAP_RESUME() bench_heap_resume()
 #define BENCH_HEAP_INC(size) bench_heap_inc(size)
 #define BENCH_HEAP_DEC(size) bench_heap_dec(size)
 #else
 // stubs if not measuring heap
+#define BENCH_HEAP_RESET()
 #define BENCH_HEAP_PAUSE()
 #define BENCH_HEAP_RESUME()
 #define BENCH_HEAP_INC(size)

runners/test_runner.c

@@ -24,20 +24,6 @@
 #include <stddef.h>
 
 
-// some common types
-#ifndef TEST_KIWIBD
-typedef lfs3_emubd_ns_t   test_ns_t;
-typedef lfs3_emubd_sns_t  test_sns_t;
-typedef lfs3_emubd_powercycles_t  test_powercycles_t;
-typedef lfs3_emubd_spowercycles_t test_spowercycles_t;
-#else
-typedef lfs3_kiwibd_ns_t  test_ns_t;
-typedef lfs3_kiwibd_sns_t test_sns_t;
-typedef void              test_powercycles_t;
-typedef void              test_spowercycles_t;
-#endif
-
-
 // some helpers
 
 // append to an array with amortized doubling

runners/test_runner.h

@@ -78,6 +78,19 @@ void test_trace(const char *fmt, ...);
 #undef _STDIO_H
 
 
+// some common types
+#ifndef TEST_KIWIBD
+typedef lfs3_emubd_ns_t   test_ns_t;
+typedef lfs3_emubd_sns_t  test_sns_t;
+typedef lfs3_emubd_powercycles_t  test_powercycles_t;
+typedef lfs3_emubd_spowercycles_t test_spowercycles_t;
+#else
+typedef lfs3_kiwibd_ns_t  test_ns_t;
+typedef lfs3_kiwibd_sns_t test_sns_t;
+typedef void              test_powercycles_t;
+typedef void              test_spowercycles_t;
+#endif
+
 // generated test configurations
 struct lfs3_cfg;