Reworked test_ck_spam* tests to rely on gcksums

Now that gcksums are working and we can detect rollback issues, it's
worth revisiting our most aggressive bit-error tests.

Unfortunately, I think due to focusing on ckprogs, these were a bit less
ready-to-go than I had hoped. We still have the read-hole, so the sort
of errors we can expect to detect is a bit limited.

Still, managed to come up with some schemes that I think are
interesting:

- ckprogs - Limited to catching bit-errors during progs, but these tests
  work great.

- ckdata - Limited to manual bit-errors, but can detect both metdata +
  data errors.

- ckmeta+ckfetches - Limited to manual bit-errors, ckmeta detects
  mtree errors, while ckfetches detects btree + data errors.

- ckmeta+ckdatacksums - Limited to manual bit-errors, ckmeta detects
  metadata errors, while ckdatacksums detects data errors.

To make testing manual bit-errors a bit easier, and to avoid
reimplementing the bit randomizer in emubd, I added
LFS_EMUBD_BADBLOCK_MANUAL and lfs_emubd_flip to let the tests manually
control when bits flip.

---

Unfortunately open files are proving to be an issue for these tests,
since we don't really expect corrupted metadata after lfsr_file_open (
assuming no read-hole).

For now I've limited these new ck-modes to the tests without open files,
but we should probably revisit this.

bd/lfs_emubd.c

@@ -912,10 +912,7 @@ int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block) {
         // flipping bits? if we're not manually overridden, choose a
         // new bad bit on erase, this makes it more likely to
         // eventually cause problems
-        } else if (bd->cfg->badblock_behavior
+        } else {
-                    == LFS_EMUBD_BADBLOCK_PROGFLIP
-                || bd->cfg->badblock_behavior
-                    == LFS_EMUBD_BADBLOCK_READFLIP) {
             if (!(0x80000000 & b->bad_bit)) {
                 b->bad_bit = lfs_emubd_prng_(&bd->prng)
                         % (cfg->block_size*8);
@@ -1118,6 +1115,12 @@ int lfs_emubd_markbad(const struct lfs_config *cfg,
     // set the wear
     b->wear = -1;
 
+    // choose a bad bit now in case this block is never erased
+    if (!(0x80000000 & b->bad_bit)) {
+        b->bad_bit = lfs_emubd_prng_(&bd->prng)
+                % (cfg->block_size*8);
+    }
+
     LFS_EMUBD_TRACE("lfs_emubd_markbad -> %d", 0);
     return 0;
 }
@@ -1241,10 +1244,8 @@ int lfs_emubd_markbadbit(const struct lfs_config *cfg,
     return 0;
 }
 
-int lfs_emubd_flipbit(const struct lfs_config *cfg,
+int lfs_emubd_flipbit_(const struct lfs_config *cfg,
         lfs_block_t block, lfs_size_t bit) {
-    LFS_EMUBD_TRACE("lfs_emubd_flipbit(%p, %"PRIu32", %"PRIu32")",
-            (void*)cfg, block, bit);
     lfs_emubd_t *bd = cfg->context;
 
     // check if block is valid
@@ -1253,7 +1254,6 @@ int lfs_emubd_flipbit(const struct lfs_config *cfg,
     // mutate the block
     lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, bd->blocks[block]);
     if (!b) {
-        LFS_EMUBD_TRACE("lfs_emubd_flipbit -> %d", LFS_ERR_NOMEM);
         return LFS_ERR_NOMEM;
     }
     bd->blocks[block] = b;
@@ -1268,22 +1268,57 @@ int lfs_emubd_flipbit(const struct lfs_config *cfg,
                 SEEK_SET);
         if (res1 < 0) {
             int err = -errno;
-            LFS_EMUBD_TRACE("lfs_emubd_flipbit -> %d", err);
             return err;
         }
 
         ssize_t res2 = write(bd->disk->fd, &b->data[bit/8], 1);
         if (res2 < 0) {
             int err = -errno;
-            LFS_EMUBD_TRACE("lfs_emubd_flipbit -> %d", err);
             return err;
         }
     }
 
+    return 0;
+}
+
+
+int lfs_emubd_flipbit(const struct lfs_config *cfg,
+        lfs_block_t block, lfs_size_t bit) {
+    LFS_EMUBD_TRACE("lfs_emubd_flipbit(%p, %"PRIu32", %"PRIu32")",
+            (void*)cfg, block, bit);
+
+    // flip the bit
+    int err = lfs_emubd_flipbit_(cfg, block, bit);
+    if (err) {
+        LFS_EMUBD_TRACE("lfs_emubd_flipbit -> %d", err);
+        return err;
+    }
+
     LFS_EMUBD_TRACE("lfs_emubd_flipbit -> %d", 0);
     return 0;
 }
 
+int lfs_emubd_flip(const struct lfs_config *cfg) {
+    LFS_EMUBD_TRACE("lfs_emubd_flip(%p)", (void*)cfg);
+    lfs_emubd_t *bd = cfg->context;
+
+    // flip all bits in bad blocks, make sure not to allocate blocks we
+    // don't need
+    for (lfs_block_t i = 0; i < cfg->block_count; i++) {
+        const lfs_emubd_block_t *b = bd->blocks[i];
+        if (b && b->wear > bd->cfg->erase_cycles) {
+            int err = lfs_emubd_flipbit_(cfg, i, b->bad_bit & 0x7fffffff);
+            if (err) {
+                LFS_EMUBD_TRACE("lfs_emubd_flip -> %d", err);
+                return err;
+            }
+        }
+    }
+
+    LFS_EMUBD_TRACE("lfs_emubd_flip -> %d", 0);
+    return 0;
+}
+
 lfs_emubd_spowercycles_t lfs_emubd_powercycles(
         const struct lfs_config *cfg) {
     LFS_EMUBD_TRACE("lfs_emubd_powercycles(%p)", (void*)cfg);

bd/lfs_emubd.h

@@ -41,6 +41,7 @@ typedef enum lfs_emubd_badblock_behavior {
     LFS_EMUBD_BADBLOCK_ERASENOOP    = 4, // Erase does nothing silently
     LFS_EMUBD_BADBLOCK_PROGFLIP     = 5, // Prog flips a bit
     LFS_EMUBD_BADBLOCK_READFLIP     = 6, // Read flips a bit sometimes
+    LFS_EMUBD_BADBLOCK_MANUAL       = 7, // Bits require manual flipping
 } lfs_emubd_badblock_behavior_t;
 
 // Mode determining how power-loss behaves during testing.
@@ -248,6 +249,9 @@ int lfs_emubd_markbadbit(const struct lfs_config *cfg,
 int lfs_emubd_flipbit(const struct lfs_config *cfg,
         lfs_block_t block, lfs_size_t bit);
 
+// Flip all bits marked as bad
+int lfs_emubd_flip(const struct lfs_config *cfg);
+
 // Get the remaining power-cycles
 lfs_emubd_spowercycles_t lfs_emubd_powercycles(
         const struct lfs_config *cfg);

tests/test_ck.toml

@@ -2429,22 +2429,31 @@ code = '''
 # we basically just throw errors at filesystem operations until they
 # error with either LFS_ERR_CORRUPT or LFS_ERR_NOSPC
 #
-# TODO this is all basically in stasis until rollback is solved
+# TODO revisit these when ckredund is implemented, ckredund should
+# finally close the ckread hole
 
 # fuzz errors with fuzz dirs
 [cases.test_ck_spam_dir_fuzz]
-# TODO enable other methods once rollback protection is in place
 # METHOD=0 => ckprogs
-# METHOD=1 => ckfetches
+# METHOD=1 => ckdata
-# METHOD=2 => ckparity
+# METHOD=2 => ckmeta+ckfetches
-# METHOD=3 => ckdatacksums
+# METHOD=3 => ckmeta+ckdatacksums
-defines.METHOD = [0]
+defines.METHOD = [0, 1, 2, 3]
 defines.PERIOD = 10
+# protecting the mrootanchor encourages more interesting failures, and
+# simulates storage with hardened {0,1} blocks
 defines.PROTECTED_MROOTANCHOR = [false, true]
-defines.BADBLOCK_BEHAVIOR = 'LFS_EMUBD_BADBLOCK_PROGFLIP'
+# we can't reliably detect bit errors in erased blocks, we rely on
+# future progs failing if this happens
+defines.ERASE_VALUE = -1
+defines.BADBLOCK_BEHAVIOR = '''
+    (METHOD == 0)
+        ? LFS_EMUBD_BADBLOCK_PROGFLIP
+        : LFS_EMUBD_BADBLOCK_MANUAL
+'''
 defines.CKPROGS       = 'METHOD == 0'
-defines.CKFETCHES = 'METHOD == 1'
+defines.CKFETCHES     = 'METHOD == 2'
-defines.CKPARITY  = 'METHOD == 2'
+defines.CKPARITY      = false
 defines.CKDATACKSUMS  = 'METHOD == 3'
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256]
 defines.SEED = 'range(10)'
@@ -2493,13 +2502,14 @@ code = '''
     lfs_size_t *sim = malloc(N*sizeof(lfs_size_t));
     lfs_size_t sim_size = 0;
 
-    // keep adding block errors until we either run out of blocks or
+    // keep adding errors until we either run out of blocks or detect
-    // our errors
+    // corruption
     lfs_size_t i = 0;
     for (; i < PERIOD*BLOCK_COUNT; i++) {
+        // add an error?
         if (i % PERIOD == 0
-                && !(PROTECTED_MROOTANCHOR
+                // protected mrootanchor? (just makes things more interesting)
-                    && badblocks[i/PERIOD] < 2)) {
+                && !(PROTECTED_MROOTANCHOR && badblocks[i/PERIOD] < 2)) {
             lfs_block_t badblock = badblocks[i/PERIOD];
             printf("badblock: 0x%x\n", badblock);
 
@@ -2507,16 +2517,52 @@ code = '''
             // types of errors, so we implement errors for each one a
             // bit differently
 
-            // ckprogs? ckparity? ckdatacksums?
-            if (METHOD == 0 || METHOD == 2 || METHOD == 3) {
             // mark our badblock as bad
             lfs_emubd_markbad(CFG, badblock) => 0;
 
-            // ckfetches?
+            // manually flipping? flip all badbits in badblocks
-            } else if (METHOD == 1) {
+            if (BADBLOCK_BEHAVIOR == LFS_EMUBD_BADBLOCK_MANUAL) {
-                // flip a bit
+                lfs_emubd_flip(CFG) => 0;
-                lfs_emubd_flipbit(CFG, badblock,
+            }
-                        lfs_emubd_prng(CFG) % (BLOCK_SIZE*8)) => 0;
+
+            // run ckdata?
+            if (METHOD == 1) {
+                int err = lfsr_fs_ckdata(&lfs);
+                assert(!err || err == LFS_ERR_CORRUPT);
+                if (err == LFS_ERR_CORRUPT) {
+                    goto corrupt_mounted;
+                }
+            // run ckmeta?
+            } else if (METHOD == 3) {
+                int err = lfsr_fs_ckmeta(&lfs);
+                assert(!err || err == LFS_ERR_CORRUPT);
+                if (err == LFS_ERR_CORRUPT) {
+                    goto corrupt_mounted;
+                }
+            // run ckmeta mtreeonly?
+            } else if (METHOD == 2) {
+                // need an explicit traversal for this
+                lfsr_traversal_t t;
+                lfsr_traversal_open(&lfs, &t,
+                        LFS_T_MTREEONLY | LFS_T_CKMETA) => 0;
+                for (lfs_block_t i = 0;; i++) {
+                    // a bit hacky, but this catches infinite loops
+                    LFS_ASSERT(i < 2*BLOCK_COUNT);
+
+                    struct lfs_tinfo tinfo;
+                    int err = lfsr_traversal_read(&lfs, &t, &tinfo);
+                    assert(!err
+                            || err == LFS_ERR_NOENT
+                            || err == LFS_ERR_CORRUPT);
+                    if (err == LFS_ERR_NOENT) {
+                        break;
+                    }
+                    if (err == LFS_ERR_CORRUPT) {
+                        lfsr_traversal_close(&lfs, &t) => 0;
+                        goto corrupt_mounted;
+                    }
+                }
+                lfsr_traversal_close(&lfs, &t) => 0;
             }
         }
 
@@ -2686,18 +2732,26 @@ corrupt_mounted:;
 
 # fuzz errors with fuzz files
 [cases.test_ck_spam_file_fuzz]
-# TODO enable other methods once rollback protection is in place
 # METHOD=0 => ckprogs
-# METHOD=1 => ckfetches
+# METHOD=1 => ckdata
-# METHOD=2 => ckparity
+# METHOD=2 => ckmeta+ckfetches
-# METHOD=3 => ckdatacksums
+# METHOD=3 => ckmeta+ckdatacksums
-defines.METHOD = [0]
+defines.METHOD = [0, 1, 2, 3]
 defines.PERIOD = 10
+# protecting the mrootanchor encourages more interesting failures, and
+# simulates storage with hardened {0,1} blocks
 defines.PROTECTED_MROOTANCHOR = [false, true]
-defines.BADBLOCK_BEHAVIOR = 'LFS_EMUBD_BADBLOCK_PROGFLIP'
+# we can't reliably detect bit errors in erased blocks, we rely on
+# future progs failing if this happens
+defines.ERASE_VALUE = -1
+defines.BADBLOCK_BEHAVIOR = '''
+    (METHOD == 0)
+        ? LFS_EMUBD_BADBLOCK_PROGFLIP
+        : LFS_EMUBD_BADBLOCK_MANUAL
+'''
 defines.CKPROGS       = 'METHOD == 0'
-defines.CKFETCHES = 'METHOD == 1'
+defines.CKFETCHES     = 'METHOD == 2'
-defines.CKPARITY  = 'METHOD == 2'
+defines.CKPARITY      = false
 defines.CKDATACKSUMS  = 'METHOD == 3'
 defines.N = [1, 2, 4, 8, 16, 32, 64]
 defines.SIZE = [
@@ -2757,13 +2811,14 @@ code = '''
     uint32_t *sim_prngs = malloc(N*sizeof(uint32_t));
     lfs_size_t sim_size = 0;
 
-    // keep adding block errors until we either run out of blocks or
+    // keep adding errors until we either run out of blocks or detect
-    // our errors
+    // corruption
     lfs_size_t i = 0;
     for (; i < PERIOD*BLOCK_COUNT; i++) {
+        // add an error?
         if (i % PERIOD == 0
-                && !(PROTECTED_MROOTANCHOR
+                // protected mrootanchor? (just makes things more interesting)
-                    && badblocks[i/PERIOD] < 2)) {
+                && !(PROTECTED_MROOTANCHOR && badblocks[i/PERIOD] < 2)) {
             lfs_block_t badblock = badblocks[i/PERIOD];
             printf("badblock: 0x%x\n", badblock);
 
@@ -2771,16 +2826,52 @@ code = '''
             // types of errors, so we implement errors for each one a
             // bit differently
 
-            // ckprogs? ckparity? ckdatacksums?
-            if (METHOD == 0 || METHOD == 2 || METHOD == 3) {
             // mark our badblock as bad
             lfs_emubd_markbad(CFG, badblock) => 0;
 
-            // ckfetches?
+            // manually flipping? flip all badbits in badblocks
-            } else if (METHOD == 1) {
+            if (BADBLOCK_BEHAVIOR == LFS_EMUBD_BADBLOCK_MANUAL) {
-                // flip a bit
+                lfs_emubd_flip(CFG) => 0;
-                lfs_emubd_flipbit(CFG, badblock,
+            }
-                        lfs_emubd_prng(CFG) % (BLOCK_SIZE*8)) => 0;
+
+            // run ckdata?
+            if (METHOD == 1) {
+                int err = lfsr_fs_ckdata(&lfs);
+                assert(!err || err == LFS_ERR_CORRUPT);
+                if (err == LFS_ERR_CORRUPT) {
+                    goto corrupt_mounted;
+                }
+            // run ckmeta?
+            } else if (METHOD == 3) {
+                int err = lfsr_fs_ckmeta(&lfs);
+                assert(!err || err == LFS_ERR_CORRUPT);
+                if (err == LFS_ERR_CORRUPT) {
+                    goto corrupt_mounted;
+                }
+            // run ckmeta mtreeonly?
+            } else if (METHOD == 2) {
+                // need an explicit traversal for this
+                lfsr_traversal_t t;
+                lfsr_traversal_open(&lfs, &t,
+                        LFS_T_MTREEONLY | LFS_T_CKMETA) => 0;
+                for (lfs_block_t i = 0;; i++) {
+                    // a bit hacky, but this catches infinite loops
+                    LFS_ASSERT(i < 2*BLOCK_COUNT);
+
+                    struct lfs_tinfo tinfo;
+                    int err = lfsr_traversal_read(&lfs, &t, &tinfo);
+                    assert(!err
+                            || err == LFS_ERR_NOENT
+                            || err == LFS_ERR_CORRUPT);
+                    if (err == LFS_ERR_NOENT) {
+                        break;
+                    }
+                    if (err == LFS_ERR_CORRUPT) {
+                        lfsr_traversal_close(&lfs, &t) => 0;
+                        goto corrupt_mounted;
+                    }
+                }
+                lfsr_traversal_close(&lfs, &t) => 0;
             }
         }
 
@@ -2993,7 +3084,13 @@ code = '''
             }
 
             uint8_t rbuf[SIZE];
-            lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
+            lfs_ssize_t d = lfsr_file_read(&lfs, &file, rbuf, SIZE);
+            assert(d == SIZE
+                    || (d == LFS_ERR_CORRUPT && (METHOD == 2 || METHOD == 3)));
+            if (d == LFS_ERR_CORRUPT) {
+                lfsr_file_close(&lfs, &file) => 0;
+                goto corrupt_mounted;
+            }
             assert(memcmp(rbuf, wbuf, SIZE) == 0);
             lfsr_file_close(&lfs, &file) => 0;
         }
@@ -3011,18 +3108,27 @@ corrupt_mounted:;
 
 # fuzz errors with more complex file writes
 [cases.test_ck_spam_fwrite_fuzz]
-# TODO enable other methods once rollback protection is in place
 # METHOD=0 => ckprogs
-# METHOD=1 => ckfetches
+# METHOD=1 => ckdata
-# METHOD=2 => ckparity
+# METHOD=2 => ckmeta+ckfetches
-# METHOD=3 => ckdatacksums
+# METHOD=3 => ckmeta+ckdatacksums
+# note only ckprogs works with open files
 defines.METHOD = [0]
 defines.PERIOD = 10
+# protecting the mrootanchor encourages more interesting failures, and
+# simulates storage with hardened {0,1} blocks
 defines.PROTECTED_MROOTANCHOR = [false, true]
-defines.BADBLOCK_BEHAVIOR = 'LFS_EMUBD_BADBLOCK_PROGFLIP'
+# we can't reliably detect bit errors in erased blocks, we rely on
+# future progs failing if this happens
+defines.ERASE_VALUE = -1
+defines.BADBLOCK_BEHAVIOR = '''
+    (METHOD == 0)
+        ? LFS_EMUBD_BADBLOCK_PROGFLIP
+        : LFS_EMUBD_BADBLOCK_MANUAL
+'''
 defines.CKPROGS       = 'METHOD == 0'
-defines.CKFETCHES = 'METHOD == 1'
+defines.CKFETCHES     = 'METHOD == 2'
-defines.CKPARITY  = 'METHOD == 2'
+defines.CKPARITY      = false
 defines.CKDATACKSUMS  = 'METHOD == 3'
 defines.SIZE = [
     'FILE_BUFFER_SIZE/2',
@@ -3111,13 +3217,14 @@ code = '''
         lfsr_file_sync(&lfs, &file) => 0;
     }
 
-    // keep adding block errors until we either run out of blocks or
+    // keep adding errors until we either run out of blocks or detect
-    // our errors
+    // corruption
     lfs_size_t i = 0;
     for (; i < PERIOD*BLOCK_COUNT; i++) {
+        // add an error?
         if (i % PERIOD == 0
-                && !(PROTECTED_MROOTANCHOR
+                // protected mrootanchor? (just makes things more interesting)
-                    && badblocks[i/PERIOD] < 2)) {
+                && !(PROTECTED_MROOTANCHOR && badblocks[i/PERIOD] < 2)) {
             lfs_block_t badblock = badblocks[i/PERIOD];
             printf("badblock: 0x%x\n", badblock);
 
@@ -3125,16 +3232,52 @@ code = '''
             // types of errors, so we implement errors for each one a
             // bit differently
 
-            // ckprogs? ckparity? ckdatacksums?
-            if (METHOD == 0 || METHOD == 2 || METHOD == 3) {
             // mark our badblock as bad
             lfs_emubd_markbad(CFG, badblock) => 0;
 
-            // ckfetches?
+            // manually flipping? flip all badbits in badblocks
-            } else if (METHOD == 1) {
+            if (BADBLOCK_BEHAVIOR == LFS_EMUBD_BADBLOCK_MANUAL) {
-                // flip a bit
+                lfs_emubd_flip(CFG) => 0;
-                lfs_emubd_flipbit(CFG, badblock,
+            }
-                        lfs_emubd_prng(CFG) % (BLOCK_SIZE*8)) => 0;
+
+            // run ckdata?
+            if (METHOD == 1) {
+                int err = lfsr_fs_ckdata(&lfs);
+                assert(!err || err == LFS_ERR_CORRUPT);
+                if (err == LFS_ERR_CORRUPT) {
+                    goto corrupt_open;
+                }
+            // run ckmeta?
+            } else if (METHOD == 3) {
+                int err = lfsr_fs_ckmeta(&lfs);
+                assert(!err || err == LFS_ERR_CORRUPT);
+                if (err == LFS_ERR_CORRUPT) {
+                    goto corrupt_open;
+                }
+            // run ckmeta mtreeonly?
+            } else if (METHOD == 2) {
+                // need an explicit traversal for this
+                lfsr_traversal_t t;
+                lfsr_traversal_open(&lfs, &t,
+                        LFS_T_MTREEONLY | LFS_T_CKMETA) => 0;
+                for (lfs_block_t i = 0;; i++) {
+                    // a bit hacky, but this catches infinite loops
+                    LFS_ASSERT(i < 2*BLOCK_COUNT);
+
+                    struct lfs_tinfo tinfo;
+                    int err = lfsr_traversal_read(&lfs, &t, &tinfo);
+                    assert(!err
+                            || err == LFS_ERR_NOENT
+                            || err == LFS_ERR_CORRUPT);
+                    if (err == LFS_ERR_NOENT) {
+                        break;
+                    }
+                    if (err == LFS_ERR_CORRUPT) {
+                        lfsr_traversal_close(&lfs, &t) => 0;
+                        goto corrupt_open;
+                    }
+                }
+                lfsr_traversal_close(&lfs, &t) => 0;
             }
         }
 
@@ -3158,8 +3301,10 @@ code = '''
         // update file
         lfsr_file_seek(&lfs, &file, off, LFS_SEEK_SET) => off;
         lfs_ssize_t d = lfsr_file_write(&lfs, &file, &sim[off], chunk);
-        assert(d == (lfs_ssize_t)chunk || d == LFS_ERR_NOSPC);
+        assert(d == (lfs_ssize_t)chunk
-        if (d == LFS_ERR_NOSPC) {
+                || d == LFS_ERR_NOSPC
+                || (d == LFS_ERR_CORRUPT && (METHOD == 2 || METHOD == 3)));
+        if (d == LFS_ERR_NOSPC || d == LFS_ERR_CORRUPT) {
             goto corrupt_open;
         }
 
@@ -3250,19 +3395,27 @@ corrupt_mounted:;
 
 # fuzz errors with uncreats, zombies, etc
 [cases.test_ck_spam_uz_fuzz]
-defines.BADBLOCK = -1
-# TODO enable other methods once rollback protection is in place
 # METHOD=0 => ckprogs
-# METHOD=1 => ckfetches
+# METHOD=1 => ckdata
-# METHOD=2 => ckparity
+# METHOD=2 => ckmeta+ckfetches
-# METHOD=3 => ckdatacksums
+# METHOD=3 => ckmeta+ckdatacksums
+# note only ckprogs works with open files
 defines.METHOD = [0]
 defines.PERIOD = 10
+# protecting the mrootanchor encourages more interesting failures, and
+# simulates storage with hardened {0,1} blocks
 defines.PROTECTED_MROOTANCHOR = [false, true]
-defines.BADBLOCK_BEHAVIOR = 'LFS_EMUBD_BADBLOCK_PROGFLIP'
+# we can't reliably detect bit errors in erased blocks, we rely on
+# future progs failing if this happens
+defines.ERASE_VALUE = -1
+defines.BADBLOCK_BEHAVIOR = '''
+    (METHOD == 0)
+        ? LFS_EMUBD_BADBLOCK_PROGFLIP
+        : LFS_EMUBD_BADBLOCK_MANUAL
+'''
 defines.CKPROGS       = 'METHOD == 0'
-defines.CKFETCHES = 'METHOD == 1'
+defines.CKFETCHES     = 'METHOD == 2'
-defines.CKPARITY  = 'METHOD == 2'
+defines.CKPARITY      = false
 defines.CKDATACKSUMS  = 'METHOD == 3'
 defines.N = [1, 2, 4, 8, 16, 32, 64]
 defines.SIZE = [
@@ -3332,13 +3485,14 @@ code = '''
     sim_file_t **sim_files = malloc(N*sizeof(sim_file_t*));
     lfs_size_t sim_file_count = 0;
 
-    // keep adding block errors until we either run out of blocks or
+    // keep adding errors until we either run out of blocks or detect
-    // our errors
+    // corruption
     lfs_size_t i = 0;
     for (; i < PERIOD*BLOCK_COUNT; i++) {
+        // add an error?
         if (i % PERIOD == 0
-                && !(PROTECTED_MROOTANCHOR
+                // protected mrootanchor? (just makes things more interesting)
-                    && badblocks[i/PERIOD] < 2)) {
+                && !(PROTECTED_MROOTANCHOR && badblocks[i/PERIOD] < 2)) {
             lfs_block_t badblock = badblocks[i/PERIOD];
             printf("badblock: 0x%x\n", badblock);
 
@@ -3346,16 +3500,52 @@ code = '''
             // types of errors, so we implement errors for each one a
             // bit differently
 
-            // ckprogs? ckparity? ckdatacksums?
-            if (METHOD == 0 || METHOD == 2 || METHOD == 3) {
             // mark our badblock as bad
             lfs_emubd_markbad(CFG, badblock) => 0;
 
-            // ckfetches?
+            // manually flipping? flip all badbits in badblocks
-            } else if (METHOD == 1) {
+            if (BADBLOCK_BEHAVIOR == LFS_EMUBD_BADBLOCK_MANUAL) {
-                // flip a bit
+                lfs_emubd_flip(CFG) => 0;
-                lfs_emubd_flipbit(CFG, badblock,
+            }
-                        lfs_emubd_prng(CFG) % (BLOCK_SIZE*8)) => 0;
+
+            // run ckdata?
+            if (METHOD == 1) {
+                int err = lfsr_fs_ckdata(&lfs);
+                assert(!err || err == LFS_ERR_CORRUPT);
+                if (err == LFS_ERR_CORRUPT) {
+                    goto corrupt_mounted;
+                }
+            // run ckmeta?
+            } else if (METHOD == 3) {
+                int err = lfsr_fs_ckmeta(&lfs);
+                assert(!err || err == LFS_ERR_CORRUPT);
+                if (err == LFS_ERR_CORRUPT) {
+                    goto corrupt_mounted;
+                }
+            // run ckmeta mtreeonly?
+            } else if (METHOD == 2) {
+                // need an explicit traversal for this
+                lfsr_traversal_t t;
+                lfsr_traversal_open(&lfs, &t,
+                        LFS_T_MTREEONLY | LFS_T_CKMETA) => 0;
+                for (lfs_block_t i = 0;; i++) {
+                    // a bit hacky, but this catches infinite loops
+                    LFS_ASSERT(i < 2*BLOCK_COUNT);
+
+                    struct lfs_tinfo tinfo;
+                    int err = lfsr_traversal_read(&lfs, &t, &tinfo);
+                    assert(!err
+                            || err == LFS_ERR_NOENT
+                            || err == LFS_ERR_CORRUPT);
+                    if (err == LFS_ERR_NOENT) {
+                        break;
+                    }
+                    if (err == LFS_ERR_CORRUPT) {
+                        lfsr_traversal_close(&lfs, &t) => 0;
+                        goto corrupt_mounted;
+                    }
+                }
+                lfsr_traversal_close(&lfs, &t) => 0;
             }
         }
 
@@ -3473,8 +3663,10 @@ code = '''
             }
             lfs_ssize_t d = lfsr_file_write(&lfs, &sim_files[j]->file,
                     wbuf, SIZE);
-            assert(d == SIZE || d == LFS_ERR_NOSPC);
+            assert(d == SIZE
-            if (d == LFS_ERR_NOSPC) {
+                    || d == LFS_ERR_NOSPC
+                    || (d == LFS_ERR_CORRUPT && (METHOD == 2 || METHOD == 3)));
+            if (d == LFS_ERR_NOSPC || d == LFS_ERR_CORRUPT) {
                 goto corrupt_mounted;
             }
             int err = lfsr_file_sync(&lfs, &sim_files[j]->file);
@@ -3657,7 +3849,13 @@ code = '''
         }
 
         uint8_t rbuf[SIZE];
-        lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
+        lfs_ssize_t d = lfsr_file_read(&lfs, &file, rbuf, SIZE);
+        assert(d == SIZE
+                || (d == LFS_ERR_CORRUPT && (METHOD == 2 || METHOD == 3)));
+        if (d == LFS_ERR_CORRUPT) {
+            lfsr_file_close(&lfs, &file) => 0;
+            goto corrupt_mounted;
+        }
         assert(memcmp(rbuf, wbuf, SIZE) == 0);
         lfsr_file_close(&lfs, &file) => 0;
     }
@@ -3672,7 +3870,12 @@ code = '''
 
         lfsr_file_rewind(&lfs, &sim_files[j]->file) => 0;
         uint8_t rbuf[SIZE];
-        lfsr_file_read(&lfs, &sim_files[j]->file, rbuf, SIZE) => SIZE;
+        lfs_ssize_t d = lfsr_file_read(&lfs, &sim_files[j]->file, rbuf, SIZE);
+        assert(d == SIZE
+                || (d == LFS_ERR_CORRUPT && (METHOD == 2 || METHOD == 3)));
+        if (d == LFS_ERR_CORRUPT) {
+            goto corrupt_mounted;
+        }
         assert(memcmp(rbuf, wbuf, SIZE) == 0);
     }
 
@@ -3694,18 +3897,27 @@ corrupt_mounted:;
 
 # fuzz errors with uncreats, zombies, dirs, etc
 [cases.test_ck_spam_uzd_fuzz]
-# TODO enable other methods once rollback protection is in place
 # METHOD=0 => ckprogs
-# METHOD=1 => ckfetches
+# METHOD=1 => ckdata
-# METHOD=2 => ckparity
+# METHOD=2 => ckmeta+ckfetches
-# METHOD=3 => ckdatacksums
+# METHOD=3 => ckmeta+ckdatacksums
+# note only ckprogs works with open files
 defines.METHOD = [0]
 defines.PERIOD = 10
+# protecting the mrootanchor encourages more interesting failures, and
+# simulates storage with hardened {0,1} blocks
 defines.PROTECTED_MROOTANCHOR = [false, true]
-defines.BADBLOCK_BEHAVIOR = 'LFS_EMUBD_BADBLOCK_PROGFLIP'
+# we can't reliably detect bit errors in erased blocks, we rely on
+# future progs failing if this happens
+defines.ERASE_VALUE = -1
+defines.BADBLOCK_BEHAVIOR = '''
+    (METHOD == 0)
+        ? LFS_EMUBD_BADBLOCK_PROGFLIP
+        : LFS_EMUBD_BADBLOCK_MANUAL
+'''
 defines.CKPROGS       = 'METHOD == 0'
-defines.CKFETCHES = 'METHOD == 1'
+defines.CKFETCHES     = 'METHOD == 2'
-defines.CKPARITY  = 'METHOD == 2'
+defines.CKPARITY      = false
 defines.CKDATACKSUMS  = 'METHOD == 3'
 defines.N = [1, 2, 4, 8, 16, 32, 64]
 defines.SIZE = [
@@ -3776,13 +3988,14 @@ code = '''
     sim_file_t **sim_files = malloc(N*sizeof(sim_file_t*));
     lfs_size_t sim_file_count = 0;
 
-    // keep adding block errors until we either run out of blocks or
+    // keep adding errors until we either run out of blocks or detect
-    // our errors
+    // corruption
     lfs_size_t i = 0;
     for (; i < PERIOD*BLOCK_COUNT; i++) {
+        // add an error?
         if (i % PERIOD == 0
-                && !(PROTECTED_MROOTANCHOR
+                // protected mrootanchor? (just makes things more interesting)
-                    && badblocks[i/PERIOD] < 2)) {
+                && !(PROTECTED_MROOTANCHOR && badblocks[i/PERIOD] < 2)) {
             lfs_block_t badblock = badblocks[i/PERIOD];
             printf("badblock: 0x%x\n", badblock);
 
@@ -3790,16 +4003,52 @@ code = '''
             // types of errors, so we implement errors for each one a
             // bit differently
 
-            // ckprogs? ckparity? ckdatacksums?
-            if (METHOD == 0 || METHOD == 2 || METHOD == 3) {
             // mark our badblock as bad
             lfs_emubd_markbad(CFG, badblock) => 0;
 
-            // ckfetches?
+            // manually flipping? flip all badbits in badblocks
-            } else if (METHOD == 1) {
+            if (BADBLOCK_BEHAVIOR == LFS_EMUBD_BADBLOCK_MANUAL) {
-                // flip a bit
+                lfs_emubd_flip(CFG) => 0;
-                lfs_emubd_flipbit(CFG, badblock,
+            }
-                        lfs_emubd_prng(CFG) % (BLOCK_SIZE*8)) => 0;
+
+            // run ckdata?
+            if (METHOD == 1) {
+                int err = lfsr_fs_ckdata(&lfs);
+                assert(!err || err == LFS_ERR_CORRUPT);
+                if (err == LFS_ERR_CORRUPT) {
+                    goto corrupt_mounted;
+                }
+            // run ckmeta?
+            } else if (METHOD == 3) {
+                int err = lfsr_fs_ckmeta(&lfs);
+                assert(!err || err == LFS_ERR_CORRUPT);
+                if (err == LFS_ERR_CORRUPT) {
+                    goto corrupt_mounted;
+                }
+            // run ckmeta mtreeonly?
+            } else if (METHOD == 2) {
+                // need an explicit traversal for this
+                lfsr_traversal_t t;
+                lfsr_traversal_open(&lfs, &t,
+                        LFS_T_MTREEONLY | LFS_T_CKMETA) => 0;
+                for (lfs_block_t i = 0;; i++) {
+                    // a bit hacky, but this catches infinite loops
+                    LFS_ASSERT(i < 2*BLOCK_COUNT);
+
+                    struct lfs_tinfo tinfo;
+                    int err = lfsr_traversal_read(&lfs, &t, &tinfo);
+                    assert(!err
+                            || err == LFS_ERR_NOENT
+                            || err == LFS_ERR_CORRUPT);
+                    if (err == LFS_ERR_NOENT) {
+                        break;
+                    }
+                    if (err == LFS_ERR_CORRUPT) {
+                        lfsr_traversal_close(&lfs, &t) => 0;
+                        goto corrupt_mounted;
+                    }
+                }
+                lfsr_traversal_close(&lfs, &t) => 0;
             }
         }
 
@@ -3923,8 +4172,10 @@ code = '''
             }
             lfs_ssize_t d = lfsr_file_write(&lfs, &sim_files[j]->file,
                     wbuf, SIZE);
-            assert(d == SIZE || d == LFS_ERR_NOSPC);
+            assert(d == SIZE
-            if (d == LFS_ERR_NOSPC) {
+                    || d == LFS_ERR_NOSPC
+                    || (d == LFS_ERR_CORRUPT && (METHOD == 2 || METHOD == 3)));
+            if (d == LFS_ERR_NOSPC || d == LFS_ERR_CORRUPT) {
                 goto corrupt_mounted;
             }
             int err = lfsr_file_sync(&lfs, &sim_files[j]->file);
@@ -4182,7 +4433,13 @@ code = '''
             }
 
             uint8_t rbuf[SIZE];
-            lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
+            lfs_ssize_t d = lfsr_file_read(&lfs, &file, rbuf, SIZE);
+            assert(d == SIZE
+                    || (d == LFS_ERR_CORRUPT && (METHOD == 2 || METHOD == 3)));
+            if (d == LFS_ERR_CORRUPT) {
+                lfsr_file_close(&lfs, &file) => 0;
+                goto corrupt_mounted;
+            }
             assert(memcmp(rbuf, wbuf, SIZE) == 0);
             lfsr_file_close(&lfs, &file) => 0;
         }
@@ -4198,7 +4455,12 @@ code = '''
 
         lfsr_file_rewind(&lfs, &sim_files[j]->file) => 0;
         uint8_t rbuf[SIZE];
-        lfsr_file_read(&lfs, &sim_files[j]->file, rbuf, SIZE) => SIZE;
+        lfs_ssize_t d = lfsr_file_read(&lfs, &sim_files[j]->file, rbuf, SIZE);
+        assert(d == SIZE
+                || (d == LFS_ERR_CORRUPT && (METHOD == 2 || METHOD == 3)));
+        if (d == LFS_ERR_CORRUPT) {
+            goto corrupt_mounted;
+        }
         assert(memcmp(rbuf, wbuf, SIZE) == 0);
     }