Changed how fuzz tests are iterated to allow powerloss-fuzz testing

Instead of iterating over a number of seeds in the test itself, the
seeds are now permuted as a part of normal test defines.

This lets each seed take advantage of other test features, mainly the
ability to test powerlosses heuristically.

This is probably how it should have been done in the first place, but
the permutation tests can't do this since the number of permutations
changes as the size of the test input changes. The test define system
can't handle that very well.

The tradeoffs here are:

- We can't do cross-fuzz checks, such as the balance checks in the rbyd
  tests, though those really should be moved to benchmarks anyways.

- The large number of cheap fuzz permutations skews the total
  permutation count, though I'm not sure this matters.

  before: 3083 permutations (-Gnor)
  after: 409893 permutations (-Gnor)

scripts/bench.py

@@ -758,20 +758,22 @@ def find_ids(runner, bench_ids=[], **args):
     # find suite/case by id
     bench_ids_ = []
     for id in bench_ids:
+        # strip permutation
+        name, *_ = id.split(':', 1)
         bench_ids__ = []
         # resolve globs
-        if '*' in id:
+        if '*' in name:
             bench_ids__.extend(suite
                 for suite in expected_suite_perms.keys()
-                if fnmatch.fnmatch(suite, id))
+                if fnmatch.fnmatch(suite, name))
             bench_ids__.extend(case_
                 for case_ in expected_case_perms.keys()
-                if fnmatch.fnmatch(case_, id))
+                if fnmatch.fnmatch(case_, name))
         # literal suite
-        elif id in expected_suite_perms:
+        elif name in expected_suite_perms:
             bench_ids__.append(id)
         # literal case
-        elif id in expected_case_perms:
+        elif name in expected_case_perms:
             bench_ids__.append(id)
 
         # no suite/case found? error

scripts/test.py

@@ -767,20 +767,22 @@ def find_ids(runner, test_ids=[], **args):
     # find suite/case by id
     test_ids_ = []
     for id in test_ids:
+        # strip permutation
+        name, *_ = id.split(':', 1)
         test_ids__ = []
         # resolve globs
-        if '*' in id:
+        if '*' in name:
             test_ids__.extend(suite
                 for suite in expected_suite_perms.keys()
-                if fnmatch.fnmatch(suite, id))
+                if fnmatch.fnmatch(suite, name))
             test_ids__.extend(case_
                 for case_ in expected_case_perms.keys()
-                if fnmatch.fnmatch(case_, id))
+                if fnmatch.fnmatch(case_, name))
         # literal suite
-        elif id in expected_suite_perms:
+        elif name in expected_suite_perms:
             test_ids__.append(id)
         # literal case
-        elif id in expected_case_perms:
+        elif name in expected_case_perms:
             test_ids__.append(id)
 
         # no suite/case found? error

tests/t1_rbyd.toml

@@ -3542,10 +3542,7 @@ code = '''
 # balancing algorithm
 [cases.t1_rbyd_fuzz_append_removes]
 defines.N = 'range(1, 33)'
-defines.SAMPLES = 1000
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(1000)'
 # large progs take too long for now
 if = 'PROG_SIZE < 512'
 in = 'lfs.c'
@@ -3564,17 +3561,8 @@ code = '''
     const char *alpha = "abcdefghijklmnopqrstuvwxyz";
     uint8_t buffer[4];
 
-    // keep track of the worst case size and seed
-    lfs_size_t worst_size = 0;
-    uint32_t worst_seed = 0;
-
-    // iterate through seeds so we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
     printf("perm: [");
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (unsigned i = 0; i < N; i++) {
         // choose an attr
         uint8_t attr = TEST_PRNG(&prng) % N;
@@ -3598,7 +3586,7 @@ code = '''
     rbyd = init_rbyd;
     lfs_bd_erase(&lfs, rbyd.block) => 0;
 
-        prng = seed;
+    prng = SEED;
     for (unsigned i = 0; i < N; i++) {
         // choose an attr
         uint8_t attr = TEST_PRNG(&prng) % N;
@@ -3657,26 +3645,8 @@ code = '''
         }
     }
 
-        // keep track of the worst seed
-        if (rbyd.off > worst_size) {
-            worst_size = rbyd.off;
-            worst_seed = seed;
-        }
-
     // cleanup
     free(sim);
-    }
-
-    // print the worst seed + size
-    printf("--- summary ---\n");
-    printf("worst seed: %d\n", worst_seed);
-    printf("worst size: %d\n", worst_size);
-
-    // our tree should be strictly <= 2*log(n)+1, assume tags are strictly
-    // <=12 bytes, note this only holds true with byte-level progs
-    if (PROG_SIZE == 1) {
-        assert(worst_size <= N*12*(2*lfs_nlog2(N)+1)+1);
-    }
 '''
 
 
@@ -8279,10 +8249,7 @@ code = '''
 # balancing algorithm
 [cases.t1_rbyd_fuzz_create_deletes]
 defines.N = 'range(1, 33)'
-defines.SAMPLES = 1000
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(1000)'
 # large progs take too long for now
 if = 'PROG_SIZE < 512'
 in = 'lfs.c'
@@ -8301,17 +8268,8 @@ code = '''
     const char *alpha = "abcdefghijklmnopqrstuvwxyz";
     uint8_t buffer[4];
 
-    // keep track of the worst case size and seed
-    lfs_size_t worst_size = 0;
-    uint32_t worst_seed = 0;
-
-    // iterate through seeds so we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
     printf("perm: [");
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     lfs_size_t count = 0;
     for (unsigned i = 0; i < N; i++) {
         // choose an id
@@ -8339,7 +8297,7 @@ code = '''
     rbyd = init_rbyd;
     lfs_bd_erase(&lfs, rbyd.block) => 0;
 
-        prng = seed;
+    prng = SEED;
     count = 0;
     for (unsigned i = 0; i < N; i++) {
         // choose an id
@@ -8393,27 +8351,8 @@ code = '''
         assert(memcmp(&sim[id], buffer, 1) == 0);
     }
 
-        // keep track of the worst seed
-        if (rbyd.off > worst_size) {
-            worst_size = rbyd.off;
-            worst_seed = seed;
-        }
-
     // cleanup
     free(sim);
-    }
-
-    // print the worst seed + size, and rerun it so it's left on the disk
-    // if used with -ddisk
-    printf("--- summary ---\n");
-    printf("worst_seed: %d\n", worst_seed);
-    printf("worst_size: %d\n", worst_size);
-
-    // our tree should be strictly <= 2*log(n)+1, assume tags are strictly
-    // <=12 bytes, note this only holds true with byte-level progs
-    if (PROG_SIZE == 1) {
-        assert(worst_size <= 2*N*12*(2*lfs_nlog2(N)+1)+1);
-    }
 '''
 
 
@@ -11239,10 +11178,7 @@ code = '''
 [cases.t1_rbyd_fuzz_mixed]
 defines.N = 'range(1, 33)'
 defines.M = 3
-defines.SAMPLES = 1000
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(1000)'
 # large progs take too long for now
 if = 'PROG_SIZE < 512'
 in = 'lfs.c'
@@ -11261,17 +11197,8 @@ code = '''
     const char *alpha = "abcdefghijklmnopqrstuvwxyz";
     uint8_t buffer[4];
 
-    // keep track of the worst case size and seed
-    lfs_size_t worst_size = 0;
-    uint32_t worst_seed = 0;
-
-    // iterate through seeds so we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
     printf("perm: [");
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     lfs_size_t count = 0;
     for (unsigned i = 0; i < N; i++) {
         // choose create/delete or attr append/remove
@@ -11307,7 +11234,7 @@ code = '''
     rbyd = init_rbyd;
     lfs_bd_erase(&lfs, rbyd.block) => 0;
 
-        prng = seed;
+    prng = SEED;
     count = 0;
     for (unsigned i = 0; i < N; i++) {
         // choose create/delete or attr append/remove
@@ -11395,36 +11322,14 @@ code = '''
         assert(memcmp(&sim[id*(M+1)], buffer, 1) == 0);
     }
 
-        // keep track of the worst seed
-        if (rbyd.off > worst_size) {
-            worst_size = rbyd.off;
-            worst_seed = seed;
-        }
-
     // cleanup
     free(sim);
-    }
-
-    // print the worst seed + size, and rerun it so it's left on the disk
-    // if used with -ddisk
-    printf("--- summary ---\n");
-    printf("worst_seed: %d\n", worst_seed);
-    printf("worst_size: %d\n", worst_size);
-
-    // our tree should be strictly <= 2*log(n)+1, assume tags are strictly
-    // <=12 bytes, note this only holds true with byte-level progs
-    if (PROG_SIZE == 1) {
-        assert(worst_size <= 2*N*12*(2*lfs_nlog2(N)+1)+1);
-    }
 '''
 
 [cases.t1_rbyd_fuzz_sparse]
 defines.N = 'range(1, 33)'
 defines.W = 5
-defines.SAMPLES = 1000
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(1000)'
 # large progs take too long for now
 if = 'PROG_SIZE < 512'
 in = 'lfs.c'
@@ -11447,17 +11352,8 @@ code = '''
     const char *alpha = "abcdefghijklmnopqrstuvwxyz";
     uint8_t buffer[4];
 
-    // keep track of the worst case size and seed
-    lfs_size_t worst_size = 0;
-    uint32_t worst_seed = 0;
-
-    // iterate through seeds so we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
     printf("perm: [");
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     lfs_size_t count = 0;
     for (unsigned i = 0; i < N; i++) {
         // choose create/delete/grow/shrink
@@ -11495,7 +11391,7 @@ code = '''
     rbyd = init_rbyd;
     lfs_bd_erase(&lfs, rbyd.block) => 0;
 
-        prng = seed;
+    prng = SEED;
     count = 0;
     for (unsigned i = 0; i < N; i++) {
         // choose create/delete/grow/shrink
@@ -11617,28 +11513,9 @@ code = '''
         assert(memcmp(&sim[id], buffer, 1) == 0);
     }
 
-        // keep track of the worst seed
-        if (rbyd.off > worst_size) {
-            worst_size = rbyd.off;
-            worst_seed = seed;
-        }
-
     // cleanup
     free(sim);
     free(sim_weights);
-    }
-
-    // print the worst seed + size, and rerun it so it's left on the disk
-    // if used with -ddisk
-    printf("--- summary ---\n");
-    printf("worst_seed: %d\n", worst_seed);
-    printf("worst_size: %d\n", worst_size);
-
-    // our tree should be strictly <= 2*log(n)+1, assume tags are strictly
-    // <=12 bytes, note this only holds true with byte-level progs
-    if (PROG_SIZE == 1) {
-        assert(worst_size <= 2*N*12*(2*lfs_nlog2(N)+1)+1);
-    }
 '''
 
 

tests/t2_btree.toml

@@ -384,21 +384,11 @@ code = '''
 
 [cases.t2_btree_push_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
-defines.SAMPLES = 10
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(10)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfs_init(&lfs, cfg) => 0;
     // create free lookahead
@@ -420,7 +410,7 @@ code = '''
     lfs_size_t sim_size = 0;
     memset(sim, 0, N);
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < N; i++) {
         // choose a pseudo-random id
         lfs_size_t id = TEST_PRNG(&prng) % (sim_size+1);
@@ -475,7 +465,6 @@ code = '''
     // clean up sim
     free(sim);
     lfs_deinit(&lfs) => 0;
-    }
 '''
 
 [cases.t2_btree_push_sparse]
@@ -550,21 +539,11 @@ code = '''
 [cases.t2_btree_push_sparse_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
 defines.W = 5
-defines.SAMPLES = 10
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(10)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfs_init(&lfs, cfg) => 0;
     // create free lookahead
@@ -588,7 +567,7 @@ code = '''
     memset(sim, 0, N);
     memset(sim_weights, 0, N*sizeof(lfs_size_t));
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < N; i++) {
         // choose a pseudo-random id
         lfs_size_t id = TEST_PRNG(&prng) % (sim_size+1);
@@ -694,7 +673,6 @@ code = '''
 
     // clean up sim
     free(sim);
-    }
 '''
 
 
@@ -926,21 +904,12 @@ code = '''
 [cases.t2_btree_update_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
 defines.SAMPLES = 10
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(10)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
     const char *uppers = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfs_init(&lfs, cfg) => 0;
     // create free lookahead
@@ -976,7 +945,7 @@ code = '''
         sim[i] = alphas[i % 26];
     }
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < N; i++) {
         // choose a pseudo-random id
         lfs_size_t id = TEST_PRNG(&prng) % N;
@@ -1028,7 +997,6 @@ code = '''
 
     // clean up sim
     free(sim);
-    }
 '''
 
 [cases.t2_btree_update_sparse]
@@ -1120,22 +1088,12 @@ code = '''
 [cases.t2_btree_update_sparse_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
 defines.W = 5
-defines.SAMPLES = 10
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(10)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
     const char *uppers = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfs_init(&lfs, cfg) => 0;
     // create free lookahead
@@ -1173,7 +1131,7 @@ code = '''
         sim_weights[i] = W;
     }
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < N; i++) {
         // choose a pseudo-random id
         lfs_size_t id = TEST_PRNG(&prng) % N;
@@ -1275,7 +1233,6 @@ code = '''
 
     // clean up sim
     free(sim);
-    }
 '''
 
 
@@ -1735,22 +1692,12 @@ code = '''
 [cases.t2_btree_pop_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
 defines.REMAINING = [64, 2, 1, 0]
-defines.SAMPLES = 10
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(10)'
 if = 'N > REMAINING'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfs_init(&lfs, cfg) => 0;
     // create free lookahead
@@ -1787,7 +1734,7 @@ code = '''
         sim[i] = alphas[i % 26];
     }
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < (N-REMAINING); i++) {
         // choose a pseudo-random id
         lfs_size_t id = TEST_PRNG(&prng) % sim_size;
@@ -1839,7 +1786,6 @@ code = '''
 
     // clean up sim
     free(sim);
-    }
 '''
 
 [cases.t2_btree_pop_sparse]
@@ -1963,22 +1909,12 @@ code = '''
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
 defines.W = 5
 defines.REMAINING = [64, 2, 1, 0]
-defines.SAMPLES = 10
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(10)'
 if = 'N > REMAINING'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfs_init(&lfs, cfg) => 0;
     // create free lookahead
@@ -2001,7 +1937,7 @@ code = '''
     lfs_size_t sim_size = 0;
 
     // set up simulation and btree with pseudo-random weights
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < N; i++) {
         // choose a pseudo-random weight
         lfs_size_t weight = 1 + (TEST_PRNG(&prng) % W);
@@ -2130,7 +2066,6 @@ code = '''
 
     // clean up sim
     free(sim);
-    }
 '''
 
 
@@ -2192,22 +2127,12 @@ code = '''
 
 [cases.t2_btree_split_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
-defines.SAMPLES = 10
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(10)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
     const char *uppers = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfs_init(&lfs, cfg) => 0;
     // create free lookahead
@@ -2232,7 +2157,7 @@ code = '''
     memset(sim, 0, N);
     sim[0] = '_';
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 1; i < N; i++) {
         // choose a pseudo-random id
         lfs_size_t id = TEST_PRNG(&prng) % sim_size;
@@ -2289,7 +2214,6 @@ code = '''
     // clean up sim
     free(sim);
     lfs_deinit(&lfs) => 0;
-    }
 '''
 
 [cases.t2_btree_split_sparse]
@@ -2351,22 +2275,12 @@ code = '''
 [cases.t2_btree_split_sparse_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
 defines.W = 5
-defines.SAMPLES = 10
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(10)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
     const char *uppers = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfs_init(&lfs, cfg) => 0;
     // create free lookahead
@@ -2394,7 +2308,7 @@ code = '''
     sim[0] = '_';
     sim_weights[0] = W;
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 1; i < N; i++) {
         // choose a pseudo-random id
         lfs_size_t id = TEST_PRNG(&prng) % sim_size;
@@ -2506,7 +2420,6 @@ code = '''
 
     // clean up sim
     free(sim);
-    }
 '''
 
 
@@ -2514,21 +2427,11 @@ code = '''
 # Some more general fuzz testing
 [cases.t2_btree_general_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
-defines.SAMPLES = 100
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(100)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfs_init(&lfs, cfg) => 0;
     // create free lookahead
@@ -2550,7 +2453,7 @@ code = '''
     lfs_size_t sim_size = 0;
     memset(sim, 0, N);
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < N; i++) {
         // choose a pseudo-random op
         uint8_t op = TEST_PRNG(&prng) % 3;
@@ -2636,27 +2539,16 @@ code = '''
 
     // clean up sim
     free(sim);
-    }
 '''
 
 [cases.t2_btree_general_sparse_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
 defines.W = 5
-defines.SAMPLES = 100
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(100)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfs_init(&lfs, cfg) => 0;
     // create free lookahead
@@ -2680,7 +2572,7 @@ code = '''
     memset(sim, 0, N);
     memset(sim_weights, 0, N*sizeof(lfs_size_t));
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < N; i++) {
         // choose a pseudo-random op
         uint8_t op = TEST_PRNG(&prng) % 3;
@@ -2821,7 +2713,6 @@ code = '''
 
     // clean up sim
     free(sim);
-    }
 '''
 
 
@@ -3176,22 +3067,12 @@ code = '''
 
 [cases.t2_btree_find_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
-defines.SAMPLES = 10
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(10)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
     const char *nums = "0123456789";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfs_init(&lfs, cfg) => 0;
     // create free lookahead
@@ -3219,7 +3100,7 @@ code = '''
     sim[0] = '_';
     memcpy(&sim_names[0], "___", 3);
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 1; i < N; i++) {
         // choose a pseudo-random name
         lfs_size_t x = TEST_PRNG(&prng) % (26*26*26);
@@ -3293,7 +3174,6 @@ code = '''
     free(sim);
     free(sim_names);
     lfs_deinit(&lfs) => 0;
-    }
 '''
 
 [cases.t2_btree_find_sparse]
@@ -3366,22 +3246,12 @@ code = '''
 [cases.t2_btree_find_sparse_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
 defines.W = 5
-defines.SAMPLES = 10
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(10)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
     const char *nums = "0123456789";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfs_init(&lfs, cfg) => 0;
     // create free lookahead
@@ -3412,7 +3282,7 @@ code = '''
     memcpy(&sim_names[0], "___", 3);
     sim_weights[0] = W;
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 1; i < N; i++) {
         // choose a pseudo-random name
         lfs_size_t x = TEST_PRNG(&prng) % (26*26*26);
@@ -3522,28 +3392,17 @@ code = '''
     free(sim_names);
     free(sim_weights);
     lfs_deinit(&lfs) => 0;
-    }
 '''
 
 # make sure we test finds with other operations
 [cases.t2_btree_find_general_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
-defines.SAMPLES = 100
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(100)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
     const char *nums = "0123456789";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfs_init(&lfs, cfg) => 0;
     // create free lookahead
@@ -3571,7 +3430,7 @@ code = '''
     sim[0] = '_';
     memcpy(&sim_names[0], "___", 3);
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < N; i++) {
         // choose a pseudo-random op
         uint8_t op = TEST_PRNG(&prng) % 3;
@@ -3708,28 +3567,17 @@ code = '''
     // clean up sim
     free(sim);
     free(sim_names);
-    }
 '''
 
 [cases.t2_btree_find_general_sparse_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
 defines.W = 5
-defines.SAMPLES = 100
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(100)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
     const char *nums = "0123456789";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfs_init(&lfs, cfg) => 0;
     // create free lookahead
@@ -3760,7 +3608,7 @@ code = '''
     memcpy(&sim_names[0], "___", 3);
     sim_weights[0] = W;
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < N; i++) {
         // choose a pseudo-random op
         uint8_t op = TEST_PRNG(&prng) % 3;
@@ -3941,7 +3789,6 @@ code = '''
     free(sim);
     free(sim_names);
     free(sim_weights);
-    }
 '''
 
 
@@ -4068,21 +3915,11 @@ code = '''
 
 [cases.t2_btree_traversal_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
-defines.SAMPLES = 10
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(10)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfs_init(&lfs, cfg) => 0;
     // create free lookahead
@@ -4104,7 +3941,7 @@ code = '''
     lfs_size_t sim_size = 0;
     memset(sim, 0, N);
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < N; i++) {
         // choose a pseudo-random id
         lfs_size_t id = TEST_PRNG(&prng) % (sim_size+1);
@@ -4242,6 +4079,5 @@ code = '''
     // clean up sim
     free(sim);
     lfs_deinit(&lfs) => 0;
-    }
 '''
 

tests/t3_mtree.toml

@@ -474,21 +474,11 @@ code = '''
 [cases.t3_mtree_split_fuzz]
 defines.N = [5, 10, 20, 40, 80, 160]
 defines.FORCE_COMPACTION = [false, true]
-defines.SAMPLES = 100
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(100)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfsr_format(&lfs, cfg) => 0;
     lfsr_mount(&lfs, cfg) => 0;
@@ -500,7 +490,7 @@ code = '''
     // at least keep track of the number of entries we expect
     lfs_size_t count = 0;
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < N; i++) {
         // choose a pseudo-random mid
         lfs_ssize_t mid = lfsr_mtree_weight(&lfs) == 0
@@ -584,7 +574,6 @@ code = '''
     assert(count_ == count);
 
     lfsr_unmount(&lfs) => 0;
-    }
 '''
 
 
@@ -1441,21 +1430,11 @@ code = '''
 [cases.t3_mtree_drop_fuzz]
 defines.N = [5, 10, 20, 40, 80, 160]
 defines.FORCE_COMPACTION = [false, true]
-defines.SAMPLES = 100
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(100)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfsr_format(&lfs, cfg) => 0;
     lfsr_mount(&lfs, cfg) => 0;
@@ -1467,7 +1446,7 @@ code = '''
     // at least keep track of the number of entries we expect
     lfs_size_t count = 0;
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < N; i++) {
         // choose a pseudo-random mid
         lfs_ssize_t mid = lfsr_mtree_weight(&lfs) == 0
@@ -1574,7 +1553,6 @@ code = '''
     assert(count_ == count);
 
     lfsr_unmount(&lfs) => 0;
-    }
 '''
 
 
@@ -2540,21 +2518,11 @@ code = '''
 defines.N = [5, 10, 20, 40]
 defines.FORCE_COMPACTION = [false, true]
 defines.BLOCK_CYCLES = [5, 2, 1]
-defines.SAMPLES = 500
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(500)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfsr_format(&lfs, cfg) => 0;
     lfsr_mount(&lfs, cfg) => 0;
@@ -2566,7 +2534,7 @@ code = '''
     // at least keep track of the number of entries we expect
     lfs_size_t count = 0;
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < N; i++) {
         // choose a pseudo-random mid
         lfs_ssize_t mid = lfsr_mtree_weight(&lfs) == 0
@@ -2686,7 +2654,6 @@ code = '''
     assert(count_ == count);
 
     lfsr_unmount(&lfs) => 0;
-    }
 '''
 
 
@@ -4041,21 +4008,11 @@ code = '''
 defines.N = [5, 10, 20, 40, 80, 160]
 defines.VALIDATE = [false, true]
 defines.FORCE_COMPACTION = [false, true]
-defines.SAMPLES = 100
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(100)'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // create lfs here since we need to reset each iteration, we're
-        // space constrained and we can't expect gc to work at this point
     lfs_t lfs;
     lfsr_format(&lfs, cfg) => 0;
     lfsr_mount(&lfs, cfg) => 0;
@@ -4067,7 +4024,7 @@ code = '''
     // at least keep track of the number of entries we expect
     lfs_size_t count = 0;
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < N; i++) {
         // choose a pseudo-random mid
         lfs_ssize_t mid = lfsr_mtree_weight(&lfs) == 0
@@ -4209,7 +4166,6 @@ code = '''
     assert(count_ == count);
 
     lfsr_unmount(&lfs) => 0;
-    }
 '''
 
 

tests/t5_dirs.toml

@@ -706,29 +706,27 @@ code = '''
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
 defines.PARENT = [false, true]
 defines.REMOUNT = [false, true]
-defines.SAMPLES = 10
-# -1 => all pseudo-random seeds
-#  n => reproduce a specific seed
-defines.SEED = -1
+defines.SEED = 'range(10)'
+reentrant = true
 code = '''
-    // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
-            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
-            seed++) {
-        printf("--- seed: %d ---\n", seed);
-        // reset lfs here each iteration
+    // format once per test
     lfs_t lfs;
+    int err = lfsr_mount(&lfs, cfg);
+    if (err) {
         lfsr_format(&lfs, cfg) => 0;
         lfsr_mount(&lfs, cfg) => 0;
+    }
+
     if (PARENT) {
-            lfsr_mkdir(&lfs, "parent") => 0;
+        err = lfsr_mkdir(&lfs, "parent");
+        assert(!err || (TEST_PL && err == LFS_ERR_EXIST));
     }
 
     // set up a simulation to compare against
     lfs_size_t *sim = malloc(N*sizeof(lfs_size_t));
     lfs_size_t sim_size = 0;
 
-        uint32_t prng = seed;
+    uint32_t prng = SEED;
     for (lfs_size_t i = 0; i < N; i++) {
         // choose a pseudo-random number, truncate to 4 decimals
         lfs_size_t x = TEST_PRNG(&prng) % 1000;
@@ -737,7 +735,7 @@ code = '''
         for (lfs_size_t j = 0;; j++) {
             if (j >= sim_size || sim[j] >= x) {
                 // already seen? skip
-                    if (sim[j] == x) {
+                if (j < sim_size && sim[j] == x) {
                     goto next;
                 }
 
@@ -746,14 +744,6 @@ code = '''
                         (sim_size-j)*sizeof(lfs_size_t));
                 sim_size += 1;
                 sim[j] = x;
-
-                    // remount?
-                    if (REMOUNT) {
-                        lfsr_unmount(&lfs) => 0;
-                        lfsr_mount(&lfs, cfg) => 0;
-                        // grm should be zero here
-                        assert(lfs.grm[0] == 0);
-                    }
                 break;
             }
         }
@@ -761,10 +751,19 @@ code = '''
         // create a directory here
         char name[256];
         sprintf(name, "%s/dir%04d", (PARENT ? "parent" : ""), x);
-            lfsr_mkdir(&lfs, name) => 0;
+        err = lfsr_mkdir(&lfs, name);
+        assert(!err || (TEST_PL && err == LFS_ERR_EXIST));
     next:;
     }
 
+    // remount?
+    if (REMOUNT) {
+        lfsr_unmount(&lfs) => 0;
+        lfsr_mount(&lfs, cfg) => 0;
+        // grm should be zero here
+        assert(lfs.grm[0] == 0);
+    }
+
     // test that our directories match our simulation
     for (lfs_size_t j = 0; j < sim_size; j++) {
         char name[256];
@@ -798,7 +797,6 @@ code = '''
     // clean up sim/lfs
     free(sim);
     lfsr_unmount(&lfs) => 0;
-    }
 '''