Added TEST_PERMUTATION, made it easier to reproduce perm/fuzz failures

TEST_PERMUTATION/BENCH_PERMUTATION make it possible to map an integer to
a specific permutation efficiently. This is helpful since our testing
framework really only parameterizes single integers.

The exact implementation took a bit of trial and error. It's based on
https://stackoverflow.com/a/7919887 and
https://stackoverflow.com/a/24257996, but modified to run in O(n) with
no extra memory. In the discussion it seemed like this may not actually
be possible for lexicographic ordering of permutations, but fortunately
we don't care about the specific ordering, only the reproducibility.

Here's how it works:

1. First populate an array with all numbers 0-n.

2. Iterate through each index, selecting only from the remaining
   numbers based on our current permutation.

          .- i%rem --.
          v     .----+----.
     [p0 p1 |-> r0 r1 r2 r3]

   Normally to maintain lexicographic ordering you should have to do a O(n)
   shift at this step as you remove each number. But instead we can just swap
   the removed number and number under the index. This effectively
   shrinks the remaining part of the array, but permutes the numbers
   a bit. Fortunately, since each successive permutation swaps
   at the same location, the resulting permutations will be both
   exhaustive and reproducible, if unintuitive.

Now permutation/fuzz tests can reproduce specific failures by defining
either -DPERMUTATION=x or -DSEED=x.

runners/bench_runner.c

@@ -603,6 +603,42 @@ uint32_t bench_prng(uint32_t *state) {
     return x;
 }
 
+// bench factorial
+size_t bench_factorial(size_t x) {
+    size_t y = 1;
+    for (size_t i = 2; i <= x; i++) {
+        y *= i;
+    }
+    return y;
+}
+
+// bench array permutations
+void bench_permutation(size_t i, uint32_t *buffer, size_t size) {
+    // https://stackoverflow.com/a/7919887 and
+    // https://stackoverflow.com/a/24257996 helped a lot with this, but
+    // changed to run in O(n) with no extra memory. This has a tradeoff
+    // of generating the permutations in an unintuitive order.
+
+    // initialize array
+    for (size_t j = 0; j < size; j++) {
+        buffer[j] = j;
+    }
+
+    for (size_t j = 0; j < size; j++) {
+        // swap index with digit
+        //
+        //      .- i%rem --.
+        //      v     .----+----.
+        // [p0 p1 |-> r0 r1 r2 r3]
+        //
+        size_t t = buffer[j + (i % (size-j))];
+        buffer[j + (i % (size-j))] = buffer[j];
+        buffer[j] = t;
+        // update i
+        i /= (size-j);
+    }
+}
+
 
 // bench recording state
 static struct lfs_config *bench_cfg = NULL;

runners/bench_runner.h

@@ -80,6 +80,13 @@ uint32_t bench_prng(uint32_t *state);
 
 #define BENCH_PRNG(state) bench_prng(state)
 
+// generation of specific permutations of an array for exhaustive benching
+size_t bench_factorial(size_t x);
+void bench_permutation(size_t i, uint32_t *buffer, size_t size);
+
+#define BENCH_FACTORIAL(x) bench_factorial(x)
+#define BENCH_PERMUTATION(i, buffer, size) bench_permutation(i, buffer, size)
+
 
 // access generated bench defines
 intmax_t bench_define(size_t define);

runners/test_runner.c

@@ -619,6 +619,42 @@ uint32_t test_prng(uint32_t *state) {
     return x;
 }
 
+// test factorial
+size_t test_factorial(size_t x) {
+    size_t y = 1;
+    for (size_t i = 2; i <= x; i++) {
+        y *= i;
+    }
+    return y;
+}
+
+// test array permutations
+void test_permutation(size_t i, uint32_t *buffer, size_t size) {
+    // https://stackoverflow.com/a/7919887 and
+    // https://stackoverflow.com/a/24257996 helped a lot with this, but
+    // changed to run in O(n) with no extra memory. This has a tradeoff
+    // of generating the permutations in an unintuitive order.
+
+    // initialize array
+    for (size_t j = 0; j < size; j++) {
+        buffer[j] = j;
+    }
+
+    for (size_t j = 0; j < size; j++) {
+        // swap index with digit
+        //
+        //      .- i%rem --.
+        //      v     .----+----.
+        // [p0 p1 |-> r0 r1 r2 r3]
+        //
+        size_t t = buffer[j + (i % (size-j))];
+        buffer[j + (i % (size-j))] = buffer[j];
+        buffer[j] = t;
+        // update i
+        i /= (size-j);
+    }
+}
+
 
 // encode our permutation into a reusable id
 static void perm_printid(

runners/test_runner.h

@@ -73,6 +73,13 @@ uint32_t test_prng(uint32_t *state);
 
 #define TEST_PRNG(state) test_prng(state)
 
+// generation of specific permutations of an array for exhaustive testing
+size_t test_factorial(size_t x);
+void test_permutation(size_t i, uint32_t *buffer, size_t size);
+
+#define TEST_FACTORIAL(x) test_factorial(x)
+#define TEST_PERMUTATION(i, buffer, size) test_permutation(i, buffer, size)
+
 
 // access generated test defines
 intmax_t test_define(size_t define);

tests/test_btree.toml

@@ -409,13 +409,19 @@ code = '''
 
 [cases.test_btree_push_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
-defines.ITER = 10
+defines.SAMPLES = 10
+# -1 => all pseudo-random seeds
+#  n => reproduce a specific seed
+defines.SEED = -1
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
     // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = 1; seed < ITER+1; seed++) {
+    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
+            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
+            seed++) {
+        printf("--- seed: %d ---\n", seed);
         // create lfs here since we need to reset each iteration, we're
         // space constrained and we can't expect gc to work at this point
         lfs_t lfs;
@@ -566,13 +572,19 @@ code = '''
 [cases.test_btree_push_sparse_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
 defines.W = 5
-defines.ITER = 10
+defines.SAMPLES = 10
+# -1 => all pseudo-random seeds
+#  n => reproduce a specific seed
+defines.SEED = -1
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
     // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = 1; seed < ITER+1; seed++) {
+    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
+            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
+            seed++) {
+        printf("--- seed: %d ---\n", seed);
         // create lfs here since we need to reset each iteration, we're
         // space constrained and we can't expect gc to work at this point
         lfs_t lfs;
@@ -929,14 +941,20 @@ code = '''
 
 [cases.test_btree_update_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
-defines.ITER = 10
+defines.SAMPLES = 10
+# -1 => all pseudo-random seeds
+#  n => reproduce a specific seed
+defines.SEED = -1
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
     const char *uppers = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
 
     // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = 1; seed < ITER+1; seed++) {
+    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
+            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
+            seed++) {
+        printf("--- seed: %d ---\n", seed);
         // create lfs here since we need to reset each iteration, we're
         // space constrained and we can't expect gc to work at this point
         lfs_t lfs;
@@ -1095,14 +1113,20 @@ code = '''
 [cases.test_btree_update_sparse_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
 defines.W = 5
-defines.ITER = 10
+defines.SAMPLES = 10
+# -1 => all pseudo-random seeds
+#  n => reproduce a specific seed
+defines.SEED = -1
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
     const char *uppers = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
 
     // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = 1; seed < ITER+1; seed++) {
+    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
+            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
+            seed++) {
+        printf("--- seed: %d ---\n", seed);
         // create lfs here since we need to reset each iteration, we're
         // space constrained and we can't expect gc to work at this point
         lfs_t lfs;
@@ -1711,14 +1735,20 @@ code = '''
 [cases.test_btree_pop_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
 defines.REMAINING = [64, 2, 1, 0]
-defines.ITER = 10
+defines.SAMPLES = 10
+# -1 => all pseudo-random seeds
+#  n => reproduce a specific seed
+defines.SEED = -1
 if = 'N > REMAINING'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
     // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = 1; seed < ITER+1; seed++) {
+    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
+            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
+            seed++) {
+        printf("--- seed: %d ---\n", seed);
         // create lfs here since we need to reset each iteration, we're
         // space constrained and we can't expect gc to work at this point
         lfs_t lfs;
@@ -1915,14 +1945,20 @@ code = '''
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
 defines.REMAINING = [64, 2, 1, 0]
 defines.W = 5
-defines.ITER = 10
+defines.SAMPLES = 10
+# -1 => all pseudo-random seeds
+#  n => reproduce a specific seed
+defines.SEED = -1
 if = 'N > REMAINING'
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
     // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = 1; seed < ITER+1; seed++) {
+    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
+            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
+            seed++) {
+        printf("--- seed: %d ---\n", seed);
         // create lfs here since we need to reset each iteration, we're
         // space constrained and we can't expect gc to work at this point
         lfs_t lfs;
@@ -2124,14 +2160,20 @@ code = '''
 
 [cases.test_btree_split_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
-defines.ITER = 10
+defines.SAMPLES = 10
+# -1 => all pseudo-random seeds
+#  n => reproduce a specific seed
+defines.SEED = -1
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
     const char *uppers = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
 
     // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = 1; seed < ITER+1; seed++) {
+    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
+            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
+            seed++) {
+        printf("--- seed: %d ---\n", seed);
         // create lfs here since we need to reset each iteration, we're
         // space constrained and we can't expect gc to work at this point
         lfs_t lfs;
@@ -2275,14 +2317,20 @@ code = '''
 [cases.test_btree_split_sparse_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
 defines.W = 5
-defines.ITER = 10
+defines.SAMPLES = 10
+# -1 => all pseudo-random seeds
+#  n => reproduce a specific seed
+defines.SEED = -1
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
     const char *uppers = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
 
     // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = 1; seed < ITER+1; seed++) {
+    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
+            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
+            seed++) {
+        printf("--- seed: %d ---\n", seed);
         // create lfs here since we need to reset each iteration, we're
         // space constrained and we can't expect gc to work at this point
         lfs_t lfs;
@@ -2428,13 +2476,19 @@ code = '''
 # Some more general fuzz testing
 [cases.test_btree_general_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
-defines.ITER = 100
+defines.SAMPLES = 100
+# -1 => all pseudo-random seeds
+#  n => reproduce a specific seed
+defines.SEED = -1
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
     // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = 1; seed < ITER+1; seed++) {
+    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
+            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
+            seed++) {
+        printf("--- seed: %d ---\n", seed);
         // create lfs here since we need to reset each iteration, we're
         // space constrained and we can't expect gc to work at this point
         lfs_t lfs;
@@ -2540,13 +2594,19 @@ code = '''
 [cases.test_btree_general_sparse_fuzz]
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
 defines.W = 5
-defines.ITER = 100
+defines.SAMPLES = 100
+# -1 => all pseudo-random seeds
+#  n => reproduce a specific seed
+defines.SEED = -1
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
 
     // iterate through severals seeds that we can reproduce easily
-    for (uint32_t seed = 1; seed < ITER+1; seed++) {
+    for (uint32_t seed = (SEED == -1 ? 1 : SEED);
+            (SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
+            seed++) {
+        printf("--- seed: %d ---\n", seed);
         // create lfs here since we need to reset each iteration, we're
         // space constrained and we can't expect gc to work at this point
         lfs_t lfs;