Added sparse B-tree name tests, fixed weight-calculation bug during merge

More proof the tests are working.

This bug was in the code that does an extra lookup for the was-split entry
during merge, so we make sure we have the right id to attach the split
name to. Humorously, this code was already set up correctly, the
"split_id" just wasn't actually used. Unfortunately since
lfsr_rbyd_lookup uses out-pointers to return multiple things the
compiler couldn't detect the unused variable.

lfs.c

@@ -3603,7 +3603,7 @@ static int lfsr_btree_commit(lfs_t *lfs,
             }
 
             err = lfsr_rbyd_append(lfs, &rbyd_,
-                    LFSR_TAG_MKBRANCH, (sdelta == 0 ? sweight : rweight),
+                    LFSR_TAG_MKBRANCH, split_id,
                     LFSR_DATA_DISK(parent.block, split_off, split_size));
             if (err) {
                 assert(!err);

tests/test_btree.toml

@@ -1622,7 +1622,7 @@ code = '''
             &tag_, &id_, &weight_,
             buffer, 4) => LFS_ERR_NOENT;
 
-    // try recovering 
+    // try recovering
     lfsr_btree_push(&lfs, &btree, REMAINING, LFSR_TAG_INLINED, 1,
             "R", 1) => 0;
 
@@ -1705,7 +1705,7 @@ code = '''
             &tag_, &id_, &weight_,
             buffer, 4) => LFS_ERR_NOENT;
 
-    // try recovering 
+    // try recovering
     lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1,
             "R", 1) => 0;
 
@@ -1890,7 +1890,7 @@ code = '''
             &tag_, &id_, &weight_,
             buffer, 4) => LFS_ERR_NOENT;
 
-    // try recovering 
+    // try recovering
     lfsr_btree_push(&lfs, &btree, REMAINING*W, LFSR_TAG_INLINED, W,
             "R", 1) => 0;
 
@@ -1993,7 +1993,7 @@ code = '''
             for (lfs_size_t j = 0; j < i; j++) {
                 weighted_id += sim_weights[j];
             }
-            
+
             lfsr_btree_push(&lfs, &btree, weighted_id, LFSR_TAG_INLINED, weight,
                     &alphas[i % 26], 1) => 0;
 
@@ -2189,7 +2189,7 @@ code = '''
         // create a btree
         lfsr_btree_t btree = LFSR_BTREE_NULL;
         lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1,
-                &alphas[0 % 26], 1) => 0;
+                "_", 1) => 0;
 
         // set up a simulation to compare against
         //
@@ -2198,7 +2198,7 @@ code = '''
         char *sim = malloc(N);
         lfs_size_t sim_size = 1;
         memset(sim, 0, N);
-        sim[0] = alphas[0 % 26];
+        sim[0] = '_';
 
         uint32_t prng = seed;
         for (lfs_size_t i = 1; i < N; i++) {
@@ -2346,7 +2346,7 @@ code = '''
         // create a btree
         lfsr_btree_t btree = LFSR_BTREE_NULL;
         lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W,
-                &alphas[0 % 26], 1) => 0;
+                "_", 1) => 0;
 
         // set up a simulation to compare against
         //
@@ -2357,7 +2357,7 @@ code = '''
         lfs_size_t sim_size = 1;
         memset(sim, 0, N);
         memset(sim_weights, 0, N*sizeof(lfs_size_t));
-        sim[0] = alphas[0 % 26];
+        sim[0] = '_';
         sim_weights[0] = W;
 
         uint32_t prng = seed;
@@ -2416,7 +2416,7 @@ code = '''
             btree.weight);
 
         lfs_size_t total_weight = 0;
-        for (lfs_size_t j = 0; j < N; j++) {
+        for (lfs_size_t j = 0; j < sim_size; j++) {
             total_weight += sim_weights[j];
         }
         assert(btree.weight == total_weight);
@@ -2689,11 +2689,18 @@ code = '''
         printf("expd: [");
         bool first = true;
         for (lfs_size_t i = 0; i < sim_size; i++) {
+            // calculate actual id in btree space
+            lfs_size_t weighted_id = 0;
+            for (lfs_size_t j = 0; j < i; j++) {
+                weighted_id += sim_weights[j];
+            }
+
             if (!first) {
                 printf(", ");
             }
             first = false;
-            printf("%c", sim[i]);
+            printf("%dw%d=%c", weighted_id+sim_weights[i]-1,
+                    sim_weights[i], sim[i]);
         }
         printf("]\n");
         printf("btree: 0x%x.%x 0x%x w%d\n",
@@ -3087,7 +3094,7 @@ code = '''
         // create a btree
         lfsr_btree_t btree = LFSR_BTREE_NULL;
         lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1,
-                &alphas[0 % 26], 1) => 0;
+                "_", 1) => 0;
 
         // set up a simulation to compare against
         //
@@ -3098,7 +3105,7 @@ code = '''
         lfs_size_t sim_size = 1;
         memset(sim, 0, N);
         memset(sim_names, 0, N*3);
-        sim[0] = alphas[0 % 26];
+        sim[0] = '_';
         memcpy(&sim_names[0], "___", 3);
 
         uint32_t prng = seed;
@@ -3261,7 +3268,7 @@ code = '''
         // create a btree
         lfsr_btree_t btree = LFSR_BTREE_NULL;
         lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W,
-                &alphas[0 % 26], 1) => 0;
+                "_", 1) => 0;
 
         // set up a simulation to compare against
         //
@@ -3274,7 +3281,7 @@ code = '''
         memset(sim, 0, N);
         memset(sim_names, 0, N*3);
         memset(sim_weights, 0, N*sizeof(lfs_size_t));
-        sim[0] = alphas[0 % 26];
+        sim[0] = '_';
         memcpy(&sim_names[0], "___", 3);
         sim_weights[0] = W;
 
@@ -3352,7 +3359,7 @@ code = '''
             btree.weight);
 
         lfs_size_t total_weight = 0;
-        for (lfs_size_t j = 0; j < N; j++) {
+        for (lfs_size_t j = 0; j < sim_size; j++) {
             total_weight += sim_weights[j];
         }
         assert(btree.weight == total_weight);
@@ -3417,7 +3424,7 @@ code = '''
         // create a btree
         lfsr_btree_t btree = LFSR_BTREE_NULL;
         lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1,
-                &alphas[0 % 26], 1) => 0;
+                "_", 1) => 0;
 
         // set up a simulation to compare against
         //
@@ -3428,7 +3435,7 @@ code = '''
         lfs_size_t sim_size = 1;
         memset(sim, 0, N);
         memset(sim_names, 0, N*3);
-        sim[0] = alphas[0 % 26];
+        sim[0] = '_';
         memcpy(&sim_names[0], "___", 3);
 
         uint32_t prng = seed;
@@ -3446,7 +3453,6 @@ code = '''
             // don't let sim drop below one element
             if (op == 0 || sim_size <= 1) {
                 // find where to split
-                printf("- split(\"%.3s\", \"%c\")\n", name, nums[i % 10]);
                 lfs_size_t id = 0;
                 while (id+1 < sim_size
                         && memcmp(sim_names[id+1], name, 3) <= 0) {
@@ -3464,7 +3470,7 @@ code = '''
                         NULL, &split_id, NULL, split_buf, 4) => 1;
                 if (split_id > id) {
                     lfsr_btree_split(&lfs, &btree,
-                            split_id, sim_names[split_id], 3,
+                            split_id, sim_names[id+1], 3,
                             LFSR_TAG_INLINED, 1, &nums[i % 10], 1,
                             LFSR_TAG_INLINED, 1, split_buf,     1) => 0;
                 } else {
@@ -3483,7 +3489,6 @@ code = '''
 
             } else if (op == 1) {
                 // update btree
-                printf("- update(%d, \"%c\")\n", id, nums[i % 10]);
                 lfsr_btree_update(&lfs, &btree, id,
                         LFSR_TAG_INLINED, 1,
                         &nums[i % 10], 1) => 0;
@@ -3493,7 +3498,6 @@ code = '''
 
             } else {
                 // pop from btree
-                printf("- pop(%d)\n", id);
                 lfsr_btree_pop(&lfs, &btree, id) => 0;
 
                 // pop from sim
@@ -3557,6 +3561,7 @@ defines.SEED = -1
 in = 'lfs.c'
 code = '''
     const char *alphas = "abcdefghijklmnopqrstuvwxyz";
+    const char *nums = "0123456789";
 
     // iterate through severals seeds that we can reproduce easily
     for (uint32_t seed = (SEED == -1 ? 1 : SEED);
@@ -3577,23 +3582,35 @@ code = '''
 
         // create a btree
         lfsr_btree_t btree = LFSR_BTREE_NULL;
+        lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W,
+                "_", 1) => 0;
 
         // set up a simulation to compare against
         //
         // fun fact this is slower than our actual tree! unfun fact this is
         // starting to be a problem...
         char *sim = malloc(N);
+        char (*sim_names)[3] = malloc(N*3);
         lfs_size_t *sim_weights = malloc(N*sizeof(lfs_size_t));
-        lfs_size_t sim_size = 0;
+        lfs_size_t sim_size = 1;
         memset(sim, 0, N);
+        memset(sim_names, 0, N*3);
         memset(sim_weights, 0, N*sizeof(lfs_size_t));
+        sim[0] = '_';
+        memcpy(&sim_names[0], "___", 3);
+        sim_weights[0] = W;
 
         uint32_t prng = seed;
         for (lfs_size_t i = 0; i < N; i++) {
             // choose a pseudo-random op
             uint8_t op = TEST_PRNG(&prng) % 3;
             // choose a pseudo-random id
-            lfs_size_t id = TEST_PRNG(&prng) % (sim_size+1);
+            lfs_size_t id = TEST_PRNG(&prng) % (sim_size == 0 ? 1 : sim_size);
+            // choose a pseudo-random name
+            lfs_size_t x = TEST_PRNG(&prng) % (26*26*26);
+            char name[3] = {
+                alphas[(x/26/26) % 26], alphas[(x/26) % 26], alphas[x % 26]
+            };
             // choose a pseudo-random weight
             lfs_size_t weight = 1 + (TEST_PRNG(&prng) % W);
 
@@ -3603,40 +3620,82 @@ code = '''
                 weighted_id += sim_weights[j];
             }
 
-            if (op == 0 || id == sim_size) {
+            // don't let sim drop below one element
-                // push to btree
+            if (op == 0 || sim_size <= 1) {
-                lfsr_btree_push(&lfs, &btree, weighted_id,
+                // find where to split
-                        LFSR_TAG_INLINED, weight,
+                lfs_size_t id = 0;
-                        &alphas[i % 26], 1) => 0;
+                while (id+1 < sim_size
+                        && memcmp(sim_names[id+1], name, 3) <= 0) {
+                    id += 1;
+                }
+                // just skip exact matches for now
+                if (memcmp(sim_names[id], name, 3) == 0) {
+                    continue;
+                }
 
-                // push to sim
+                // calculate actual id in btree space
+                lfs_size_t weighted_id = 0;
+                for (lfs_size_t j = 0; j < id; j++) {
+                    weighted_id += sim_weights[j];
+                }
+
+                // split btree
+                lfs_size_t split_id;
+                lfs_size_t split_weight;
+                uint8_t split_buf[4];
+                lfsr_btree_find(&lfs, &btree, name, 3,
+                        NULL, &split_id, &split_weight, split_buf, 4) => 1;
+                if (split_id > weighted_id+sim_weights[id]-1) {
+                    lfsr_btree_split(&lfs, &btree,
+                            split_id, sim_names[id+1], 3,
+                            LFSR_TAG_INLINED, weight,       &nums[i % 10], 1,
+                            LFSR_TAG_INLINED, split_weight, split_buf,     1)
+                            => 0;
+                } else {
+                    lfsr_btree_split(&lfs, &btree,
+                            split_id, name, 3,
+                            LFSR_TAG_INLINED, split_weight, split_buf,     1,
+                            LFSR_TAG_INLINED, weight,       &nums[i % 10], 1)
+                            => 0;
+                }
+
+                // split sim
                 memmove(&sim[id+1], &sim[id], sim_size-id);
+                memmove(&sim_names[id+1], &sim_names[id], (sim_size-id)*3);
                 memmove(&sim_weights[id+1], &sim_weights[id],
                         (sim_size-id)*sizeof(lfs_size_t));
-                sim[id] = alphas[i % 26];
+                sim[id+1] = nums[i % 10];
-                sim_weights[id] = weight;
+                memcpy(&sim_names[id+1], name, 3);
+                sim_weights[id+1] = weight;
                 sim_size += 1;
 
             } else if (op == 1) {
                 // update btree
                 lfsr_btree_update(&lfs, &btree,
                         weighted_id+sim_weights[id]-1, LFSR_TAG_INLINED, weight,
-                        &alphas[i % 26], 1) => 0;
+                        &nums[i % 10], 1) => 0;
 
                 // update sim
-                sim[id] = alphas[i % 26];
+                sim[id] = nums[i % 10];
                 sim_weights[id] = weight;
 
             } else {
-                // remove from btree
+                // pop from btree
                 lfsr_btree_pop(&lfs, &btree,
                         weighted_id+sim_weights[id]-1) => 0;
 
-                // remove from sim
+                // pop from sim
                 memmove(&sim[id], &sim[id+1], sim_size-(id+1));
+                memmove(&sim_names[id], &sim_names[id+1], (sim_size-(id+1))*3);
                 memmove(&sim_weights[id], &sim_weights[id+1],
                         (sim_size-(id+1))*sizeof(lfs_size_t));
                 sim_size -= 1;
+
+                // our B-tree doesn't actually track the name of id0, so we need
+                // mirror this in our sim
+                if (id == 0) {
+                    memcpy(&sim_names[0], "___", 3);
+                }
             }
         }
 
@@ -3644,11 +3703,21 @@ code = '''
         printf("expd: [");
         bool first = true;
         for (lfs_size_t i = 0; i < sim_size; i++) {
+            // calculate actual id in btree space
+            lfs_size_t weighted_id = 0;
+            for (lfs_size_t j = 0; j < i; j++) {
+                weighted_id += sim_weights[j];
+            }
+
             if (!first) {
                 printf(", ");
             }
             first = false;
-            printf("%c", sim[i]);
+            printf("%.3sid%dw%d=%c",
+                    sim_names[i],
+                    weighted_id+sim_weights[i]-1,
+                    sim_weights[i],
+                    sim[i]);
         }
         printf("]\n");
         printf("btree: 0x%x.%x 0x%x w%d\n",
@@ -3674,7 +3743,7 @@ code = '''
                 weighted_id += sim_weights[j];
             }
 
-            lfsr_btree_get(&lfs, &btree, weighted_id+sim_weights[i]-1,
+            lfsr_btree_find(&lfs, &btree, sim_names[i], 3,
                     &tag_, &id_, &weight_,
                     buffer, 4) => 1;
             assert(tag_ == LFSR_TAG_INLINED);
@@ -3683,34 +3752,10 @@ code = '''
             assert(memcmp(buffer, &sim[i], 1) == 0);
         }
 
-        // and no extra elements
-        lfsr_btree_get(&lfs, &btree, total_weight,
-                &tag_, &id_, &weight_,
-                buffer, 4) => LFS_ERR_NOENT;
-
-        // also test that we can traverse the tree without prior knowledge
-        id_ = -1;
-        for (lfs_size_t i = 0; i < sim_size; i++) {
-            // calculate actual id in btree space
-            lfs_size_t weighted_id = 0;
-            for (lfs_size_t j = 0; j < i; j++) {
-                weighted_id += sim_weights[j];
-            }
-
-            lfsr_btree_get(&lfs, &btree, id_+1,
-                    &tag_, &id_, &weight_,
-                    buffer, 4) => 1;
-            assert(tag_ == LFSR_TAG_INLINED);
-            assert(id_ == weighted_id+sim_weights[i]-1);
-            assert(weight_ == sim_weights[i]);
-            assert(memcmp(buffer, &sim[i], 1) == 0);
-        }
-        lfsr_btree_get(&lfs, &btree, id_+1,
-                &tag_, &id_, &weight_,
-                buffer, 4) => LFS_ERR_NOENT;
-
         // clean up sim
         free(sim);
+        free(sim_names);
+        free(sim_weights);
     }
 '''