From 23956cc25baaecd3b83f67d155c1390b30155aa8 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 20 Mar 2023 22:00:24 -0500 Subject: [PATCH] Added sparse B-tree name tests, fixed weight-calculation bug during merge More proof the tests are working. This bug was in the code that does an extra lookup for the was-split entry during merge, so we make sure we have the right id to attach the split name to. Humorously, this code was already set up correctly, the "split_id" just wasn't actually used. Unfortunately since lfsr_rbyd_lookup uses out-pointers to return multiple things the compiler couldn't detect the unused variable. --- lfs.c | 2 +- tests/test_btree.toml | 171 ++++++++++++++++++++++++++---------------- 2 files changed, 109 insertions(+), 64 deletions(-) diff --git a/lfs.c b/lfs.c index bc09f6d8..fdd2d26e 100644 --- a/lfs.c +++ b/lfs.c @@ -3603,7 +3603,7 @@ static int lfsr_btree_commit(lfs_t *lfs, } err = lfsr_rbyd_append(lfs, &rbyd_, - LFSR_TAG_MKBRANCH, (sdelta == 0 ? sweight : rweight), + LFSR_TAG_MKBRANCH, split_id, LFSR_DATA_DISK(parent.block, split_off, split_size)); if (err) { assert(!err); diff --git a/tests/test_btree.toml b/tests/test_btree.toml index d6889f50..ce951fd4 100644 --- a/tests/test_btree.toml +++ b/tests/test_btree.toml @@ -1622,7 +1622,7 @@ code = ''' &tag_, &id_, &weight_, buffer, 4) => LFS_ERR_NOENT; - // try recovering + // try recovering lfsr_btree_push(&lfs, &btree, REMAINING, LFSR_TAG_INLINED, 1, "R", 1) => 0; @@ -1705,7 +1705,7 @@ code = ''' &tag_, &id_, &weight_, buffer, 4) => LFS_ERR_NOENT; - // try recovering + // try recovering lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, "R", 1) => 0; @@ -1890,7 +1890,7 @@ code = ''' &tag_, &id_, &weight_, buffer, 4) => LFS_ERR_NOENT; - // try recovering + // try recovering lfsr_btree_push(&lfs, &btree, REMAINING*W, LFSR_TAG_INLINED, W, "R", 1) => 0; @@ -1993,7 +1993,7 @@ code = ''' for (lfs_size_t j = 0; j < i; j++) { weighted_id += sim_weights[j]; } - + lfsr_btree_push(&lfs, &btree, weighted_id, LFSR_TAG_INLINED, weight, &alphas[i % 26], 1) => 0; @@ -2189,7 +2189,7 @@ code = ''' // create a btree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - &alphas[0 % 26], 1) => 0; + "_", 1) => 0; // set up a simulation to compare against // @@ -2198,7 +2198,7 @@ code = ''' char *sim = malloc(N); lfs_size_t sim_size = 1; memset(sim, 0, N); - sim[0] = alphas[0 % 26]; + sim[0] = '_'; uint32_t prng = seed; for (lfs_size_t i = 1; i < N; i++) { @@ -2346,7 +2346,7 @@ code = ''' // create a btree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W, - &alphas[0 % 26], 1) => 0; + "_", 1) => 0; // set up a simulation to compare against // @@ -2357,7 +2357,7 @@ code = ''' lfs_size_t sim_size = 1; memset(sim, 0, N); memset(sim_weights, 0, N*sizeof(lfs_size_t)); - sim[0] = alphas[0 % 26]; + sim[0] = '_'; sim_weights[0] = W; uint32_t prng = seed; @@ -2416,7 +2416,7 @@ code = ''' btree.weight); lfs_size_t total_weight = 0; - for (lfs_size_t j = 0; j < N; j++) { + for (lfs_size_t j = 0; j < sim_size; j++) { total_weight += sim_weights[j]; } assert(btree.weight == total_weight); @@ -2689,11 +2689,18 @@ code = ''' printf("expd: ["); bool first = true; for (lfs_size_t i = 0; i < sim_size; i++) { + // calculate actual id in btree space + lfs_size_t weighted_id = 0; + for (lfs_size_t j = 0; j < i; j++) { + weighted_id += sim_weights[j]; + } + if (!first) { printf(", "); } first = false; - printf("%c", sim[i]); + printf("%dw%d=%c", weighted_id+sim_weights[i]-1, + sim_weights[i], sim[i]); } printf("]\n"); printf("btree: 0x%x.%x 0x%x w%d\n", @@ -3087,7 +3094,7 @@ code = ''' // create a btree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - &alphas[0 % 26], 1) => 0; + "_", 1) => 0; // set up a simulation to compare against // @@ -3098,7 +3105,7 @@ code = ''' lfs_size_t sim_size = 1; memset(sim, 0, N); memset(sim_names, 0, N*3); - sim[0] = alphas[0 % 26]; + sim[0] = '_'; memcpy(&sim_names[0], "___", 3); uint32_t prng = seed; @@ -3261,7 +3268,7 @@ code = ''' // create a btree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W, - &alphas[0 % 26], 1) => 0; + "_", 1) => 0; // set up a simulation to compare against // @@ -3274,7 +3281,7 @@ code = ''' memset(sim, 0, N); memset(sim_names, 0, N*3); memset(sim_weights, 0, N*sizeof(lfs_size_t)); - sim[0] = alphas[0 % 26]; + sim[0] = '_'; memcpy(&sim_names[0], "___", 3); sim_weights[0] = W; @@ -3352,7 +3359,7 @@ code = ''' btree.weight); lfs_size_t total_weight = 0; - for (lfs_size_t j = 0; j < N; j++) { + for (lfs_size_t j = 0; j < sim_size; j++) { total_weight += sim_weights[j]; } assert(btree.weight == total_weight); @@ -3417,7 +3424,7 @@ code = ''' // create a btree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - &alphas[0 % 26], 1) => 0; + "_", 1) => 0; // set up a simulation to compare against // @@ -3428,7 +3435,7 @@ code = ''' lfs_size_t sim_size = 1; memset(sim, 0, N); memset(sim_names, 0, N*3); - sim[0] = alphas[0 % 26]; + sim[0] = '_'; memcpy(&sim_names[0], "___", 3); uint32_t prng = seed; @@ -3446,7 +3453,6 @@ code = ''' // don't let sim drop below one element if (op == 0 || sim_size <= 1) { // find where to split - printf("- split(\"%.3s\", \"%c\")\n", name, nums[i % 10]); lfs_size_t id = 0; while (id+1 < sim_size && memcmp(sim_names[id+1], name, 3) <= 0) { @@ -3464,7 +3470,7 @@ code = ''' NULL, &split_id, NULL, split_buf, 4) => 1; if (split_id > id) { lfsr_btree_split(&lfs, &btree, - split_id, sim_names[split_id], 3, + split_id, sim_names[id+1], 3, LFSR_TAG_INLINED, 1, &nums[i % 10], 1, LFSR_TAG_INLINED, 1, split_buf, 1) => 0; } else { @@ -3483,7 +3489,6 @@ code = ''' } else if (op == 1) { // update btree - printf("- update(%d, \"%c\")\n", id, nums[i % 10]); lfsr_btree_update(&lfs, &btree, id, LFSR_TAG_INLINED, 1, &nums[i % 10], 1) => 0; @@ -3493,7 +3498,6 @@ code = ''' } else { // pop from btree - printf("- pop(%d)\n", id); lfsr_btree_pop(&lfs, &btree, id) => 0; // pop from sim @@ -3557,6 +3561,7 @@ defines.SEED = -1 in = 'lfs.c' code = ''' const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + const char *nums = "0123456789"; // iterate through severals seeds that we can reproduce easily for (uint32_t seed = (SEED == -1 ? 1 : SEED); @@ -3577,23 +3582,35 @@ code = ''' // create a btree lfsr_btree_t btree = LFSR_BTREE_NULL; + lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W, + "_", 1) => 0; // set up a simulation to compare against // // fun fact this is slower than our actual tree! unfun fact this is // starting to be a problem... char *sim = malloc(N); + char (*sim_names)[3] = malloc(N*3); lfs_size_t *sim_weights = malloc(N*sizeof(lfs_size_t)); - lfs_size_t sim_size = 0; + lfs_size_t sim_size = 1; memset(sim, 0, N); + memset(sim_names, 0, N*3); memset(sim_weights, 0, N*sizeof(lfs_size_t)); + sim[0] = '_'; + memcpy(&sim_names[0], "___", 3); + sim_weights[0] = W; uint32_t prng = seed; for (lfs_size_t i = 0; i < N; i++) { // choose a pseudo-random op uint8_t op = TEST_PRNG(&prng) % 3; // choose a pseudo-random id - lfs_size_t id = TEST_PRNG(&prng) % (sim_size+1); + lfs_size_t id = TEST_PRNG(&prng) % (sim_size == 0 ? 1 : sim_size); + // choose a pseudo-random name + lfs_size_t x = TEST_PRNG(&prng) % (26*26*26); + char name[3] = { + alphas[(x/26/26) % 26], alphas[(x/26) % 26], alphas[x % 26] + }; // choose a pseudo-random weight lfs_size_t weight = 1 + (TEST_PRNG(&prng) % W); @@ -3603,40 +3620,82 @@ code = ''' weighted_id += sim_weights[j]; } - if (op == 0 || id == sim_size) { - // push to btree - lfsr_btree_push(&lfs, &btree, weighted_id, - LFSR_TAG_INLINED, weight, - &alphas[i % 26], 1) => 0; + // don't let sim drop below one element + if (op == 0 || sim_size <= 1) { + // find where to split + lfs_size_t id = 0; + while (id+1 < sim_size + && memcmp(sim_names[id+1], name, 3) <= 0) { + id += 1; + } + // just skip exact matches for now + if (memcmp(sim_names[id], name, 3) == 0) { + continue; + } - // push to sim + // calculate actual id in btree space + lfs_size_t weighted_id = 0; + for (lfs_size_t j = 0; j < id; j++) { + weighted_id += sim_weights[j]; + } + + // split btree + lfs_size_t split_id; + lfs_size_t split_weight; + uint8_t split_buf[4]; + lfsr_btree_find(&lfs, &btree, name, 3, + NULL, &split_id, &split_weight, split_buf, 4) => 1; + if (split_id > weighted_id+sim_weights[id]-1) { + lfsr_btree_split(&lfs, &btree, + split_id, sim_names[id+1], 3, + LFSR_TAG_INLINED, weight, &nums[i % 10], 1, + LFSR_TAG_INLINED, split_weight, split_buf, 1) + => 0; + } else { + lfsr_btree_split(&lfs, &btree, + split_id, name, 3, + LFSR_TAG_INLINED, split_weight, split_buf, 1, + LFSR_TAG_INLINED, weight, &nums[i % 10], 1) + => 0; + } + + // split sim memmove(&sim[id+1], &sim[id], sim_size-id); + memmove(&sim_names[id+1], &sim_names[id], (sim_size-id)*3); memmove(&sim_weights[id+1], &sim_weights[id], (sim_size-id)*sizeof(lfs_size_t)); - sim[id] = alphas[i % 26]; - sim_weights[id] = weight; + sim[id+1] = nums[i % 10]; + memcpy(&sim_names[id+1], name, 3); + sim_weights[id+1] = weight; sim_size += 1; } else if (op == 1) { // update btree lfsr_btree_update(&lfs, &btree, weighted_id+sim_weights[id]-1, LFSR_TAG_INLINED, weight, - &alphas[i % 26], 1) => 0; + &nums[i % 10], 1) => 0; // update sim - sim[id] = alphas[i % 26]; + sim[id] = nums[i % 10]; sim_weights[id] = weight; } else { - // remove from btree + // pop from btree lfsr_btree_pop(&lfs, &btree, weighted_id+sim_weights[id]-1) => 0; - // remove from sim + // pop from sim memmove(&sim[id], &sim[id+1], sim_size-(id+1)); + memmove(&sim_names[id], &sim_names[id+1], (sim_size-(id+1))*3); memmove(&sim_weights[id], &sim_weights[id+1], (sim_size-(id+1))*sizeof(lfs_size_t)); sim_size -= 1; + + // our B-tree doesn't actually track the name of id0, so we need + // mirror this in our sim + if (id == 0) { + memcpy(&sim_names[0], "___", 3); + } } } @@ -3644,11 +3703,21 @@ code = ''' printf("expd: ["); bool first = true; for (lfs_size_t i = 0; i < sim_size; i++) { + // calculate actual id in btree space + lfs_size_t weighted_id = 0; + for (lfs_size_t j = 0; j < i; j++) { + weighted_id += sim_weights[j]; + } + if (!first) { printf(", "); } first = false; - printf("%c", sim[i]); + printf("%.3sid%dw%d=%c", + sim_names[i], + weighted_id+sim_weights[i]-1, + sim_weights[i], + sim[i]); } printf("]\n"); printf("btree: 0x%x.%x 0x%x w%d\n", @@ -3674,7 +3743,7 @@ code = ''' weighted_id += sim_weights[j]; } - lfsr_btree_get(&lfs, &btree, weighted_id+sim_weights[i]-1, + lfsr_btree_find(&lfs, &btree, sim_names[i], 3, &tag_, &id_, &weight_, buffer, 4) => 1; assert(tag_ == LFSR_TAG_INLINED); @@ -3683,34 +3752,10 @@ code = ''' assert(memcmp(buffer, &sim[i], 1) == 0); } - // and no extra elements - lfsr_btree_get(&lfs, &btree, total_weight, - &tag_, &id_, &weight_, - buffer, 4) => LFS_ERR_NOENT; - - // also test that we can traverse the tree without prior knowledge - id_ = -1; - for (lfs_size_t i = 0; i < sim_size; i++) { - // calculate actual id in btree space - lfs_size_t weighted_id = 0; - for (lfs_size_t j = 0; j < i; j++) { - weighted_id += sim_weights[j]; - } - - lfsr_btree_get(&lfs, &btree, id_+1, - &tag_, &id_, &weight_, - buffer, 4) => 1; - assert(tag_ == LFSR_TAG_INLINED); - assert(id_ == weighted_id+sim_weights[i]-1); - assert(weight_ == sim_weights[i]); - assert(memcmp(buffer, &sim[i], 1) == 0); - } - lfsr_btree_get(&lfs, &btree, id_+1, - &tag_, &id_, &weight_, - buffer, 4) => LFS_ERR_NOENT; - // clean up sim free(sim); + free(sim_names); + free(sim_weights); } '''