From 15e27f92af3b56a7b88a5d8f2c907d3db7deee38 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 13 Mar 2023 14:18:22 -0500 Subject: [PATCH] Implemented lfsr_btree_split, a shortcut for pop+push+push Another straightforward exercise of making sure the pending attributes are setup correctly. If you think this isn't worth its own function, consider how much overhead the 3x commits for pop+push+push would add, especially for large-prog devices. Worst case this can be dropped in the future. --- lfs.c | 68 ++++++++ tests/test_btree.toml | 372 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 430 insertions(+), 10 deletions(-) diff --git a/lfs.c b/lfs.c index 3d96de0a..9608852f 100644 --- a/lfs.c +++ b/lfs.c @@ -4225,6 +4225,74 @@ static int lfsr_btree_pop(lfs_t *lfs, lfsr_btree_t *btree, lfs_size_t id) { } } +// lfsr_btree_split can be done with a pop+push+push, but this function +// does all this in one commit, which is much more efficient +static int lfsr_btree_split(lfs_t *lfs, lfsr_btree_t *btree, + lfs_size_t id, + lfsr_tag_t tag1, lfs_size_t weight1, + const void *buffer1, lfs_size_t size1, + lfsr_tag_t tag2, lfs_size_t weight2, + const void *buffer2, lfs_size_t size2) { + LFS_ASSERT(id < btree->weight); + + // inlined btree, need to expand into an rbyd + if (btree->tag) { + lfsr_rbyd_t rbyd; + int err = lfsr_rbyd_alloc(lfs, &rbyd, 1); + if (err) { + return err; + } + + // commit our entries + err = lfsr_rbyd_commit(lfs, &rbyd, + LFSR_ATTR(GROW, 0, NULL, weight1, + LFSR_ATTR(MKBRANCH, 0+weight1-1, NULL, 0, + LFSR_ATTR_(tag1, 0+weight1-1, + buffer1, size1, + LFSR_ATTR(GROW, weight1, NULL, weight2, + LFSR_ATTR(MKBRANCH, weight1+weight2-1, NULL, 0, + LFSR_ATTR_(tag2, weight1+weight2-1, + buffer2, size2, + NULL))))))); + if (err) { + return err; + } + + btree->tag = 0; + btree->weight = rbyd.weight; + btree->u.trunk.block = rbyd.block; + btree->u.trunk.limit = rbyd.off; + return 0; + + // a normal btree + } else { + // lookup in which leaf our id resides + lfsr_rbyd_t rbyd; + lfs_ssize_t rid; + lfs_size_t rweight; + lfs_ssize_t size = lfsr_btree_lookup(lfs, btree, id, + NULL, NULL, &rbyd, &rid, &rweight, NULL, 0); + if (size < 0) { + return size; + } + + // commit our id into the tree, letting lfsr_btree_commit take care + // of the rest + return lfsr_btree_commit(lfs, btree, id, &rbyd, + LFSR_ATTR(SHRINK, rid-(rweight-1), NULL, rweight, + LFSR_ATTR(GROW, rid-(rweight-1), NULL, weight1, + LFSR_ATTR(MKBRANCH, rid-(rweight-1)+weight1-1, NULL, 0, + LFSR_ATTR_(tag1, rid-(rweight-1)+weight1-1, + buffer1, size1, + LFSR_ATTR(GROW, rid-(rweight-1)+weight1, NULL, weight2, + LFSR_ATTR(MKBRANCH, rid-(rweight-1)+weight1+weight2-1, NULL, 0, + LFSR_ATTR_(tag2, rid-(rweight-1)+weight1+weight2-1, + buffer2, size2, + NULL)))))))); + } +} + + diff --git a/tests/test_btree.toml b/tests/test_btree.toml index 0424a453..5e740d82 100644 --- a/tests/test_btree.toml +++ b/tests/test_btree.toml @@ -457,7 +457,7 @@ code = ''' // check that btree matches sim printf("expd: ["); bool first = true; - for (lfs_size_t i = 0; i < N; i++) { + for (lfs_size_t i = 0; i < sim_size; i++) { if (!first) { printf(", "); } @@ -470,13 +470,13 @@ code = ''' btree.u.trunk.limit, btree.tag, btree.weight); - assert(btree.weight == N); + assert(btree.weight == sim_size); uint8_t buffer[4]; lfsr_tag_t tag_; lfs_size_t id_; lfs_size_t weight_; - for (lfs_size_t i = 0; i < N; i++) { + for (lfs_size_t i = 0; i < sim_size; i++) { lfsr_btree_get(&lfs, &btree, i, &tag_, &id_, &weight_, buffer, 4) => 1; @@ -487,7 +487,7 @@ code = ''' } // and no extra elements - lfsr_btree_get(&lfs, &btree, N, + lfsr_btree_get(&lfs, &btree, sim_size, &tag_, &id_, &weight_, buffer, 4) => LFS_ERR_NOENT; @@ -627,7 +627,7 @@ code = ''' // check that btree matches sim printf("expd: ["); bool first = true; - for (lfs_size_t i = 0; i < N; i++) { + for (lfs_size_t i = 0; i < sim_size; i++) { // calculate actual id in btree space lfs_size_t weighted_id = 0; for (lfs_size_t j = 0; j < i; j++) { @@ -649,7 +649,7 @@ code = ''' btree.weight); lfs_size_t total_weight = 0; - for (lfs_size_t j = 0; j < N; j++) { + for (lfs_size_t j = 0; j < sim_size; j++) { total_weight += sim_weights[j]; } assert(btree.weight == total_weight); @@ -658,7 +658,7 @@ code = ''' lfsr_tag_t tag_; lfs_size_t id_; lfs_size_t weight_; - for (lfs_size_t i = 0; i < N; i++) { + for (lfs_size_t i = 0; i < sim_size; i++) { // calculate actual id in btree space lfs_size_t weighted_id = 0; for (lfs_size_t j = 0; j < i; j++) { @@ -681,7 +681,7 @@ code = ''' // also test that we can traverse the tree without prior knowledge id_ = -1; - for (lfs_size_t i = 0; i < N; i++) { + for (lfs_size_t i = 0; i < sim_size; i++) { // calculate actual id in btree space lfs_size_t weighted_id = 0; for (lfs_size_t j = 0; j < i; j++) { @@ -2068,8 +2068,360 @@ code = ''' ''' -# TODO [cases.test_btree_split] -# TODO [cases.test_btree_split_fuzz] +# test btree splits +[cases.test_btree_split] +defines.N = [4, 8, 16, 32, 64, 128, 256, 512, 1024] +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfs_init(&lfs, cfg) => 0; + // create free lookahead + memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size); + lfs.free.off = 0; + lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size, + lfs.cfg->block_count); + lfs.free.i = 0; + lfs_alloc_ack(&lfs); + + // create a tree with N elements + lfsr_btree_t btree = LFSR_BTREE_NULL; + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, + &alphas[0 % 26], 1) => 0; + for (lfs_size_t i = 1; i < N; i++) { + lfsr_btree_split(&lfs, &btree, i-1, + LFSR_TAG_INLINED, 1, &alphas[(i-1) % 26], 1, + LFSR_TAG_INLINED, 1, &alphas[(i-0) % 26], 1) => 0; + } + printf("btree: 0x%x.%x 0x%x w%d\n", + btree.u.trunk.block, + btree.u.trunk.limit, + btree.tag, + btree.weight); + assert(btree.weight == N); + + // check that the elements are in the tree + uint8_t buffer[4]; + lfsr_tag_t tag_; + lfs_size_t id_; + lfs_size_t weight_; + + for (lfs_size_t i = 0; i < N; i++) { + lfsr_btree_get(&lfs, &btree, i, + &tag_, &id_, &weight_, + buffer, 4) => 1; + assert(tag_ == LFSR_TAG_INLINED); + assert(id_ == i); + assert(weight_ == 1); + assert(memcmp(buffer, &alphas[i % 26], 1) == 0); + } + + // and check that we can't lookup elements that aren't in the tree + lfsr_btree_get(&lfs, &btree, N, + &tag_, &id_, &weight_, + buffer, 4) => LFS_ERR_NOENT; +''' + +[cases.test_btree_split_fuzz] +defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] +defines.ITER = 10 +in = 'lfs.c' +code = ''' + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + const char *uppers = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + // iterate through severals seeds that we can reproduce easily + for (uint32_t seed = 1; seed < ITER+1; seed++) { + // create lfs here since we need to reset each iteration, we're + // space constrained and we can't expect gc to work at this point + lfs_t lfs; + lfs_init(&lfs, cfg) => 0; + // create free lookahead + memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size); + lfs.free.off = 0; + lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size, + lfs.cfg->block_count); + lfs.free.i = 0; + lfs_alloc_ack(&lfs); + + // create a btree + lfsr_btree_t btree = LFSR_BTREE_NULL; + lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, + &alphas[0 % 26], 1) => 0; + + // set up a simulation to compare against + // + // fun fact this is slower than our actual tree! unfun fact this is + // starting to be a problem... + char *sim = malloc(N); + lfs_size_t sim_size = 1; + memset(sim, 0, N); + sim[0] = alphas[0 % 26]; + + uint32_t prng = seed; + for (lfs_size_t i = 1; i < N; i++) { + // choose a pseudo-random id + lfs_size_t id = TEST_PRNG(&prng) % sim_size; + + // split btree + lfsr_btree_split(&lfs, &btree, id, + LFSR_TAG_INLINED, 1, &alphas[i % 26], 1, + LFSR_TAG_INLINED, 1, &uppers[i % 26], 1) => 0; + + // split sim + memmove(&sim[id+1], &sim[id], sim_size-id); + sim[id+0] = alphas[i % 26]; + sim[id+1] = uppers[i % 26]; + sim_size += 1; + } + + // check that btree matches sim + printf("expd: ["); + bool first = true; + for (lfs_size_t i = 0; i < sim_size; i++) { + if (!first) { + printf(", "); + } + first = false; + printf("%c", sim[i]); + } + printf("]\n"); + printf("btree: 0x%x.%x 0x%x w%d\n", + btree.u.trunk.block, + btree.u.trunk.limit, + btree.tag, + btree.weight); + assert(btree.weight == sim_size); + + uint8_t buffer[4]; + lfsr_tag_t tag_; + lfs_size_t id_; + lfs_size_t weight_; + for (lfs_size_t i = 0; i < sim_size; i++) { + lfsr_btree_get(&lfs, &btree, i, + &tag_, &id_, &weight_, + buffer, 4) => 1; + assert(tag_ == LFSR_TAG_INLINED); + assert(id_ == i); + assert(weight_ == 1); + assert(memcmp(buffer, &sim[i], 1) == 0); + } + + // and no extra elements + lfsr_btree_get(&lfs, &btree, sim_size, + &tag_, &id_, &weight_, + buffer, 4) => LFS_ERR_NOENT; + + // clean up sim + free(sim); + lfs_deinit(&lfs) => 0; + } +''' + +[cases.test_btree_split_sparse] +defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] +defines.W = 5 +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfs_init(&lfs, cfg) => 0; + // create free lookahead + memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size); + lfs.free.off = 0; + lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size, + lfs.cfg->block_count); + lfs.free.i = 0; + lfs_alloc_ack(&lfs); + + // create a tree with N elements + lfsr_btree_t btree = LFSR_BTREE_NULL; + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W, + &alphas[0 % 26], 1) => 0; + for (lfs_size_t i = 1; i < N; i++) { + lfsr_btree_split(&lfs, &btree, (i-1)*W+W-1, + LFSR_TAG_INLINED, W, &alphas[(i-1) % 26], 1, + LFSR_TAG_INLINED, W, &alphas[(i-0) % 26], 1) => 0; + } + printf("btree: 0x%x.%x 0x%x w%d\n", + btree.u.trunk.block, + btree.u.trunk.limit, + btree.tag, + btree.weight); + assert(btree.weight == N*W); + + // check that the elements are in the tree + uint8_t buffer[4]; + lfsr_tag_t tag_; + lfs_size_t id_; + lfs_size_t weight_; + + for (lfs_size_t i = 0; i < N; i++) { + lfsr_btree_get(&lfs, &btree, i*W+W-1, + &tag_, &id_, &weight_, + buffer, 4) => 1; + assert(tag_ == LFSR_TAG_INLINED); + assert(id_ == i*W+W-1); + assert(weight_ == W); + assert(memcmp(buffer, &alphas[i % 26], 1) == 0); + } + + // and check that we can't lookup elements that aren't in the tree + lfsr_btree_get(&lfs, &btree, N*W, + &tag_, &id_, &weight_, + buffer, 4) => LFS_ERR_NOENT; +''' + +[cases.test_btree_split_sparse_fuzz] +defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] +defines.W = 5 +defines.ITER = 10 +in = 'lfs.c' +code = ''' + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + const char *uppers = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + // iterate through severals seeds that we can reproduce easily + for (uint32_t seed = 1; seed < ITER+1; seed++) { + // create lfs here since we need to reset each iteration, we're + // space constrained and we can't expect gc to work at this point + lfs_t lfs; + lfs_init(&lfs, cfg) => 0; + // create free lookahead + memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size); + lfs.free.off = 0; + lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size, + lfs.cfg->block_count); + lfs.free.i = 0; + lfs_alloc_ack(&lfs); + + // create a btree + lfsr_btree_t btree = LFSR_BTREE_NULL; + lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W, + &alphas[0 % 26], 1) => 0; + + // set up a simulation to compare against + // + // fun fact this is slower than our actual tree! unfun fact this is + // starting to be a problem... + char *sim = malloc(N); + lfs_size_t *sim_weights = malloc(N*sizeof(lfs_size_t)); + lfs_size_t sim_size = 1; + memset(sim, 0, N); + memset(sim_weights, 0, N*sizeof(lfs_size_t)); + sim[0] = alphas[0 % 26]; + sim_weights[0] = W; + + uint32_t prng = seed; + for (lfs_size_t i = 1; i < N; i++) { + // choose a pseudo-random id + lfs_size_t id = TEST_PRNG(&prng) % sim_size; + // choose pseudo-random weights + lfs_size_t weight1 = 1 + (TEST_PRNG(&prng) % W); + lfs_size_t weight2 = 1 + (TEST_PRNG(&prng) % W); + + // calculate actual id in btree space + lfs_size_t weighted_id = 0; + for (lfs_size_t j = 0; j < id; j++) { + weighted_id += sim_weights[j]; + } + + // split btree + lfsr_btree_split(&lfs, &btree, weighted_id+sim_weights[id]-1, + LFSR_TAG_INLINED, weight1, &alphas[i % 26], 1, + LFSR_TAG_INLINED, weight2, &uppers[i % 26], 1) => 0; + + // add to sim + memmove(&sim[id+1], &sim[id], sim_size-id); + memmove(&sim_weights[id+1], &sim_weights[id], + (sim_size-id)*sizeof(lfs_size_t)); + sim[id+0] = alphas[i % 26]; + sim[id+1] = uppers[i % 26]; + sim_weights[id+0] = weight1; + sim_weights[id+1] = weight2; + sim_size += 1; + } + + // check that btree matches sim + printf("expd: ["); + bool first = true; + for (lfs_size_t i = 0; i < sim_size; i++) { + // calculate actual id in btree space + lfs_size_t weighted_id = 0; + for (lfs_size_t j = 0; j < i; j++) { + weighted_id += sim_weights[j]; + } + + if (!first) { + printf(", "); + } + first = false; + printf("%dw%d=%c", weighted_id+sim_weights[i]-1, + sim_weights[i], sim[i]); + } + printf("]\n"); + printf("btree: 0x%x.%x 0x%x w%d\n", + btree.u.trunk.block, + btree.u.trunk.limit, + btree.tag, + btree.weight); + + lfs_size_t total_weight = 0; + for (lfs_size_t j = 0; j < N; j++) { + total_weight += sim_weights[j]; + } + assert(btree.weight == total_weight); + + uint8_t buffer[4]; + lfsr_tag_t tag_; + lfs_size_t id_; + lfs_size_t weight_; + for (lfs_size_t i = 0; i < N; i++) { + // calculate actual id in btree space + lfs_size_t weighted_id = 0; + for (lfs_size_t j = 0; j < i; j++) { + weighted_id += sim_weights[j]; + } + + lfsr_btree_get(&lfs, &btree, weighted_id+sim_weights[i]-1, + &tag_, &id_, &weight_, + buffer, 4) => 1; + assert(tag_ == LFSR_TAG_INLINED); + assert(id_ == weighted_id+sim_weights[i]-1); + assert(weight_ == sim_weights[i]); + assert(memcmp(buffer, &sim[i], 1) == 0); + } + + // and no extra elements + lfsr_btree_get(&lfs, &btree, total_weight, + &tag_, &id_, &weight_, + buffer, 4) => LFS_ERR_NOENT; + + // also test that we can traverse the tree without prior knowledge + id_ = -1; + for (lfs_size_t i = 0; i < N; i++) { + // calculate actual id in btree space + lfs_size_t weighted_id = 0; + for (lfs_size_t j = 0; j < i; j++) { + weighted_id += sim_weights[j]; + } + + lfsr_btree_get(&lfs, &btree, id_+1, + &tag_, &id_, &weight_, + buffer, 4) => 1; + assert(tag_ == LFSR_TAG_INLINED); + assert(id_ == weighted_id+sim_weights[i]-1); + assert(weight_ == sim_weights[i]); + assert(memcmp(buffer, &sim[i], 1) == 0); + } + lfsr_btree_get(&lfs, &btree, id_+1, + &tag_, &id_, &weight_, + buffer, 4) => LFS_ERR_NOENT; + + // clean up sim + free(sim); + } +'''