diff --git a/lfs.c b/lfs.c index b2c37543..b340aaf8 100644 --- a/lfs.c +++ b/lfs.c @@ -3137,11 +3137,11 @@ static int lfsr_btree_lookupnext_(lfs_t *lfs, } } + // adjust rid with subtree's weight + rid -= (rid__ - (weight__-1)); + // found another branch if (tag__ == LFSR_TAG_BTREE) { - // adjust rid with subtree's weight - rid -= (rid__ - (weight__-1)); - // fetch the next branch lfs_ssize_t d = lfsr_branch_fromdisk(lfs, &branch, data__); if (d < 0) { @@ -3153,7 +3153,7 @@ static int lfsr_btree_lookupnext_(lfs_t *lfs, } else { // TODO how many of these should be conditional? if (bid_) { - *bid_ = bid + (rid__ - rid); + *bid_ = bid - (rid - (weight__-1)); } if (rbyd_) { *rbyd_ = branch; @@ -3396,11 +3396,11 @@ static lfs_ssize_t lfsr_btree_namelookupnext(lfs_t *lfs, } } + // update our bid + bid += rid__ - (weight__-1); + // found another branch if (tag__ == LFSR_TAG_BTREE) { - // update our id - bid += rid__-(weight__-1); - // fetch the next branch lfs_ssize_t d = lfsr_branch_fromdisk(lfs, &branch, data__); if (d < 0) { @@ -3412,7 +3412,7 @@ static lfs_ssize_t lfsr_btree_namelookupnext(lfs_t *lfs, } else { // TODO how many of these should be conditional? if (bid_) { - *bid_ = bid + rid__; + *bid_ = bid + (weight__-1); } if (rbyd_) { *rbyd_ = branch; @@ -4353,6 +4353,174 @@ static int lfsr_btree_split(lfs_t *lfs, lfsr_btree_t *btree, } +// incremental btree traversal +// +// note this is different from iteration, iteration should use +// lfsr_btree_lookupnext, traversal includes inner entries +typedef struct lfsr_btree_traversal { + lfs_size_t bid; + lfs_ssize_t rid; + lfsr_rbyd_t branch; +} lfsr_btree_traversal_t; + +static int lfsr_btree_traversal_start(lfs_t *lfs, + const lfsr_btree_t *btree, + lfsr_btree_traversal_t *traversal) { + (void)lfs; + (void)btree; + // setup traversal to fetch the root next call + traversal->bid = 0; + traversal->rid = 0; + traversal->branch.trunk = 0; + traversal->branch.weight = 0; + return 0; +} + +static int lfsr_btree_traversal_next(lfs_t *lfs, + const lfsr_btree_t *btree, + lfsr_btree_traversal_t *traversal, + lfs_size_t *bid_, lfsr_tag_t *tag_, lfs_size_t *weight_, + lfsr_data_t *data_) { + while (true) { + // in range? + if (traversal->bid >= lfsr_btree_weight(btree)) { + return LFS_ERR_NOENT; + } + + // inlined? + if (lfsr_btree_isinlined(btree)) { + // setup traversal to terminate next call + traversal->bid = lfsr_btree_weight(btree); + + // TODO how many of these should be conditional? + if (bid_) { + *bid_ = lfsr_btree_weight(btree)-1; + } + if (tag_) { + *tag_ = btree->inlined.tag; + } + if (weight_) { + *weight_ = lfsr_btree_weight(btree); + } + if (data_) { + *data_ = LFSR_DATA_BUF(btree->inlined.buffer, + btree->inlined.size); + } + return 0; + } + + // make sure we traverse the root + if (traversal->branch.trunk == 0) { + traversal->bid += traversal->branch.weight; + traversal->rid = traversal->bid; + traversal->branch = btree->root; + + if (traversal->rid == 0) { + // TODO how many of these should be conditional? + if (bid_) { + *bid_ = lfsr_btree_weight(btree)-1; + } + if (tag_) { + *tag_ = LFSR_TAG_BTREE; + } + if (weight_) { + *weight_ = lfsr_btree_weight(btree); + } + if (data_) { + // note btrees are returned decoded + *data_ = LFSR_DATA_BUF(&traversal->branch, + sizeof(lfsr_rbyd_t)); + } + return 0; + } + + // continue, mostly for range check + continue; + } + + // descend down the tree + lfs_ssize_t rid__; + lfsr_tag_t tag__; + lfs_size_t weight__; + lfsr_data_t data__; + int err = lfsr_rbyd_lookupnext(lfs, &traversal->branch, + traversal->rid, 0, + &rid__, &tag__, &weight__, &data__); + if (err) { + LFS_ASSERT(err != LFS_ERR_NOENT); + return err; + } + + if (lfsr_tag_suptype(tag__) == LFSR_TAG_NAME) { + err = lfsr_rbyd_lookupnext(lfs, &traversal->branch, + rid__, LFSR_TAG_STRUCT, + NULL, &tag__, NULL, &data__); + if (err) { + LFS_ASSERT(err != LFS_ERR_NOENT); + return err; + } + } + + // adjust rid with subtree's weight + traversal->rid -= (rid__ - (weight__-1)); + + // found another branch + if (tag__ == LFSR_TAG_BTREE) { + // fetch the next branch + lfs_ssize_t d = lfsr_branch_fromdisk(lfs, + &traversal->branch, data__); + if (d < 0) { + return d; + } + LFS_ASSERT(traversal->branch.weight == weight__); + + // return inner btree nodes if this is the first time we've + // seen them + if (traversal->rid == 0) { + // TODO how many of these should be conditional? + if (bid_) { + *bid_ = traversal->bid - ( + traversal->rid - (traversal->branch.weight-1)); + } + if (tag_) { + *tag_ = LFSR_TAG_BTREE; + } + if (weight_) { + *weight_ = traversal->branch.weight; + } + if (data_) { + // note btrees are returned decoded + *data_ = LFSR_DATA_BUF( + &traversal->branch, sizeof(lfsr_rbyd_t)); + } + return 0; + } + + // found our bid + } else { + // update traversal + traversal->branch.trunk = 0; + traversal->branch.weight = weight__; + + // TODO how many of these should be conditional? + if (bid_) { + *bid_ = traversal->bid - (traversal->rid - (weight__-1)); + } + if (tag_) { + *tag_ = tag__; + } + if (weight_) { + *weight_ = weight__; + } + if (data_) { + *data_ = data__; + } + return 0; + } + } +} + + /// Metadata pair operations /// diff --git a/tests/test_btree.toml b/tests/test_btree.toml index c3e7e68b..b1a9ee07 100644 --- a/tests/test_btree.toml +++ b/tests/test_btree.toml @@ -4056,3 +4056,319 @@ code = ''' } ''' + +## B-tree traversal tests ## + +# some simple btree traversals +[cases.test_btree_traversal] +defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfs_init(&lfs, cfg) => 0; + // create free lookahead + memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size); + lfs.free.off = 0; + lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size, + lfs.cfg->block_count); + lfs.free.i = 0; + lfs_alloc_ack(&lfs); + + // create a tree with N elements + lfsr_btree_t btree = LFSR_BTREE_NULL; + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + lfs_size_t n = 0; + for (lfs_size_t i = 0; i < N; i++) { + int err = lfsr_btree_push(&lfs, &btree, i, LFSR_TAG_INLINED, 1, + LFSR_DATA_BUF(&alphas[i % 26], 1)); + // ignore space issues + if (err == LFS_ERR_NOSPC) { + break; + } + assert(err == 0); + n += 1; + } + printf("btree: w%d 0x%x.%x\n", + btree.weight, + btree.root.block, + btree.root.trunk); + assert(lfsr_btree_weight(&btree) == n); + + // check that the elements are in the tree + uint8_t buffer[4]; + lfsr_tag_t tag_; + lfs_size_t weight_; + + for (lfs_size_t i = 0; i < n; i++) { + lfsr_btree_get(&lfs, &btree, i, + &tag_, &weight_, + buffer, 4, false) => 1; + assert(tag_ == LFSR_TAG_INLINED); + assert(weight_ == 1); + assert(memcmp(buffer, &alphas[i % 26], 1) == 0); + } + + // and check that we can't lookup elements that aren't in the tree + lfsr_btree_get(&lfs, &btree, n, + &tag_, &weight_, + buffer, 4, false) => LFS_ERR_NOENT; + + // test that we can traverse the tree, keeping track of all blocks we see + uint8_t *seen = malloc((BLOCK_COUNT+7)/8); + memset(seen, 0, (BLOCK_COUNT+7)/8); + + lfsr_btree_traversal_t traversal; + lfsr_btree_traversal_start(&lfs, &btree, &traversal) => 0; + + for (lfs_block_t i = 0;; i++) { + // a bit hacky, but catch infinite loops + assert(i < 2*N); + + lfs_size_t bid_; + lfsr_tag_t tag_; + lfs_size_t weight_; + lfsr_data_t data_; + int err = lfsr_btree_traversal_next(&lfs, &btree, &traversal, + &bid_, &tag_, &weight_, &data_); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + break; + } + + if (tag_ == LFSR_TAG_BTREE) { + const lfsr_rbyd_t *branch = (const lfsr_rbyd_t *)data_.buf.buffer; + printf("traversal: %d 0x%x w%d btree 0x%x.%x\n", + bid_, + tag_, + weight_, + branch->block, branch->trunk); + + // keep track of seen blocks + seen[branch->block / 8] |= 1 << (branch->block % 8); + } else { + printf("traversal: %d 0x%x w%d %d\n", + bid_, + tag_, + weight_, + lfsr_data_size(data_)); + } + } + + // if traversal worked, we should be able to clobber all other blocks + uint8_t buffer_[BLOCK_SIZE]; + memset(buffer_, 0xcc, BLOCK_SIZE); + for (lfs_block_t block = 0; block < BLOCK_COUNT; block++) { + if (!(seen[block / 8] & (1 << (block % 8)))) { + cfg->erase(cfg, block) => 0; + cfg->prog(cfg, block, 0, buffer_, BLOCK_SIZE) => 0; + } + } + + // and the tree should still work + + // check that the elements are in the tree + for (lfs_size_t i = 0; i < n; i++) { + lfsr_btree_get(&lfs, &btree, i, + &tag_, &weight_, + buffer, 4, false) => 1; + assert(tag_ == LFSR_TAG_INLINED); + assert(weight_ == 1); + assert(memcmp(buffer, &alphas[i % 26], 1) == 0); + } + + // and check that we can't lookup elements that aren't in the tree + lfsr_btree_get(&lfs, &btree, n, + &tag_, &weight_, + buffer, 4, false) => LFS_ERR_NOENT; + + // clean up traversal stuff + free(seen); +''' + +[cases.test_btree_traversal_fuzz] +defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] +defines.SAMPLES = 10 +# -1 => all pseudo-random seeds +# n => reproduce a specific seed +defines.SEED = -1 +in = 'lfs.c' +code = ''' + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + + // iterate through severals seeds that we can reproduce easily + for (uint32_t seed = (SEED == -1 ? 1 : SEED); + (SEED == -1 ? seed < SAMPLES+1 : seed == SEED); + seed++) { + printf("--- seed: %d ---\n", seed); + // create lfs here since we need to reset each iteration, we're + // space constrained and we can't expect gc to work at this point + lfs_t lfs; + lfs_init(&lfs, cfg) => 0; + // create free lookahead + memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size); + lfs.free.off = 0; + lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size, + lfs.cfg->block_count); + lfs.free.i = 0; + lfs_alloc_ack(&lfs); + + // create a btree + lfsr_btree_t btree = LFSR_BTREE_NULL; + + // set up a simulation to compare against + // + // fun fact this is slower than our actual tree! unfun fact this is + // starting to be a problem... + char *sim = malloc(N); + lfs_size_t sim_size = 0; + memset(sim, 0, N); + + uint32_t prng = seed; + for (lfs_size_t i = 0; i < N; i++) { + // choose a pseudo-random id + lfs_size_t id = TEST_PRNG(&prng) % (sim_size+1); + + // add to btree + int err = lfsr_btree_push(&lfs, &btree, id, LFSR_TAG_INLINED, 1, + LFSR_DATA_BUF(&alphas[i % 26], 1)); + // ignore space issues + if (err == LFS_ERR_NOSPC) { + break; + } + assert(err == 0); + + // add to sim + memmove(&sim[id+1], &sim[id], sim_size-id); + sim[id] = alphas[i % 26]; + sim_size += 1; + } + + // check that btree matches sim + printf("expd: ["); + bool first = true; + for (lfs_size_t i = 0; i < sim_size; i++) { + if (!first) { + printf(", "); + } + first = false; + printf("%c", sim[i]); + } + printf("]\n"); + printf("btree: w%d 0x%x.%x\n", + btree.weight, + btree.root.block, + btree.root.trunk); + assert(lfsr_btree_weight(&btree) == sim_size); + + uint8_t buffer[4]; + lfsr_tag_t tag_; + lfs_size_t weight_; + for (lfs_size_t i = 0; i < sim_size; i++) { + lfsr_btree_get(&lfs, &btree, i, + &tag_, &weight_, + buffer, 4, false) => 1; + assert(tag_ == LFSR_TAG_INLINED); + assert(weight_ == 1); + assert(memcmp(buffer, &sim[i], 1) == 0); + } + + // and no extra elements + lfsr_btree_get(&lfs, &btree, sim_size, + &tag_, &weight_, + buffer, 4, false) => LFS_ERR_NOENT; + + // test that we can traverse the tree, keeping track of all blocks + // we see + uint8_t *seen = malloc((BLOCK_COUNT+7)/8); + memset(seen, 0, (BLOCK_COUNT+7)/8); + + lfsr_btree_traversal_t traversal; + lfsr_btree_traversal_start(&lfs, &btree, &traversal) => 0; + + for (lfs_block_t i = 0;; i++) { + // a bit hacky, but catch infinite loops + assert(i < 2*N); + + lfs_size_t bid_; + lfsr_tag_t tag_; + lfs_size_t weight_; + lfsr_data_t data_; + int err = lfsr_btree_traversal_next(&lfs, &btree, &traversal, + &bid_, &tag_, &weight_, &data_); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + break; + } + + if (tag_ == LFSR_TAG_BTREE) { + const lfsr_rbyd_t *branch = ( + (const lfsr_rbyd_t *)data_.buf.buffer); + printf("traversal: %d 0x%x w%d btree 0x%x.%x\n", + bid_, + tag_, + weight_, + branch->block, branch->trunk); + + // keep track of seen blocks + seen[branch->block / 8] |= 1 << (branch->block % 8); + } else { + printf("traversal: %d 0x%x w%d %d\n", + bid_, + tag_, + weight_, + lfsr_data_size(data_)); + } + } + + // if traversal worked, we should be able to clobber all other blocks + uint8_t buffer_[BLOCK_SIZE]; + memset(buffer_, 0xcc, BLOCK_SIZE); + for (lfs_block_t block = 0; block < BLOCK_COUNT; block++) { + if (!(seen[block / 8] & (1 << (block % 8)))) { + cfg->erase(cfg, block) => 0; + cfg->prog(cfg, block, 0, buffer_, BLOCK_SIZE) => 0; + } + } + + // and the tree should still work + + // check that btree matches sim + printf("expd: ["); + first = true; + for (lfs_size_t i = 0; i < sim_size; i++) { + if (!first) { + printf(", "); + } + first = false; + printf("%c", sim[i]); + } + printf("]\n"); + printf("btree: w%d 0x%x.%x\n", + btree.weight, + btree.root.block, + btree.root.trunk); + assert(lfsr_btree_weight(&btree) == sim_size); + + for (lfs_size_t i = 0; i < sim_size; i++) { + lfsr_btree_get(&lfs, &btree, i, + &tag_, &weight_, + buffer, 4, false) => 1; + assert(tag_ == LFSR_TAG_INLINED); + assert(weight_ == 1); + assert(memcmp(buffer, &sim[i], 1) == 0); + } + + // and no extra elements + lfsr_btree_get(&lfs, &btree, sim_size, + &tag_, &weight_, + buffer, 4, false) => LFS_ERR_NOENT; + + // clean up traversal stuff + free(seen); + + // clean up sim + free(sim); + lfs_deinit(&lfs) => 0; + } +''' +