Added lfsr_btree_traversal_t, incremental traversal of btree nodes
The main thing to note is that traversal here != iteration. Thanks to the right-leaning nature of our btrees, iteration is already provided by lfsr_btree_lookupnext, using the bid as the current iteration state. What btree traversal provides is traversal over every rbyd + entries used in the btree, include the inner btree nodes. This is useful for things like garbage collection and error detection that need to operate on the raw rbyds. Note that both btree traversal and iteration are still O(n log_b(n)). We can't do any better than that without recursion. One non-intuitive implementation detail, we return a tag describing each entry, but instead of returning an on-disk data reference for inner btree nodes, we return a pointer to a temporarily decoded rbyd struct. This simplifies root handling, and we probably want the decoded version anyways: - tag=LFSR_TAG_BTREE => lfsr_rbyd_t - tag=anything else => lfsr_data_t The reason for making btree traversal incremental, and not just use a callback like we've done previously, is to eventually use this as a part of high-level incremental garbage-collection/error-correction. For this to work, all of the lower-levels also need to be incremental.
This commit is contained in:
Christopher Haster
@@ -3137,11 +3137,11 @@ static int lfsr_btree_lookupnext_(lfs_t *lfs,
|
||||
}
|
||||
}
|
||||
|
||||
// adjust rid with subtree's weight
|
||||
rid -= (rid__ - (weight__-1));
|
||||
|
||||
// found another branch
|
||||
if (tag__ == LFSR_TAG_BTREE) {
|
||||
// adjust rid with subtree's weight
|
||||
rid -= (rid__ - (weight__-1));
|
||||
|
||||
// fetch the next branch
|
||||
lfs_ssize_t d = lfsr_branch_fromdisk(lfs, &branch, data__);
|
||||
if (d < 0) {
|
||||
@@ -3153,7 +3153,7 @@ static int lfsr_btree_lookupnext_(lfs_t *lfs,
|
||||
} else {
|
||||
// TODO how many of these should be conditional?
|
||||
if (bid_) {
|
||||
*bid_ = bid + (rid__ - rid);
|
||||
*bid_ = bid - (rid - (weight__-1));
|
||||
}
|
||||
if (rbyd_) {
|
||||
*rbyd_ = branch;
|
||||
@@ -3396,11 +3396,11 @@ static lfs_ssize_t lfsr_btree_namelookupnext(lfs_t *lfs,
|
||||
}
|
||||
}
|
||||
|
||||
// update our bid
|
||||
bid += rid__ - (weight__-1);
|
||||
|
||||
// found another branch
|
||||
if (tag__ == LFSR_TAG_BTREE) {
|
||||
// update our id
|
||||
bid += rid__-(weight__-1);
|
||||
|
||||
// fetch the next branch
|
||||
lfs_ssize_t d = lfsr_branch_fromdisk(lfs, &branch, data__);
|
||||
if (d < 0) {
|
||||
@@ -3412,7 +3412,7 @@ static lfs_ssize_t lfsr_btree_namelookupnext(lfs_t *lfs,
|
||||
} else {
|
||||
// TODO how many of these should be conditional?
|
||||
if (bid_) {
|
||||
*bid_ = bid + rid__;
|
||||
*bid_ = bid + (weight__-1);
|
||||
}
|
||||
if (rbyd_) {
|
||||
*rbyd_ = branch;
|
||||
@@ -4353,6 +4353,174 @@ static int lfsr_btree_split(lfs_t *lfs, lfsr_btree_t *btree,
|
||||
}
|
||||
|
||||
|
||||
// incremental btree traversal
|
||||
//
|
||||
// note this is different from iteration, iteration should use
|
||||
// lfsr_btree_lookupnext, traversal includes inner entries
|
||||
typedef struct lfsr_btree_traversal {
|
||||
lfs_size_t bid;
|
||||
lfs_ssize_t rid;
|
||||
lfsr_rbyd_t branch;
|
||||
} lfsr_btree_traversal_t;
|
||||
|
||||
static int lfsr_btree_traversal_start(lfs_t *lfs,
|
||||
const lfsr_btree_t *btree,
|
||||
lfsr_btree_traversal_t *traversal) {
|
||||
(void)lfs;
|
||||
(void)btree;
|
||||
// setup traversal to fetch the root next call
|
||||
traversal->bid = 0;
|
||||
traversal->rid = 0;
|
||||
traversal->branch.trunk = 0;
|
||||
traversal->branch.weight = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int lfsr_btree_traversal_next(lfs_t *lfs,
|
||||
const lfsr_btree_t *btree,
|
||||
lfsr_btree_traversal_t *traversal,
|
||||
lfs_size_t *bid_, lfsr_tag_t *tag_, lfs_size_t *weight_,
|
||||
lfsr_data_t *data_) {
|
||||
while (true) {
|
||||
// in range?
|
||||
if (traversal->bid >= lfsr_btree_weight(btree)) {
|
||||
return LFS_ERR_NOENT;
|
||||
}
|
||||
|
||||
// inlined?
|
||||
if (lfsr_btree_isinlined(btree)) {
|
||||
// setup traversal to terminate next call
|
||||
traversal->bid = lfsr_btree_weight(btree);
|
||||
|
||||
// TODO how many of these should be conditional?
|
||||
if (bid_) {
|
||||
*bid_ = lfsr_btree_weight(btree)-1;
|
||||
}
|
||||
if (tag_) {
|
||||
*tag_ = btree->inlined.tag;
|
||||
}
|
||||
if (weight_) {
|
||||
*weight_ = lfsr_btree_weight(btree);
|
||||
}
|
||||
if (data_) {
|
||||
*data_ = LFSR_DATA_BUF(btree->inlined.buffer,
|
||||
btree->inlined.size);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// make sure we traverse the root
|
||||
if (traversal->branch.trunk == 0) {
|
||||
traversal->bid += traversal->branch.weight;
|
||||
traversal->rid = traversal->bid;
|
||||
traversal->branch = btree->root;
|
||||
|
||||
if (traversal->rid == 0) {
|
||||
// TODO how many of these should be conditional?
|
||||
if (bid_) {
|
||||
*bid_ = lfsr_btree_weight(btree)-1;
|
||||
}
|
||||
if (tag_) {
|
||||
*tag_ = LFSR_TAG_BTREE;
|
||||
}
|
||||
if (weight_) {
|
||||
*weight_ = lfsr_btree_weight(btree);
|
||||
}
|
||||
if (data_) {
|
||||
// note btrees are returned decoded
|
||||
*data_ = LFSR_DATA_BUF(&traversal->branch,
|
||||
sizeof(lfsr_rbyd_t));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// continue, mostly for range check
|
||||
continue;
|
||||
}
|
||||
|
||||
// descend down the tree
|
||||
lfs_ssize_t rid__;
|
||||
lfsr_tag_t tag__;
|
||||
lfs_size_t weight__;
|
||||
lfsr_data_t data__;
|
||||
int err = lfsr_rbyd_lookupnext(lfs, &traversal->branch,
|
||||
traversal->rid, 0,
|
||||
&rid__, &tag__, &weight__, &data__);
|
||||
if (err) {
|
||||
LFS_ASSERT(err != LFS_ERR_NOENT);
|
||||
return err;
|
||||
}
|
||||
|
||||
if (lfsr_tag_suptype(tag__) == LFSR_TAG_NAME) {
|
||||
err = lfsr_rbyd_lookupnext(lfs, &traversal->branch,
|
||||
rid__, LFSR_TAG_STRUCT,
|
||||
NULL, &tag__, NULL, &data__);
|
||||
if (err) {
|
||||
LFS_ASSERT(err != LFS_ERR_NOENT);
|
||||
return err;
|
||||
}
|
||||
}
|
||||
|
||||
// adjust rid with subtree's weight
|
||||
traversal->rid -= (rid__ - (weight__-1));
|
||||
|
||||
// found another branch
|
||||
if (tag__ == LFSR_TAG_BTREE) {
|
||||
// fetch the next branch
|
||||
lfs_ssize_t d = lfsr_branch_fromdisk(lfs,
|
||||
&traversal->branch, data__);
|
||||
if (d < 0) {
|
||||
return d;
|
||||
}
|
||||
LFS_ASSERT(traversal->branch.weight == weight__);
|
||||
|
||||
// return inner btree nodes if this is the first time we've
|
||||
// seen them
|
||||
if (traversal->rid == 0) {
|
||||
// TODO how many of these should be conditional?
|
||||
if (bid_) {
|
||||
*bid_ = traversal->bid - (
|
||||
traversal->rid - (traversal->branch.weight-1));
|
||||
}
|
||||
if (tag_) {
|
||||
*tag_ = LFSR_TAG_BTREE;
|
||||
}
|
||||
if (weight_) {
|
||||
*weight_ = traversal->branch.weight;
|
||||
}
|
||||
if (data_) {
|
||||
// note btrees are returned decoded
|
||||
*data_ = LFSR_DATA_BUF(
|
||||
&traversal->branch, sizeof(lfsr_rbyd_t));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// found our bid
|
||||
} else {
|
||||
// update traversal
|
||||
traversal->branch.trunk = 0;
|
||||
traversal->branch.weight = weight__;
|
||||
|
||||
// TODO how many of these should be conditional?
|
||||
if (bid_) {
|
||||
*bid_ = traversal->bid - (traversal->rid - (weight__-1));
|
||||
}
|
||||
if (tag_) {
|
||||
*tag_ = tag__;
|
||||
}
|
||||
if (weight_) {
|
||||
*weight_ = weight__;
|
||||
}
|
||||
if (data_) {
|
||||
*data_ = data__;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/// Metadata pair operations ///
|
||||
|
||||
|
||||
@@ -4056,3 +4056,319 @@ code = '''
|
||||
}
|
||||
'''
|
||||
|
||||
|
||||
## B-tree traversal tests ##
|
||||
|
||||
# some simple btree traversals
|
||||
[cases.test_btree_traversal]
|
||||
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
|
||||
in = 'lfs.c'
|
||||
code = '''
|
||||
lfs_t lfs;
|
||||
lfs_init(&lfs, cfg) => 0;
|
||||
// create free lookahead
|
||||
memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size);
|
||||
lfs.free.off = 0;
|
||||
lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size,
|
||||
lfs.cfg->block_count);
|
||||
lfs.free.i = 0;
|
||||
lfs_alloc_ack(&lfs);
|
||||
|
||||
// create a tree with N elements
|
||||
lfsr_btree_t btree = LFSR_BTREE_NULL;
|
||||
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
|
||||
lfs_size_t n = 0;
|
||||
for (lfs_size_t i = 0; i < N; i++) {
|
||||
int err = lfsr_btree_push(&lfs, &btree, i, LFSR_TAG_INLINED, 1,
|
||||
LFSR_DATA_BUF(&alphas[i % 26], 1));
|
||||
// ignore space issues
|
||||
if (err == LFS_ERR_NOSPC) {
|
||||
break;
|
||||
}
|
||||
assert(err == 0);
|
||||
n += 1;
|
||||
}
|
||||
printf("btree: w%d 0x%x.%x\n",
|
||||
btree.weight,
|
||||
btree.root.block,
|
||||
btree.root.trunk);
|
||||
assert(lfsr_btree_weight(&btree) == n);
|
||||
|
||||
// check that the elements are in the tree
|
||||
uint8_t buffer[4];
|
||||
lfsr_tag_t tag_;
|
||||
lfs_size_t weight_;
|
||||
|
||||
for (lfs_size_t i = 0; i < n; i++) {
|
||||
lfsr_btree_get(&lfs, &btree, i,
|
||||
&tag_, &weight_,
|
||||
buffer, 4, false) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
|
||||
}
|
||||
|
||||
// and check that we can't lookup elements that aren't in the tree
|
||||
lfsr_btree_get(&lfs, &btree, n,
|
||||
&tag_, &weight_,
|
||||
buffer, 4, false) => LFS_ERR_NOENT;
|
||||
|
||||
// test that we can traverse the tree, keeping track of all blocks we see
|
||||
uint8_t *seen = malloc((BLOCK_COUNT+7)/8);
|
||||
memset(seen, 0, (BLOCK_COUNT+7)/8);
|
||||
|
||||
lfsr_btree_traversal_t traversal;
|
||||
lfsr_btree_traversal_start(&lfs, &btree, &traversal) => 0;
|
||||
|
||||
for (lfs_block_t i = 0;; i++) {
|
||||
// a bit hacky, but catch infinite loops
|
||||
assert(i < 2*N);
|
||||
|
||||
lfs_size_t bid_;
|
||||
lfsr_tag_t tag_;
|
||||
lfs_size_t weight_;
|
||||
lfsr_data_t data_;
|
||||
int err = lfsr_btree_traversal_next(&lfs, &btree, &traversal,
|
||||
&bid_, &tag_, &weight_, &data_);
|
||||
assert(!err || err == LFS_ERR_NOENT);
|
||||
if (err == LFS_ERR_NOENT) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (tag_ == LFSR_TAG_BTREE) {
|
||||
const lfsr_rbyd_t *branch = (const lfsr_rbyd_t *)data_.buf.buffer;
|
||||
printf("traversal: %d 0x%x w%d btree 0x%x.%x\n",
|
||||
bid_,
|
||||
tag_,
|
||||
weight_,
|
||||
branch->block, branch->trunk);
|
||||
|
||||
// keep track of seen blocks
|
||||
seen[branch->block / 8] |= 1 << (branch->block % 8);
|
||||
} else {
|
||||
printf("traversal: %d 0x%x w%d %d\n",
|
||||
bid_,
|
||||
tag_,
|
||||
weight_,
|
||||
lfsr_data_size(data_));
|
||||
}
|
||||
}
|
||||
|
||||
// if traversal worked, we should be able to clobber all other blocks
|
||||
uint8_t buffer_[BLOCK_SIZE];
|
||||
memset(buffer_, 0xcc, BLOCK_SIZE);
|
||||
for (lfs_block_t block = 0; block < BLOCK_COUNT; block++) {
|
||||
if (!(seen[block / 8] & (1 << (block % 8)))) {
|
||||
cfg->erase(cfg, block) => 0;
|
||||
cfg->prog(cfg, block, 0, buffer_, BLOCK_SIZE) => 0;
|
||||
}
|
||||
}
|
||||
|
||||
// and the tree should still work
|
||||
|
||||
// check that the elements are in the tree
|
||||
for (lfs_size_t i = 0; i < n; i++) {
|
||||
lfsr_btree_get(&lfs, &btree, i,
|
||||
&tag_, &weight_,
|
||||
buffer, 4, false) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
|
||||
}
|
||||
|
||||
// and check that we can't lookup elements that aren't in the tree
|
||||
lfsr_btree_get(&lfs, &btree, n,
|
||||
&tag_, &weight_,
|
||||
buffer, 4, false) => LFS_ERR_NOENT;
|
||||
|
||||
// clean up traversal stuff
|
||||
free(seen);
|
||||
'''
|
||||
|
||||
[cases.test_btree_traversal_fuzz]
|
||||
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
|
||||
defines.SAMPLES = 10
|
||||
# -1 => all pseudo-random seeds
|
||||
# n => reproduce a specific seed
|
||||
defines.SEED = -1
|
||||
in = 'lfs.c'
|
||||
code = '''
|
||||
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
|
||||
|
||||
// iterate through severals seeds that we can reproduce easily
|
||||
for (uint32_t seed = (SEED == -1 ? 1 : SEED);
|
||||
(SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
|
||||
seed++) {
|
||||
printf("--- seed: %d ---\n", seed);
|
||||
// create lfs here since we need to reset each iteration, we're
|
||||
// space constrained and we can't expect gc to work at this point
|
||||
lfs_t lfs;
|
||||
lfs_init(&lfs, cfg) => 0;
|
||||
// create free lookahead
|
||||
memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size);
|
||||
lfs.free.off = 0;
|
||||
lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size,
|
||||
lfs.cfg->block_count);
|
||||
lfs.free.i = 0;
|
||||
lfs_alloc_ack(&lfs);
|
||||
|
||||
// create a btree
|
||||
lfsr_btree_t btree = LFSR_BTREE_NULL;
|
||||
|
||||
// set up a simulation to compare against
|
||||
//
|
||||
// fun fact this is slower than our actual tree! unfun fact this is
|
||||
// starting to be a problem...
|
||||
char *sim = malloc(N);
|
||||
lfs_size_t sim_size = 0;
|
||||
memset(sim, 0, N);
|
||||
|
||||
uint32_t prng = seed;
|
||||
for (lfs_size_t i = 0; i < N; i++) {
|
||||
// choose a pseudo-random id
|
||||
lfs_size_t id = TEST_PRNG(&prng) % (sim_size+1);
|
||||
|
||||
// add to btree
|
||||
int err = lfsr_btree_push(&lfs, &btree, id, LFSR_TAG_INLINED, 1,
|
||||
LFSR_DATA_BUF(&alphas[i % 26], 1));
|
||||
// ignore space issues
|
||||
if (err == LFS_ERR_NOSPC) {
|
||||
break;
|
||||
}
|
||||
assert(err == 0);
|
||||
|
||||
// add to sim
|
||||
memmove(&sim[id+1], &sim[id], sim_size-id);
|
||||
sim[id] = alphas[i % 26];
|
||||
sim_size += 1;
|
||||
}
|
||||
|
||||
// check that btree matches sim
|
||||
printf("expd: [");
|
||||
bool first = true;
|
||||
for (lfs_size_t i = 0; i < sim_size; i++) {
|
||||
if (!first) {
|
||||
printf(", ");
|
||||
}
|
||||
first = false;
|
||||
printf("%c", sim[i]);
|
||||
}
|
||||
printf("]\n");
|
||||
printf("btree: w%d 0x%x.%x\n",
|
||||
btree.weight,
|
||||
btree.root.block,
|
||||
btree.root.trunk);
|
||||
assert(lfsr_btree_weight(&btree) == sim_size);
|
||||
|
||||
uint8_t buffer[4];
|
||||
lfsr_tag_t tag_;
|
||||
lfs_size_t weight_;
|
||||
for (lfs_size_t i = 0; i < sim_size; i++) {
|
||||
lfsr_btree_get(&lfs, &btree, i,
|
||||
&tag_, &weight_,
|
||||
buffer, 4, false) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, &sim[i], 1) == 0);
|
||||
}
|
||||
|
||||
// and no extra elements
|
||||
lfsr_btree_get(&lfs, &btree, sim_size,
|
||||
&tag_, &weight_,
|
||||
buffer, 4, false) => LFS_ERR_NOENT;
|
||||
|
||||
// test that we can traverse the tree, keeping track of all blocks
|
||||
// we see
|
||||
uint8_t *seen = malloc((BLOCK_COUNT+7)/8);
|
||||
memset(seen, 0, (BLOCK_COUNT+7)/8);
|
||||
|
||||
lfsr_btree_traversal_t traversal;
|
||||
lfsr_btree_traversal_start(&lfs, &btree, &traversal) => 0;
|
||||
|
||||
for (lfs_block_t i = 0;; i++) {
|
||||
// a bit hacky, but catch infinite loops
|
||||
assert(i < 2*N);
|
||||
|
||||
lfs_size_t bid_;
|
||||
lfsr_tag_t tag_;
|
||||
lfs_size_t weight_;
|
||||
lfsr_data_t data_;
|
||||
int err = lfsr_btree_traversal_next(&lfs, &btree, &traversal,
|
||||
&bid_, &tag_, &weight_, &data_);
|
||||
assert(!err || err == LFS_ERR_NOENT);
|
||||
if (err == LFS_ERR_NOENT) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (tag_ == LFSR_TAG_BTREE) {
|
||||
const lfsr_rbyd_t *branch = (
|
||||
(const lfsr_rbyd_t *)data_.buf.buffer);
|
||||
printf("traversal: %d 0x%x w%d btree 0x%x.%x\n",
|
||||
bid_,
|
||||
tag_,
|
||||
weight_,
|
||||
branch->block, branch->trunk);
|
||||
|
||||
// keep track of seen blocks
|
||||
seen[branch->block / 8] |= 1 << (branch->block % 8);
|
||||
} else {
|
||||
printf("traversal: %d 0x%x w%d %d\n",
|
||||
bid_,
|
||||
tag_,
|
||||
weight_,
|
||||
lfsr_data_size(data_));
|
||||
}
|
||||
}
|
||||
|
||||
// if traversal worked, we should be able to clobber all other blocks
|
||||
uint8_t buffer_[BLOCK_SIZE];
|
||||
memset(buffer_, 0xcc, BLOCK_SIZE);
|
||||
for (lfs_block_t block = 0; block < BLOCK_COUNT; block++) {
|
||||
if (!(seen[block / 8] & (1 << (block % 8)))) {
|
||||
cfg->erase(cfg, block) => 0;
|
||||
cfg->prog(cfg, block, 0, buffer_, BLOCK_SIZE) => 0;
|
||||
}
|
||||
}
|
||||
|
||||
// and the tree should still work
|
||||
|
||||
// check that btree matches sim
|
||||
printf("expd: [");
|
||||
first = true;
|
||||
for (lfs_size_t i = 0; i < sim_size; i++) {
|
||||
if (!first) {
|
||||
printf(", ");
|
||||
}
|
||||
first = false;
|
||||
printf("%c", sim[i]);
|
||||
}
|
||||
printf("]\n");
|
||||
printf("btree: w%d 0x%x.%x\n",
|
||||
btree.weight,
|
||||
btree.root.block,
|
||||
btree.root.trunk);
|
||||
assert(lfsr_btree_weight(&btree) == sim_size);
|
||||
|
||||
for (lfs_size_t i = 0; i < sim_size; i++) {
|
||||
lfsr_btree_get(&lfs, &btree, i,
|
||||
&tag_, &weight_,
|
||||
buffer, 4, false) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, &sim[i], 1) == 0);
|
||||
}
|
||||
|
||||
// and no extra elements
|
||||
lfsr_btree_get(&lfs, &btree, sim_size,
|
||||
&tag_, &weight_,
|
||||
buffer, 4, false) => LFS_ERR_NOENT;
|
||||
|
||||
// clean up traversal stuff
|
||||
free(seen);
|
||||
|
||||
// clean up sim
|
||||
free(sim);
|
||||
lfs_deinit(&lfs) => 0;
|
||||
}
|
||||
'''
|
||||
|
||||
|
||||
Reference in New Issue
Block a user