From 135bb174091a5760f51b85caa219105daf4790f2 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 8 Nov 2023 22:27:46 -0600 Subject: [PATCH] Tweaked named btrees to support strict key->value mapping We don't strictly need this for the mtree, but its impact is pretty minimal, and it's useful for some future plans. It also makes low-level benchmarks a bit easier to write. The main change involves subtleties around vestigial names in leaf rbyds (the bottom most layer of btree inner nodes). Since the mtree terminates in mdirs, the left-most mdir in each leaf rbyd in the mtree never actually needs a name. But in a hypothetical strict key->value tree, every entry in the leaf rbyds need a name, and this name needs to be respected during btree operations (mainly merges). As a side-effect, our named btrees now require vestigial names for every inner btree node, with the exception of the left-most inner nodes since those can't be merged left with anything. On the bright side, being able to assume a vestigial name on every mergable node does simplify merge operations a bit. It's worth noting that despite these changes, we still update vestigial names on inner btree nodes lazily. It isn't super clear that this should work, but it turns out that even though a leaf nodes may diverge from the vestigial name in it's parent, it must still following the bounds of the parent's vestigial name because of how btree lookups work. And this property propagates up though each layer in the btree: .---------------. |a: |h: |-> | '--|---|--------' .---' '----------. v v .---------------. .---------------. |a: |c: | | |i: |m: |-> | '--|---|--------' '--|---|--------' ...--' | | '--------... v v .---------------. .---------------. |d:0|e:1|f:2|-> | |j:3|k:4|l:5|-> | '---------------' '---------------' The exception are the left-most inner nodes, but these can never merge left, so it doesn't really matter. The vestigial names on the left-most inner nodes are truly vestigial: .---------------. |c: |e: |-> | '--|---|--------' .---' '--------... v .---------------. |b: |d: | | '--|---|--------' .---' '-------... v .---------------. |a:0|b:1|c:2|-> | '---------------' An alternative implementation may prefer to update these names eagerly, but this would increase the amount of data written to each inner node during btree commits. mdir updates are lazy by necessity, so even if you adopted eager updates, the names of deleted files would still stick around. --- lfs.c | 146 +++++++++++++++---------------- tests/test_btree.toml | 197 ++++++++++++++++++++++-------------------- tests/test_rbyd.toml | 1 + 3 files changed, 175 insertions(+), 169 deletions(-) diff --git a/lfs.c b/lfs.c index 8d4c250b..0c3a159b 100644 --- a/lfs.c +++ b/lfs.c @@ -3565,29 +3565,21 @@ static lfs_ssize_t lfsr_rbyd_estimate(lfs_t *lfs, const lfsr_rbyd_t *rbyd, // // names in littlefs are tuples of directory-ids + ascii/utf8 strings -// binary search an rbyd for a name, leaving the rid_/weight_ with the best -// matching name if not found -static int lfsr_rbyd_namelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd, +// binary search an rbyd for a name, leaving the rid_/tag_/weight_/data_ +// with the best matching name if not found +static lfs_scmp_t lfsr_rbyd_namelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd, lfsr_did_t did, const char *name, lfs_size_t name_size, lfsr_srid_t *rid_, lfsr_tag_t *tag_, lfsr_rid_t *weight_, lfsr_data_t *data_) { - // if we have an empty mdir, default to rid = -1 - if (rid_) { - *rid_ = -1; - } - if (tag_) { - *tag_ = 0; - } - if (weight_) { - *weight_ = 0; - } - if (data_) { - *data_ = LFSR_DATA_NULL; + // empty rbyd? leave it up to upper layers to handle this + if (rbyd->weight == 0) { + return LFS_ERR_NOENT; } // binary search for our name lfsr_srid_t lower = 0; lfsr_srid_t upper = rbyd->weight; + lfs_scmp_t cmp; while (lower < upper) { lfsr_tag_t tag__; lfsr_srid_t rid__; @@ -3603,10 +3595,8 @@ static int lfsr_rbyd_namelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd, return err; } - // if we have no name or a vestigial name, treat this rid as always lt - lfs_scmp_t cmp; - if ((tag__ == LFSR_TAG_NAME && rid__-(weight__-1) == 0) - || lfsr_tag_suptype(tag__) != LFSR_TAG_NAME) { + // if we have no name, treat this rid as always lt + if (lfsr_tag_suptype(tag__) != LFSR_TAG_NAME) { cmp = LFS_CMP_LT; // compare names @@ -3621,10 +3611,27 @@ static int lfsr_rbyd_namelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd, if (lfs_cmp(cmp) > 0) { upper = rid__ - (weight__-1); + // only keep track of best-match rids > our target if we haven't + // seen an rid < our target + if (lower == 0) { + if (rid_) { + *rid_ = rid__; + } + if (tag_) { + *tag_ = tag__; + } + if (weight_) { + *weight_ = weight__; + } + if (data_) { + *data_ = data__; + } + } + } else if (lfs_cmp(cmp) < 0) { lower = rid__ + 1; - // keep track of best-matching rid >= our target + // keep track of best-matching rid < our target if (rid_) { *rid_ = rid__; } @@ -3652,13 +3659,14 @@ static int lfsr_rbyd_namelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd, if (data_) { *data_ = data__; } - return 0; + return LFS_CMP_EQ; } } - // no match, at least update rid_/tag_/weight_/data_ with the best - // match so far - return LFS_ERR_NOENT; + // no match, return if found name was lt/gt expect + // + // this will always be lt unless all rids are gt + return (lower == 0) ? LFS_CMP_GT : LFS_CMP_LT; } @@ -4362,33 +4370,6 @@ static int lfsr_btree_commit(lfs_t *lfs, lfsr_btree_t *btree, return err; } - // bring in name that previously split the siblings - err = lfsr_rbyd_lookupnext(lfs, &parent, - rid+1, LFSR_TAG_NAME, - NULL, &split_tag, NULL, &split_data); - if (err) { - return err; - } - - if (lfsr_tag_suptype(split_tag) == LFSR_TAG_NAME) { - // lookup the rid (weight really) of the previously-split entry - lfsr_srid_t split_rid; - err = lfsr_rbyd_lookupnext(lfs, &rbyd_, - rbyd.weight, LFSR_TAG_NAME, - &split_rid, NULL, NULL, NULL); - if (err) { - LFS_ASSERT(err != LFS_ERR_NOENT); - return err; - } - - err = lfsr_rbyd_appendattr(lfs, &rbyd_, - split_rid, split_tag, 0, split_data); - if (err) { - LFS_ASSERT(err != LFS_ERR_RANGE); - return err; - } - } - // append any pending attrs, it's up to upper // layers to make sure these always fit err = lfsr_rbyd_appendattrs(lfs, &rbyd_, bid, -1, @@ -4467,7 +4448,7 @@ static int lfsr_btree_commit(lfs_t *lfs, lfsr_btree_t *btree, } // lookup in a btree by name -static int lfsr_btree_namelookup(lfs_t *lfs, const lfsr_btree_t *btree, +static lfs_scmp_t lfsr_btree_namelookup(lfs_t *lfs, const lfsr_btree_t *btree, lfsr_did_t did, const char *name, lfs_size_t name_size, lfsr_bid_t *bid_, lfsr_tag_t *tag_, lfsr_bid_t *weight_, lfsr_data_t *data_) { @@ -4483,16 +4464,18 @@ static int lfsr_btree_namelookup(lfs_t *lfs, const lfsr_btree_t *btree, // lookup our name in the rbyd via binary search lfsr_srid_t rid__; lfsr_rid_t weight__; - int err = lfsr_rbyd_namelookup(lfs, &branch, did, name, name_size, + lfs_scmp_t cmp = lfsr_rbyd_namelookup(lfs, &branch, + did, name, name_size, &rid__, NULL, &weight__, NULL); - if (err && err != LFS_ERR_NOENT) { - return err; + if (cmp < 0) { + LFS_ASSERT(cmp != LFS_ERR_NOENT); + return cmp; } // the name may not match exactly, but indicates which branch to follow lfsr_tag_t tag__; lfsr_data_t data__; - err = lfsr_rbyd_lookup(lfs, &branch, rid__, LFSR_TAG_WIDE(STRUCT), + int err = lfsr_rbyd_lookup(lfs, &branch, rid__, LFSR_TAG_WIDE(STRUCT), &tag__, &data__); if (err) { LFS_ASSERT(err != LFS_ERR_NOENT); @@ -4525,7 +4508,7 @@ static int lfsr_btree_namelookup(lfs_t *lfs, const lfsr_btree_t *btree, if (data_) { *data_ = data__; } - return 0; + return cmp; } } } @@ -6366,23 +6349,39 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, } -// lookup names in our mtree +// lookup names in an mdir +// +// if not found, rid will be the best place to insert +// static int lfsr_mdir_namelookup(lfs_t *lfs, const lfsr_mdir_t *mdir, lfsr_did_t did, const char *name, lfs_size_t name_size, lfsr_srid_t *rid_, lfsr_tag_t *tag_, lfsr_data_t *data_) { - int err = lfsr_rbyd_namelookup(lfs, &mdir->u.rbyd, - did, name, name_size, - rid_, tag_, NULL, data_); - - // When not found, lfsr_rbyd_namelookup returns the rid smaller than our - // expected name. This is correct for btree lookups, but not correct for - // mdir insertions. For mdirs we need to adjust this by 1 so we insert - // _after_ the smaller rid. - if (rid_ && err == LFS_ERR_NOENT) { - *rid_ += 1; + // empty mdir? make sure rid_ = 0 at least + if (mdir->u.m.weight == 0) { + if (rid_) { + *rid_ = 0; + } + return LFS_ERR_NOENT; } - return err; + lfsr_srid_t rid; + lfs_scmp_t cmp = lfsr_rbyd_namelookup(lfs, &mdir->u.rbyd, + did, name, name_size, + &rid, tag_, NULL, data_); + if (cmp < 0) { + LFS_ASSERT(cmp != LFS_ERR_NOENT); + return cmp; + } + + // adjust rid if necessary + if (lfs_cmp(cmp) < 0) { + rid += 1; + } + + if (rid_) { + *rid_ = rid; + } + return (lfs_cmp(cmp) == 0) ? 0 : LFS_ERR_NOENT; } // note if we fail, we at least leave mdir_/rid_ with the best place to insert @@ -6409,17 +6408,18 @@ static int lfsr_mtree_namelookup(lfs_t *lfs, const lfsr_mtree_t *mtree, lfsr_tag_t tag; lfsr_bid_t weight; lfsr_data_t data; - int err = lfsr_btree_namelookup(lfs, &lfs->mtree.u.btree, + lfs_scmp_t cmp = lfsr_btree_namelookup(lfs, &lfs->mtree.u.btree, did, name, name_size, &bid, &tag, &weight, &data); - if (err) { - return err; + if (cmp < 0) { + LFS_ASSERT(cmp != LFS_ERR_NOENT); + return cmp; } LFS_ASSERT(tag == LFSR_TAG_MDIR); LFS_ASSERT(weight == lfsr_mleafweight(lfs)); // decode mdir - err = lfsr_data_readmptr(lfs, &data, mdir.u.m.blocks); + int err = lfsr_data_readmptr(lfs, &data, mdir.u.m.blocks); if (err) { return err; } diff --git a/tests/test_btree.toml b/tests/test_btree.toml index bd8e7b72..a472fa1a 100644 --- a/tests/test_btree.toml +++ b/tests/test_btree.toml @@ -5,6 +5,8 @@ after = 'test_rbyd' # of the disk for these tests defines.LOOKAHEAD_SIZE = 'lfs_alignup(BLOCK_COUNT / 8, 8)' +# TODO we should eventually replace these with +# lfsr_btree_commit/lfsr_btree_lookup # helper functions in = 'lfs.c' code = ''' @@ -3180,8 +3182,10 @@ code = ''' // create a single-entry tree lfsr_btree_t btree; lfsr_btree_alloc(&lfs, &btree) => 0; - lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, 1, - LFSR_DATA_BUF("0", 1)) => 0; + lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS( + LFSR_ATTR(0, + NAME, +1, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("aaa", 3))), + LFSR_ATTR(0, DATA, 0, BUF("0", 1)))) => 0; printf("btree: w%d 0x%x.%x\n", btree.weight, btree.block, @@ -3196,7 +3200,7 @@ code = ''' lfsr_data_t data_; lfsr_btree_namelookup(&lfs, &btree, 0*DID, "aaa", 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == 0); assert(weight_ == 1); @@ -3204,7 +3208,7 @@ code = ''' assert(memcmp(buffer, "0", 1) == 0); lfsr_btree_namelookup(&lfs, &btree, 1*DID, "aab", 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_LT; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == 0); assert(weight_ == 1); @@ -3230,8 +3234,10 @@ code = ''' // create a two-entry tree lfsr_btree_t btree; lfsr_btree_alloc(&lfs, &btree) => 0; - lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, 1, - LFSR_DATA_BUF("0", 1)) => 0; + lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS( + LFSR_ATTR(0, + NAME, +1, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("aaa", 3))), + LFSR_ATTR(0, DATA, 0, BUF("0", 1)))) => 0; lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("aab", 3)), LFSR_TAG_DATA, 1, LFSR_DATA_BUF("0", 1), @@ -3250,7 +3256,7 @@ code = ''' lfsr_data_t data_; lfsr_btree_namelookup(&lfs, &btree, 0, "aaa", 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == 0); assert(weight_ == 1); @@ -3258,7 +3264,7 @@ code = ''' assert(memcmp(buffer, "0", 1) == 0); lfsr_btree_namelookup(&lfs, &btree, 0, "aab", 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == 1); assert(weight_ == 1); @@ -3266,7 +3272,7 @@ code = ''' assert(memcmp(buffer, "1", 1) == 0); lfsr_btree_namelookup(&lfs, &btree, 0, "aac", 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_LT; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == 1); assert(weight_ == 1); @@ -3292,8 +3298,10 @@ code = ''' // create a two-entry tree lfsr_btree_t btree; lfsr_btree_alloc(&lfs, &btree) => 0; - lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, 1, - LFSR_DATA_BUF("0", 1)) => 0; + lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS( + LFSR_ATTR(0, + NAME, +1, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("aaa", 3))), + LFSR_ATTR(0, DATA, 0, BUF("0", 1)))) => 0; lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_CAT(LFSR_DATA_LEB128(1*DID), LFSR_DATA_BUF("aab", 3)), LFSR_TAG_DATA, 1, LFSR_DATA_BUF("0", 1), @@ -3316,7 +3324,7 @@ code = ''' lfsr_data_t data_; lfsr_btree_namelookup(&lfs, &btree, 0*DID, "aaa", 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == 0); assert(weight_ == 1); @@ -3324,7 +3332,7 @@ code = ''' assert(memcmp(buffer, "0", 1) == 0); lfsr_btree_namelookup(&lfs, &btree, 1*DID, "aab", 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == 1); assert(weight_ == 1); @@ -3332,7 +3340,7 @@ code = ''' assert(memcmp(buffer, "1", 1) == 0); lfsr_btree_namelookup(&lfs, &btree, 2*DID, "aac", 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == 2); assert(weight_ == 1); @@ -3340,7 +3348,7 @@ code = ''' assert(memcmp(buffer, "2", 1) == 0); lfsr_btree_namelookup(&lfs, &btree, 3*DID, "aad", 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_LT; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == 2); assert(weight_ == 1); @@ -3366,8 +3374,10 @@ code = ''' // create a two-entry tree lfsr_btree_t btree; lfsr_btree_alloc(&lfs, &btree) => 0; - lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, 1, - LFSR_DATA_BUF("0", 1)) => 0; + lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS( + LFSR_ATTR(0, + NAME, +1, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("aaa", 3))), + LFSR_ATTR(0, DATA, 0, BUF("0", 1)))) => 0; lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_CAT(LFSR_DATA_LEB128(2*DID), LFSR_DATA_BUF("aac", 3)), LFSR_TAG_DATA, 1, LFSR_DATA_BUF("1", 1), @@ -3390,7 +3400,7 @@ code = ''' lfsr_data_t data_; lfsr_btree_namelookup(&lfs, &btree, 0*DID, "aaa", 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == 0); assert(weight_ == 1); @@ -3398,7 +3408,7 @@ code = ''' assert(memcmp(buffer, "0", 1) == 0); lfsr_btree_namelookup(&lfs, &btree, 1*DID, "aab", 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == 1); assert(weight_ == 1); @@ -3406,7 +3416,7 @@ code = ''' assert(memcmp(buffer, "1", 1) == 0); lfsr_btree_namelookup(&lfs, &btree, 2*DID, "aac", 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == 2); assert(weight_ == 1); @@ -3414,7 +3424,7 @@ code = ''' assert(memcmp(buffer, "2", 1) == 0); lfsr_btree_namelookup(&lfs, &btree, 3*DID, "aad", 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_LT; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == 2); assert(weight_ == 1); @@ -3443,8 +3453,13 @@ code = ''' lfsr_btree_alloc(&lfs, &btree) => 0; const char *alphas = "abcdefghijklmnopqrstuvwxyz"; const char *nums = "0123456789"; - lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, 1, - LFSR_DATA_BUF(&nums[0 % 10], 1)) => 0; + char name[3] = { + alphas[(0/26/26) % 26], alphas[(0/26) % 26], alphas[0 % 26] + }; + lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS( + LFSR_ATTR(0, + NAME, +1, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF(name, 3))), + LFSR_ATTR(0, DATA, 0, BUF(&nums[0 % 10], 1)))) => 0; lfs_size_t n = 1; for (lfs_size_t i = 1; i < N; i++) { char name[3] = { @@ -3480,7 +3495,7 @@ code = ''' }; lfsr_btree_namelookup(&lfs, &btree, i*DID, name, 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == i); assert(weight_ == 1); @@ -3510,8 +3525,10 @@ code = ''' // create a btree lfsr_btree_t btree; lfsr_btree_alloc(&lfs, &btree) => 0; - lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, 1, - LFSR_DATA_BUF("_", 1)) => 0; + lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS( + LFSR_ATTR(0, + NAME, +1, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("___", 3))), + LFSR_ATTR(0, DATA, 0, BUF("_", 1)))) => 0; // set up a simulation to compare against // @@ -3587,7 +3604,7 @@ code = ''' lfsr_data_t data_; for (lfs_size_t i = 0; i < sim_size; i++) { lfsr_btree_namelookup(&lfs, &btree, 0, sim_names[i], 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == i); assert(weight_ == 1); @@ -3623,8 +3640,13 @@ code = ''' lfsr_btree_alloc(&lfs, &btree) => 0; const char *alphas = "abcdefghijklmnopqrstuvwxyz"; const char *nums = "0123456789"; - lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, W, - LFSR_DATA_BUF(&nums[0 % 10], 1)) => 0; + char name[3] = { + alphas[(0/26/26) % 26], alphas[(0/26) % 26], alphas[0 % 26] + }; + lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS( + LFSR_ATTR(0, + NAME, +W, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF(name, 3))), + LFSR_ATTR(W-1, DATA, 0, BUF(&nums[0 % 10], 1)))) => 0; lfs_size_t n = 1; for (lfs_size_t i = 1; i < N; i++) { char name[3] = { @@ -3660,7 +3682,7 @@ code = ''' }; lfsr_btree_namelookup(&lfs, &btree, i*DID, name, 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == i*W+W-1); assert(weight_ == W); @@ -3691,8 +3713,10 @@ code = ''' // create a btree lfsr_btree_t btree; lfsr_btree_alloc(&lfs, &btree) => 0; - lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, W, - LFSR_DATA_BUF("_", 1)) => 0; + lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS( + LFSR_ATTR(0, + NAME, +W, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("___", 3))), + LFSR_ATTR(W-1, DATA, 0, BUF("_", 1)))) => 0; // set up a simulation to compare against // @@ -3806,7 +3830,7 @@ code = ''' } lfsr_btree_namelookup(&lfs, &btree, 0, sim_names[i], 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == weighted_bid+sim_weights[i]-1); assert(weight_ == sim_weights[i]); @@ -3843,8 +3867,10 @@ code = ''' // create a btree lfsr_btree_t btree; lfsr_btree_alloc(&lfs, &btree) => 0; - lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, 1, - LFSR_DATA_BUF("_", 1)) => 0; + lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS( + LFSR_ATTR(0, + NAME, +1, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("___", 3))), + LFSR_ATTR(0, DATA, 0, BUF("_", 1)))) => 0; // set up a simulation to compare against // @@ -3874,30 +3900,28 @@ code = ''' if (op == 0 || sim_size <= 1) { // find where to split lfs_size_t bid = 0; - while (bid+1 < sim_size - && memcmp(sim_names[bid+1], name, 3) <= 0) { + while (bid < sim_size && memcmp(name, sim_names[bid], 3) > 0) { bid += 1; } // just skip exact matches for now - if (memcmp(sim_names[bid], name, 3) == 0) { + if (memcmp(name, sim_names[bid], 3) == 0) { continue; } // split btree lfs_size_t split_bid; lfsr_data_t split_data; - lfsr_btree_namelookup(&lfs, &btree, 0, name, 3, - &split_bid, NULL, NULL, &split_data) => 0; - uint8_t split_buf[4]; - lfsr_data_read(&lfs, &split_data, split_buf, 4) => 1; - if (split_bid > bid) { - int err = lfsr_btree_split(&lfs, &btree, - split_bid, - LFSR_DATA_CAT( - LFSR_DATA_LEB128(0), - LFSR_DATA_BUF(sim_names[bid+1], 3)), - LFSR_TAG_DATA, 1, LFSR_DATA_BUF(&nums[i % 10], 1), - LFSR_TAG_DATA, 1, LFSR_DATA_BUF(split_buf, 1)); + lfs_scmp_t cmp = lfsr_btree_namelookup(&lfs, &btree, 0, name, 3, + &split_bid, NULL, NULL, &split_data); + assert(cmp >= 0); + assert(lfs_cmp(cmp) != 0); + if (lfs_cmp(cmp) > 0) { + int err = lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS( + LFSR_ATTR(split_bid, + NAME, +1, CAT( + LFSR_DATA_LEB128(0), + LFSR_DATA_BUF(name, 3))), + LFSR_ATTR(split_bid, DATA, 0, BUF(&nums[i % 10], 1)))); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -3908,7 +3932,7 @@ code = ''' split_bid, LFSR_DATA_CAT( LFSR_DATA_LEB128(0), LFSR_DATA_BUF(name, 3)), - LFSR_TAG_DATA, 1, LFSR_DATA_BUF(split_buf, 1), + LFSR_TAG_DATA, 1, split_data, LFSR_TAG_DATA, 1, LFSR_DATA_BUF(&nums[i % 10], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { @@ -3920,8 +3944,8 @@ code = ''' // split sim memmove(&sim[bid+1], &sim[bid], sim_size-bid); memmove(&sim_names[bid+1], &sim_names[bid], (sim_size-bid)*3); - sim[bid+1] = nums[i % 10]; - memcpy(&sim_names[bid+1], name, 3); + sim[bid] = nums[i % 10]; + memcpy(&sim_names[bid], name, 3); sim_size += 1; } else if (op == 1) { @@ -3951,12 +3975,6 @@ code = ''' memmove(&sim[bid], &sim[bid+1], sim_size-(bid+1)); memmove(&sim_names[bid], &sim_names[bid+1], (sim_size-(bid+1))*3); sim_size -= 1; - - // our B-tree doesn't actually track the name of id0, so we need - // mirror this in our sim - if (bid == 0) { - memcpy(&sim_names[0], "___", 3); - } } } @@ -3984,7 +4002,7 @@ code = ''' lfsr_data_t data_; for (lfs_size_t i = 0; i < sim_size; i++) { lfsr_btree_namelookup(&lfs, &btree, 0, sim_names[i], 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == i); assert(weight_ == 1); @@ -4019,8 +4037,10 @@ code = ''' // create a btree lfsr_btree_t btree; lfsr_btree_alloc(&lfs, &btree) => 0; - lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, W, - LFSR_DATA_BUF("_", 1)) => 0; + lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS( + LFSR_ATTR(0, + NAME, +W, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("___", 3))), + LFSR_ATTR(W-1, DATA, 0, BUF("_", 1)))) => 0; // set up a simulation to compare against // @@ -4061,39 +4081,30 @@ code = ''' if (op == 0 || sim_size <= 1) { // find where to split lfs_size_t bid = 0; - while (bid+1 < sim_size - && memcmp(sim_names[bid+1], name, 3) <= 0) { + while (bid < sim_size && memcmp(name, sim_names[bid], 3) > 0) { bid += 1; } // just skip exact matches for now - if (memcmp(sim_names[bid], name, 3) == 0) { + if (memcmp(name, sim_names[bid], 3) == 0) { continue; } - // calculate actual bid in btree space - lfs_size_t weighted_bid = 0; - for (lfs_size_t j = 0; j < bid; j++) { - weighted_bid += sim_weights[j]; - } - // split btree lfs_size_t split_bid; lfs_size_t split_weight; lfsr_data_t split_data; - lfsr_btree_namelookup(&lfs, &btree, 0, name, 3, - &split_bid, NULL, &split_weight, - &split_data) => 0; - uint8_t split_buf[4]; - lfsr_data_read(&lfs, &split_data, split_buf, 4) => 1; - if (split_bid > weighted_bid+sim_weights[bid]-1) { - int err = lfsr_btree_split(&lfs, &btree, split_bid, - LFSR_DATA_CAT( - LFSR_DATA_LEB128(0), - LFSR_DATA_BUF(sim_names[bid+1], 3)), - LFSR_TAG_DATA, weight, - LFSR_DATA_BUF(&nums[i % 10], 1), - LFSR_TAG_DATA, split_weight, - LFSR_DATA_BUF(split_buf, 1)); + lfs_scmp_t cmp = lfsr_btree_namelookup(&lfs, &btree, 0, name, 3, + &split_bid, NULL, &split_weight, &split_data); + assert(cmp >= 0); + assert(lfs_cmp(cmp) != 0); + if (lfs_cmp(cmp) > 0) { + int err = lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS( + LFSR_ATTR(split_bid-(split_weight-1), + NAME, +weight, CAT( + LFSR_DATA_LEB128(0), + LFSR_DATA_BUF(name, 3))), + LFSR_ATTR(split_bid-(split_weight-1)+(weight-1), + DATA, 0, BUF(&nums[i % 10], 1)))); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -4105,7 +4116,7 @@ code = ''' LFSR_DATA_LEB128(0), LFSR_DATA_BUF(name, 3)), LFSR_TAG_DATA, split_weight, - LFSR_DATA_BUF(split_buf, 1), + split_data, LFSR_TAG_DATA, weight, LFSR_DATA_BUF(&nums[i % 10], 1)); // ignore space issues @@ -4120,9 +4131,9 @@ code = ''' memmove(&sim_names[bid+1], &sim_names[bid], (sim_size-bid)*3); memmove(&sim_weights[bid+1], &sim_weights[bid], (sim_size-bid)*sizeof(lfs_size_t)); - sim[bid+1] = nums[i % 10]; - memcpy(&sim_names[bid+1], name, 3); - sim_weights[bid+1] = weight; + sim[bid] = nums[i % 10]; + memcpy(&sim_names[bid], name, 3); + sim_weights[bid] = weight; sim_size += 1; } else if (op == 1) { @@ -4156,12 +4167,6 @@ code = ''' memmove(&sim_weights[bid], &sim_weights[bid+1], (sim_size-(bid+1))*sizeof(lfs_size_t)); sim_size -= 1; - - // our B-tree doesn't actually track the name of id0, so we need - // mirror this in our sim - if (bid == 0) { - memcpy(&sim_names[0], "___", 3); - } } } @@ -4210,7 +4215,7 @@ code = ''' } lfsr_btree_namelookup(&lfs, &btree, 0, sim_names[i], 3, - &bid_, &tag_, &weight_, &data_) => 0; + &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ; assert(tag_ == LFSR_TAG_DATA); assert(bid_ == weighted_bid+sim_weights[i]-1); assert(weight_ == sim_weights[i]); diff --git a/tests/test_rbyd.toml b/tests/test_rbyd.toml index 82fe854f..e258d5b3 100644 --- a/tests/test_rbyd.toml +++ b/tests/test_rbyd.toml @@ -13,6 +13,7 @@ defines.ERASE_VALUE = [0xff, 0x00, 0x1b] # waste time when testing defines.BLOCK_SIZE = 32768 +# TODO we should eventually replace these with lfsr_rbyd_lookup # some internal helpers in = 'lfs.c' code = '''