Github/littlefs
1
Fork 0
Code Actions Activity

Tweaked named btrees to support strict key->value mapping

Browse Source 
We don't strictly need this for the mtree, but its impact is pretty
minimal, and it's useful for some future plans. It also makes low-level
benchmarks a bit easier to write.

The main change involves subtleties around vestigial names in leaf
rbyds (the bottom most layer of btree inner nodes). Since the mtree
terminates in mdirs, the left-most mdir in each leaf rbyd in the mtree
never actually needs a name. But in a hypothetical strict key->value
tree, every entry in the leaf rbyds need a name, and this name needs to
be respected during btree operations (mainly merges).

As a side-effect, our named btrees now require vestigial names for every
inner btree node, with the exception of the left-most inner nodes since
those can't be merged left with anything. On the bright side, being able
to assume a vestigial name on every mergable node does simplify merge
operations a bit.

It's worth noting that despite these changes, we still update vestigial
names on inner btree nodes lazily. It isn't super clear that this should
work, but it turns out that even though a leaf nodes may diverge from
the vestigial name in it's parent, it must still following the bounds of
the parent's vestigial name because of how btree lookups work. And this
property propagates up though each layer in the btree:

             .---------------.
             |a: |h: |->     |
             '--|---|--------'
            .---'   '----------.
            v                  v
    .---------------.  .---------------.
    |a: |c: |       |  |i: |m: |->     |
    '--|---|--------'  '--|---|--------'
  ...--'   |              |   '--------...
           v              v
    .---------------.  .---------------.
    |d:0|e:1|f:2|-> |  |j:3|k:4|l:5|-> |
    '---------------'  '---------------'

The exception are the left-most inner nodes, but these can never merge
left, so it doesn't really matter. The vestigial names on the left-most
inner nodes are truly vestigial:

                    .---------------.
                    |c: |e: |->     |
                    '--|---|--------'
                   .---'   '--------...
                   v
           .---------------.
           |b: |d: |       |
           '--|---|--------'
          .---'   '-------...
          v
  .---------------.
  |a:0|b:1|c:2|-> |
  '---------------'

An alternative implementation may prefer to update these names eagerly,
but this would increase the amount of data written to each inner node
during btree commits. mdir updates are lazy by necessity, so even if you
adopted eager updates, the names of deleted files would still stick
around.
This commit is contained in:
Christopher Haster
2023-11-08 22:27:46 -06:00
parent f9a38756ca
commit 135bb17409
3 changed files with 175 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+73 -73
View File
@@ -3565,29 +3565,21 @@ static lfs_ssize_t lfsr_rbyd_estimate(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
// //
// names in littlefs are tuples of directory-ids + ascii/utf8 strings // names in littlefs are tuples of directory-ids + ascii/utf8 strings
// binary search an rbyd for a name, leaving the rid_/weight_ with the best // binary search an rbyd for a name, leaving the rid_/tag_/weight_/data_
// matching name if not found // with the best matching name if not found
static int lfsr_rbyd_namelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd, static lfs_scmp_t lfsr_rbyd_namelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
lfsr_did_t did, const char *name, lfs_size_t name_size, lfsr_did_t did, const char *name, lfs_size_t name_size,
lfsr_srid_t *rid_, lfsr_srid_t *rid_,
lfsr_tag_t *tag_, lfsr_rid_t *weight_, lfsr_data_t *data_) { lfsr_tag_t *tag_, lfsr_rid_t *weight_, lfsr_data_t *data_) {
// if we have an empty mdir, default to rid = -1 // empty rbyd? leave it up to upper layers to handle this
if (rid_) { if (rbyd->weight == 0) {
*rid_ = -1; return LFS_ERR_NOENT;
}
if (tag_) {
*tag_ = 0;
}
if (weight_) {
*weight_ = 0;
}
if (data_) {
*data_ = LFSR_DATA_NULL;
} }
// binary search for our name // binary search for our name
lfsr_srid_t lower = 0; lfsr_srid_t lower = 0;
lfsr_srid_t upper = rbyd->weight; lfsr_srid_t upper = rbyd->weight;
lfs_scmp_t cmp;
while (lower < upper) { while (lower < upper) {
lfsr_tag_t tag__; lfsr_tag_t tag__;
lfsr_srid_t rid__; lfsr_srid_t rid__;
@@ -3603,10 +3595,8 @@ static int lfsr_rbyd_namelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
return err; return err;
} }
// if we have no name or a vestigial name, treat this rid as always lt // if we have no name, treat this rid as always lt
lfs_scmp_t cmp; if (lfsr_tag_suptype(tag__) != LFSR_TAG_NAME) {
if ((tag__ == LFSR_TAG_NAME && rid__-(weight__-1) == 0)
|| lfsr_tag_suptype(tag__) != LFSR_TAG_NAME) {
cmp = LFS_CMP_LT; cmp = LFS_CMP_LT;
// compare names // compare names
@@ -3621,10 +3611,27 @@ static int lfsr_rbyd_namelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
if (lfs_cmp(cmp) > 0) { if (lfs_cmp(cmp) > 0) {
upper = rid__ - (weight__-1); upper = rid__ - (weight__-1);
// only keep track of best-match rids > our target if we haven't
// seen an rid < our target
if (lower == 0) {
if (rid_) {
*rid_ = rid__;
}
if (tag_) {
*tag_ = tag__;
}
if (weight_) {
*weight_ = weight__;
}
if (data_) {
*data_ = data__;
}
}
} else if (lfs_cmp(cmp) < 0) { } else if (lfs_cmp(cmp) < 0) {
lower = rid__ + 1; lower = rid__ + 1;
// keep track of best-matching rid >= our target // keep track of best-matching rid < our target
if (rid_) { if (rid_) {
*rid_ = rid__; *rid_ = rid__;
} }
@@ -3652,13 +3659,14 @@ static int lfsr_rbyd_namelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
if (data_) { if (data_) {
*data_ = data__; *data_ = data__;
} }
return 0; return LFS_CMP_EQ;
} }
} }
// no match, at least update rid_/tag_/weight_/data_ with the best // no match, return if found name was lt/gt expect
// match so far //
return LFS_ERR_NOENT; // this will always be lt unless all rids are gt
return (lower == 0) ? LFS_CMP_GT : LFS_CMP_LT;
} }
@@ -4362,33 +4370,6 @@ static int lfsr_btree_commit(lfs_t *lfs, lfsr_btree_t *btree,
return err; return err;
} }
// bring in name that previously split the siblings
err = lfsr_rbyd_lookupnext(lfs, &parent,
rid+1, LFSR_TAG_NAME,
NULL, &split_tag, NULL, &split_data);
if (err) {
return err;
}
if (lfsr_tag_suptype(split_tag) == LFSR_TAG_NAME) {
// lookup the rid (weight really) of the previously-split entry
lfsr_srid_t split_rid;
err = lfsr_rbyd_lookupnext(lfs, &rbyd_,
rbyd.weight, LFSR_TAG_NAME,
&split_rid, NULL, NULL, NULL);
if (err) {
LFS_ASSERT(err != LFS_ERR_NOENT);
return err;
}
err = lfsr_rbyd_appendattr(lfs, &rbyd_,
split_rid, split_tag, 0, split_data);
if (err) {
LFS_ASSERT(err != LFS_ERR_RANGE);
return err;
}
}
// append any pending attrs, it's up to upper // append any pending attrs, it's up to upper
// layers to make sure these always fit // layers to make sure these always fit
err = lfsr_rbyd_appendattrs(lfs, &rbyd_, bid, -1, err = lfsr_rbyd_appendattrs(lfs, &rbyd_, bid, -1,
@@ -4467,7 +4448,7 @@ static int lfsr_btree_commit(lfs_t *lfs, lfsr_btree_t *btree,
} }
// lookup in a btree by name // lookup in a btree by name
static int lfsr_btree_namelookup(lfs_t *lfs, const lfsr_btree_t *btree, static lfs_scmp_t lfsr_btree_namelookup(lfs_t *lfs, const lfsr_btree_t *btree,
lfsr_did_t did, const char *name, lfs_size_t name_size, lfsr_did_t did, const char *name, lfs_size_t name_size,
lfsr_bid_t *bid_, lfsr_bid_t *bid_,
lfsr_tag_t *tag_, lfsr_bid_t *weight_, lfsr_data_t *data_) { lfsr_tag_t *tag_, lfsr_bid_t *weight_, lfsr_data_t *data_) {
@@ -4483,16 +4464,18 @@ static int lfsr_btree_namelookup(lfs_t *lfs, const lfsr_btree_t *btree,
// lookup our name in the rbyd via binary search // lookup our name in the rbyd via binary search
lfsr_srid_t rid__; lfsr_srid_t rid__;
lfsr_rid_t weight__; lfsr_rid_t weight__;
int err = lfsr_rbyd_namelookup(lfs, &branch, did, name, name_size, lfs_scmp_t cmp = lfsr_rbyd_namelookup(lfs, &branch,
did, name, name_size,
&rid__, NULL, &weight__, NULL); &rid__, NULL, &weight__, NULL);
if (err && err != LFS_ERR_NOENT) { if (cmp < 0) {
return err; LFS_ASSERT(cmp != LFS_ERR_NOENT);
return cmp;
} }
// the name may not match exactly, but indicates which branch to follow // the name may not match exactly, but indicates which branch to follow
lfsr_tag_t tag__; lfsr_tag_t tag__;
lfsr_data_t data__; lfsr_data_t data__;
err = lfsr_rbyd_lookup(lfs, &branch, rid__, LFSR_TAG_WIDE(STRUCT), int err = lfsr_rbyd_lookup(lfs, &branch, rid__, LFSR_TAG_WIDE(STRUCT),
&tag__, &data__); &tag__, &data__);
if (err) { if (err) {
LFS_ASSERT(err != LFS_ERR_NOENT); LFS_ASSERT(err != LFS_ERR_NOENT);
@@ -4525,7 +4508,7 @@ static int lfsr_btree_namelookup(lfs_t *lfs, const lfsr_btree_t *btree,
if (data_) { if (data_) {
*data_ = data__; *data_ = data__;
} }
return 0; return cmp;
} }
} }
} }
@@ -6366,23 +6349,39 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir,
} }
// lookup names in our mtree // lookup names in an mdir
//
// if not found, rid will be the best place to insert
//
static int lfsr_mdir_namelookup(lfs_t *lfs, const lfsr_mdir_t *mdir, static int lfsr_mdir_namelookup(lfs_t *lfs, const lfsr_mdir_t *mdir,
lfsr_did_t did, const char *name, lfs_size_t name_size, lfsr_did_t did, const char *name, lfs_size_t name_size,
lfsr_srid_t *rid_, lfsr_tag_t *tag_, lfsr_data_t *data_) { lfsr_srid_t *rid_, lfsr_tag_t *tag_, lfsr_data_t *data_) {
int err = lfsr_rbyd_namelookup(lfs, &mdir->u.rbyd, // empty mdir? make sure rid_ = 0 at least
did, name, name_size, if (mdir->u.m.weight == 0) {
rid_, tag_, NULL, data_); if (rid_) {
*rid_ = 0;
// When not found, lfsr_rbyd_namelookup returns the rid smaller than our }
// expected name. This is correct for btree lookups, but not correct for return LFS_ERR_NOENT;
// mdir insertions. For mdirs we need to adjust this by 1 so we insert
// _after_ the smaller rid.
if (rid_ && err == LFS_ERR_NOENT) {
*rid_ += 1;
} }
return err; lfsr_srid_t rid;
lfs_scmp_t cmp = lfsr_rbyd_namelookup(lfs, &mdir->u.rbyd,
did, name, name_size,
&rid, tag_, NULL, data_);
if (cmp < 0) {
LFS_ASSERT(cmp != LFS_ERR_NOENT);
return cmp;
}
// adjust rid if necessary
if (lfs_cmp(cmp) < 0) {
rid += 1;
}
if (rid_) {
*rid_ = rid;
}
return (lfs_cmp(cmp) == 0) ? 0 : LFS_ERR_NOENT;
} }
// note if we fail, we at least leave mdir_/rid_ with the best place to insert // note if we fail, we at least leave mdir_/rid_ with the best place to insert
@@ -6409,17 +6408,18 @@ static int lfsr_mtree_namelookup(lfs_t *lfs, const lfsr_mtree_t *mtree,
lfsr_tag_t tag; lfsr_tag_t tag;
lfsr_bid_t weight; lfsr_bid_t weight;
lfsr_data_t data; lfsr_data_t data;
int err = lfsr_btree_namelookup(lfs, &lfs->mtree.u.btree, lfs_scmp_t cmp = lfsr_btree_namelookup(lfs, &lfs->mtree.u.btree,
did, name, name_size, did, name, name_size,
&bid, &tag, &weight, &data); &bid, &tag, &weight, &data);
if (err) { if (cmp < 0) {
return err; LFS_ASSERT(cmp != LFS_ERR_NOENT);
return cmp;
} }
LFS_ASSERT(tag == LFSR_TAG_MDIR); LFS_ASSERT(tag == LFSR_TAG_MDIR);
LFS_ASSERT(weight == lfsr_mleafweight(lfs)); LFS_ASSERT(weight == lfsr_mleafweight(lfs));
// decode mdir // decode mdir
err = lfsr_data_readmptr(lfs, &data, mdir.u.m.blocks); int err = lfsr_data_readmptr(lfs, &data, mdir.u.m.blocks);
if (err) { if (err) {
return err; return err;
} }
tests/test_btree.toml
+99 -94
View File
@@ -5,6 +5,8 @@ after = 'test_rbyd'
# of the disk for these tests # of the disk for these tests
defines.LOOKAHEAD_SIZE = 'lfs_alignup(BLOCK_COUNT / 8, 8)' defines.LOOKAHEAD_SIZE = 'lfs_alignup(BLOCK_COUNT / 8, 8)'
# TODO we should eventually replace these with
# lfsr_btree_commit/lfsr_btree_lookup
# helper functions # helper functions
in = 'lfs.c' in = 'lfs.c'
code = ''' code = '''
@@ -3180,8 +3182,10 @@ code = '''
// create a single-entry tree // create a single-entry tree
lfsr_btree_t btree; lfsr_btree_t btree;
lfsr_btree_alloc(&lfs, &btree) => 0; lfsr_btree_alloc(&lfs, &btree) => 0;
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, 1, lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS(
LFSR_DATA_BUF("0", 1)) => 0; LFSR_ATTR(0,
NAME, +1, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("aaa", 3))),
LFSR_ATTR(0, DATA, 0, BUF("0", 1)))) => 0;
printf("btree: w%d 0x%x.%x\n", printf("btree: w%d 0x%x.%x\n",
btree.weight, btree.weight,
btree.block, btree.block,
@@ -3196,7 +3200,7 @@ code = '''
lfsr_data_t data_; lfsr_data_t data_;
lfsr_btree_namelookup(&lfs, &btree, 0*DID, "aaa", 3, lfsr_btree_namelookup(&lfs, &btree, 0*DID, "aaa", 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == 0); assert(bid_ == 0);
assert(weight_ == 1); assert(weight_ == 1);
@@ -3204,7 +3208,7 @@ code = '''
assert(memcmp(buffer, "0", 1) == 0); assert(memcmp(buffer, "0", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 1*DID, "aab", 3, lfsr_btree_namelookup(&lfs, &btree, 1*DID, "aab", 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_LT;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == 0); assert(bid_ == 0);
assert(weight_ == 1); assert(weight_ == 1);
@@ -3230,8 +3234,10 @@ code = '''
// create a two-entry tree // create a two-entry tree
lfsr_btree_t btree; lfsr_btree_t btree;
lfsr_btree_alloc(&lfs, &btree) => 0; lfsr_btree_alloc(&lfs, &btree) => 0;
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, 1, lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS(
LFSR_DATA_BUF("0", 1)) => 0; LFSR_ATTR(0,
NAME, +1, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("aaa", 3))),
LFSR_ATTR(0, DATA, 0, BUF("0", 1)))) => 0;
lfsr_btree_split(&lfs, &btree, 0, lfsr_btree_split(&lfs, &btree, 0,
LFSR_DATA_CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("aab", 3)), LFSR_DATA_CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("aab", 3)),
LFSR_TAG_DATA, 1, LFSR_DATA_BUF("0", 1), LFSR_TAG_DATA, 1, LFSR_DATA_BUF("0", 1),
@@ -3250,7 +3256,7 @@ code = '''
lfsr_data_t data_; lfsr_data_t data_;
lfsr_btree_namelookup(&lfs, &btree, 0, "aaa", 3, lfsr_btree_namelookup(&lfs, &btree, 0, "aaa", 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == 0); assert(bid_ == 0);
assert(weight_ == 1); assert(weight_ == 1);
@@ -3258,7 +3264,7 @@ code = '''
assert(memcmp(buffer, "0", 1) == 0); assert(memcmp(buffer, "0", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 0, "aab", 3, lfsr_btree_namelookup(&lfs, &btree, 0, "aab", 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == 1); assert(bid_ == 1);
assert(weight_ == 1); assert(weight_ == 1);
@@ -3266,7 +3272,7 @@ code = '''
assert(memcmp(buffer, "1", 1) == 0); assert(memcmp(buffer, "1", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 0, "aac", 3, lfsr_btree_namelookup(&lfs, &btree, 0, "aac", 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_LT;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == 1); assert(bid_ == 1);
assert(weight_ == 1); assert(weight_ == 1);
@@ -3292,8 +3298,10 @@ code = '''
// create a two-entry tree // create a two-entry tree
lfsr_btree_t btree; lfsr_btree_t btree;
lfsr_btree_alloc(&lfs, &btree) => 0; lfsr_btree_alloc(&lfs, &btree) => 0;
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, 1, lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS(
LFSR_DATA_BUF("0", 1)) => 0; LFSR_ATTR(0,
NAME, +1, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("aaa", 3))),
LFSR_ATTR(0, DATA, 0, BUF("0", 1)))) => 0;
lfsr_btree_split(&lfs, &btree, 0, lfsr_btree_split(&lfs, &btree, 0,
LFSR_DATA_CAT(LFSR_DATA_LEB128(1*DID), LFSR_DATA_BUF("aab", 3)), LFSR_DATA_CAT(LFSR_DATA_LEB128(1*DID), LFSR_DATA_BUF("aab", 3)),
LFSR_TAG_DATA, 1, LFSR_DATA_BUF("0", 1), LFSR_TAG_DATA, 1, LFSR_DATA_BUF("0", 1),
@@ -3316,7 +3324,7 @@ code = '''
lfsr_data_t data_; lfsr_data_t data_;
lfsr_btree_namelookup(&lfs, &btree, 0*DID, "aaa", 3, lfsr_btree_namelookup(&lfs, &btree, 0*DID, "aaa", 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == 0); assert(bid_ == 0);
assert(weight_ == 1); assert(weight_ == 1);
@@ -3324,7 +3332,7 @@ code = '''
assert(memcmp(buffer, "0", 1) == 0); assert(memcmp(buffer, "0", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 1*DID, "aab", 3, lfsr_btree_namelookup(&lfs, &btree, 1*DID, "aab", 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == 1); assert(bid_ == 1);
assert(weight_ == 1); assert(weight_ == 1);
@@ -3332,7 +3340,7 @@ code = '''
assert(memcmp(buffer, "1", 1) == 0); assert(memcmp(buffer, "1", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 2*DID, "aac", 3, lfsr_btree_namelookup(&lfs, &btree, 2*DID, "aac", 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == 2); assert(bid_ == 2);
assert(weight_ == 1); assert(weight_ == 1);
@@ -3340,7 +3348,7 @@ code = '''
assert(memcmp(buffer, "2", 1) == 0); assert(memcmp(buffer, "2", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 3*DID, "aad", 3, lfsr_btree_namelookup(&lfs, &btree, 3*DID, "aad", 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_LT;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == 2); assert(bid_ == 2);
assert(weight_ == 1); assert(weight_ == 1);
@@ -3366,8 +3374,10 @@ code = '''
// create a two-entry tree // create a two-entry tree
lfsr_btree_t btree; lfsr_btree_t btree;
lfsr_btree_alloc(&lfs, &btree) => 0; lfsr_btree_alloc(&lfs, &btree) => 0;
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, 1, lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS(
LFSR_DATA_BUF("0", 1)) => 0; LFSR_ATTR(0,
NAME, +1, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("aaa", 3))),
LFSR_ATTR(0, DATA, 0, BUF("0", 1)))) => 0;
lfsr_btree_split(&lfs, &btree, 0, lfsr_btree_split(&lfs, &btree, 0,
LFSR_DATA_CAT(LFSR_DATA_LEB128(2*DID), LFSR_DATA_BUF("aac", 3)), LFSR_DATA_CAT(LFSR_DATA_LEB128(2*DID), LFSR_DATA_BUF("aac", 3)),
LFSR_TAG_DATA, 1, LFSR_DATA_BUF("1", 1), LFSR_TAG_DATA, 1, LFSR_DATA_BUF("1", 1),
@@ -3390,7 +3400,7 @@ code = '''
lfsr_data_t data_; lfsr_data_t data_;
lfsr_btree_namelookup(&lfs, &btree, 0*DID, "aaa", 3, lfsr_btree_namelookup(&lfs, &btree, 0*DID, "aaa", 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == 0); assert(bid_ == 0);
assert(weight_ == 1); assert(weight_ == 1);
@@ -3398,7 +3408,7 @@ code = '''
assert(memcmp(buffer, "0", 1) == 0); assert(memcmp(buffer, "0", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 1*DID, "aab", 3, lfsr_btree_namelookup(&lfs, &btree, 1*DID, "aab", 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == 1); assert(bid_ == 1);
assert(weight_ == 1); assert(weight_ == 1);
@@ -3406,7 +3416,7 @@ code = '''
assert(memcmp(buffer, "1", 1) == 0); assert(memcmp(buffer, "1", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 2*DID, "aac", 3, lfsr_btree_namelookup(&lfs, &btree, 2*DID, "aac", 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == 2); assert(bid_ == 2);
assert(weight_ == 1); assert(weight_ == 1);
@@ -3414,7 +3424,7 @@ code = '''
assert(memcmp(buffer, "2", 1) == 0); assert(memcmp(buffer, "2", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 3*DID, "aad", 3, lfsr_btree_namelookup(&lfs, &btree, 3*DID, "aad", 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_LT;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == 2); assert(bid_ == 2);
assert(weight_ == 1); assert(weight_ == 1);
@@ -3443,8 +3453,13 @@ code = '''
lfsr_btree_alloc(&lfs, &btree) => 0; lfsr_btree_alloc(&lfs, &btree) => 0;
const char *alphas = "abcdefghijklmnopqrstuvwxyz"; const char *alphas = "abcdefghijklmnopqrstuvwxyz";
const char *nums = "0123456789"; const char *nums = "0123456789";
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, 1, char name[3] = {
LFSR_DATA_BUF(&nums[0 % 10], 1)) => 0; alphas[(0/26/26) % 26], alphas[(0/26) % 26], alphas[0 % 26]
};
lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS(
LFSR_ATTR(0,
NAME, +1, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF(name, 3))),
LFSR_ATTR(0, DATA, 0, BUF(&nums[0 % 10], 1)))) => 0;
lfs_size_t n = 1; lfs_size_t n = 1;
for (lfs_size_t i = 1; i < N; i++) { for (lfs_size_t i = 1; i < N; i++) {
char name[3] = { char name[3] = {
@@ -3480,7 +3495,7 @@ code = '''
}; };
lfsr_btree_namelookup(&lfs, &btree, i*DID, name, 3, lfsr_btree_namelookup(&lfs, &btree, i*DID, name, 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == i); assert(bid_ == i);
assert(weight_ == 1); assert(weight_ == 1);
@@ -3510,8 +3525,10 @@ code = '''
// create a btree // create a btree
lfsr_btree_t btree; lfsr_btree_t btree;
lfsr_btree_alloc(&lfs, &btree) => 0; lfsr_btree_alloc(&lfs, &btree) => 0;
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, 1, lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS(
LFSR_DATA_BUF("_", 1)) => 0; LFSR_ATTR(0,
NAME, +1, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("___", 3))),
LFSR_ATTR(0, DATA, 0, BUF("_", 1)))) => 0;
// set up a simulation to compare against // set up a simulation to compare against
// //
@@ -3587,7 +3604,7 @@ code = '''
lfsr_data_t data_; lfsr_data_t data_;
for (lfs_size_t i = 0; i < sim_size; i++) { for (lfs_size_t i = 0; i < sim_size; i++) {
lfsr_btree_namelookup(&lfs, &btree, 0, sim_names[i], 3, lfsr_btree_namelookup(&lfs, &btree, 0, sim_names[i], 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == i); assert(bid_ == i);
assert(weight_ == 1); assert(weight_ == 1);
@@ -3623,8 +3640,13 @@ code = '''
lfsr_btree_alloc(&lfs, &btree) => 0; lfsr_btree_alloc(&lfs, &btree) => 0;
const char *alphas = "abcdefghijklmnopqrstuvwxyz"; const char *alphas = "abcdefghijklmnopqrstuvwxyz";
const char *nums = "0123456789"; const char *nums = "0123456789";
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, W, char name[3] = {
LFSR_DATA_BUF(&nums[0 % 10], 1)) => 0; alphas[(0/26/26) % 26], alphas[(0/26) % 26], alphas[0 % 26]
};
lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS(
LFSR_ATTR(0,
NAME, +W, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF(name, 3))),
LFSR_ATTR(W-1, DATA, 0, BUF(&nums[0 % 10], 1)))) => 0;
lfs_size_t n = 1; lfs_size_t n = 1;
for (lfs_size_t i = 1; i < N; i++) { for (lfs_size_t i = 1; i < N; i++) {
char name[3] = { char name[3] = {
@@ -3660,7 +3682,7 @@ code = '''
}; };
lfsr_btree_namelookup(&lfs, &btree, i*DID, name, 3, lfsr_btree_namelookup(&lfs, &btree, i*DID, name, 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == i*W+W-1); assert(bid_ == i*W+W-1);
assert(weight_ == W); assert(weight_ == W);
@@ -3691,8 +3713,10 @@ code = '''
// create a btree // create a btree
lfsr_btree_t btree; lfsr_btree_t btree;
lfsr_btree_alloc(&lfs, &btree) => 0; lfsr_btree_alloc(&lfs, &btree) => 0;
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, W, lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS(
LFSR_DATA_BUF("_", 1)) => 0; LFSR_ATTR(0,
NAME, +W, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("___", 3))),
LFSR_ATTR(W-1, DATA, 0, BUF("_", 1)))) => 0;
// set up a simulation to compare against // set up a simulation to compare against
// //
@@ -3806,7 +3830,7 @@ code = '''
} }
lfsr_btree_namelookup(&lfs, &btree, 0, sim_names[i], 3, lfsr_btree_namelookup(&lfs, &btree, 0, sim_names[i], 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == weighted_bid+sim_weights[i]-1); assert(bid_ == weighted_bid+sim_weights[i]-1);
assert(weight_ == sim_weights[i]); assert(weight_ == sim_weights[i]);
@@ -3843,8 +3867,10 @@ code = '''
// create a btree // create a btree
lfsr_btree_t btree; lfsr_btree_t btree;
lfsr_btree_alloc(&lfs, &btree) => 0; lfsr_btree_alloc(&lfs, &btree) => 0;
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, 1, lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS(
LFSR_DATA_BUF("_", 1)) => 0; LFSR_ATTR(0,
NAME, +1, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("___", 3))),
LFSR_ATTR(0, DATA, 0, BUF("_", 1)))) => 0;
// set up a simulation to compare against // set up a simulation to compare against
// //
@@ -3874,30 +3900,28 @@ code = '''
if (op == 0 || sim_size <= 1) { if (op == 0 || sim_size <= 1) {
// find where to split // find where to split
lfs_size_t bid = 0; lfs_size_t bid = 0;
while (bid+1 < sim_size while (bid < sim_size && memcmp(name, sim_names[bid], 3) > 0) {
&& memcmp(sim_names[bid+1], name, 3) <= 0) {
bid += 1; bid += 1;
} }
// just skip exact matches for now // just skip exact matches for now
if (memcmp(sim_names[bid], name, 3) == 0) { if (memcmp(name, sim_names[bid], 3) == 0) {
continue; continue;
} }
// split btree // split btree
lfs_size_t split_bid; lfs_size_t split_bid;
lfsr_data_t split_data; lfsr_data_t split_data;
lfsr_btree_namelookup(&lfs, &btree, 0, name, 3, lfs_scmp_t cmp = lfsr_btree_namelookup(&lfs, &btree, 0, name, 3,
&split_bid, NULL, NULL, &split_data) => 0; &split_bid, NULL, NULL, &split_data);
uint8_t split_buf[4]; assert(cmp >= 0);
lfsr_data_read(&lfs, &split_data, split_buf, 4) => 1; assert(lfs_cmp(cmp) != 0);
if (split_bid > bid) { if (lfs_cmp(cmp) > 0) {
int err = lfsr_btree_split(&lfs, &btree, int err = lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS(
split_bid, LFSR_ATTR(split_bid,
LFSR_DATA_CAT( NAME, +1, CAT(
LFSR_DATA_LEB128(0), LFSR_DATA_LEB128(0),
LFSR_DATA_BUF(sim_names[bid+1], 3)), LFSR_DATA_BUF(name, 3))),
LFSR_TAG_DATA, 1, LFSR_DATA_BUF(&nums[i % 10], 1), LFSR_ATTR(split_bid, DATA, 0, BUF(&nums[i % 10], 1))));
LFSR_TAG_DATA, 1, LFSR_DATA_BUF(split_buf, 1));
// ignore space issues // ignore space issues
if (err == LFS_ERR_NOSPC) { if (err == LFS_ERR_NOSPC) {
break; break;
@@ -3908,7 +3932,7 @@ code = '''
split_bid, LFSR_DATA_CAT( split_bid, LFSR_DATA_CAT(
LFSR_DATA_LEB128(0), LFSR_DATA_LEB128(0),
LFSR_DATA_BUF(name, 3)), LFSR_DATA_BUF(name, 3)),
LFSR_TAG_DATA, 1, LFSR_DATA_BUF(split_buf, 1), LFSR_TAG_DATA, 1, split_data,
LFSR_TAG_DATA, 1, LFSR_DATA_BUF(&nums[i % 10], 1)); LFSR_TAG_DATA, 1, LFSR_DATA_BUF(&nums[i % 10], 1));
// ignore space issues // ignore space issues
if (err == LFS_ERR_NOSPC) { if (err == LFS_ERR_NOSPC) {
@@ -3920,8 +3944,8 @@ code = '''
// split sim // split sim
memmove(&sim[bid+1], &sim[bid], sim_size-bid); memmove(&sim[bid+1], &sim[bid], sim_size-bid);
memmove(&sim_names[bid+1], &sim_names[bid], (sim_size-bid)*3); memmove(&sim_names[bid+1], &sim_names[bid], (sim_size-bid)*3);
sim[bid+1] = nums[i % 10]; sim[bid] = nums[i % 10];
memcpy(&sim_names[bid+1], name, 3); memcpy(&sim_names[bid], name, 3);
sim_size += 1; sim_size += 1;
} else if (op == 1) { } else if (op == 1) {
@@ -3951,12 +3975,6 @@ code = '''
memmove(&sim[bid], &sim[bid+1], sim_size-(bid+1)); memmove(&sim[bid], &sim[bid+1], sim_size-(bid+1));
memmove(&sim_names[bid], &sim_names[bid+1], (sim_size-(bid+1))*3); memmove(&sim_names[bid], &sim_names[bid+1], (sim_size-(bid+1))*3);
sim_size -= 1; sim_size -= 1;
// our B-tree doesn't actually track the name of id0, so we need
// mirror this in our sim
if (bid == 0) {
memcpy(&sim_names[0], "___", 3);
}
} }
} }
@@ -3984,7 +4002,7 @@ code = '''
lfsr_data_t data_; lfsr_data_t data_;
for (lfs_size_t i = 0; i < sim_size; i++) { for (lfs_size_t i = 0; i < sim_size; i++) {
lfsr_btree_namelookup(&lfs, &btree, 0, sim_names[i], 3, lfsr_btree_namelookup(&lfs, &btree, 0, sim_names[i], 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == i); assert(bid_ == i);
assert(weight_ == 1); assert(weight_ == 1);
@@ -4019,8 +4037,10 @@ code = '''
// create a btree // create a btree
lfsr_btree_t btree; lfsr_btree_t btree;
lfsr_btree_alloc(&lfs, &btree) => 0; lfsr_btree_alloc(&lfs, &btree) => 0;
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_DATA, W, lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS(
LFSR_DATA_BUF("_", 1)) => 0; LFSR_ATTR(0,
NAME, +W, CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("___", 3))),
LFSR_ATTR(W-1, DATA, 0, BUF("_", 1)))) => 0;
// set up a simulation to compare against // set up a simulation to compare against
// //
@@ -4061,39 +4081,30 @@ code = '''
if (op == 0 || sim_size <= 1) { if (op == 0 || sim_size <= 1) {
// find where to split // find where to split
lfs_size_t bid = 0; lfs_size_t bid = 0;
while (bid+1 < sim_size while (bid < sim_size && memcmp(name, sim_names[bid], 3) > 0) {
&& memcmp(sim_names[bid+1], name, 3) <= 0) {
bid += 1; bid += 1;
} }
// just skip exact matches for now // just skip exact matches for now
if (memcmp(sim_names[bid], name, 3) == 0) { if (memcmp(name, sim_names[bid], 3) == 0) {
continue; continue;
} }
// calculate actual bid in btree space
lfs_size_t weighted_bid = 0;
for (lfs_size_t j = 0; j < bid; j++) {
weighted_bid += sim_weights[j];
}
// split btree // split btree
lfs_size_t split_bid; lfs_size_t split_bid;
lfs_size_t split_weight; lfs_size_t split_weight;
lfsr_data_t split_data; lfsr_data_t split_data;
lfsr_btree_namelookup(&lfs, &btree, 0, name, 3, lfs_scmp_t cmp = lfsr_btree_namelookup(&lfs, &btree, 0, name, 3,
&split_bid, NULL, &split_weight, &split_bid, NULL, &split_weight, &split_data);
&split_data) => 0; assert(cmp >= 0);
uint8_t split_buf[4]; assert(lfs_cmp(cmp) != 0);
lfsr_data_read(&lfs, &split_data, split_buf, 4) => 1; if (lfs_cmp(cmp) > 0) {
if (split_bid > weighted_bid+sim_weights[bid]-1) { int err = lfsr_btree_commit(&lfs, &btree, LFSR_ATTRS(
int err = lfsr_btree_split(&lfs, &btree, split_bid, LFSR_ATTR(split_bid-(split_weight-1),
LFSR_DATA_CAT( NAME, +weight, CAT(
LFSR_DATA_LEB128(0), LFSR_DATA_LEB128(0),
LFSR_DATA_BUF(sim_names[bid+1], 3)), LFSR_DATA_BUF(name, 3))),
LFSR_TAG_DATA, weight, LFSR_ATTR(split_bid-(split_weight-1)+(weight-1),
LFSR_DATA_BUF(&nums[i % 10], 1), DATA, 0, BUF(&nums[i % 10], 1))));
LFSR_TAG_DATA, split_weight,
LFSR_DATA_BUF(split_buf, 1));
// ignore space issues // ignore space issues
if (err == LFS_ERR_NOSPC) { if (err == LFS_ERR_NOSPC) {
break; break;
@@ -4105,7 +4116,7 @@ code = '''
LFSR_DATA_LEB128(0), LFSR_DATA_LEB128(0),
LFSR_DATA_BUF(name, 3)), LFSR_DATA_BUF(name, 3)),
LFSR_TAG_DATA, split_weight, LFSR_TAG_DATA, split_weight,
LFSR_DATA_BUF(split_buf, 1), split_data,
LFSR_TAG_DATA, weight, LFSR_TAG_DATA, weight,
LFSR_DATA_BUF(&nums[i % 10], 1)); LFSR_DATA_BUF(&nums[i % 10], 1));
// ignore space issues // ignore space issues
@@ -4120,9 +4131,9 @@ code = '''
memmove(&sim_names[bid+1], &sim_names[bid], (sim_size-bid)*3); memmove(&sim_names[bid+1], &sim_names[bid], (sim_size-bid)*3);
memmove(&sim_weights[bid+1], &sim_weights[bid], memmove(&sim_weights[bid+1], &sim_weights[bid],
(sim_size-bid)*sizeof(lfs_size_t)); (sim_size-bid)*sizeof(lfs_size_t));
sim[bid+1] = nums[i % 10]; sim[bid] = nums[i % 10];
memcpy(&sim_names[bid+1], name, 3); memcpy(&sim_names[bid], name, 3);
sim_weights[bid+1] = weight; sim_weights[bid] = weight;
sim_size += 1; sim_size += 1;
} else if (op == 1) { } else if (op == 1) {
@@ -4156,12 +4167,6 @@ code = '''
memmove(&sim_weights[bid], &sim_weights[bid+1], memmove(&sim_weights[bid], &sim_weights[bid+1],
(sim_size-(bid+1))*sizeof(lfs_size_t)); (sim_size-(bid+1))*sizeof(lfs_size_t));
sim_size -= 1; sim_size -= 1;
// our B-tree doesn't actually track the name of id0, so we need
// mirror this in our sim
if (bid == 0) {
memcpy(&sim_names[0], "___", 3);
}
} }
} }
@@ -4210,7 +4215,7 @@ code = '''
} }
lfsr_btree_namelookup(&lfs, &btree, 0, sim_names[i], 3, lfsr_btree_namelookup(&lfs, &btree, 0, sim_names[i], 3,
&bid_, &tag_, &weight_, &data_) => 0; &bid_, &tag_, &weight_, &data_) => LFS_CMP_EQ;
assert(tag_ == LFSR_TAG_DATA); assert(tag_ == LFSR_TAG_DATA);
assert(bid_ == weighted_bid+sim_weights[i]-1); assert(bid_ == weighted_bid+sim_weights[i]-1);
assert(weight_ == sim_weights[i]); assert(weight_ == sim_weights[i]);
tests/test_rbyd.toml
+1
View File
@@ -13,6 +13,7 @@ defines.ERASE_VALUE = [0xff, 0x00, 0x1b]
# waste time when testing # waste time when testing
defines.BLOCK_SIZE = 32768 defines.BLOCK_SIZE = 32768
# TODO we should eventually replace these with lfsr_rbyd_lookup
# some internal helpers # some internal helpers
in = 'lfs.c' in = 'lfs.c'
code = ''' code = '''