From 89d5a5ef80955d9791dd6d763573e1f65c2b6ba2 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 20 Mar 2023 19:23:15 -0500 Subject: [PATCH] Working implementation of B-tree name split/lookup with vestigial names B-trees with names are now working, though this required a number of changes to the B-tree layout: 1. B-tree no-longer require name entries (LFSR_TAG_MK) on each branch. This is a nice optimization to the design, since these name entries just waste space in purely weight-based B-trees, which are probably going to be most B-trees in the filesystem. If a name entry is missing, the struct entry, which is required, should have the effective weight of the entry. The first entry in every rbyd block is expected to be have no name entry, since this is the default path for B-tree lookups. 2. The first entry in every rbyd block _may_ have a name entry, which is ignored. I'm calling these "vestigial names" to make them sound cooler than they actually are. These vestigial names show up in a couple complicated B-tree operations: - During B-tree split, since pending attributes are calculated before the split, we need to play out pending attributes into the rbyd before deciding what name becomes the name of entry in the parent. This creates a vestigial name which we _could_ immediately remove, but the remove adds additional size to the must-fit split operation - During B-tree pop/merge, if we remove the leading no-name entry, the second, named entry becomes the leading entry. This creates a vestigial name that _looks_ easy enough to remove when making the pending attributes for pop/merge, but turns out the be surprisingly tricky if the parent undergoes a split/merge at the same time. It may be possible to remove all these vestigial names proactively, but this adds additional rbyd lookups to figure out the exact tag to remove, complicates things in a fragile way, and doesn't actually reduce storage costs until the rbyd is compacted. The main downside is that these B-trees may be a bit more confusing to debug. --- lfs.c | 203 +++++++++++++++++++++++++++++------------- scripts/dbgbtree.py | 4 +- scripts/dbgrbyd.py | 3 +- tests/test_btree.toml | 19 +++- 4 files changed, 161 insertions(+), 68 deletions(-) diff --git a/lfs.c b/lfs.c index 00f93a59..bc09f6d8 100644 --- a/lfs.c +++ b/lfs.c @@ -453,34 +453,35 @@ static inline lfs_size_t lfs_tag_dsize(lfs_tag_t tag) { // on-disk, these are encoded in leb128, so smaller constants are prefered // enum lfsr_tag_type { - LFSR_TAG_MK = 0x0400, - LFSR_TAG_MKBRANCH = 0x0400, - LFSR_TAG_MKREG = 0x0410, - LFSR_TAG_MKDIR = 0x0420, + LFSR_TAG_MK = 0x0400, + LFSR_TAG_MKBRANCH = 0x0400, + LFSR_TAG_MKREG = 0x0410, + LFSR_TAG_MKDIR = 0x0420, + LFSR_TAG_RMMKBRANCH = 0x0402, - LFSR_TAG_STRUCT = 0x0800, - LFSR_TAG_INLINED = 0x0800, - LFSR_TAG_BLOCK = 0x0810, - LFSR_TAG_BTREE = 0x0820, - LFSR_TAG_BRANCH = 0x0830, + LFSR_TAG_STRUCT = 0x0800, + LFSR_TAG_INLINED = 0x0800, + LFSR_TAG_BLOCK = 0x0810, + LFSR_TAG_BTREE = 0x0820, + LFSR_TAG_BRANCH = 0x0830, - LFSR_TAG_UATTR = 0x2000, - LFSR_TAG_RMUATTR = 0x2002, + LFSR_TAG_UATTR = 0x2000, + LFSR_TAG_RMUATTR = 0x2002, - LFSR_TAG_GROW = 0x0006, - LFSR_TAG_SHRINK = 0x0016, - LFSR_TAG_FROM = 0x0026, // in-device only + LFSR_TAG_GROW = 0x0006, + LFSR_TAG_SHRINK = 0x0016, + LFSR_TAG_FROM = 0x0026, // in-device only - LFSR_TAG_ALT = 0x0008, - LFSR_TAG_ALTBLE = 0x0008, - LFSR_TAG_ALTRLE = 0x000a, - LFSR_TAG_ALTBGT = 0x000c, - LFSR_TAG_ALTRGT = 0x000e, + LFSR_TAG_ALT = 0x0008, + LFSR_TAG_ALTBLE = 0x0008, + LFSR_TAG_ALTRLE = 0x000a, + LFSR_TAG_ALTBGT = 0x000c, + LFSR_TAG_ALTRGT = 0x000e, - LFSR_TAG_CRC = 0x0004, - LFSR_TAG_CRC0 = 0x0004, - LFSR_TAG_CRC1 = 0x0014, - LFSR_TAG_FCRC = 0x0024, + LFSR_TAG_CRC = 0x0004, + LFSR_TAG_CRC0 = 0x0004, + LFSR_TAG_CRC1 = 0x0014, + LFSR_TAG_FCRC = 0x0024, }; #define LFSR_TAG_ALT_(color, dir, key) \ @@ -771,6 +772,18 @@ struct lfsr_attr { #define LFSR_ATTR_IF(_pred, _type, _id, _buf, _len, _next) \ LFSR_ATTR_IF_(_pred, LFSR_TAG_##_type, _id, _buf, _len, _next) +#define LFSR_ATTR_DISK_IF_(_pred, _tag, _id, _block, _off, _len, _next) \ + LFSR_ATTR_DISK_( \ + (_pred) ? (_tag) : LFSR_TAG_GROW, \ + _id, \ + _block, \ + _off, \ + (_pred) ? (_len) : 0, \ + _next) + +#define LFSR_ATTR_DISK_IF(_pred, _type, _id, _block, _off, _len, _next) \ + LFSR_ATTR_DISK_IF_(_pred, LFSR_TAG_##_type, _id, _block, _off, _len, _next) + struct lfsr_attr_from { const lfsr_rbyd_t *rbyd; const struct lfsr_attr *attrs; @@ -1337,9 +1350,10 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, if (find && find->predicted_id >= id) { if (find->predicted_id < id+(lfs_ssize_t)size) { find->predicted_tag = 0; - find->predicted_id = id; + find->predicted_id = id-1; + } else { + find->predicted_id -= size; } - find->predicted_id -= size; } } @@ -1988,11 +2002,11 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, lfsr_tag_t other_tag_; lfs_ssize_t other_id_; if (tag == LFSR_TAG_GROW) { - LFS_ASSERT(id <= rbyd->weight); // noop? if (lfsr_data_len(data) == 0) { return 0; } + LFS_ASSERT(id <= rbyd->weight); rbyd->weight += lfsr_data_len(data); tag_ = 0; @@ -2000,12 +2014,12 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, other_tag_ = tag_; other_id_ = id_; } else if (tag == LFSR_TAG_SHRINK) { - LFS_ASSERT(id < rbyd->weight); - LFS_ASSERT(lfsr_data_len(data) <= rbyd->weight); // noop? if (lfsr_data_len(data) == 0) { return 0; } + LFS_ASSERT(id < rbyd->weight); + LFS_ASSERT(lfsr_data_len(data) <= rbyd->weight); rbyd->weight -= lfsr_data_len(data); tag_ = 0; @@ -2904,6 +2918,7 @@ static lfs_ssize_t lfsr_btree_find_(lfs_t *lfs, // the find may not match exactly, but it will indicate which id we // should follow + // TODO can we actually get weight in follow? lfs_ssize_t rid__; lfs_size_t weight__; err = lfsr_rbyd_lookup(lfs, &rbyd, LFSR_TAG_MK, find.found_id, @@ -3292,26 +3307,45 @@ static int lfsr_btree_commit(lfs_t *lfs, } } - // finalize commit - err = lfsr_rbyd_commit(lfs, &sibling, NULL); - if (err) { - assert(!err); - return err; - } - // lookup first name in sibling to use as the split name // // note we need to do this after playing out pending attrs in case // they introduce a new name! + lfsr_tag_t stag; + lfs_ssize_t sid; lfs_off_t soff; lfs_size_t ssize; err = lfsr_rbyd_lookup(lfs, &sibling, LFSR_TAG_MK, 0, - NULL, NULL, NULL, &soff, &ssize); + &stag, &sid, NULL, &soff, &ssize); if (err && err != LFS_ERR_NOENT) { assert(!err); return err; } +// // remove the first name since we are moving it up as part of our split +// // +// // note if we wanted to just not write out the first name it would +// // checking the attr last for mks that insert a new first name last +// // minute, this is just easier as long as split fixups are not the block +// // size bottleneck +// if (lfsr_tag_ismk(stag)) { +// err = lfsr_rbyd_append(lfs, &sibling, +// lfsr_tag_mkrm(stag), sid, +// LFSR_DATA_NULL); +// if (err) { +// assert(!err); +// return err; +// } +// } + + // TODO can move this up? + // finalize commit + err = lfsr_rbyd_commit(lfs, &sibling, NULL); + if (err) { + assert(!err); + return err; + } + // no parent? introduce a new trunk if (rid == -1) { int err = lfsr_rbyd_alloc(lfs, &parent, 1); @@ -3341,23 +3375,20 @@ static int lfsr_btree_commit(lfs_t *lfs, NULL, rbyd_.weight, &scratch_attrs[1]); scratch_attrs[1] = *LFSR_ATTR( - MKBRANCH, 0+rbyd_.weight-1, - NULL, 0, - &scratch_attrs[2]); - scratch_attrs[2] = *LFSR_ATTR( BRANCH, 0+rbyd_.weight-1, scratch_buf1, delta1, - &scratch_attrs[3]); + &scratch_attrs[2]); - scratch_attrs[3] = *LFSR_ATTR( + scratch_attrs[2] = *LFSR_ATTR( GROW, 0+rbyd_.weight, NULL, sibling.weight, - &scratch_attrs[4]); - scratch_attrs[4] = *LFSR_ATTR_DISK( + &scratch_attrs[3]); + scratch_attrs[3] = *LFSR_ATTR_DISK_IF( + lfsr_tag_ismk(stag), MKBRANCH, 0+rbyd_.weight+sibling.weight-1, sibling.block, soff, ssize, - &scratch_attrs[5]); - scratch_attrs[5] = *LFSR_ATTR( + &scratch_attrs[4]); + scratch_attrs[4] = *LFSR_ATTR( BRANCH, 0+rbyd_.weight+sibling.weight-1, scratch_buf2, delta2, NULL); @@ -3397,7 +3428,8 @@ static int lfsr_btree_commit(lfs_t *lfs, GROW, rid-(rweight-1)+rbyd_.weight, NULL, sibling.weight, &scratch_attrs[3]); - scratch_attrs[3] = *LFSR_ATTR_DISK( + scratch_attrs[3] = *LFSR_ATTR_DISK_IF( + lfsr_tag_ismk(stag), MKBRANCH, rid-(rweight-1)+rbyd_.weight +sibling.weight-1, sibling.block, soff, ssize, @@ -3425,7 +3457,7 @@ static int lfsr_btree_commit(lfs_t *lfs, } // last child? try the left sibling - lfs_ssize_t sid; + // lfs_ssize_t sid; lfs_ssize_t sdelta; if ((lfs_size_t)rid == parent.weight-1) { sid = rid-rweight; @@ -3438,8 +3470,13 @@ static int lfsr_btree_commit(lfs_t *lfs, // try looking up the sibling lfs_size_t sweight; + // keep track of our siblings name + lfsr_tag_t split_tag; + lfs_off_t split_off; + lfs_size_t split_size; + err = lfsr_rbyd_lookup(lfs, &parent, LFSR_TAG_MK, sid, - NULL, &sid, &sweight, NULL, NULL); + &split_tag, &sid, &sweight, &split_off, &split_size); if (err && err != LFS_ERR_NOENT) { assert(!err); return err; @@ -3450,7 +3487,7 @@ static int lfsr_btree_commit(lfs_t *lfs, goto abort; } - lfsr_tag_t stag; + // lfsr_tag_t stag; lfs_off_t off; lfs_size_t size; err = lfsr_rbyd_lookup(lfs, &parent, LFSR_TAG_BRANCH, sid, @@ -3542,6 +3579,38 @@ static int lfsr_btree_commit(lfs_t *lfs, } } + // bring in name that previously split the siblings + if (sdelta == 0) { + // if we're merging left we need to actually use our name + err = lfsr_rbyd_lookup(lfs, &parent, LFSR_TAG_MK, rid, + &split_tag, NULL, NULL, &split_off, &split_size); + if (err) { + assert(!err); + return err; + } + } + + if (lfsr_tag_ismk(split_tag)) { + // TODO can we avoid this? + // lookup the id of the previously-split entry + lfs_ssize_t split_id; + err = lfsr_rbyd_lookup(lfs, &rbyd_, + LFSR_TAG_MK, (sdelta == 0 ? sweight : rweight), + NULL, &split_id, NULL, NULL, NULL); + if (err) { + assert(!err); + return err; + } + + err = lfsr_rbyd_append(lfs, &rbyd_, + LFSR_TAG_MKBRANCH, (sdelta == 0 ? sweight : rweight), + LFSR_DATA_DISK(parent.block, split_off, split_size)); + if (err) { + assert(!err); + return err; + } + } + // finalize commit with new attrs, it's up to upper // layers to make sure these always fit for (const struct lfsr_attr *attr = attrs; attr; attr = attr->next) { @@ -3651,14 +3720,12 @@ static int lfsr_btree_push(lfs_t *lfs, lfsr_btree_t *btree, // commit our entries err = lfsr_rbyd_commit(lfs, &rbyd, LFSR_ATTR(GROW, 0, NULL, btree->weight, - LFSR_ATTR(MKBRANCH, 0+btree->weight-1, NULL, 0, LFSR_ATTR_(btree->tag, 0+btree->weight-1, btree->u.inlined.buf, btree->u.inlined.size, LFSR_ATTR(GROW, id, NULL, weight, - LFSR_ATTR(MKBRANCH, id+weight-1, NULL, 0, LFSR_ATTR_(tag, id+weight-1, buffer, size, - NULL))))))); + NULL))))); if (err) { return err; } @@ -3694,9 +3761,8 @@ static int lfsr_btree_push(lfs_t *lfs, lfsr_btree_t *btree, return lfsr_btree_commit(lfs, btree, lfs_min32(id, btree->weight-1), &rbyd, LFSR_ATTR(GROW, rid, NULL, weight, - LFSR_ATTR(MKBRANCH, rid+weight-1, NULL, 0, LFSR_ATTR_(tag, rid+weight-1, buffer, size, - NULL)))); + NULL))); } } @@ -3828,6 +3894,10 @@ static int lfsr_btree_pop(lfs_t *lfs, lfsr_btree_t *btree, lfs_size_t id) { // of the rest return lfsr_btree_commit(lfs, btree, id, &rbyd, LFSR_ATTR(SHRINK, rid-(rweight-1), NULL, rweight, +// // TODO conditional? merge with above? +// LFSR_ATTR_IF( +// rid-(rweight-1) == 0 && rweight < rbyd.weight, +// RMMKBRANCH, 0, NULL, 0, NULL)); } } @@ -3858,7 +3928,6 @@ static int lfsr_btree_split(lfs_t *lfs, lfsr_btree_t *btree, // commit our entries err = lfsr_rbyd_commit(lfs, &rbyd, LFSR_ATTR(GROW, 0, NULL, weight1, - LFSR_ATTR(MKBRANCH, 0+weight1-1, NULL, 0, LFSR_ATTR_(tag1, 0+weight1-1, buffer1, size1, LFSR_ATTR(GROW, weight1, NULL, weight2, @@ -3866,7 +3935,7 @@ static int lfsr_btree_split(lfs_t *lfs, lfsr_btree_t *btree, name, name_len, LFSR_ATTR_(tag2, weight1+weight2-1, buffer2, size2, - NULL))))))); + NULL)))))); if (err) { return err; } @@ -3883,11 +3952,21 @@ static int lfsr_btree_split(lfs_t *lfs, lfsr_btree_t *btree, lfsr_rbyd_t rbyd; lfs_ssize_t rid; lfs_size_t rweight; - lfs_ssize_t size = lfsr_btree_lookup(lfs, btree, id, - NULL, NULL, &rbyd, &rid, &rweight, NULL, 0); - if (size < 0) { - return size; - } +// // TODO should we have two split functions? +// // TODO should we bother with id splits if we usually split on names? +// if (name_len == 0) { + lfs_ssize_t size = lfsr_btree_lookup(lfs, btree, id, + NULL, NULL, &rbyd, &rid, &rweight, NULL, 0); + if (size < 0) { + return size; + } +// } else { +// lfs_ssize_t size = lfsr_btree_find_(lfs, btree, name, name_len, +// NULL, &id, &rbyd, &rid, &rweight, NULL, 0); +// if (size < 0) { +// return size; +// } +// } // commit our id into the tree, letting lfsr_btree_commit take care // of the rest diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index 3a492e45..60745b21 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -283,10 +283,12 @@ def main(disk, block_size=None, trunk=0, limit=None, *, while True: # first lookup id/name - (done, _, rid_, w, + (done, name_tag, rid_, w, name_j, name_d, name) = rbyd.lookup(0x400, rid) if done: return True, id, 0, rbyd, -1, 0, 0, 0, b'', 0, 0, b'', path + if name_tag & 0xf00 != 0x400: + name_j, name_d, name = name_j, 0, b'' # then lookup struct (done, tag, _, _, diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index c3d72e77..4e75be6e 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -73,7 +73,8 @@ def xxd(data, width=16, crc=False): def tagrepr(tag, id, size, off=None): if (tag & ~0x3f0) == 0x0400: - return 'mk%s id%d %d' % ( + return '%smk%s id%d %d' % ( + 'rm' if tag & 0x2 else '', 'branch' if ((tag & 0x3f0) >> 4) == 0x00 else 'reg' if ((tag & 0x3f0) >> 4) == 0x01 else 'dir' if ((tag & 0x3f0) >> 4) == 0x02 diff --git a/tests/test_btree.toml b/tests/test_btree.toml index b8ef30f0..d6889f50 100644 --- a/tests/test_btree.toml +++ b/tests/test_btree.toml @@ -3458,14 +3458,25 @@ code = ''' } // split btree - lfsr_btree_split(&lfs, &btree, id, name, 3, - LFSR_TAG_INLINED, 1, &nums[i % 10], 1, - LFSR_TAG_INLINED, 1, &nums[i % 10], 1) => 0; + lfs_size_t split_id; + uint8_t split_buf[4]; + lfsr_btree_find(&lfs, &btree, name, 3, + NULL, &split_id, NULL, split_buf, 4) => 1; + if (split_id > id) { + lfsr_btree_split(&lfs, &btree, + split_id, sim_names[split_id], 3, + LFSR_TAG_INLINED, 1, &nums[i % 10], 1, + LFSR_TAG_INLINED, 1, split_buf, 1) => 0; + } else { + lfsr_btree_split(&lfs, &btree, + split_id, name, 3, + LFSR_TAG_INLINED, 1, split_buf, 1, + LFSR_TAG_INLINED, 1, &nums[i % 10], 1) => 0; + } // split sim memmove(&sim[id+1], &sim[id], sim_size-id); memmove(&sim_names[id+1], &sim_names[id], (sim_size-id)*3); - sim[id+0] = nums[i % 10]; sim[id+1] = nums[i % 10]; memcpy(&sim_names[id+1], name, 3); sim_size += 1;