From a57e79bc68bc8ebfebaeb3ed110ba2c4d39d03de Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 9 May 2023 12:50:55 -0500 Subject: [PATCH] Some cleanup, reverted merge of rbyd_estimate/isdegenerate Unfortunately due to different early-exit conditions, estimate/isdegenerate isn't trivially compatible. The previous, merged implementation missed the opportunity to inline btrees with two large entries undergoing compaction. It's unlikely to hit this, but splitting these back into two separate passes simplifies the code and avoids the potential for other bugs from this combination of unrelated pieces of logic. Keep in mind lfsr_rbyd_isdegenerate is cheap: 1. Only ran when compacting the root of a btree. 2. Cutoff is usually small, at the moment requires at most 2 ids, or 2*2 rbyd lookups with the current btree implementation. --- lfs.c | 1048 ++++++++++++--------------------------------------------- 1 file changed, 224 insertions(+), 824 deletions(-) diff --git a/lfs.c b/lfs.c index e8dd640d..7cb4341f 100644 --- a/lfs.c +++ b/lfs.c @@ -2678,15 +2678,8 @@ failed:; // the following are mostly btree helpers, but since they operate on rbyds, // exist in the rbyd namespace -enum { - LFSR_ESTIMATE_OVERFLOWS = 0, - LFSR_ESTIMATE_FITS = 1, - LFSR_ESTIMATE_DEGENERATE = 2, -}; - static int lfsr_rbyd_estimate(lfs_t *lfs, const lfsr_rbyd_t *rbyd, - lfs_ssize_t id, lfs_ssize_t cutoff, - lfs_size_t init_tcount, lfs_size_t init_dsize, + lfs_ssize_t init_id, lfs_size_t init_tcount, lfs_size_t init_dsize, lfs_size_t *lower_id_, lfs_size_t *lower_dsize_) { // determine if a given rbyd will be within the compaction threshold (1/2) // after compaction, note this uses a conservative estimate so the actual @@ -2704,12 +2697,8 @@ static int lfsr_rbyd_estimate(lfs_t *lfs, const lfsr_rbyd_t *rbyd, lfs_size_t tcount = init_tcount; lfs_size_t dsize = 0; lfs_size_t real_dsize = init_dsize; - // note id count != tag count, and id count != id+1 - // - // we use this to test for degenerate rbyds, where a degenerate rbyd is - // defined as an rbyd with id count < cutoff, irregardless of weight - lfs_size_t idcount = 0; + lfs_ssize_t id = init_id; lfsr_tag_t tag = 0; while (true) { lfs_size_t w; @@ -2720,15 +2709,7 @@ static int lfsr_rbyd_estimate(lfs_t *lfs, const lfsr_rbyd_t *rbyd, return err; } if (err == LFS_ERR_NOENT) { - if (cutoff >= 0 && idcount <= (lfs_size_t)cutoff) { - return LFSR_ESTIMATE_DEGENERATE; - } - return LFSR_ESTIMATE_FITS; - } - - // count ids to determine if we're without our cutoff - if (w > 0) { - idcount += 1; + return true; } // Exhibit A. Why I really didn't want to estimate the rbyd threshold: @@ -2762,111 +2743,11 @@ static int lfsr_rbyd_estimate(lfs_t *lfs, const lfsr_rbyd_t *rbyd, *lower_dsize_ = dsize; } - return LFSR_ESTIMATE_OVERFLOWS; + return false; } } } -//enum { -// LFSR_INTHRESH_NO = 0, -// LFSR_INTHRESH_YES = 1, -// LFSR_INTHRESH_UNINLINED = 2, -// LFSR_INTHRESH_DEGENERATE = 3, -//}; -// -//static int lfsr_rbyd_inthresh(lfs_t *lfs, const lfsr_rbyd_t *rbyd, -// bool inlined, lfs_ssize_t cutoff, lfs_size_t dcount, lfs_size_t dsize, -// lfs_size_t *lower_id_, lfs_size_t *lower_dsize_) { -// // determine if a given rbyd will be within the compaction threshold (1/2) -// // after compaction, note this uses a conservative estimate so the actual -// // on-disk cost may be smaller -// // -// // returns the id/dsize where the threshold failed, this isn't that useful -// // on its own, but can be used to find a good split_id with lfsr_rbyd_bisect -// -// // TODO should we store this in lfs_t somewhere? -// // assume a tighter bound on size/jump leb128 encoding if we know -// // our block_size -// const lfs_size_t tag_dsize = 2 -// + 5 -// + (lfs_nlog2(lfs->cfg->block_size)+7-1)/7; -// lfs_size_t count = 0; -// lfs_size_t uninlined_dcount = 0; -// lfs_size_t uninlined_dsize = 0; -// lfs_size_t altless_dsize = 0; -// -// lfs_ssize_t id = -1; -// lfsr_tag_t tag = 0; -// while (true) { -// lfs_size_t w; -// lfsr_data_t data; -// int err = lfsr_rbyd_lookupnext(lfs, rbyd, id, lfsr_tag_next(tag), -// &id, &tag, &w, &data); -// if (err && err != LFS_ERR_NOENT) { -// return err; -// } -// if (err == LFS_ERR_NOENT) { -// if (cutoff >= 0 && count <= (lfs_size_t)cutoff) { -// return LFSR_INTHRESH_DEGENERATE; -// } -// return LFSR_INTHRESH_YES; -// } -// -// // count ids to determine if we're without our cutoff -// if (w > 0) { -// count += 1; -// } -// -// // Exhibit A. Why I really didn't want to estimate the rbyd threshold: -// -// // TODO do we really need this tight a bound? this might be the only -// // place we divide by a non-power-of-two -// -// // determine the upper-bound of our alt pointers, tag, and data -// // -// // fortunately rybd gives us a tight bound on the number of alt -// // pointers -// dcount += 1; -// dsize += (2*lfs_nlog2(dcount+1)+1) * tag_dsize -// + tag_dsize -// + lfsr_data_size(data); -// -// // also keep track of upper-bound if we ignore any -1 ids, this -// // is useful for lfsr_mdir_commit to see if we don't need to split -// // when uninlining -// if (id >= 0) { -// uninlined_dcount += 1; -// uninlined_dsize += (2*lfs_nlog2(uninlined_dcount+1)+1) * tag_dsize -// + tag_dsize -// + lfsr_data_size(data); -// } -// -// // also keep track of alt-less dsize in case we need to split, -// // assume a worst-case tag size because it's cheaper and this -// // matters less -// if (!inlined || id >= 0) { -// altless_dsize += LFSR_TAG_DSIZE + lfsr_data_size(data); -// } -// -// // exceeded our compaction threshold? -// if (dsize > lfs->cfg->block_size/2) { -// // TODO do these need to be conditional? -// if (lower_id_) { -// *lower_id_ = id; -// } -// if (lower_dsize_) { -// *lower_dsize_ = altless_dsize; -// } -// -// if (inlined && uninlined_dsize <= lfs->cfg->block_size/2) { -// return LFSR_INTHRESH_UNINLINED; -// } else { -// return LFSR_INTHRESH_NO; -// } -// } -// } -//} - static lfs_ssize_t lfsr_rbyd_bisect(lfs_t *lfs, const lfsr_rbyd_t *rbyd, lfs_size_t lower_id, lfs_size_t lower_dsize) { // find the best id to split the rbyd evenly @@ -2938,39 +2819,39 @@ static lfs_ssize_t lfsr_rbyd_bisect(lfs_t *lfs, const lfsr_rbyd_t *rbyd, return lower_id_; } -//static int lfsr_rbyd_incutoff(lfs_t *lfs, const lfsr_rbyd_t *rbyd, -// lfs_ssize_t cutoff) { -// // determine if there are fewer than "cutoff" unique ids in the rbyd, -// // this is used to determine if the underlying rbyd is degenerate and can -// // be reverted to an inlined btree -// // -// // note cutoff is expected to be quite small, <= 2, so we should make sure -// // to exit our traverse early -// -// // cutoff=-1 => no cutoff -// if (cutoff < 0) { -// return false; -// } -// -// // count ids until we exceed our cutoff -// lfs_ssize_t id = -1; -// lfs_size_t count = 0; -// while (true) { -// int err = lfsr_rbyd_lookupnext(lfs, rbyd, id+1, 0, -// &id, NULL, NULL, NULL); -// if (err && err != LFS_ERR_NOENT) { -// return err; -// } -// if (err == LFS_ERR_NOENT) { -// return true; -// } -// -// count += 1; -// if (count > (lfs_size_t)cutoff) { -// return false; -// } -// } -//} +static int lfsr_rbyd_isdegenerate(lfs_t *lfs, const lfsr_rbyd_t *rbyd, + lfs_ssize_t cutoff) { + // determine if there are fewer than "cutoff" unique ids in the rbyd, + // this is used to determine if the underlying rbyd is degenerate and can + // be reverted to an inlined btree + // + // note cutoff is expected to be quite small, <= 2, so we should make sure + // to exit our traverse early + + // cutoff=-1 => no cutoff + if (cutoff < 0) { + return false; + } + + // count ids until we exceed our cutoff + lfs_ssize_t id = -1; + lfs_size_t count = 0; + while (true) { + int err = lfsr_rbyd_lookupnext(lfs, rbyd, id+1, 0, + &id, NULL, NULL, NULL); + if (err && err != LFS_ERR_NOENT) { + return err; + } + if (err == LFS_ERR_NOENT) { + return true; + } + + count += 1; + if (count > (lfs_size_t)cutoff) { + return false; + } + } +} @@ -3569,163 +3450,8 @@ static int lfsr_btree_commit(lfs_t *lfs, // TODO wait should we also move if there is corruption here? return err; } - - #ifndef LFSR_BTREE_NOTHRESH - // can't commit, try to compact - lfsr_rbyd_t rbyd_; - lfs_size_t lower_id; - lfs_size_t lower_dsize; - lfs_size_t tcount; if (err) { -// // TODO can we combine this with lfsr_rbyd_inthresh? -// // -// // first check if we are a degenerate root and can be reverted to -// // an inlined btree -// // -// // This gets a bit weird since we're defering our pending -// // attributes to after the compaction. When we can/can't be inlined -// // depends on those attributes, but trying to evaluate attributes -// // is complicated and expensive. -// // -// // Instead we just let the upper layers indicate a cutoff for when -// // an rbyd can be inlined, and leave the inlining work up to the -// // upper layers. -// if (pid == -1) { -// int degenerate = lfsr_rbyd_incutoff(lfs, rbyd, cutoff); -// if (degenerate) { -// return degenerate; -// } -// } - - // check if we're within our compaction threshold, otherwise we - // need to split - // - // while we're doing this we also check if we are a degenerate root - // and can be reverted to an inlined btree, though this gets a bit - // weird since we won't know the exact id count until after - // appending attributes, so we leave the cutoff _before_ pending - // attributes up to upper layers - // - // note we account for the revision count here - int estimate = lfsr_rbyd_estimate(lfs, rbyd, - -1, (pid == -1 ? cutoff : -1), 0, sizeof(uint32_t), - &lower_id, &lower_dsize); - if (estimate < 0) { - return estimate; - } - - if (estimate == LFSR_ESTIMATE_DEGENERATE) { - // TODO LFSR_BTREE_DEGENERATE? just propagate LFSR_ESTIMATE_DEGENERATE? - return true; - } else if (estimate == LFSR_ESTIMATE_OVERFLOWS) { - goto split; - } - -// int inthresh = lfsr_rbyd_inthresh(lfs, rbyd, -// false, (pid == -1 ? cutoff : -1), 0, sizeof(uint32_t), -// &lower_id, &lower_dsize); -// if (inthresh < 0) { -// return inthresh; -// } -// -// if (inthresh == LFSR_INTHRESH_DEGENERATE) { -// return true; -// } else if (!inthresh) { -// LFS_ASSERT(lower_id > 0); -// goto split; -// } - - // TODO were we doing something funky with rev? - // allocate a new rbyd - err = lfsr_rbyd_alloc(lfs, &rbyd_, rbyd->rev+1); - if (err) { - LFS_ASSERT(err != LFS_ERR_RANGE); - return err; - } - - // try to copy over tags - lfs_ssize_t id = 0; - lfsr_tag_t tag = 0; - while (true) { - lfsr_data_t data; - err = lfsr_rbyd_lookupnext(lfs, rbyd, id, lfsr_tag_next(tag), - &id, &tag, NULL, &data); - if (err && err != LFS_ERR_NOENT) { - LFS_ASSERT(err != LFS_ERR_RANGE); - return err; - } - if (err == LFS_ERR_NOENT) { - break; - } - - // Because it makes a lot of the split-sensitive cross-id - // operations easier, we can end up with an occasional - // "vestigial" name tag on the first id in a block. We make - // sure to ignore these during lookup, but it would be more - // complicated then it's worth to clean these up proactively. - // - // Discarding these during compaction is easy and prevents any - // real storage cost. - if (lfsr_tag_suptype(tag) == LFSR_TAG_NAME - && rbyd_.weight == 0) { - continue; - } - - // note we need to account for the missing weight of vestigial - // name tags in the following branch tag, which is why we - // calculate weight like this - lfs_size_t w = id+1 - rbyd_.weight; - - // append the attr - err = lfsr_rbyd_append(lfs, &rbyd_, - id-lfs_smax32(w-1, 0), lfsr_tag_setmk(tag), +w, - data); - if (err) { - LFS_ASSERT(err != LFS_ERR_RANGE); - return err; - } - - // keep track of the number of tags we've written in case we - // try to merge - tcount += 1; - } - - // append any pending attrs, it's up to upper - // layers to make sure these always fit - for (lfs_size_t i = 0; i < attr_count; i++) { - err = lfsr_rbyd_append(lfs, &rbyd_, - attrs[i].id, attrs[i].tag, attrs[i].delta, - attrs[i].data); - if (err) { - LFS_ASSERT(err != LFS_ERR_RANGE); - return err; - } - - // keep track of the number of tags we've written in case we - // try to merge - tcount += 1; - } - - // TODO do we really need a threshold for this? should we just - // always try since this only happens on compaction and our merges - // are defered? - // TODO should we allow merging both siblings? - // TODO we should have a benchmark for how removes affect tree size - // is our compacted size too small? try to merge with one of - // our siblings - if (rbyd_.off < lfs->cfg->block_size/4) { - goto merge; - merge_abort:; - } - - // finalize commit - err = lfsr_rbyd_commit(lfs, &rbyd_, NULL, 0); - if (err) { - LFS_ASSERT(err != LFS_ERR_RANGE); - return err; - } - - *rbyd = rbyd_; + goto compact; } // done? @@ -3760,6 +3486,173 @@ static int lfsr_btree_commit(lfs_t *lfs, cutoff = -1; continue; + compact:; + // can't commit, try to compact + lfsr_rbyd_t rbyd_; + lfs_size_t lower_id; + lfs_size_t lower_dsize; + lfs_size_t tcount; + + // first check if we are a degenerate root and can be reverted to + // an inlined btree + // + // This gets a bit weird since we're defering our pending + // attributes to after the compaction. When we can/can't be inlined + // depends on those attributes, but trying to evaluate attributes + // is complicated and expensive. + // + // Instead we just let the upper layers indicate a cutoff for when + // an rbyd can be inlined, and leave the inlining work up to the + // upper layers. + if (pid == -1) { + int degenerate = lfsr_rbyd_isdegenerate(lfs, rbyd, cutoff); + if (degenerate) { + return degenerate; + } + } + + // check if we're within our compaction threshold, otherwise we + // need to split + // + // note we account for the revision count here + // TODO ugh, need to move cutoff out again, it doesn't make sense here + // with the early threshold terminate + int fits = lfsr_rbyd_estimate(lfs, rbyd, + -1, 0, sizeof(uint32_t), + &lower_id, &lower_dsize); + if (fits < 0) { + return fits; + } + + if (!fits) { + // need to split + goto split; + } + + // TODO were we doing something funky with rev? + // allocate a new rbyd + err = lfsr_rbyd_alloc(lfs, &rbyd_, rbyd->rev+1); + if (err) { + LFS_ASSERT(err != LFS_ERR_RANGE); + return err; + } + + // try to copy over tags + lfs_ssize_t id = 0; + lfsr_tag_t tag = 0; + while (true) { + lfsr_data_t data; + err = lfsr_rbyd_lookupnext(lfs, rbyd, id, lfsr_tag_next(tag), + &id, &tag, NULL, &data); + if (err && err != LFS_ERR_NOENT) { + LFS_ASSERT(err != LFS_ERR_RANGE); + return err; + } + if (err == LFS_ERR_NOENT) { + break; + } + + // Because it makes a lot of the split-sensitive cross-id + // operations easier, we can end up with an occasional + // "vestigial" name tag on the first id in a block. We make + // sure to ignore these during lookup, but it would be more + // complicated then it's worth to clean these up proactively. + // + // Discarding these during compaction is easy and prevents any + // real storage cost. + if (lfsr_tag_suptype(tag) == LFSR_TAG_NAME + && rbyd_.weight == 0) { + continue; + } + + // note we need to account for the missing weight of vestigial + // name tags in the following branch tag, which is why we + // calculate weight like this + lfs_size_t w = id+1 - rbyd_.weight; + + // append the attr + err = lfsr_rbyd_append(lfs, &rbyd_, + id-lfs_smax32(w-1, 0), lfsr_tag_setmk(tag), +w, + data); + if (err) { + LFS_ASSERT(err != LFS_ERR_RANGE); + return err; + } + + // keep track of the number of tags we've written in case we + // try to merge + tcount += 1; + } + + // append any pending attrs, it's up to upper + // layers to make sure these always fit + for (lfs_size_t i = 0; i < attr_count; i++) { + err = lfsr_rbyd_append(lfs, &rbyd_, + attrs[i].id, attrs[i].tag, attrs[i].delta, + attrs[i].data); + if (err) { + LFS_ASSERT(err != LFS_ERR_RANGE); + return err; + } + + // keep track of the number of tags we've written in case we + // try to merge + tcount += 1; + } + + // TODO do we really need a threshold for this? should we just + // always try since this only happens on compaction and our merges + // are defered? + // TODO should we allow merging both siblings? + // TODO we should have a benchmark for how removes affect tree size + // is our compacted size too small? try to merge with one of + // our siblings + if (rbyd_.off < lfs->cfg->block_size/4) { + goto merge; + merge_abort:; + } + + // finalize commit + err = lfsr_rbyd_commit(lfs, &rbyd_, NULL, 0); + if (err) { + LFS_ASSERT(err != LFS_ERR_RANGE); + return err; + } + + *rbyd = rbyd_; + + // done? + if (pid == -1) { + break; + } + + // cannibalize some attributes in our attr list to store + // our branch + scratch_buf = (uint8_t*)&attrs[2]; + d = lfsr_branch_todisk(lfs, rbyd, scratch_buf); + if (d < 0) { + return d; + } + + // prepare commit to parent, tail recursing upwards + // + // note that since we defer merges to compaction time, we can + // end up removing an rbyd here + if (rbyd->weight == 0) { + attrs[0] = LFSR_ATTR(pid, MKUNR, +rbyd->weight-pweight, + scratch_buf, d); + attr_count = 1; + } else { + attrs[0] = LFSR_ATTR(pid, UNR, +rbyd->weight-pweight, NULL, 0); + attrs[1] = LFSR_ATTR(pid+rbyd->weight-pweight, BTREE, 0, + scratch_buf, d); + attr_count = 2; + } + + *rbyd = parent; + cutoff = -1; + continue; + split:; // first figure out which id we need to split around LFS_ASSERT(lower_id > 0); @@ -3784,8 +3677,8 @@ static int lfsr_btree_commit(lfs_t *lfs, } // copy over tags < split_id - lfs_ssize_t id = 0; - lfsr_tag_t tag = 0; + id = 0; + tag = 0; while (true) { lfs_size_t w; lfsr_data_t data; @@ -4041,15 +3934,15 @@ static int lfsr_btree_commit(lfs_t *lfs, LFS_ASSERT(sibling.weight == sweight); // estimate if our sibling will fit - int estimate = lfsr_rbyd_estimate(lfs, &sibling, - -1, -1, tcount, rbyd_.off, + int fits = lfsr_rbyd_estimate(lfs, &sibling, + -1, tcount, rbyd_.off, NULL, NULL); - if (estimate < 0) { - return estimate; + if (fits < 0) { + return fits; } // don't fit? can't merge - if (estimate == LFSR_ESTIMATE_OVERFLOWS) { + if (!fits) { continue; } @@ -4156,499 +4049,6 @@ static int lfsr_btree_commit(lfs_t *lfs, *rbyd = parent; cutoff = -1; continue; - - #else -// -// // can't commit, try to compact -// lfsr_rbyd_t rbyd_; -// lfs_size_t lower_dsize = 0; -// if (err) { -// // TODO can we combine this with lfsr_rbyd_inthresh? -// // -// // first check if we are a degenerate root and can be reverted to -// // an inlined btree -// // -// // This gets a bit weird since we're defering our pending -// // attributes to after the compaction. When we can/can't be inlined -// // depends on those attributes, but trying to evaluate attributes -// // is complicated and expensive. -// // -// // Instead we just let the upper layers indicate a cutoff for when -// // an rbyd can be inlined, and leave the inlining work up to the -// // upper layers. -// if (pid == -1) { -// int degenerate = lfsr_rbyd_incutoff(lfs, rbyd, cutoff); -// if (degenerate) { -// return degenerate; -// } -// } -// -// // TODO were we doing something funky with rev? -// // allocate a new rbyd -// err = lfsr_rbyd_alloc(lfs, &rbyd_, rbyd->rev+1); -// if (err) { -// return err; -// } -// -// // try to copy over ids -// lfs_ssize_t id = 0; -// lfsr_tag_t tag = 0; -// while (true) { -// lfsr_data_t data; -// err = lfsr_rbyd_lookupnext(lfs, rbyd, id, lfsr_tag_next(tag), -// &id, &tag, NULL, &data); -// if (err && err != LFS_ERR_NOENT) { -// return err; -// } -// if (err == LFS_ERR_NOENT) { -// break; -// } -// -// // Because it makes a lot of the split-sensitive cross-id -// // operations easier, we can end up with an occasional -// // "vestigial" name tag on the first id in a block. We make -// // sure to ignore these during lookup, but it would be more -// // complicated then it's worth to clean these up proactively. -// // -// // Discarding these during compaction is easy and prevents any -// // real storage cost. -// if (lfsr_tag_suptype(tag) == LFSR_TAG_NAME -// && rbyd_.weight == 0) { -// continue; -// } -// -// // note we need to account for the missing weight of vestigial -// // name tags in the following branch tag, which is why we -// // calculate weight like this -// lfs_size_t w = id+1 - rbyd_.weight; -// -// // keep track of worst-case encoding size in case we need to -// // split -// lower_dsize += LFSR_TAG_DSIZE + lfsr_data_size(data); -// -// // append the attr -// err = lfsr_rbyd_append(lfs, &rbyd_, -// id-lfs_smax32(w-1, 0), lfsr_tag_setmk(tag), +w, -// data); -// if (err) { -// return err; -// } -// -// // keep rbyd < our compaction threshold (1/2) to avoid -// // degenerate cases -// if (rbyd_.off > lfs->cfg->block_size/2) { -// goto split; -// } -// } -// -// // append any pending attrs, it's up to upper -// // layers to make sure these always fit -// for (lfs_size_t i = 0; i < attr_count; i++) { -// err = lfsr_rbyd_append(lfs, &rbyd_, -// attrs[i].id, attrs[i].tag, attrs[i].delta, -// attrs[i].data); -// if (err) { -// LFS_ASSERT(err != LFS_ERR_RANGE); -// return err; -// } -// } -// -// // is our compacted size too small? try to merge with one of -// // our siblings -// if (rbyd_.off < lfs->cfg->block_size/4) { -// goto merge; -// merge_abort:; -// } -// -// // finalize commit -// err = lfsr_rbyd_commit(lfs, &rbyd_, NULL, 0); -// if (err) { -// LFS_ASSERT(err != LFS_ERR_RANGE); -// return err; -// } -// -// *rbyd = rbyd_; -// } -// -// // done? -// if (pid == -1) { -// break; -// } -// -// // cannibalize some attributes in our attr list to store -// // our branch -// uint8_t *scratch_buf = (uint8_t*)&attrs[2]; -// lfs_ssize_t d = lfsr_branch_todisk(lfs, rbyd, scratch_buf); -// if (d < 0) { -// return d; -// } -// -// // prepare commit to parent, tail recursing upwards -// // -// // note that since we defer merges to compaction time, we can -// // end up removing an rbyd here -// if (rbyd->weight == 0) { -// attrs[0] = LFSR_ATTR(pid, MKUNR, +rbyd->weight-pweight, -// scratch_buf, d); -// attr_count = 1; -// } else { -// attrs[0] = LFSR_ATTR(pid, UNR, +rbyd->weight-pweight, NULL, 0); -// attrs[1] = LFSR_ATTR(pid+rbyd->weight-pweight, BTREE, 0, -// scratch_buf, d); -// attr_count = 2; -// } -// -// *rbyd = parent; -// cutoff = -1; -// continue; -// -// split:; -// // first figure out which id we need to split around -// lfs_ssize_t split_id = lfsr_rbyd_bisect(lfs, rbyd, -// rbyd_.weight, lower_dsize); -// if (split_id < 0) { -// return split_id; -// } -// -// // we can keep our attempted compact, we just need to remove any -// // ids that belong in the sibling -// LFS_ASSERT((lfs_size_t)split_id < rbyd_.weight); -// err = lfsr_rbyd_append(lfs, &rbyd_, -// rbyd_.weight-1, LFSR_TAG_MKUNR, -(rbyd_.weight-split_id), -// LFSR_DATA_NULL); -// if (err) { -// return err; -// } -// -// // commit pending attrs, these may need to go into both rbyds, -// // upper layers should make sure this can't fail by limiting the -// // maximum commit size -// // TODO filter-like tag? "from" but from device? -// lfs_size_t split_id_ = split_id; -// for (lfs_size_t i = 0; i < attr_count; i++) { -// if (attrs[i].id < (lfs_ssize_t)split_id_) { -// err = lfsr_rbyd_append(lfs, &rbyd_, -// attrs[i].id, attrs[i].tag, attrs[i].delta, -// attrs[i].data); -// if (err) { -// LFS_ASSERT(err != LFS_ERR_RANGE); -// return err; -// } -// } -// -// // we need to make sure we keep split_id updated with weight changes -// if (attrs[i].id < (lfs_ssize_t)split_id_) { -// split_id_ += attrs[i].delta; -// } -// } -// -// // finalize commit -// err = lfsr_rbyd_commit(lfs, &rbyd_, NULL, 0); -// if (err) { -// LFS_ASSERT(err != LFS_ERR_RANGE); -// return err; -// } -// -// // create a sibling and copy remaining ids there, upper layers -// // should make sure this can't fail by limiting the maximum -// // commit size -// lfsr_rbyd_t sibling; -// err = lfsr_rbyd_alloc(lfs, &sibling, rbyd->rev+1); -// if (err) { -// return err; -// } -// -// lfs_ssize_t id = split_id; -// lfsr_tag_t tag = 0; -// while (true) { -// lfs_size_t w; -// lfsr_data_t data; -// err = lfsr_rbyd_lookupnext(lfs, rbyd, id, lfsr_tag_next(tag), -// &id, &tag, &w, &data); -// if (err && err != LFS_ERR_NOENT) { -// return err; -// } -// if (err == LFS_ERR_NOENT) { -// break; -// } -// -// // append the attr -// err = lfsr_rbyd_append(lfs, &sibling, -// id-split_id-lfs_smax32(w-1, 0), lfsr_tag_setmk(tag), +w, -// data); -// if (err) { -// return err; -// } -// } -// -// // commit pending attrs, these may need to go into both rbyds, -// // upper layers should make sure this can't fail by limiting the -// // maximum commit size -// split_id_ = split_id; -// for (lfs_size_t i = 0; i < attr_count; i++) { -// if (attrs[i].id >= (lfs_ssize_t)split_id_) { -// err = lfsr_rbyd_append(lfs, &sibling, -// attrs[i].id-split_id_, attrs[i].tag, attrs[i].delta, -// attrs[i].data); -// if (err) { -// LFS_ASSERT(err != LFS_ERR_RANGE); -// return err; -// } -// } -// -// // we need to make sure we keep split_id updated with weight changes -// if (attrs[i].id < (lfs_ssize_t)split_id_) { -// split_id_ += attrs[i].delta; -// } -// } -// -// // finalize commit -// err = lfsr_rbyd_commit(lfs, &sibling, NULL, 0); -// if (err) { -// LFS_ASSERT(err != LFS_ERR_RANGE); -// return err; -// } -// -// // lookup first name in sibling to use as the split name -// // -// // note we need to do this after playing out pending attrs in case -// // they introduce a new name! -// lfsr_tag_t stag; -// lfsr_data_t sdata; -// err = lfsr_rbyd_lookupnext(lfs, &sibling, 0, LFSR_TAG_NAME, -// NULL, &stag, NULL, &sdata); -// if (err) { -// LFS_ASSERT(err != LFS_ERR_NOENT); -// return err; -// } -// -// // cannibalize some attributes in our attr list to store -// // our branches -// uint8_t *scratch_buf1 = (uint8_t*)&attrs[4]; -// uint8_t *scratch_buf2 = (uint8_t*)&attrs[4] + LFSR_BRANCH_DSIZE; -// lfs_ssize_t d1 = lfsr_branch_todisk(lfs, &rbyd_, scratch_buf1); -// if (d1 < 0) { -// return d1; -// } -// lfs_ssize_t d2 = lfsr_branch_todisk(lfs, &sibling, scratch_buf2); -// if (d2 < 0) { -// return d2; -// } -// -// // no parent? introduce a new trunk -// if (pid == -1) { -// int err = lfsr_rbyd_alloc(lfs, &parent, 1); -// if (err) { -// return err; -// } -// -// // prepare commit to parent, tail recursing upwards -// attrs[0] = LFSR_ATTR(0, MKBTREE, +rbyd_.weight, -// scratch_buf1, d1); -// attrs[1] = (lfsr_tag_suptype(stag) == LFSR_TAG_NAME -// ? LFSR_ATTR_DATA(rbyd_.weight, MKBRANCH, +sibling.weight, -// sdata) -// : LFSR_ATTR_NOOP); -// attrs[2] = (lfsr_tag_suptype(stag) == LFSR_TAG_NAME -// ? LFSR_ATTR(0+rbyd_.weight+sibling.weight-1, BTREE, 0, -// scratch_buf2, d2) -// : LFSR_ATTR(0+rbyd_.weight, MKBTREE, +sibling.weight, -// scratch_buf2, d2)); -// attr_count = 3; -// -// // yes parent? push up split -// } else { -// // prepare commit to parent, tail recursing upwards -// attrs[0] = LFSR_ATTR(pid, UNR, +rbyd_.weight-pweight, NULL, 0); -// attrs[1] = LFSR_ATTR(pid-(pweight-1)+rbyd_.weight-1, BTREE, 0, -// scratch_buf1, d1); -// attrs[2] = (lfsr_tag_suptype(stag) == LFSR_TAG_NAME -// ? LFSR_ATTR_DATA(pid-(pweight-1)+rbyd_.weight, -// MKBRANCH, +sibling.weight, -// sdata) -// : LFSR_ATTR_NOOP); -// attrs[3] = (lfsr_tag_suptype(stag) == LFSR_TAG_NAME -// ? LFSR_ATTR(pid-(pweight-1)+rbyd_.weight+sibling.weight-1, -// BTREE, 0, -// scratch_buf2, d2) -// : LFSR_ATTR(pid-(pweight-1)+rbyd_.weight, -// MKBTREE, +sibling.weight, -// scratch_buf2, d2)); -// attr_count = 4; -// } -// -// *rbyd = parent; -// cutoff = -1; -// continue; -// -// merge:; -// // no parent? can't merge -// if (pid == -1) { -// goto merge_abort; -// } -// -// // only child? can't merge -// if (pweight == parent.weight) { -// goto merge_abort; -// } -// -// // last child? try the left sibling -// lfs_ssize_t sid; -// lfs_ssize_t sdelta; -// if ((lfs_size_t)pid == parent.weight-1) { -// sid = pid-pweight; -// sdelta = 0; -// // not last child? try the right sibling -// } else { -// sid = pid+1; -// sdelta = rbyd_.weight; -// } -// -// // try looking up the sibling -// // TODO do we really need to fetch sweight if we get it in our -// // btree struct? -// lfs_size_t sweight; -// err = lfsr_rbyd_lookupnext(lfs, &parent, sid, LFSR_TAG_NAME, -// &sid, &stag, &sweight, &sdata); -// if (err) { -// // no sibling? can't merge -// if (err == LFS_ERR_NOENT) { -// goto merge_abort; -// } -// return err; -// } -// -// if (stag == LFSR_TAG_NAME) { -// err = lfsr_rbyd_lookupnext(lfs, &parent, sid, LFSR_TAG_STRUCT, -// NULL, &stag, NULL, &sdata); -// if (err) { -// LFS_ASSERT(err != LFS_ERR_NOENT); -// return err; -// } -// } -// -// // no sibling? can't merge -// if (stag != LFSR_TAG_BTREE) { -// goto merge_abort; -// } -// -// d = lfsr_branch_fromdisk(lfs, &sibling, sdata); -// if (d < 0) { -// return d; -// } -// LFS_ASSERT(sibling.weight == sweight); -// -// // try to add our sibling's tags to our rbyd -// lfs_size_t rweight_ = rbyd_.weight; -// id = 0; -// tag = 0; -// while (true) { -// lfs_size_t w; -// lfsr_data_t data; -// err = lfsr_rbyd_lookupnext(lfs, &sibling, id, lfsr_tag_next(tag), -// &id, &tag, &w, &data); -// if (err && err != LFS_ERR_NOENT) { -// return err; -// } -// if (err == LFS_ERR_NOENT) { -// break; -// } -// -// // append the attr -// err = lfsr_rbyd_append(lfs, &rbyd_, -// sdelta+id-lfs_smax32(w-1, 0), lfsr_tag_setmk(tag), +w, -// data); -// if (err) { -// return err; -// } -// -// // if we exceed our compaction threshold our merge has -// // failed, clean up ids and merge_abort -// if (rbyd_.off > lfs->cfg->block_size/2) { -// err = lfsr_rbyd_append(lfs, &rbyd_, -// sdelta+(rbyd_.weight-rweight_)-1, -// LFSR_TAG_MKUNR, -(rbyd_.weight-rweight_), -// LFSR_DATA_NULL); -// if (err) { -// return err; -// } -// -// goto merge_abort; -// } -// } -// -// if (sweight > 0 && rweight_ > 0) { -// // bring in name that previously split the siblings -// lfsr_tag_t split_tag; -// lfsr_data_t split_data; -// err = lfsr_rbyd_lookupnext(lfs, &parent, -// (sdelta == 0 ? pid : sid), LFSR_TAG_NAME, -// NULL, &split_tag, NULL, &split_data); -// if (err) { -// LFS_ASSERT(err != LFS_ERR_RANGE); -// return err; -// } -// -// if (lfsr_tag_suptype(split_tag) == LFSR_TAG_NAME) { -// // lookup the id (weight really) of the previously-split entry -// lfs_ssize_t split_id; -// err = lfsr_rbyd_lookupnext(lfs, &rbyd_, -// (sdelta == 0 ? sweight : rweight_), LFSR_TAG_NAME, -// &split_id, NULL, NULL, NULL); -// if (err) { -// LFS_ASSERT(err != LFS_ERR_NOENT); -// return err; -// } -// -// err = lfsr_rbyd_append(lfs, &rbyd_, -// split_id, LFSR_TAG_BRANCH, 0, split_data); -// if (err) { -// LFS_ASSERT(err != LFS_ERR_RANGE); -// return err; -// } -// } -// } -// -// err = lfsr_rbyd_commit(lfs, &rbyd_, NULL, 0); -// if (err) { -// LFS_ASSERT(err != LFS_ERR_RANGE); -// return err; -// } -// -// // we must have a parent at this point, but is our parent degenerate? -// LFS_ASSERT(pid != -1); -// if (pweight+sweight == lfsr_btree_weight(btree)) { -// // collapse our parent, decreasing the height of the tree -// *rbyd = rbyd_; -// break; -// -// } else { -// // make pid the lower child so the following math is easier -// if (pid > sid) { -// lfs_sswap32(&pid, &sid); -// lfs_swap32(&pweight, &sweight); -// } -// -// // cannibalize some attributes in our attr list to store -// // our branch -// uint8_t *scratch_buf = (uint8_t*)&attrs[3]; -// lfs_ssize_t d = lfsr_branch_todisk(lfs, &rbyd_, scratch_buf); -// if (d < 0) { -// return d; -// } -// -// // prepare commit to parent, tail recursing upwards -// attrs[0] = LFSR_ATTR(sid, MKUNR, -sweight, NULL, 0); -// attrs[1] = LFSR_ATTR(pid, UNR, +rbyd_.weight-pweight, NULL, 0); -// attrs[2] = LFSR_ATTR(pid+rbyd_.weight-pweight, BTREE, 0, -// scratch_buf, d); -// attr_count = 3; -// } -// -// *rbyd = parent; -// cutoff = -1; -// continue; - #endif } // at this point rbyd should be the trunk of our tree @@ -5282,14 +4682,14 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid, // need to split // // note we account for the revision count here - int estimate = lfsr_rbyd_estimate(lfs, &mdir->rbyd, - -1, -1, 0, sizeof(uint32_t), + int fits = lfsr_rbyd_estimate(lfs, &mdir->rbyd, + -1, 0, sizeof(uint32_t), &lower_id, &lower_dsize); - if (estimate < 0) { - return estimate; + if (fits < 0) { + return fits; } - if (estimate != LFSR_ESTIMATE_FITS) { + if (!fits) { // are we inlined into the mroot? we need to uninline // before we split, and it's possible uninlining makes the mdir // small enough that we don't even need to split @@ -5297,17 +4697,17 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid, uninlining = true; // do we still need to split? - estimate = lfsr_rbyd_estimate(lfs, &mdir->rbyd, - // note id was changed to 0 here - 0, -1, 0, sizeof(uint32_t), + fits = lfsr_rbyd_estimate(lfs, &mdir->rbyd, + // note init_id was changed to 0 here + 0, 0, sizeof(uint32_t), &lower_id, &lower_dsize); - if (estimate < 0) { - return estimate; + if (fits < 0) { + return fits; } } - if (estimate == LFSR_ESTIMATE_OVERFLOWS) { - // needs a split + if (!fits) { + // needs to split goto split; }