From 5636895eee69142bbf9ffe295725a3eaa2643304 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 3 Dec 2023 21:09:24 -0600 Subject: [PATCH] Significantly improved lfsr_btree_carve The main optimization here is to try the minimize the number of individual btree/bshrub commits. We can't always perform lfsr_btree_carve in a single commit unfortunately, due to unbounded crystal fragments and needing to commit to different leaf rbyds, but we can combine attrs into single commits more than we were previously. This is especially important for bshrubs, where commits can trigger full mdir compactions. Additionally, the method we use for breaking up small blocks into fragments has been changed to never "lose" the underlying block pointer until the fragmentation is complete. This prevents the block pointer misallocation bug found earlier. In theory this won't be necessary once file writes create on-stack bshrub copies for error recovery, but it's at least nice to prove that this is possible in case we ever want to not maintain on-stack bshrub copies (code savings?). code stack before: 31512 2648 after: 31280 (-0.7%) 2736 (+3.3%) --- lfs.c | 520 +++++++++++++++++++++++++--------------------------------- 1 file changed, 219 insertions(+), 301 deletions(-) diff --git a/lfs.c b/lfs.c index 268a3bc1..f5244ef2 100644 --- a/lfs.c +++ b/lfs.c @@ -1161,7 +1161,8 @@ static lfsr_data_t lfsr_data_add(lfsr_data_t data, lfs_size_t off) { } static lfsr_data_t lfsr_data_truncate(lfsr_data_t data, lfs_size_t size) { - LFS_ASSERT(size <= lfsr_data_size(&data)); + // limit size to our data range + size = lfs_min32(size, lfsr_data_size(&data)); // on-disk? update size if (lfsr_data_ondisk(&data)) { @@ -1188,7 +1189,8 @@ static lfsr_data_t lfsr_data_truncate(lfsr_data_t data, lfs_size_t size) { } static lfsr_data_t lfsr_data_fruncate(lfsr_data_t data, lfs_size_t size) { - LFS_ASSERT(size <= lfsr_data_size(&data)); + // limit size to our data range + size = lfs_min32(size, lfsr_data_size(&data)); // lfsr_data_fruncate and lfsr_data_add are basically the same operation return lfsr_data_add(data, lfsr_data_size(&data) - size); @@ -4860,6 +4862,11 @@ static int lfsr_bshrub_commit(lfs_t *lfs, } } + // bshrubs need to be manually staged if they aren't in our opened + // mdir list, though this is only allowed for new bshrubs due to + // mdir compactions + bshrub->rbyd_ = bshrub->rbyd; + // commit to shrub err = lfsr_mdir_commit(lfs, mdir, LFSR_ATTRS( LFSR_ATTR(mdir->mid, @@ -4869,6 +4876,7 @@ static int lfsr_bshrub_commit(lfs_t *lfs, return err; } + bshrub->rbyd = bshrub->rbyd_; bshrub->progged += progged; } @@ -9645,52 +9653,61 @@ lfs_ssize_t lfsr_file_read(lfs_t *lfs, lfsr_file_t *file, static int lfsr_file_carve(lfs_t *lfs, lfsr_file_t *file, lfs_off_t pos, lfs_off_t weight, lfs_soff_t delta, lfsr_tag_t tag, lfsr_data_t data) { - // note! we take special care to make sure our btree size doesn't - // overflow, even temporarily + // Note! This function has some rather special constraints: + // + // 1. We must never allow our btree size to overflow, even temporarily. + // + // 2. We must not lose track of bptrs until we no longer need them, to + // prevent incorrect allocation from the block allocator. + // + // 3. We should avoid copying data fragments as much as possible. + // + // These requirements end up conflicting a bit... + // + // The second requirement isn't strictly necessary if we track temporary + // copies during file writes, but it is nice to prove this constraint is + // possible in case we ever don't track temporary copies. TODO this + // currently isn't implemented. // TODO do we ever create direct bptrs with this strategy? // always convert to bshrub/btree when this function is called if (!lfsr_file_isbshruborbtree(file)) { - // note bshrub commits must always be in our opened mdir list - // - // TODO is this a reasonable design? - // - lfsr_tag_t tag = 0; - lfs_off_t weight; - lfsr_data_t data; - uint8_t bptr_buf[LFSR_BPTR_DSIZE]; - // have data? - if (lfsr_file_isbsprout(file)) { - tag = LFSR_TAG_DATA; - weight = lfsr_data_size(&file->u.bsprout.data); - data = file->u.bsprout.data; - - // have bptr? - } else if (lfsr_file_isbptr(file)) { - tag = LFSR_TAG_BLOCK; - weight = lfsr_bptr_size(&file->u.bptr); - data = lfsr_data_frombptr(&file->u.bptr, bptr_buf); - } - - int err = lfsr_bshrub_alloc(lfs, &file->mdir, &file->u.bshrub); + lfsr_bshrub_t bshrub; + int err = lfsr_bshrub_alloc(lfs, &file->mdir, &bshrub); if (err) { return err; } - if (tag) { + if (lfsr_file_uweight(file) > 0) { + uint8_t bptr_buf[LFSR_BPTR_DSIZE]; err = lfsr_bshrub_commit(lfs, - &file->mdir, &file->u.bshrub, LFSR_ATTRS( - LFSR_ATTR(0, - TAG(tag), +weight, DATA(data)))); + &file->mdir, &bshrub, LFSR_ATTRS( + (lfsr_file_isbsprout(file)) + ? LFSR_ATTR(0, + DATA, +lfsr_file_uweight(file), + DATA(file->u.bsprout.data)) + : LFSR_ATTR(0, + BLOCK, +lfsr_file_uweight(file), + FROMBPTR(&file->u.bptr, bptr_buf)))); if (err) { return err; } } + + file->u.bshrub = bshrub; } + // TODO adopt this pattern for other scratch attrs + // + // try to merge commits where possible + lfsr_attr_t attrs[3]; + lfs_size_t attr_count = 0; + uint8_t buf[2*LFSR_BPTR_DSIZE]; + lfs_size_t buf_size = 0; + // try to carve any existing data - while (pos < lfsr_file_uweight(file) && weight > 0) { + while (pos < lfsr_file_uweight(file)) { lfsr_bid_t bid_; lfsr_tag_t tag_; lfsr_bid_t weight_; @@ -9705,294 +9722,197 @@ static int lfsr_file_carve(lfs_t *lfs, lfsr_file_t *file, LFS_ASSERT(tag_ == LFSR_TAG_DATA || tag_ == LFSR_TAG_BLOCK); - // note an entry can be both a left and right sibling! + // note, an entry can be both a left and right sibling + lfsr_data_t left_slice_ = lfsr_data_truncate(data_, + pos - (bid_-(weight_-1))); + lfsr_data_t right_slice_ = lfsr_data_add(data_, + pos+weight - (bid_-(weight_-1))); - // found left sibling? - if (pos > bid_-(weight_-1)) { - lfs_off_t overlap_ = (bid_+1) - pos; - LFS_ASSERT((lfs_soff_t)overlap_ >= 0); + // left sibling needs carving but falls underneath our + // crystallization threshold? break into fragments + while (tag_ == LFSR_TAG_BLOCK + && lfsr_data_size(&left_slice_) > lfs->cfg->fragment_size + && lfsr_data_size(&left_slice_) <= lfs->cfg->crystal_size) { + lfsr_bptr_t bptr_ = { + .block = data_.u.disk.block, + .off = data_.u.disk.off + lfs->cfg->fragment_size, + .size = lfsr_data_size(&data_) - lfs->cfg->fragment_size, + }; - lfsr_data_t slice_ = lfsr_data_truncate(data_, - lfs_min32( - weight_ - overlap_, - lfsr_data_size(&data_))); - - // we can get away with a grow attribute in some cases, avoiding - // a data copy - if (lfsr_data_size(&data_) == lfsr_data_size(&slice_)) { - err = lfsr_bshrub_commit(lfs, - &file->mdir, &file->u.bshrub, LFSR_ATTRS( - LFSR_ATTR(bid_, - GROW, -overlap_, NULL()))); - if (err) { - return err; - } - - // carve bptr? - } else if (tag_ == LFSR_TAG_BLOCK - && lfsr_data_size(&slice_) > lfs->cfg->crystal_size) { - lfsr_bptr_t bptr_ = { - .block = slice_.u.disk.block, - .off = slice_.u.disk.off, - .size = lfsr_data_size(&slice_), - }; - - uint8_t bptr_buf[LFSR_BPTR_DSIZE]; - err = lfsr_bshrub_commit(lfs, - &file->mdir, &file->u.bshrub, LFSR_ATTRS( - LFSR_ATTR(bid_, - GROW(WIDE(BLOCK)), -overlap_, - FROMBPTR(&bptr_, bptr_buf)))); - if (err) { - return err; - } - - // break into multiple fragments and carve if bptr/fragment is - // below our crystal size - } else { - // write the last fragment first to avoid overflow issues - err = lfsr_bshrub_commit(lfs, - &file->mdir, &file->u.bshrub, LFSR_ATTRS( - LFSR_ATTR(bid_, - GROW(WIDE(DATA)), -overlap_ - lfs_aligndown( - lfsr_data_size(&slice_)-1, - lfs->cfg->fragment_size), - DATA(lfsr_data_add(slice_, - lfs_aligndown( - lfsr_data_size(&slice_)-1, - lfs->cfg->fragment_size)))))); - if (err) { - return err; - } - - for (lfs_size_t i = 0; - i < lfs_aligndown( - lfsr_data_size(&slice_)-1, - lfs->cfg->fragment_size); - i += lfs->cfg->fragment_size) { - err = lfsr_bshrub_commit(lfs, - &file->mdir, &file->u.bshrub, LFSR_ATTRS( - LFSR_ATTR(bid_-(weight_-1) + i, - DATA, +lfs->cfg->fragment_size, - DISK( - slice_.u.disk.block, - slice_.u.disk.off + i, - lfs->cfg->fragment_size)))); - if (err) { - return err; - } - } - } - - // TODO adopt this logic in carveshrub? it avoids a redundant - // lookup - // - // found a split? (left sibing == right sibling) - if (overlap_ > weight) { - lfs_off_t overlap_ = (pos + weight) - (bid_-(weight_-1)); - LFS_ASSERT((lfs_soff_t)overlap_ >= 0); - - lfsr_data_t slice_ = lfsr_data_fruncate(data_, - lfsr_data_size(&data_) - lfs_min32( - overlap_, - lfsr_data_size(&data_))); - - // can we coalesce a hole? - if (lfsr_data_size(&slice_) == 0) { - delta += bid_+1 - (pos + weight); - - // carve bptr? - } else if (tag_ == LFSR_TAG_BLOCK - && lfsr_data_size(&slice_) > lfs->cfg->crystal_size) { - lfsr_bptr_t bptr_ = { - .block = slice_.u.disk.block, - .off = slice_.u.disk.off, - .size = lfsr_data_size(&slice_), - }; - - uint8_t bptr_buf[LFSR_BPTR_DSIZE]; - err = lfsr_bshrub_commit(lfs, - &file->mdir, &file->u.bshrub, LFSR_ATTRS( - LFSR_ATTR(pos, - BLOCK, +(weight_ - overlap_), - FROMBPTR(&bptr_, bptr_buf)))); - if (err) { - return err; - } - - // break into multiple fragments and carve if bptr/fragment is - // below our crystal size - } else { - // TODO can this be simplified a bit? - // write the last fragment first to avoid overflow issues - err = lfsr_bshrub_commit(lfs, - &file->mdir, &file->u.bshrub, LFSR_ATTRS( - LFSR_ATTR(pos, - DATA, +weight_ - overlap_ - - lfs_aligndown( - lfsr_data_size(&slice_)-1, - lfs->cfg->fragment_size), - DATA(lfsr_data_add(slice_, - lfs_aligndown( - lfsr_data_size(&slice_)-1, - lfs->cfg->fragment_size)))))); - if (err) { - return err; - } - - for (lfs_size_t i = 0; - i < lfs_aligndown( - lfsr_data_size(&slice_)-1, - lfs->cfg->fragment_size); - i += lfs->cfg->fragment_size) { - err = lfsr_bshrub_commit(lfs, - &file->mdir, &file->u.bshrub, LFSR_ATTRS( - LFSR_ATTR(pos + i, - DATA, +lfs->cfg->fragment_size, - DISK( - slice_.u.disk.block, - slice_.u.disk.off + i, - lfs->cfg->fragment_size)))); - if (err) { - return err; - } - } - } - } - - // found right sibling? - } else if (pos + weight < bid_+1) { - lfs_off_t overlap_ = (pos + weight) - (bid_-(weight_-1)); - LFS_ASSERT((lfs_soff_t)overlap_ >= 0); - - lfsr_data_t slice_ = lfsr_data_fruncate(data_, - lfsr_data_size(&data_) - lfs_min32( - overlap_, - lfsr_data_size(&data_))); - - // can we coalesce a hole? - if (lfsr_data_size(&slice_) == 0) { - delta += bid_+1 - (pos + weight); - err = lfsr_bshrub_commit(lfs, - &file->mdir, &file->u.bshrub, LFSR_ATTRS( - LFSR_ATTR(bid_, - RM, -weight_, NULL()))); - if (err) { - return err; - } - - // carve bptr? - } else if (tag_ == LFSR_TAG_BLOCK - && lfsr_data_size(&slice_) > lfs->cfg->crystal_size) { - lfsr_bptr_t bptr_ = { - .block = slice_.u.disk.block, - .off = slice_.u.disk.off, - .size = lfsr_data_size(&slice_), - }; - - uint8_t bptr_buf[LFSR_BPTR_DSIZE]; - err = lfsr_bshrub_commit(lfs, - &file->mdir, &file->u.bshrub, LFSR_ATTRS( - LFSR_ATTR(bid_, - GROW(WIDE(BLOCK)), -overlap_, - FROMBPTR(&bptr_, bptr_buf)))); - if (err) { - return err; - } - - // break into multiple fragments and carve if bptr/fragment is - // below our crystal size - } else { - // write the last fragment first to avoid overflow issues - err = lfsr_bshrub_commit(lfs, - &file->mdir, &file->u.bshrub, LFSR_ATTRS( - LFSR_ATTR(bid_, - GROW(WIDE(DATA)), -overlap_ - lfs_aligndown( - lfsr_data_size(&slice_)-1, - lfs->cfg->fragment_size), - DATA(lfsr_data_add(slice_, - lfs_aligndown( - lfsr_data_size(&slice_)-1, - lfs->cfg->fragment_size)))))); - if (err) { - return err; - } - - for (lfs_size_t i = 0; - i < lfs_aligndown( - lfsr_data_size(&slice_)-1, - lfs->cfg->fragment_size); - i += lfs->cfg->fragment_size) { - err = lfsr_bshrub_commit(lfs, - &file->mdir, &file->u.bshrub, LFSR_ATTRS( - LFSR_ATTR(bid_-(weight_-1) + i, - DATA, +lfs->cfg->fragment_size, - DISK( - slice_.u.disk.block, - slice_.u.disk.off + i, - lfs->cfg->fragment_size)))); - if (err) { - return err; - } - } - } - - // found fully overwritten data? - } else { - // remove err = lfsr_bshrub_commit(lfs, &file->mdir, &file->u.bshrub, LFSR_ATTRS( LFSR_ATTR(bid_, - RM, -weight_, NULL()))); + GROW(WIDE(DATA)), + -(weight_ - lfs->cfg->fragment_size), + DATA(lfsr_data_truncate(data_, + lfs->cfg->fragment_size))), + LFSR_ATTR(bid_-(weight_ - lfs->cfg->fragment_size)+1, + BLOCK, +(weight_ - lfs->cfg->fragment_size), + FROMBPTR(&bptr_, buf)))); if (err) { return err; } + + weight_ -= lfs->cfg->fragment_size; + data_ = lfsr_data_add(data_, lfs->cfg->fragment_size); + left_slice_ = lfsr_data_truncate(data_, + pos - (bid_-(weight_-1))); + } + + // right sibling needs carving but falls underneath our + // crystallization threshold? break into fragments + while (tag_ == LFSR_TAG_BLOCK + && lfsr_data_size(&right_slice_) > lfs->cfg->fragment_size + && lfsr_data_size(&right_slice_) <= lfs->cfg->crystal_size) { + lfsr_bptr_t bptr_ = { + .block = data_.u.disk.block, + .off = data_.u.disk.off, + .size = lfsr_data_size(&data_) - lfs->cfg->fragment_size, + }; + + err = lfsr_bshrub_commit(lfs, + &file->mdir, &file->u.bshrub, LFSR_ATTRS( + LFSR_ATTR(bid_, + GROW(WIDE(BLOCK)), + -(weight_ - bptr_.size), + FROMBPTR(&bptr_, buf)), + LFSR_ATTR(bid_-(weight_ - bptr_.size)+1, + DATA, +(weight_ - bptr_.size), + DATA(lfsr_data_fruncate(data_, + lfs->cfg->fragment_size))))); + if (err) { + return err; + } + + bid_ -= (weight_-bptr_.size); + weight_ -= (weight_-bptr_.size); + data_ = lfsr_data_truncate(data_, bptr_.size); + right_slice_ = lfsr_data_add(data_, + pos+weight - (bid_-(weight_-1))); + } + + // found left sibling? + if (bid_-(weight_-1) < pos) { + // can we get away with a grow attribute? + if (lfsr_data_size(&data_) == lfsr_data_size(&left_slice_)) { + attrs[attr_count++] = LFSR_ATTR(bid_, + GROW, -(bid_+1 - pos), NULL()); + + // carve bptr? + } else if (tag_ == LFSR_TAG_BLOCK) { + lfsr_bptr_t bptr_ = { + .block = left_slice_.u.disk.block, + .off = left_slice_.u.disk.off, + .size = lfsr_data_size(&left_slice_), + }; + + attrs[attr_count++] = LFSR_ATTR(bid_, + GROW(WIDE(BLOCK)), -(bid_+1 - pos), + FROMBPTR(&bptr_, &buf[buf_size])); + buf_size += LFSR_BPTR_DSIZE; + + // carve fragment? + } else { + attrs[attr_count++] = LFSR_ATTR(bid_, + GROW(WIDE(DATA)), -(bid_+1 - pos), + DATA(left_slice_)); + } + + // completely overwriting this entry? + } else { + attrs[attr_count++] = LFSR_ATTR(bid_, + RM, -weight_, NULL()); + } + + // spans more than one entry? we can't do everything in one commit, + // so commit what we have and move on to next entry + if (pos+weight > bid_+1) { + LFS_ASSERT(lfsr_data_size(&right_slice_) == 0); + LFS_ASSERT(attr_count <= sizeof(attrs)/sizeof(lfsr_attr_t)); + LFS_ASSERT(buf_size <= sizeof(buf)); + + err = lfsr_bshrub_commit(lfs, + &file->mdir, &file->u.bshrub, + attrs, attr_count); + if (err) { + return err; + } + + delta += lfs_min32(weight, bid_+1 - pos); + weight -= lfs_min32(weight, bid_+1 - pos); + attr_count = 0; + buf_size = 0; + continue; + } + + // found right sibling? + if (pos+weight < bid_+1) { + // can we coalesce a hole? + if (lfsr_data_size(&right_slice_) == 0) { + delta += bid_+1 - (pos+weight); + + // carve bptr? + } else if (tag_ == LFSR_TAG_BLOCK) { + lfsr_bptr_t bptr_ = { + .block = right_slice_.u.disk.block, + .off = right_slice_.u.disk.off, + .size = lfsr_data_size(&right_slice_), + }; + + attrs[attr_count++] = LFSR_ATTR(pos, + BLOCK, +(bid_+1 - (pos+weight)), + FROMBPTR(&bptr_, &buf[buf_size])); + buf_size += LFSR_BPTR_DSIZE; + + // carve fragment? + } else { + attrs[attr_count++] = LFSR_ATTR(pos, + DATA, +(bid_+1 - (pos+weight)), + DATA(right_slice_)); + } } delta += lfs_min32(weight, bid_+1 - pos); weight -= lfs_min32(weight, bid_+1 - pos); + break; } // need a hole? - if (pos > lfsr_file_uweight(file) - // if we have no data we can coalesce our hole here - || (weight + delta > 0 && lfsr_data_size(&data) == 0)) { - lfs_off_t pos_ = lfs_min32(pos, lfsr_file_uweight(file)); - lfs_off_t hole = pos - pos_ - + ((lfsr_data_size(&data) == 0) ? weight + delta : 0); + if (pos > lfsr_file_uweight(file)) { + // can we coalesce? + if (lfsr_file_uweight(file) > 0) { + attrs[attr_count++] = LFSR_ATTR(lfsr_file_uweight(file)-1, + GROW, +(pos - lfsr_file_uweight(file)), NULL()); - // we can usually get away with a simple grow attribute - if (pos_ > 0) { - int err = lfsr_bshrub_commit(lfs, - &file->mdir, &file->u.bshrub, LFSR_ATTRS( - LFSR_ATTR(pos_-1, - GROW, +hole, NULL()))); - if (err) { - return err; - } - - // otherwise we need a hole attr + // new hole } else { - int err = lfsr_bshrub_commit(lfs, - &file->mdir, &file->u.bshrub, LFSR_ATTRS( - LFSR_ATTR(pos_, - DATA, +hole, NULL()))); - if (err) { - return err; - } + attrs[attr_count++] = LFSR_ATTR(lfsr_file_uweight(file), + DATA, +(pos - lfsr_file_uweight(file)), NULL()); } } - // TODO should both carveshrub and carvetree be optimized so overwriting - // a perfectly aligned entry is one tag? -- this is actually very common - // since we coalesce one layer up... - // finally append our data - if (weight + delta > 0 && lfsr_data_size(&data) != 0) { + if (weight + delta > 0) { + // can we coalesce a hole? + if (pos > 0 && lfsr_data_size(&data) == 0) { + attrs[attr_count++] = LFSR_ATTR(pos-1, + GROW, +(weight + delta), NULL()); + + // append new data + } else { + attrs[attr_count++] = LFSR_ATTR(pos, + TAG(tag), +(weight + delta), DATA(data)); + } + } + + // commit pending attrs + if (attr_count > 0) { + LFS_ASSERT(attr_count <= sizeof(attrs)/sizeof(lfsr_attr_t)); + LFS_ASSERT(buf_size <= sizeof(buf)); + int err = lfsr_bshrub_commit(lfs, - &file->mdir, &file->u.bshrub, LFSR_ATTRS( - LFSR_ATTR(pos, - TAG(tag), +(weight + delta), DATA(data)))); + &file->mdir, &file->u.bshrub, + attrs, attr_count); if (err) { return err; } @@ -10277,10 +10197,8 @@ static int lfsr_file_flush(lfs_t *lfs, lfsr_file_t *file, datas[0] = lfsr_data_truncate(data_, fragment_start - (bid_-(weight_-1))); datas[1] = lfsr_data_truncate(data, - lfs_min32( - lfsr_data_size(&data), - lfs->cfg->fragment_size - - (fragment_start - (bid_-(weight_-1))))); + lfs->cfg->fragment_size + - (fragment_start - (bid_-(weight_-1)))); data_count = 2; data = lfsr_data_fromcat(datas, data_count);