From 9c3a8665080e81f700224e19ba79f5ece056ffb0 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 23 May 2025 01:10:23 -0500 Subject: [PATCH] Reworked crystallization to better use erased-state on rewrites This adopts lazy crystallization in _addition_ to lazy grafting, managed by separate LFS_o_UNCRYST and LFS_o_UNGRAFT flags: LFS_o_UNCRYST 0x00400000 File's leaf not fully crystallized LFS_o_UNGRAFT 0x00800000 File's leaf does not match bshrub/btree This lets us graft not-fully-crystallized blocks into the tree without needing to fully crystallize, avoiding repeated recrystallizations when linearly rewriting a file. Long story short, this gives file rewrites roughly the same performance as linear file writes. --- In theory you could also have fully crystallized but ungrafted blocks (UNGRAFT + ~UNCRYST), but this doesn't happen with the current logic. lfsr_file_crystallize eagerly grafts blocks once they're crystallized. Internally, lfsr_file_crystallize replaces lfsr_file_graft for the "don't care, gimme file->leaf" operation. This is analogous to lfsr_file_flush for file->cache. Note we do _not_ use LFS_o_UNCRYST to track erased-state! If we did, erased-state wouldn't survive lfsr_file_flush! --- Of course, this adds even more code. Fortunately not _that_ much considering how many lines of code changed: code stack ctx before: 37012 2304 636 after 37084 (+0.2%) 2304 (+0.0%) 636 (+0.0%) There is another downside however, and that's that our benchmarked disk usage is slightly worse during random writes. I haven't fully investigated this, but I think it's due to more temporary fragments/blocks in the B-tree before flushing. This can cause B-tree inner nodes to split earlier than when eagerly recrystallizing. This also leads to higher disk usage pre-flush since we keep both the old and new blocks around while uncrystallized, but since most rewrites are probably going to be CoW on top of committed files, I don't think this will be a big deal. Note the disk usage ends up the same after lfsr_file_flush. --- lfs.c | 181 ++++++++++++++++++++++++-------------------- lfs.h | 3 +- scripts/dbgflags.py | 3 +- 3 files changed, 104 insertions(+), 83 deletions(-) diff --git a/lfs.c b/lfs.c index 35819a6c..879706d0 100644 --- a/lfs.c +++ b/lfs.c @@ -6686,7 +6686,7 @@ static int lfsr_bshrub_commit(lfs_t *lfs, lfsr_bshrub_t *bshrub, if (lfsr_bshrub_isbtree(bshrub)) { for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { if (lfsr_o_isbshrub(o->flags) - && (lfsr_bshrub_t*)o != bshrub + && o != &bshrub->o && lfsr_bshrub_cmp((lfsr_bshrub_t*)o, bshrub) == 0) { // mark as unerased ((lfsr_bshrub_t*)o)->shrub.eoff = -1; @@ -7003,6 +7003,10 @@ static inline bool lfsr_o_isbshrub(uint32_t flags) { || lfsr_o_type(flags) == LFS_type_TRAVERSAL; } +static inline bool lfsr_o_isuncryst(uint32_t flags) { + return flags & LFS_o_UNCRYST; +} + static inline bool lfsr_o_isungraft(uint32_t flags) { return flags & LFS_o_UNGRAFT; } @@ -11251,11 +11255,13 @@ int lfsr_removeattr(lfs_t *lfs, const char *path, uint8_t type) { // file helpers static inline void lfsr_file_discardcache(lfsr_file_t *file) { + file->b.o.flags &= ~LFS_o_UNFLUSH; file->cache.pos = 0; file->cache.size = 0; } static inline void lfsr_file_discardleaf(lfsr_file_t *file) { + file->b.o.flags &= ~LFS_o_UNCRYST & ~LFS_o_UNGRAFT; file->leaf.pos = 0; file->leaf.weight = 0; lfsr_bptr_discard(&file->leaf.bptr); @@ -11295,8 +11301,6 @@ static int lfsr_file_fetch(lfs_t *lfs, lfsr_file_t *file, bool trunc) { lfsr_file_discardcache(file); // discard the current leaf lfsr_file_discardleaf(file); - // mark as flushed - file->b.o.flags &= ~LFS_o_UNFLUSH & ~LFS_o_UNGRAFT; // don't bother reading disk if we're not created or truncating if (!lfsr_o_isuncreat(file->b.o.flags) && !trunc) { @@ -11726,7 +11730,8 @@ static lfs_ssize_t lfsr_file_read_(lfs_t *lfs, lfsr_file_t *file, if (!(pos >= file->leaf.pos && pos < file->leaf.pos + file->leaf.weight)) { // leaf in use? we need to flush it - if (lfsr_o_isungraft(file->b.o.flags)) { + if (lfsr_o_isungraft(file->b.o.flags) + || lfsr_o_isuncryst(file->b.o.flags)) { int err = lfsr_file_flush(lfs, file); if (err) { return err; @@ -11897,7 +11902,7 @@ static int lfsr_file_commit(lfs_t *lfs, lfsr_file_t *file, } // graft bptr/fragments into our bshrub/btree -static int lfsr_file_graft_(lfs_t *lfs, lfsr_file_t *file, +static int lfsr_file_graft(lfs_t *lfs, lfsr_file_t *file, lfs_off_t pos, lfs_off_t weight, lfs_soff_t delta, // data_count=-1 => single bptr // data_count>=0 => list of concatenated fragments @@ -12165,35 +12170,16 @@ static int lfsr_file_graft_(lfs_t *lfs, lfsr_file_t *file, return 0; } -static int lfsr_file_graft(lfs_t *lfs, lfsr_file_t *file) { - // do nothing if already grafted - if (!lfsr_o_isungraft(file->b.o.flags)) { - return 0; - } - // ungrafted files must be unsynced - LFS_ASSERT(lfsr_o_isunsync(file->b.o.flags)); - // only blocks can be ungrafted - LFS_ASSERT(lfsr_bptr_isbptr(&file->leaf.bptr)); - - // graft our crystal - int err = lfsr_file_graft_(lfs, file, - file->leaf.pos, file->leaf.weight, 0, - &file->leaf.bptr.data, -1); - if (err) { - return err; - } - - // mark as grafted - file->b.o.flags &= ~LFS_o_UNGRAFT; - return 0; -} - -static int lfsr_file_crystallize(lfs_t *lfs, lfsr_file_t *file, +static int lfsr_file_crystallize_(lfs_t *lfs, lfsr_file_t *file, lfs_off_t block_pos, lfs_soff_t crystal_size, lfs_off_t pos, const uint8_t *buffer, lfs_size_t size) { - // limit to block_size and theoretical file size + // align to prog_size, limit to block_size and theoretical file size lfs_off_t crystal_limit = lfs_min( - block_pos + lfs->cfg->block_size, + block_pos + lfs_min( + lfs_aligndown( + (lfs_off_t)crystal_size, + lfs->cfg->prog_size), + lfs->cfg->block_size), lfs_max( pos + size, lfsr_file_weight(file))); @@ -12217,9 +12203,9 @@ static int lfsr_file_crystallize(lfs_t *lfs, lfsr_file_t *file, // before we write, claim the erased state! for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { if (lfsr_o_type(o->flags) == LFS_TYPE_REG + && o != &file->b.o && lfsr_bptr_block(&((lfsr_file_t*)o)->leaf.bptr) - == lfsr_bptr_block(&file->leaf.bptr) - && (lfsr_file_t*)o != file) { + == lfsr_bptr_block(&file->leaf.bptr)) { lfsr_bptr_claim(&((lfsr_file_t*)o)->leaf.bptr); } } @@ -12389,8 +12375,8 @@ static int lfsr_file_crystallize(lfs_t *lfs, lfsr_file_t *file, (pos_ - block_pos) | LFSR_BPTR_ISERASED, cksum_); - // mark as ungrafted - file->b.o.flags |= LFS_o_UNGRAFT; + // mark as uncrystallized and ungrafted + file->b.o.flags |= LFS_o_UNCRYST | LFS_o_UNGRAFT; return 0; relocate:; @@ -12411,6 +12397,44 @@ static int lfsr_file_crystallize(lfs_t *lfs, lfsr_file_t *file, } } +static int lfsr_file_crystallize(lfs_t *lfs, lfsr_file_t *file) { + // finish crystallizing + if (lfsr_o_isuncryst(file->b.o.flags)) { + // uncrystallized files must be unsynced + LFS_ASSERT(lfsr_o_isunsync(file->b.o.flags)); + // only blocks can be uncrystallized + LFS_ASSERT(lfsr_bptr_isbptr(&file->leaf.bptr)); + LFS_ASSERT(lfsr_bptr_iserased(&file->leaf.bptr)); + + // finish crystallizing the block + int err = lfsr_file_crystallize_(lfs, file, + file->leaf.pos - lfsr_bptr_off(&file->leaf.bptr), + -1, + file->cache.pos, file->cache.buffer, file->cache.size); + if (err) { + return err; + } + + // mark as crystallized + file->b.o.flags &= ~LFS_o_UNCRYST; + } + + // and graft into tree + if (lfsr_o_isungraft(file->b.o.flags)) { + int err = lfsr_file_graft(lfs, file, + file->leaf.pos, file->leaf.weight, 0, + &file->leaf.bptr.data, -1); + if (err) { + return err; + } + + // mark as grafted + file->b.o.flags &= ~LFS_o_UNGRAFT; + } + + return 0; +} + static int lfsr_file_flush_(lfs_t *lfs, lfsr_file_t *file, lfs_off_t pos, const uint8_t *buffer, lfs_size_t size) { // we can skip some btree lookups if we know we are aligned from a @@ -12440,7 +12464,7 @@ static int lfsr_file_flush_(lfs_t *lfs, lfsr_file_t *file, && pos - block_end < lfs->cfg->crystal_thresh // need to bail if we can't meet prog alignment && (pos + size) - block_end >= lfs->cfg->prog_size) { - int err = lfsr_file_crystallize(lfs, file, + int err = lfsr_file_crystallize_(lfs, file, file->leaf.pos - lfsr_bptr_off(&file->leaf.bptr), (pos + size) - (file->leaf.pos - lfsr_bptr_off(&file->leaf.bptr)), @@ -12561,27 +12585,15 @@ static int lfsr_file_flush_(lfs_t *lfs, lfsr_file_t *file, // if we're mid-crystallization, finish crystallizing the block // and graft it into our bshrub/btree - if (lfsr_bptr_isbptr(&file->leaf.bptr) - && lfsr_bptr_iserased(&file->leaf.bptr)) { - int err = lfsr_file_crystallize(lfs, file, - file->leaf.pos - lfsr_bptr_off(&file->leaf.bptr), - -1, - pos, buffer, size); - if (err) { - return err; - } - - // and graft the crystal into our tree - err = lfsr_file_graft(lfs, file); - if (err) { - return err; - } - - // mark as unerased so lfsr_file_crystallize doesn't try to - // continue crystallizing this block - lfsr_bptr_claim(&file->leaf.bptr); + int err = lfsr_file_crystallize(lfs, file); + if (err) { + return err; } + // mark as unerased so lfsr_file_crystallize doesn't try to + // resume crystallizing this block + lfsr_bptr_claim(&file->leaf.bptr); + // before we can crystallize we need to figure out the best // block alignment, we use the entry immediately to the left of // our crystal for this @@ -12619,8 +12631,8 @@ static int lfsr_file_flush_(lfs_t *lfs, lfsr_file_t *file, // start crystallizing! // - // lfsr_file_crystallize handles block allocation/relocation - int err = lfsr_file_crystallize(lfs, file, + // lfsr_file_crystallize_ handles block allocation/relocation + err = lfsr_file_crystallize_(lfs, file, crystal_start, crystal_end - crystal_start, pos, buffer, size); if (err) { @@ -12647,9 +12659,17 @@ fragment:; // // but note we're still tracking its erased state for future // writes! - int err = lfsr_file_graft(lfs, file); - if (err) { - return err; + if (lfsr_o_isungraft(file->b.o.flags)) { + // graft our crystal + int err = lfsr_file_graft(lfs, file, + file->leaf.pos, file->leaf.weight, 0, + &file->leaf.bptr.data, -1); + if (err) { + return err; + } + + // mark as grafted + file->b.o.flags &= ~LFS_o_UNGRAFT; } // do we need to discard our leaf? we need to discard fragments @@ -12687,7 +12707,7 @@ fragment:; lfsr_bid_t bid; lfsr_bid_t weight; lfsr_bptr_t bptr; - err = lfsr_file_lookup(lfs, file, + int err = lfsr_file_lookup(lfs, file, fragment_start-1, &bid, &weight, &bptr); if (err) { @@ -12734,7 +12754,7 @@ fragment:; lfsr_bid_t bid; lfsr_bid_t weight; lfsr_bptr_t bptr; - err = lfsr_file_lookup(lfs, file, + int err = lfsr_file_lookup(lfs, file, fragment_end, &bid, &weight, &bptr); if (err) { @@ -12774,7 +12794,7 @@ fragment:; // once we've figured out what fragment to write, graft it into // our tree - err = lfsr_file_graft_(lfs, file, + int err = lfsr_file_graft(lfs, file, fragment_start, fragment_end - fragment_start, 0, datas, data_count); if (err) { @@ -12794,6 +12814,7 @@ fragment:; return 0; } + // high-level file writing lfs_ssize_t lfsr_file_write(lfs_t *lfs, lfsr_file_t *file, @@ -12950,8 +12971,10 @@ int lfsr_file_flush(lfs_t *lfs, lfsr_file_t *file) { // can't write to readonly files LFS_ASSERT(!lfsr_o_isrdonly(file->b.o.flags)); - // do nothing if our file is already flushed and grafted + // do nothing if our file is already flushed, crystallized, + // and grafted if (!lfsr_o_isunflush(file->b.o.flags) + && !lfsr_o_isuncryst(file->b.o.flags) && !lfsr_o_isungraft(file->b.o.flags)) { return 0; } @@ -12976,17 +12999,10 @@ int lfsr_file_flush(lfs_t *lfs, lfsr_file_t *file) { file->b.o.flags &= ~LFS_o_UNFLUSH; } - // graft any ungrafted leaves - if (lfsr_o_isungraft(file->b.o.flags)) { - err = lfsr_file_graft_(lfs, file, - file->leaf.pos, file->leaf.weight, 0, - &file->leaf.bptr.data, -1); - if (err) { - goto failed; - } - - // mark as grafted - file->b.o.flags &= ~LFS_o_UNGRAFT; + // and crystallize/graft our leaf + err = lfsr_file_crystallize(lfs, file); + if (err) { + goto failed; } return 0; @@ -13174,9 +13190,8 @@ int lfsr_file_sync(lfs_t *lfs, lfsr_file_t *file) { && file->cache.size < lfs->cfg->crystal_thresh) { if (lfsr_o_isungraft(file->b.o.flags)) { file->b.o.flags |= LFS_o_UNFLUSH; - file->b.o.flags &= ~LFS_o_UNGRAFT; - lfsr_file_discardleaf(file); } + lfsr_file_discardleaf(file); } else { err = lfsr_file_flush(lfs, file); if (err) { @@ -13213,6 +13228,7 @@ int lfsr_file_sync(lfs_t *lfs, lfsr_file_t *file) { // update flags file_->b.o.flags &= ~LFS_o_UNSYNC & ~LFS_o_UNFLUSH + & ~LFS_o_UNCRYST & ~LFS_o_UNGRAFT; // update shrubs file_->b.shrub = file->b.shrub; @@ -13271,6 +13287,7 @@ int lfsr_file_sync(lfs_t *lfs, lfsr_file_t *file) { // mark as synced file->b.o.flags &= ~LFS_o_UNSYNC & ~LFS_o_UNFLUSH + & ~LFS_o_UNCRYST & ~LFS_o_UNGRAFT & ~LFS_o_UNCREAT & ~LFS_O_DESYNC; @@ -13405,7 +13422,7 @@ int lfsr_file_truncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size_) { < lfs_min( lfs->cfg->fragment_thresh, lfs->cfg->crystal_thresh)) { - err = lfsr_file_graft(lfs, file); + err = lfsr_file_crystallize(lfs, file); if (err) { goto failed; } @@ -13414,7 +13431,7 @@ int lfsr_file_truncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size_) { } // truncate our btree - err = lfsr_file_graft_(lfs, file, + err = lfsr_file_graft(lfs, file, lfs_min(size, size_), size - lfs_min(size, size_), +size_ - size, NULL, 0); @@ -13425,6 +13442,7 @@ int lfsr_file_truncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size_) { // truncate our leaf if (size_ < file->leaf.pos + lfsr_bptr_size(&file->leaf.bptr)) { lfsr_bptr_claim(&file->leaf.bptr); + file->b.o.flags &= ~LFS_o_UNCRYST; } file->leaf.bptr.data = LFSR_DATA_TRUNCATE( file->leaf.bptr.data, @@ -13487,7 +13505,7 @@ int lfsr_file_fruncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size_) { < lfs_min( lfs->cfg->fragment_thresh, lfs->cfg->crystal_thresh)) { - err = lfsr_file_graft(lfs, file); + err = lfsr_file_crystallize(lfs, file); if (err) { goto failed; } @@ -13496,7 +13514,7 @@ int lfsr_file_fruncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size_) { } // fruncate our btree - err = lfsr_file_graft_(lfs, file, + err = lfsr_file_graft(lfs, file, 0, lfs_smax(size - size_, 0), +size_ - size, NULL, 0); @@ -13509,6 +13527,7 @@ int lfsr_file_fruncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size_) { > (lfs_soff_t)(file->leaf.pos + lfsr_bptr_size(&file->leaf.bptr))) { lfsr_bptr_claim(&file->leaf.bptr); + file->b.o.flags &= ~LFS_o_UNCRYST; } file->leaf.bptr.data = LFSR_DATA_FRUNCATE( file->leaf.bptr.data, diff --git a/lfs.h b/lfs.h index 2fe2f20d..87cec4b6 100644 --- a/lfs.h +++ b/lfs.h @@ -144,7 +144,8 @@ enum lfs_type { // internally used flags, don't use these #define LFS_o_TYPE 0xf0000000 // The file's type -#define LFS_o_UNGRAFT 0x00800000 // File's leaf does not match btree +#define LFS_o_UNCRYST 0x00400000 // File's leaf not fully crystallized +#define LFS_o_UNGRAFT 0x00800000 // File's leaf does not match bshrub/btree #define LFS_o_UNFLUSH 0x01000000 // File's data does not match disk #define LFS_o_UNSYNC 0x02000000 // File's metadata does not match disk #define LFS_o_UNCREAT 0x04000000 // File does not exist yet diff --git a/scripts/dbgflags.py b/scripts/dbgflags.py index fd582a57..e78a557c 100755 --- a/scripts/dbgflags.py +++ b/scripts/dbgflags.py @@ -44,7 +44,8 @@ FLAGS = [ ('^', 'ORPHAN', 0x50000000, "Type = orphan" ), ('^', 'TRAVERSAL', 0x60000000, "Type = traversal" ), ('^', 'UNKNOWN', 0x70000000, "Type = unknown" ), - ('o', 'UNGRAFT', 0x00800000, "File's leaf does not match btree" ), + ('o', 'UNCRYST', 0x00400000, "File's leaf not fully crystallized" ), + ('o', 'UNGRAFT', 0x00800000, "File's leaf does not match bshrub/btree" ), ('o', 'UNFLUSH', 0x01000000, "File's data does not match disk" ), ('o', 'UNSYNC', 0x02000000, "File's metadata does not match disk" ), ('o', 'UNCREAT', 0x04000000, "File does not exist yet" ),