Github/littlefs
1
Fork 0
Code Actions Activity

Reworked crystallization to better use erased-state on rewrites

Browse Source 
This adopts lazy crystallization in _addition_ to lazy grafting, managed
by separate LFS_o_UNCRYST and LFS_o_UNGRAFT flags:

  LFS_o_UNCRYST  0x00400000  File's leaf not fully crystallized
  LFS_o_UNGRAFT  0x00800000  File's leaf does not match bshrub/btree

This lets us graft not-fully-crystallized blocks into the tree without
needing to fully crystallize, avoiding repeated recrystallizations when
linearly rewriting a file.

Long story short, this gives file rewrites roughly the same performance
as linear file writes.

---

In theory you could also have fully crystallized but ungrafted blocks
(UNGRAFT + ~UNCRYST), but this doesn't happen with the current logic.
lfsr_file_crystallize eagerly grafts blocks once they're crystallized.

Internally, lfsr_file_crystallize replaces lfsr_file_graft for the
"don't care, gimme file->leaf" operation. This is analogous to
lfsr_file_flush for file->cache.

Note we do _not_ use LFS_o_UNCRYST to track erased-state! If we did,
erased-state wouldn't survive lfsr_file_flush!

---

Of course, this adds even more code. Fortunately not _that_ much
considering how many lines of code changed:

           code          stack          ctx
  before: 37012           2304          636
  after   37084 (+0.2%)   2304 (+0.0%)  636 (+0.0%)

There is another downside however, and that's that our benchmarked disk
usage is slightly worse during random writes.

I haven't fully investigated this, but I think it's due to more
temporary fragments/blocks in the B-tree before flushing. This can cause
B-tree inner nodes to split earlier than when eagerly recrystallizing.

This also leads to higher disk usage pre-flush since we keep both the
old and new blocks around while uncrystallized, but since most rewrites
are probably going to be CoW on top of committed files, I don't think
this will be a big deal.

Note the disk usage ends up the same after lfsr_file_flush.
This commit is contained in:
Christopher Haster
2025-05-23 01:10:23 -05:00
parent f4c1753075
commit 9c3a866508
3 changed files with 104 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+100 -81
View File
@@ -6686,7 +6686,7 @@ static int lfsr_bshrub_commit(lfs_t *lfs, lfsr_bshrub_t *bshrub,
if (lfsr_bshrub_isbtree(bshrub)) {
for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) {
if (lfsr_o_isbshrub(o->flags)
&& (lfsr_bshrub_t*)o != bshrub
&& o != &bshrub->o
&& lfsr_bshrub_cmp((lfsr_bshrub_t*)o, bshrub) == 0) {
// mark as unerased
((lfsr_bshrub_t*)o)->shrub.eoff = -1;
@@ -7003,6 +7003,10 @@ static inline bool lfsr_o_isbshrub(uint32_t flags) {
|| lfsr_o_type(flags) == LFS_type_TRAVERSAL;
}
static inline bool lfsr_o_isuncryst(uint32_t flags) {
return flags & LFS_o_UNCRYST;
}
static inline bool lfsr_o_isungraft(uint32_t flags) {
return flags & LFS_o_UNGRAFT;
}
@@ -11251,11 +11255,13 @@ int lfsr_removeattr(lfs_t *lfs, const char *path, uint8_t type) {
// file helpers
static inline void lfsr_file_discardcache(lfsr_file_t *file) {
file->b.o.flags &= ~LFS_o_UNFLUSH;
file->cache.pos = 0;
file->cache.size = 0;
}
static inline void lfsr_file_discardleaf(lfsr_file_t *file) {
file->b.o.flags &= ~LFS_o_UNCRYST & ~LFS_o_UNGRAFT;
file->leaf.pos = 0;
file->leaf.weight = 0;
lfsr_bptr_discard(&file->leaf.bptr);
@@ -11295,8 +11301,6 @@ static int lfsr_file_fetch(lfs_t *lfs, lfsr_file_t *file, bool trunc) {
lfsr_file_discardcache(file);
// discard the current leaf
lfsr_file_discardleaf(file);
// mark as flushed
file->b.o.flags &= ~LFS_o_UNFLUSH & ~LFS_o_UNGRAFT;
// don't bother reading disk if we're not created or truncating
if (!lfsr_o_isuncreat(file->b.o.flags) && !trunc) {
@@ -11726,7 +11730,8 @@ static lfs_ssize_t lfsr_file_read_(lfs_t *lfs, lfsr_file_t *file,
if (!(pos >= file->leaf.pos
&& pos < file->leaf.pos + file->leaf.weight)) {
// leaf in use? we need to flush it
if (lfsr_o_isungraft(file->b.o.flags)) {
if (lfsr_o_isungraft(file->b.o.flags)
|| lfsr_o_isuncryst(file->b.o.flags)) {
int err = lfsr_file_flush(lfs, file);
if (err) {
return err;
@@ -11897,7 +11902,7 @@ static int lfsr_file_commit(lfs_t *lfs, lfsr_file_t *file,
}
// graft bptr/fragments into our bshrub/btree
static int lfsr_file_graft_(lfs_t *lfs, lfsr_file_t *file,
static int lfsr_file_graft(lfs_t *lfs, lfsr_file_t *file,
lfs_off_t pos, lfs_off_t weight, lfs_soff_t delta,
// data_count=-1 => single bptr
// data_count>=0 => list of concatenated fragments
@@ -12165,35 +12170,16 @@ static int lfsr_file_graft_(lfs_t *lfs, lfsr_file_t *file,
return 0;
}
static int lfsr_file_graft(lfs_t *lfs, lfsr_file_t *file) {
// do nothing if already grafted
if (!lfsr_o_isungraft(file->b.o.flags)) {
return 0;
}
// ungrafted files must be unsynced
LFS_ASSERT(lfsr_o_isunsync(file->b.o.flags));
// only blocks can be ungrafted
LFS_ASSERT(lfsr_bptr_isbptr(&file->leaf.bptr));
// graft our crystal
int err = lfsr_file_graft_(lfs, file,
file->leaf.pos, file->leaf.weight, 0,
&file->leaf.bptr.data, -1);
if (err) {
return err;
}
// mark as grafted
file->b.o.flags &= ~LFS_o_UNGRAFT;
return 0;
}
static int lfsr_file_crystallize(lfs_t *lfs, lfsr_file_t *file,
static int lfsr_file_crystallize_(lfs_t *lfs, lfsr_file_t *file,
lfs_off_t block_pos, lfs_soff_t crystal_size,
lfs_off_t pos, const uint8_t *buffer, lfs_size_t size) {
// limit to block_size and theoretical file size
// align to prog_size, limit to block_size and theoretical file size
lfs_off_t crystal_limit = lfs_min(
block_pos + lfs->cfg->block_size,
block_pos + lfs_min(
lfs_aligndown(
(lfs_off_t)crystal_size,
lfs->cfg->prog_size),
lfs->cfg->block_size),
lfs_max(
pos + size,
lfsr_file_weight(file)));
@@ -12217,9 +12203,9 @@ static int lfsr_file_crystallize(lfs_t *lfs, lfsr_file_t *file,
// before we write, claim the erased state!
for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) {
if (lfsr_o_type(o->flags) == LFS_TYPE_REG
&& o != &file->b.o
&& lfsr_bptr_block(&((lfsr_file_t*)o)->leaf.bptr)
== lfsr_bptr_block(&file->leaf.bptr)
&& (lfsr_file_t*)o != file) {
== lfsr_bptr_block(&file->leaf.bptr)) {
lfsr_bptr_claim(&((lfsr_file_t*)o)->leaf.bptr);
}
}
@@ -12389,8 +12375,8 @@ static int lfsr_file_crystallize(lfs_t *lfs, lfsr_file_t *file,
(pos_ - block_pos) | LFSR_BPTR_ISERASED,
cksum_);
// mark as ungrafted
file->b.o.flags |= LFS_o_UNGRAFT;
// mark as uncrystallized and ungrafted
file->b.o.flags |= LFS_o_UNCRYST | LFS_o_UNGRAFT;
return 0;
relocate:;
@@ -12411,6 +12397,44 @@ static int lfsr_file_crystallize(lfs_t *lfs, lfsr_file_t *file,
}
}
static int lfsr_file_crystallize(lfs_t *lfs, lfsr_file_t *file) {
// finish crystallizing
if (lfsr_o_isuncryst(file->b.o.flags)) {
// uncrystallized files must be unsynced
LFS_ASSERT(lfsr_o_isunsync(file->b.o.flags));
// only blocks can be uncrystallized
LFS_ASSERT(lfsr_bptr_isbptr(&file->leaf.bptr));
LFS_ASSERT(lfsr_bptr_iserased(&file->leaf.bptr));
// finish crystallizing the block
int err = lfsr_file_crystallize_(lfs, file,
file->leaf.pos - lfsr_bptr_off(&file->leaf.bptr),
-1,
file->cache.pos, file->cache.buffer, file->cache.size);
if (err) {
return err;
}
// mark as crystallized
file->b.o.flags &= ~LFS_o_UNCRYST;
}
// and graft into tree
if (lfsr_o_isungraft(file->b.o.flags)) {
int err = lfsr_file_graft(lfs, file,
file->leaf.pos, file->leaf.weight, 0,
&file->leaf.bptr.data, -1);
if (err) {
return err;
}
// mark as grafted
file->b.o.flags &= ~LFS_o_UNGRAFT;
}
return 0;
}
static int lfsr_file_flush_(lfs_t *lfs, lfsr_file_t *file,
lfs_off_t pos, const uint8_t *buffer, lfs_size_t size) {
// we can skip some btree lookups if we know we are aligned from a
@@ -12440,7 +12464,7 @@ static int lfsr_file_flush_(lfs_t *lfs, lfsr_file_t *file,
&& pos - block_end < lfs->cfg->crystal_thresh
// need to bail if we can't meet prog alignment
&& (pos + size) - block_end >= lfs->cfg->prog_size) {
int err = lfsr_file_crystallize(lfs, file,
int err = lfsr_file_crystallize_(lfs, file,
file->leaf.pos - lfsr_bptr_off(&file->leaf.bptr),
(pos + size)
- (file->leaf.pos - lfsr_bptr_off(&file->leaf.bptr)),
@@ -12561,27 +12585,15 @@ static int lfsr_file_flush_(lfs_t *lfs, lfsr_file_t *file,
// if we're mid-crystallization, finish crystallizing the block
// and graft it into our bshrub/btree
if (lfsr_bptr_isbptr(&file->leaf.bptr)
&& lfsr_bptr_iserased(&file->leaf.bptr)) {
int err = lfsr_file_crystallize(lfs, file,
file->leaf.pos - lfsr_bptr_off(&file->leaf.bptr),
-1,
pos, buffer, size);
if (err) {
return err;
}
// and graft the crystal into our tree
err = lfsr_file_graft(lfs, file);
if (err) {
return err;
}
// mark as unerased so lfsr_file_crystallize doesn't try to
// continue crystallizing this block
lfsr_bptr_claim(&file->leaf.bptr);
int err = lfsr_file_crystallize(lfs, file);
if (err) {
return err;
}
// mark as unerased so lfsr_file_crystallize doesn't try to
// resume crystallizing this block
lfsr_bptr_claim(&file->leaf.bptr);
// before we can crystallize we need to figure out the best
// block alignment, we use the entry immediately to the left of
// our crystal for this
@@ -12619,8 +12631,8 @@ static int lfsr_file_flush_(lfs_t *lfs, lfsr_file_t *file,
// start crystallizing!
//
// lfsr_file_crystallize handles block allocation/relocation
int err = lfsr_file_crystallize(lfs, file,
// lfsr_file_crystallize_ handles block allocation/relocation
err = lfsr_file_crystallize_(lfs, file,
crystal_start, crystal_end - crystal_start,
pos, buffer, size);
if (err) {
@@ -12647,9 +12659,17 @@ fragment:;
//
// but note we're still tracking its erased state for future
// writes!
int err = lfsr_file_graft(lfs, file);
if (err) {
return err;
if (lfsr_o_isungraft(file->b.o.flags)) {
// graft our crystal
int err = lfsr_file_graft(lfs, file,
file->leaf.pos, file->leaf.weight, 0,
&file->leaf.bptr.data, -1);
if (err) {
return err;
}
// mark as grafted
file->b.o.flags &= ~LFS_o_UNGRAFT;
}
// do we need to discard our leaf? we need to discard fragments
@@ -12687,7 +12707,7 @@ fragment:;
lfsr_bid_t bid;
lfsr_bid_t weight;
lfsr_bptr_t bptr;
err = lfsr_file_lookup(lfs, file,
int err = lfsr_file_lookup(lfs, file,
fragment_start-1,
&bid, &weight, &bptr);
if (err) {
@@ -12734,7 +12754,7 @@ fragment:;
lfsr_bid_t bid;
lfsr_bid_t weight;
lfsr_bptr_t bptr;
err = lfsr_file_lookup(lfs, file,
int err = lfsr_file_lookup(lfs, file,
fragment_end,
&bid, &weight, &bptr);
if (err) {
@@ -12774,7 +12794,7 @@ fragment:;
// once we've figured out what fragment to write, graft it into
// our tree
err = lfsr_file_graft_(lfs, file,
int err = lfsr_file_graft(lfs, file,
fragment_start, fragment_end - fragment_start, 0,
datas, data_count);
if (err) {
@@ -12794,6 +12814,7 @@ fragment:;
return 0;
}
// high-level file writing
lfs_ssize_t lfsr_file_write(lfs_t *lfs, lfsr_file_t *file,
@@ -12950,8 +12971,10 @@ int lfsr_file_flush(lfs_t *lfs, lfsr_file_t *file) {
// can't write to readonly files
LFS_ASSERT(!lfsr_o_isrdonly(file->b.o.flags));
// do nothing if our file is already flushed and grafted
// do nothing if our file is already flushed, crystallized,
// and grafted
if (!lfsr_o_isunflush(file->b.o.flags)
&& !lfsr_o_isuncryst(file->b.o.flags)
&& !lfsr_o_isungraft(file->b.o.flags)) {
return 0;
}
@@ -12976,17 +12999,10 @@ int lfsr_file_flush(lfs_t *lfs, lfsr_file_t *file) {
file->b.o.flags &= ~LFS_o_UNFLUSH;
}
// graft any ungrafted leaves
if (lfsr_o_isungraft(file->b.o.flags)) {
err = lfsr_file_graft_(lfs, file,
file->leaf.pos, file->leaf.weight, 0,
&file->leaf.bptr.data, -1);
if (err) {
goto failed;
}
// mark as grafted
file->b.o.flags &= ~LFS_o_UNGRAFT;
// and crystallize/graft our leaf
err = lfsr_file_crystallize(lfs, file);
if (err) {
goto failed;
}
return 0;
@@ -13174,9 +13190,8 @@ int lfsr_file_sync(lfs_t *lfs, lfsr_file_t *file) {
&& file->cache.size < lfs->cfg->crystal_thresh) {
if (lfsr_o_isungraft(file->b.o.flags)) {
file->b.o.flags |= LFS_o_UNFLUSH;
file->b.o.flags &= ~LFS_o_UNGRAFT;
lfsr_file_discardleaf(file);
}
lfsr_file_discardleaf(file);
} else {
err = lfsr_file_flush(lfs, file);
if (err) {
@@ -13213,6 +13228,7 @@ int lfsr_file_sync(lfs_t *lfs, lfsr_file_t *file) {
// update flags
file_->b.o.flags &= ~LFS_o_UNSYNC
& ~LFS_o_UNFLUSH
& ~LFS_o_UNCRYST
& ~LFS_o_UNGRAFT;
// update shrubs
file_->b.shrub = file->b.shrub;
@@ -13271,6 +13287,7 @@ int lfsr_file_sync(lfs_t *lfs, lfsr_file_t *file) {
// mark as synced
file->b.o.flags &= ~LFS_o_UNSYNC
& ~LFS_o_UNFLUSH
& ~LFS_o_UNCRYST
& ~LFS_o_UNGRAFT
& ~LFS_o_UNCREAT
& ~LFS_O_DESYNC;
@@ -13405,7 +13422,7 @@ int lfsr_file_truncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size_) {
< lfs_min(
lfs->cfg->fragment_thresh,
lfs->cfg->crystal_thresh)) {
err = lfsr_file_graft(lfs, file);
err = lfsr_file_crystallize(lfs, file);
if (err) {
goto failed;
}
@@ -13414,7 +13431,7 @@ int lfsr_file_truncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size_) {
}
// truncate our btree
err = lfsr_file_graft_(lfs, file,
err = lfsr_file_graft(lfs, file,
lfs_min(size, size_), size - lfs_min(size, size_),
+size_ - size,
NULL, 0);
@@ -13425,6 +13442,7 @@ int lfsr_file_truncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size_) {
// truncate our leaf
if (size_ < file->leaf.pos + lfsr_bptr_size(&file->leaf.bptr)) {
lfsr_bptr_claim(&file->leaf.bptr);
file->b.o.flags &= ~LFS_o_UNCRYST;
}
file->leaf.bptr.data = LFSR_DATA_TRUNCATE(
file->leaf.bptr.data,
@@ -13487,7 +13505,7 @@ int lfsr_file_fruncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size_) {
< lfs_min(
lfs->cfg->fragment_thresh,
lfs->cfg->crystal_thresh)) {
err = lfsr_file_graft(lfs, file);
err = lfsr_file_crystallize(lfs, file);
if (err) {
goto failed;
}
@@ -13496,7 +13514,7 @@ int lfsr_file_fruncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size_) {
}
// fruncate our btree
err = lfsr_file_graft_(lfs, file,
err = lfsr_file_graft(lfs, file,
0, lfs_smax(size - size_, 0),
+size_ - size,
NULL, 0);
@@ -13509,6 +13527,7 @@ int lfsr_file_fruncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size_) {
> (lfs_soff_t)(file->leaf.pos
+ lfsr_bptr_size(&file->leaf.bptr))) {
lfsr_bptr_claim(&file->leaf.bptr);
file->b.o.flags &= ~LFS_o_UNCRYST;
}
file->leaf.bptr.data = LFSR_DATA_FRUNCATE(
file->leaf.bptr.data,
lfs.h
+2 -1
View File
@@ -144,7 +144,8 @@ enum lfs_type {
// internally used flags, don't use these
#define LFS_o_TYPE 0xf0000000 // The file's type
#define LFS_o_UNGRAFT 0x00800000 // File's leaf does not match btree
#define LFS_o_UNCRYST 0x00400000 // File's leaf not fully crystallized
#define LFS_o_UNGRAFT 0x00800000 // File's leaf does not match bshrub/btree
#define LFS_o_UNFLUSH 0x01000000 // File's data does not match disk
#define LFS_o_UNSYNC 0x02000000 // File's metadata does not match disk
#define LFS_o_UNCREAT 0x04000000 // File does not exist yet
scripts/dbgflags.py
+2 -1
View File
@@ -44,7 +44,8 @@ FLAGS = [
('^', 'ORPHAN', 0x50000000, "Type = orphan" ),
('^', 'TRAVERSAL', 0x60000000, "Type = traversal" ),
('^', 'UNKNOWN', 0x70000000, "Type = unknown" ),
('o', 'UNGRAFT', 0x00800000, "File's leaf does not match btree" ),
('o', 'UNCRYST', 0x00400000, "File's leaf not fully crystallized" ),
('o', 'UNGRAFT', 0x00800000, "File's leaf does not match bshrub/btree" ),
('o', 'UNFLUSH', 0x01000000, "File's data does not match disk" ),
('o', 'UNSYNC', 0x02000000, "File's metadata does not match disk" ),
('o', 'UNCREAT', 0x04000000, "File does not exist yet" ),