Github/littlefs
1
Fork 0
Code Actions Activity

bmap: The initial bmapcache algorithm seems to be working

Browse Source 
At least at a proof-of-concept level, there's still a lot of cleanup
needed.

To make things work, lfs3_alloc_ckpoint now takes an mdir, which
provides the target for gbmap gstate updates.

When the bmap is close to empty (configurable via bmap_scan_thresh), we
opportunistically rebuild it during lfs3_alloc_ckpoints. The nice thing
about lfs3_alloc_ckpoint is we know the state of all in-flight blocks,
so rebuilding the bmap just requires traversing the filesystem + in-RAM
state.

We might still fall back to the lookahead buffer, but in theory a well
tuned bmap_scan_thresh can prevent this from becoming a bottleneck (at
the cost of more frequent bmap rebuilds).

---

This is also probably a good time to resume measuring code/ram costs,
though it's worth repeating the above note about the bmap work still
needing cleanup:

             code          stack          ctx
  before:   36840           2368          684
  after:    36920 (+0.2%)   2368 (+0.0%)  684 (+0.0%)

Haha, no, the bmap isn't basically free, it's just an opt-in features.
With -DLFS3_YES_BMAP=1:

             code          stack          ctx
  no bmap:  36920           2368          684
  yes bmap: 38552 (+4.4%)   2472 (+4.4%)  812 (+18.7%)
This commit is contained in:
Christopher Haster
2025-07-28 23:27:02 -05:00
parent 8666830515
commit 316ca1cc05
17 changed files with 329 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_mtree.toml
+46 -46
View File
@@ -42,7 +42,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
for (lfs3_size_t i = 0; i < N; i++) {
lfs3_mdir_commit(&lfs3, &lfs3.mroot, LFS3_RATTRS(
@@ -86,7 +86,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
for (lfs3_size_t i = 0; i < N; i++) {
// force mroot to compact
@@ -133,7 +133,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
for (lfs3_size_t i = 0; i < N; i++) {
lfs3_mdir_commit(&lfs3, &lfs3.mroot, LFS3_RATTRS(
@@ -182,7 +182,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create a 2 large rattrs that needs to be uninlined
@@ -265,7 +265,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create 2 large entries that needs to be uninlined and split
@@ -360,7 +360,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create 2 large entries that needs to be uninlined and split
@@ -501,7 +501,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// create entries
for (lfs3_size_t i = 0; i < N; i++) {
@@ -588,7 +588,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
bool sim[N];
for (lfs3_size_t i = 0; i < N; i++) {
@@ -697,7 +697,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create 2 large entries that needs to be uninlined and split
@@ -791,7 +791,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create 2 large entries that needs to be uninlined and split
@@ -887,7 +887,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create 2 large entries that needs to be uninlined and split
@@ -964,7 +964,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create 2 large entries that needs to be uninlined and split
@@ -1084,7 +1084,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create 2 large entries that needs to be uninlined and split
@@ -1206,7 +1206,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
bool sim[N];
for (lfs3_size_t i = 0; i < N; i++) {
@@ -1343,7 +1343,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create a 2 large rattrs that needs to be uninlined
@@ -1469,7 +1469,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create 2 large entries that needs to be uninlined and split
@@ -1609,7 +1609,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create 2 large entries that needs to be uninlined and split
@@ -1749,7 +1749,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// prepare mroot with an entry
@@ -1832,7 +1832,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// prepare mroot with an entry
@@ -1970,7 +1970,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// prepare mroot with an entry
@@ -2081,7 +2081,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create 2 large entries that needs to be uninlined and split
@@ -2226,7 +2226,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create 2 large entries that needs to be uninlined and split
@@ -2388,7 +2388,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create 2 large entries that needs to be uninlined and split
@@ -2495,7 +2495,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// force mroot to compact once, so the second compact below will
// trigger a relocation
@@ -2588,7 +2588,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// force mroot to compact once, so the second compact below will
// trigger a relocation
@@ -2695,7 +2695,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
bool sim[N];
for (lfs3_size_t i = 0; i < N; i++) {
@@ -2857,7 +2857,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// setup our neighbors
lfs3_handle_t left = {.flags=0};
@@ -2917,7 +2917,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// setup our neighbors
lfs3_handle_t left = {.flags=0};
@@ -2967,7 +2967,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// setup our neighbors
lfs3_handle_t left = {.flags=0};
@@ -3019,7 +3019,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// setup our neighbors
lfs3_handle_t left = {.flags=0};
@@ -3096,7 +3096,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// setup our neighbors
lfs3_handle_t left = {.flags=0};
@@ -3187,7 +3187,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// setup our neighbors
lfs3_handle_t left = {.flags=0};
@@ -3245,7 +3245,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// setup our neighbors
lfs3_handle_t left = {.flags=0};
@@ -3342,7 +3342,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// setup our neighbors
lfs3_handle_t left = {.flags=0};
@@ -3437,7 +3437,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// setup our neighbors
lfs3_handle_t left = {.flags=0};
@@ -3547,7 +3547,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// setup our neighbors
lfs3_handle_t left = {.flags=0};
@@ -3651,7 +3651,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// insert at least one entry
lfs3_mdir_t mdir;
@@ -3757,7 +3757,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create a 2 large rattrs that needs to be uninlined
@@ -3897,7 +3897,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create 2 large entries that needs to be uninlined and split
@@ -4049,7 +4049,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// create 2 large entries that needs to be uninlined and split
@@ -4234,7 +4234,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// insert at least one entry
lfs3_mdir_t mdir;
@@ -4353,7 +4353,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
// create entries
for (lfs3_size_t i = 0; i < N; i++) {
@@ -4505,7 +4505,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
bool sim[N];
for (lfs3_size_t i = 0; i < N; i++) {
@@ -4678,7 +4678,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_mdir_commit(&lfs3, &lfs3.mroot, LFS3_RATTRS(
LFS3_RATTR_MPTR(
@@ -4998,7 +4998,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// prepare mroot with an entry
@@ -5058,7 +5058,7 @@ code = '''
lfs3_t lfs3;
lfs3_format(&lfs3, LFS3_F_RDWR | F_FLAGS, CFG) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR | M_FLAGS, CFG) => 0;
lfs3_alloc_ckpoint(&lfs3);
lfs3_alloc_ckpoint(&lfs3, NULL);
lfs3_data_t data;
// prepare mroot with an entry