Adopted file->leaf, reworked how we track crystallization

TLDR: Added file->leaf, which can track file fragments (read only) and
blocks independently from file->b.shrub. This speeds up linear
read/write performance at a heavy code/stack cost.

The jury is still out on if this ends up reverted.

---

This is another change motivated by benchmarking, specifically the
significant regression in linear reads.

The problem is that CTZ skip-lists are actually _really_ good at
appending blocks! (but only appending blocks) The entire state of the
file is contained in the last block, so file writes can resume without
any reads. With B-trees, we need at least 1 B-tree lookup to resume
appending, and this really adds up when writing extremely blocks.

To try to mitigate this, I added file->leaf, a single in-RAM bptr for
tracking the most recent leaf we've operated on. This avoids B-tree
lookups during linear reads, and allowing the leaf to fall out-of-sync
with the B-tree avoids both B-tree lookups and commits during writes.

Unfortunately this isn't a complete win for writes. If we write
fragments, i.e. cache_size < prog_size, we still need to incrementally
commit to the B-tree. Fragments are a bit annoying for caching as any
B-tree commit can discard the block they reside on.

For reading, however, this brings read performance back to roughly the
same as CTZ skip-lists.

---

This also turned into more-or-less a full rewrite of the lfsr_file_flush
-> lfsr_file_crystallize code path, which is probably a good thing. This
code needed some TLC.

file->leaf also replaces the previous eblock/eoff mechanism for
erased-state tracking via the new LFSR_BPTR_ISERASED flag. This should
be useful when exploring more erased-state tracking mechanisms (ddtree).

Unfortunately, all of this additional in-RAM state is very costly. I
think there's some cleanup that can be done (the current impl is a bit
of a mess/proof-of-concept), but this does add a significant chunk of
both code and stack:

           code          stack          ctx
  before: 36016           2296          636
  after:  37228 (+3.4%)   2328 (+1.4%)  636 (+0.0%)

file->leaf also increases the size of lfsr_file_t, but this doesn't show
up in ctx because struct lfs_info dominates:

  lfsr_file_t before: 116
  lfsr_file_t after:  136 (+17.2%)

Hm... Maybe ctx measurements should use a lower LFS_NAME_MAX?

lfs.h

@@ -144,6 +144,7 @@ enum lfs_type {
 
 // internally used flags, don't use these
 #define LFS_o_TYPE      0xf0000000  // The file's type
+#define LFS_o_UNGRAFT   0x00800000  // File's leaf does not match btree
 #define LFS_o_UNFLUSH   0x01000000  // File's data does not match disk
 #define LFS_o_UNSYNC    0x02000000  // File's metadata does not match disk
 #define LFS_o_UNCREAT   0x04000000  // File does not exist yet
@@ -639,8 +640,10 @@ typedef struct lfsr_data {
         struct {
             lfs_block_t block;
             lfs_size_t off;
-            // optional context for validating data
             #ifdef LFS_CKDATACKSUMS
+            // optional context for validating data
+            // sign(cksize)=0 => block not erased
+            // sign(cksize)=1 => block erased
             lfs_size_t cksize;
             uint32_t cksum;
             #endif
@@ -655,6 +658,8 @@ typedef struct lfsr_bptr {
     // sign2(size)=0b11 => block pointer
     lfsr_data_t data;
     #ifndef LFS_CKDATACKSUMS
+    // sign(cksize)=0 => block not erased
+    // sign(cksize)=1 => block erased
     lfs_size_t cksize;
     uint32_t cksum;
     #endif
@@ -730,11 +735,15 @@ typedef struct lfsr_bshrub {
 //} lfs_file_t;
 
 typedef struct lfsr_file {
+    // btree/bshrub stuff is in here
     lfsr_bshrub_t b;
     const struct lfs_file_config *cfg;
 
+    // current file position
     lfs_off_t pos;
 
+    // in-RAM cache
+    //
     // note this lines up with lfsr_data_t's buffer representation
     struct {
         lfs_off_t size;
@@ -742,8 +751,12 @@ typedef struct lfsr_file {
         lfs_off_t pos;
     } cache;
 
-    lfs_block_t eblock;
+    // on-disk leaf bptr
-    lfs_size_t eoff;
+    struct {
+        lfs_off_t pos;
+        lfs_off_t weight;
+        lfsr_bptr_t bptr;
+    } leaf;
 } lfsr_file_t;
 
 // littlefs directory type

lfs_util.h

@@ -258,12 +258,6 @@
 #define LFS_IFDEF_GC(a, b) (b)
 #endif
 
-#ifdef LFS_NONDAG
-#define LFS_IFDEF_NONDAG(a, b) (a)
-#else
-#define LFS_IFDEF_NONDAG(a, b) (b)
-#endif
-
 
 // Some function attributes, no way around these
 

scripts/dbgflags.py

@@ -44,6 +44,7 @@ FLAGS = [
     ('^', 'ORPHAN',    0x50000000, "Type = orphan"                            ),
     ('^', 'TRAVERSAL', 0x60000000, "Type = traversal"                         ),
     ('^', 'UNKNOWN',   0x70000000, "Type = unknown"                           ),
+    ('o', 'UNGRAFT',   0x00800000, "File's leaf does not match btree"         ),
     ('o', 'UNFLUSH',   0x01000000, "File's data does not match disk"          ),
     ('o', 'UNSYNC',    0x02000000, "File's metadata does not match disk"      ),
     ('o', 'UNCREAT',   0x04000000, "File does not exist yet"                  ),

tests/test_ck.toml

@@ -3751,13 +3751,17 @@ code = '''
                     wbuf, SIZE);
             assert(d == SIZE
                     || d == LFS_ERR_NOSPC
-                    || (d == LFS_ERR_CORRUPT && (METHOD == 2 || METHOD == 3)));
+                    || (d == LFS_ERR_CORRUPT
+                        && (METHOD == 2 || METHOD == 3)));
             if (d == LFS_ERR_NOSPC || d == LFS_ERR_CORRUPT) {
                 goto corrupt_mounted;
             }
             int err = lfsr_file_sync(&lfs, &sim_files[j]->file);
-            assert(err == 0 || err == LFS_ERR_NOSPC);
+            assert(err == 0
-            if (err == LFS_ERR_NOSPC) {
+                    || err == LFS_ERR_NOSPC
+                    || (err == LFS_ERR_CORRUPT
+                        && (METHOD == 2 || METHOD == 3)));
+            if (err == LFS_ERR_NOSPC || LFS_ERR_CORRUPT) {
                 goto corrupt_mounted;
             }
 
@@ -4349,13 +4353,17 @@ code = '''
                     wbuf, SIZE);
             assert(d == SIZE
                     || d == LFS_ERR_NOSPC
-                    || (d == LFS_ERR_CORRUPT && (METHOD == 2 || METHOD == 3)));
+                    || (d == LFS_ERR_CORRUPT
+                        && (METHOD == 2 || METHOD == 3)));
             if (d == LFS_ERR_NOSPC || d == LFS_ERR_CORRUPT) {
                 goto corrupt_mounted;
             }
             int err = lfsr_file_sync(&lfs, &sim_files[j]->file);
-            assert(err == 0 || err == LFS_ERR_NOSPC);
+            assert(err == 0
-            if (err == LFS_ERR_NOSPC) {
+                    || err == LFS_ERR_NOSPC
+                    || (err == LFS_ERR_CORRUPT
+                        && (METHOD == 2 || METHOD == 3)));
+            if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) {
                 goto corrupt_mounted;
             }
 

tests/test_traversal.toml

@@ -2131,6 +2131,7 @@ code = '''
         wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file) => 0;
     if (SYNC) {
         lfsr_file_sync(&lfs, &file) => 0;
     }
@@ -2499,6 +2500,7 @@ code = '''
         wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file1) => 0;
 
     lfsr_file_t file2;
     lfsr_file_open(&lfs, &file2, "tarantula",
@@ -2508,6 +2510,7 @@ code = '''
         wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file2) => 0;
 
     // try traversing
     lfsr_traversal_t t;
@@ -2531,6 +2534,7 @@ code = '''
         wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file1) => 0;
 
     // we should be at end of traversal now
     lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
@@ -2586,6 +2590,7 @@ code = '''
         wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file1) => 0;
 
     lfsr_file_t file2;
     lfsr_file_open(&lfs, &file2, "tarantula",
@@ -2595,6 +2600,7 @@ code = '''
         wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file2) => 0;
 
     // try traversing
     lfsr_traversal_t t;
@@ -2624,6 +2630,7 @@ code = '''
         wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file1) => 0;
 
     // traverse btree
     lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
@@ -2686,6 +2693,7 @@ code = '''
         wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file1) => 0;
 
     lfsr_file_t file2;
     lfsr_file_open(&lfs, &file2, "tarantula",
@@ -2695,6 +2703,7 @@ code = '''
         wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file2) => 0;
 
     // try traversing
     lfsr_traversal_t t;
@@ -2721,6 +2730,7 @@ code = '''
         wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file1) => 0;
 
     // traverse two data blocks
     lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
@@ -2781,6 +2791,7 @@ code = '''
         wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file1) => 0;
 
     lfsr_file_t file2;
     lfsr_file_open(&lfs, &file2, "tarantula",
@@ -2790,6 +2801,7 @@ code = '''
         wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file2) => 0;
 
     // try traversing
     lfsr_traversal_t t;
@@ -2876,6 +2888,7 @@ code = '''
         wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file1) => 0;
 
     lfsr_file_t file2;
     lfsr_file_open(&lfs, &file2, "tarantula",
@@ -2885,6 +2898,7 @@ code = '''
         wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file2) => 0;
 
     // try traversing
     lfsr_traversal_t t;
@@ -2984,6 +2998,7 @@ code = '''
         wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file1) => 0;
 
     lfsr_file_t file2;
     lfsr_file_open(&lfs, &file2, "tarantula",
@@ -2993,6 +3008,7 @@ code = '''
         wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file2) => 0;
 
     // try traversing
     lfsr_traversal_t t;
@@ -6835,6 +6851,7 @@ code = '''
         wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file1) => 0;
 
     lfsr_file_t file2;
     lfsr_file_open(&lfs, &file2, "octopus",
@@ -6844,6 +6861,7 @@ code = '''
         wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file2) => 0;
 
     // create this many orphaned files
     //
@@ -7163,6 +7181,7 @@ code = '''
         wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file1) => 0;
 
     lfsr_file_t file2;
     lfsr_file_open(&lfs, &file2, "octopus",
@@ -7172,6 +7191,7 @@ code = '''
         wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
     }
     lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
+    lfsr_file_flush(&lfs, &file2) => 0;
 
     // create this many orphaned files
     //