t: Implemented gc_compact_thresh over btree nodes

Note, gc_compact_thresh over bshrubs is not yet implemented... That's _another_ can of worms since we need to be able to commit to non-tracked bshrubs somehow... But at least this proves gc_compact_thresh over btrees is possible. Now, if LFS_T_COMPACT is provided, any btree nodes > gc_compact_thresh will be compacted during traversal/gc operations. To make this work required a rather deep modification to the lfsr_btree_commit/lfsr_bshrub_commit code paths to expose direct-rbyd commit functions that can commit to arbitrary btree nodes: - lfsr_btree_commit - bid, attrs, attr_count - lfsr_bshrub_commit - bid, attrs, attr_count - lfsr_btree_commit_ - bid, rbyd, rid, attrs, attr_count - lfsr_bshrub_commit_ - bid, rbyb, rid, attrs, attr_count - lfsr_btree_commit__ - bscratch, bid, rbyd, rid, attrs, attr_count These are good to have, and will also be useful for implementing metadata redundancy in the future. Unfortunately, all of this comes at a significant code/stack cost: code stack before: 34652 2640 after: 35268 (+1.8%) 2776 (+5.2%)
2024-06-26 17:45:36 -05:00
parent 4d06fc2e0e
commit 61ecc135dc
2 changed files with 868 additions and 128 deletions
@@ -5092,6 +5092,7 @@ code = '''
    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
    assert(tinfo.btype == LFS_BTYPE_MDIR);
    assert(tinfo.block == 0 || tinfo.block == 1);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
    lfsr_traversal_close(&lfs, &t) => 0;

    // mdir should have been compacted
@@ -5256,6 +5257,7 @@ code = '''
    assert(tinfo.btype == LFS_BTYPE_MDIR);
    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
    lfsr_traversal_close(&lfs, &t) => 0;

    // mrootanchor should have been compacted
@@ -5284,6 +5286,82 @@ code = '''
    lfsr_unmount(&lfs) => 0;
 '''

+[cases.test_traversal_compact_mroot_extend]
+defines.CKMETA = [false, true]
+defines.CK = [false, true]
+defines.LOOKAHEAD = [false, true]
+defines.SIZE = 'FILE_BUFFER_SIZE/2'
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
+# force early relocations
+defines.BLOCK_RECYCLES = 0
+in = 'lfs.c'
+code = '''
+    lfs_t lfs;
+    lfsr_format(&lfs, CFG) => 0;
+    lfsr_mount(&lfs, CFG) => 0;
+
+    uint32_t prng = 42;
+
+    // write to our mdir until >gc_compact_thresh full
+    lfsr_file_t file;
+    lfsr_file_open(&lfs, &file, "jellyfish",
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
+
+    // hack, don't use the internals like this
+    uint8_t wbuf[SIZE];
+    while ((file.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
+        lfsr_file_rewind(&lfs, &file) => 0;
+        for (lfs_size_t j = 0; j < SIZE; j++) {
+            wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
+        }
+        lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
+        lfsr_file_sync(&lfs, &file) => 0;
+    }
+
+    // try traversing and compacting
+    lfsr_traversal_t t;
+    lfsr_traversal_open(&lfs, &t,
+            LFS_T_COMPACT
+                | ((CKMETA) ? LFS_T_CKMETA : 0)
+                | ((CK) ? LFS_T_CK : 0)
+                | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
+    // it's a bit unclear if we should follow the mroot or stay on the
+    // mroot anchor during extends, so if this breaks in the future
+    // I wouldn't worry too much about it
+    //
+    // traverse mroot
+    struct lfs_tinfo tinfo;
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
+    lfsr_traversal_close(&lfs, &t) => 0;
+
+    // mdir should have been compacted
+    assert((file.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
+
+    // check we can still read the file
+    for (int remount = 0; remount < 2; remount++) {
+        // remount?
+        if (remount) {
+            lfsr_file_close(&lfs, &file) => 0;
+            lfsr_unmount(&lfs) => 0;
+            lfsr_mount(&lfs, CFG) => 0;
+            lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
+        }
+
+        lfsr_file_rewind(&lfs, &file) => 0;
+        uint8_t rbuf[SIZE];
+        lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
+        assert(memcmp(rbuf, wbuf, SIZE) == 0);
+    }
+
+    lfsr_file_close(&lfs, &file) => 0;
+    lfsr_unmount(&lfs) => 0;
+'''
+
 [cases.test_traversal_compact_mroot_split]
 defines.CKMETA = [false, true]
 defines.CK = [false, true]
@@ -5370,6 +5448,7 @@ code = '''
    assert(tinfo.btype == LFS_BTYPE_MDIR);
    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
    lfsr_traversal_close(&lfs, &t) => 0;

    // mdirs should have been compacted
@@ -5543,6 +5622,7 @@ code = '''
    assert(tinfo.btype == LFS_BTYPE_MDIR);
    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
    lfsr_traversal_close(&lfs, &t) => 0;

    // mdirs should have been compacted
@@ -5724,6 +5804,7 @@ code = '''
    assert(tinfo.btype == LFS_BTYPE_MDIR);
    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
    lfsr_traversal_close(&lfs, &t) => 0;

    // mdirs should have been compacted
@@ -5775,6 +5856,523 @@ code = '''



+# btree/bshrub compactions are quite a bit more difficult
+
+[cases.test_traversal_compact_mtree_btree]
+defines.CKMETA = [false, true]
+defines.CK = [false, true]
+defines.LOOKAHEAD = [false, true]
+defines.SIZE = 'FILE_BUFFER_SIZE/2'
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
+in = 'lfs.c'
+code = '''
+    lfs_t lfs;
+    lfsr_format(&lfs, CFG) => 0;
+    lfsr_mount(&lfs, CFG) => 0;
+
+    uint32_t prng = 42;
+
+    // create two files
+    lfsr_file_t file1;
+    lfsr_file_open(&lfs, &file1, "jellyfish",
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
+    uint8_t wbuf1[SIZE];
+    for (lfs_size_t j = 0; j < SIZE; j++) {
+        wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
+    }
+    lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+    lfsr_file_sync(&lfs, &file1) => 0;
+
+    lfsr_file_t file2;
+    lfsr_file_open(&lfs, &file2, "octopus",
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
+    uint8_t wbuf2[SIZE];
+    for (lfs_size_t j = 0; j < SIZE; j++) {
+        wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
+    }
+    lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
+    lfsr_file_sync(&lfs, &file2) => 0;
+
+    // create enough files for mroot to split
+    lfs_size_t i = 0;
+    while (lfs.mtree.u.weight == 0x80000000) {
+        char name[256];
+        sprintf(name, "medusa%03x", i);
+        lfsr_file_t file;
+        lfsr_file_open(&lfs, &file, name,
+                LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+        lfsr_file_close(&lfs, &file) => 0;
+        i += 1;
+    }
+
+    // switch to early relocations after splitting
+    lfsr_file_close(&lfs, &file1) => 0;
+    lfsr_file_close(&lfs, &file2) => 0;
+    lfsr_unmount(&lfs) => 0;
+    struct lfs_config cfg = *CFG;
+    cfg.block_recycles = 0;
+    lfsr_mount(&lfs, &cfg) => 0;
+    lfsr_file_open(&lfs, &file1, "jellyfish", LFS_O_RDWR) => 0;
+    lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDWR) => 0;
+
+    // rewrite a file until btree is >gc_compact_thresh full
+    while (true) {
+        // we need internals to check this
+        lfsr_mdir_t mdir;
+        // ckmeta needed for eoff
+        lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA);
+        lfsr_mtinfo_t mtinfo;
+        lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
+        assert(mtinfo.tag == LFSR_TAG_MDIR);
+        lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
+        assert(mtinfo.tag == LFSR_TAG_BRANCH);
+        if (lfsr_rbyd_eoff(&mtinfo.u.rbyd) > GC_COMPACT_THRESH) {
+            break;
+        }
+
+        lfsr_file_rewind(&lfs, &file1) => 0;
+        for (lfs_size_t j = 0; j < SIZE; j++) {
+            wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
+        }
+        lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+        lfsr_file_sync(&lfs, &file1) => 0;
+    }
+
+    // switch back to default relocations to avoid mroot extension issues
+    lfsr_file_close(&lfs, &file1) => 0;
+    lfsr_file_close(&lfs, &file2) => 0;
+    lfsr_unmount(&lfs) => 0;
+    lfsr_mount(&lfs, CFG) => 0;
+    lfsr_file_open(&lfs, &file1, "jellyfish", LFS_O_RDWR) => 0;
+    lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDWR) => 0;
+
+    // try traversing and compacting
+    lfsr_traversal_t t;
+    lfsr_traversal_open(&lfs, &t,
+            LFS_T_COMPACT
+                | ((CKMETA) ? LFS_T_CKMETA : 0)
+                | ((CK) ? LFS_T_CK : 0)
+                | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
+    // traverse mroot
+    struct lfs_tinfo tinfo;
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    // traverse btree
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_BTREE);
+    // traverse mdir
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    // traverse mdir
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
+    lfsr_traversal_close(&lfs, &t) => 0;
+
+    // mtree should have been compacted
+    lfsr_mdir_t mdir;
+    lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA);
+    lfsr_mtinfo_t mtinfo;
+    lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
+    assert(mtinfo.tag == LFSR_TAG_MDIR);
+    lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
+    assert(mtinfo.tag == LFSR_TAG_BRANCH);
+    assert(lfsr_rbyd_eoff(&mtinfo.u.rbyd) <= GC_COMPACT_THRESH);
+
+    // check we can still read the files
+    for (int remount = 0; remount < 2; remount++) {
+        // remount?
+        if (remount) {
+            lfsr_file_close(&lfs, &file1) => 0;
+            lfsr_file_close(&lfs, &file2) => 0;
+            lfsr_unmount(&lfs) => 0;
+            lfsr_mount(&lfs, CFG) => 0;
+            lfsr_file_open(&lfs, &file1, "jellyfish", LFS_O_RDONLY) => 0;
+            lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDONLY) => 0;
+        }
+
+        lfsr_file_rewind(&lfs, &file1) => 0;
+        uint8_t rbuf[SIZE];
+        lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE;
+        assert(memcmp(rbuf, wbuf1, SIZE) == 0);
+
+        lfsr_file_rewind(&lfs, &file2) => 0;
+        lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE;
+        assert(memcmp(rbuf, wbuf2, SIZE) == 0);
+    }
+
+    lfsr_file_close(&lfs, &file1) => 0;
+    lfsr_file_close(&lfs, &file2) => 0;
+    lfsr_unmount(&lfs) => 0;
+'''
+
+[cases.test_traversal_compact_btree]
+defines.CKMETA = [false, true]
+defines.CK = [false, true]
+defines.LOOKAHEAD = [false, true]
+# limit files to very simple btrees
+defines.INLINE_SIZE = 0
+defines.CRYSTAL_THRESH = -1
+defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8'
+defines.SIZE = '2*FRAGMENT_SIZE'
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
+code = '''
+    lfs_t lfs;
+    lfsr_format(&lfs, CFG) => 0;
+    lfsr_mount(&lfs, CFG) => 0;
+
+    uint32_t prng = 42;
+
+    // create a file
+    lfsr_file_t file;
+    lfsr_file_open(&lfs, &file, "jellyfish",
+            LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+
+    // rewrite our file until btree is >gc_compact_thresh full
+    uint8_t wbuf[SIZE];
+    while ((file.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
+        lfsr_file_rewind(&lfs, &file) => 0;
+        for (lfs_size_t j = 0; j < SIZE; j++) {
+            wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
+        }
+        lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
+    }
+
+    lfsr_file_close(&lfs, &file) => 0;
+
+    // try traversing and compacting
+    lfsr_traversal_t t;
+    lfsr_traversal_open(&lfs, &t,
+            LFS_T_COMPACT
+                | ((CKMETA) ? LFS_T_CKMETA : 0)
+                | ((CK) ? LFS_T_CK : 0)
+                | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
+    // traverse mroot
+    struct lfs_tinfo tinfo;
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    // traverse btree
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_BTREE);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
+    lfsr_traversal_close(&lfs, &t) => 0;
+
+    // btree should have been compacted
+    lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
+    assert((file.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
+
+    // check we can still read the file
+    for (int remount = 0; remount < 2; remount++) {
+        // remount?
+        if (remount) {
+            lfsr_file_close(&lfs, &file) => 0;
+            lfsr_unmount(&lfs) => 0;
+            lfsr_mount(&lfs, CFG) => 0;
+            lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
+        }
+
+        lfsr_file_rewind(&lfs, &file) => 0;
+        uint8_t rbuf[SIZE];
+        lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
+        assert(memcmp(rbuf, wbuf, SIZE) == 0);
+    }
+
+    lfsr_file_close(&lfs, &file) => 0;
+    lfsr_unmount(&lfs) => 0;
+'''
+
+[cases.test_traversal_compact_btree_open]
+defines.CKMETA = [false, true]
+defines.CK = [false, true]
+defines.LOOKAHEAD = [false, true]
+# limit files to very simple btrees
+defines.INLINE_SIZE = 0
+defines.CRYSTAL_THRESH = -1
+defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8'
+defines.SIZE = '2*FRAGMENT_SIZE'
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
+code = '''
+    lfs_t lfs;
+    lfsr_format(&lfs, CFG) => 0;
+    lfsr_mount(&lfs, CFG) => 0;
+
+    uint32_t prng = 42;
+
+    // create a file
+    lfsr_file_t file;
+    lfsr_file_open(&lfs, &file, "jellyfish",
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
+
+    // rewrite our file until btree is >gc_compact_thresh full
+    uint8_t wbuf[SIZE];
+    while ((file.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
+        lfsr_file_rewind(&lfs, &file) => 0;
+        for (lfs_size_t j = 0; j < SIZE; j++) {
+            wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
+        }
+        lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
+    }
+
+    lfsr_file_sync(&lfs, &file) => 0;
+
+    // try traversing and compacting
+    lfsr_traversal_t t;
+    lfsr_traversal_open(&lfs, &t,
+            LFS_T_COMPACT
+                | ((CKMETA) ? LFS_T_CKMETA : 0)
+                | ((CK) ? LFS_T_CK : 0)
+                | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
+    // traverse mroot
+    struct lfs_tinfo tinfo;
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    // traverse btree
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_BTREE);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
+    lfsr_traversal_close(&lfs, &t) => 0;
+
+    // btree should have been compacted
+    assert((file.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
+
+    // check we can still read the file
+    for (int remount = 0; remount < 2; remount++) {
+        // remount?
+        if (remount) {
+            lfsr_file_close(&lfs, &file) => 0;
+            lfsr_unmount(&lfs) => 0;
+            lfsr_mount(&lfs, CFG) => 0;
+            lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
+        }
+
+        lfsr_file_rewind(&lfs, &file) => 0;
+        uint8_t rbuf[SIZE];
+        lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
+        assert(memcmp(rbuf, wbuf, SIZE) == 0);
+    }
+
+    lfsr_file_close(&lfs, &file) => 0;
+    lfsr_unmount(&lfs) => 0;
+'''
+
+[cases.test_traversal_compact_btree_orphan]
+defines.CKMETA = [false, true]
+defines.CK = [false, true]
+defines.LOOKAHEAD = [false, true]
+# limit files to very simple btrees
+defines.INLINE_SIZE = 0
+defines.CRYSTAL_THRESH = -1
+defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8'
+defines.SIZE = '2*FRAGMENT_SIZE'
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
+code = '''
+    lfs_t lfs;
+    lfsr_format(&lfs, CFG) => 0;
+    lfsr_mount(&lfs, CFG) => 0;
+
+    uint32_t prng = 42;
+
+    // create a file
+    lfsr_file_t file;
+    lfsr_file_open(&lfs, &file, "jellyfish",
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
+
+    // rewrite our file until btree is >gc_compact_thresh full
+    uint8_t wbuf[SIZE];
+    while ((file.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
+        lfsr_file_rewind(&lfs, &file) => 0;
+        for (lfs_size_t j = 0; j < SIZE; j++) {
+            wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
+        }
+        lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
+    }
+
+    // try traversing and compacting
+    lfsr_traversal_t t;
+    lfsr_traversal_open(&lfs, &t,
+            LFS_T_COMPACT
+                | ((CKMETA) ? LFS_T_CKMETA : 0)
+                | ((CK) ? LFS_T_CK : 0)
+                | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
+    // traverse mroot
+    struct lfs_tinfo tinfo;
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    // traverse btree
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_BTREE);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
+    lfsr_traversal_close(&lfs, &t) => 0;
+
+    // btree should have been compacted
+    assert((file.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
+
+    // file should not have accidentally been created or anything
+    struct lfs_info info;
+    lfsr_stat(&lfs, "jellyfish", &info) => LFS_ERR_NOENT;
+
+    // check we can still read the file
+    for (int remount = 0; remount < 2; remount++) {
+        // remount?
+        if (remount) {
+            lfsr_file_close(&lfs, &file) => 0;
+            lfsr_unmount(&lfs) => 0;
+            lfsr_mount(&lfs, CFG) => 0;
+            lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
+        }
+
+        lfsr_file_rewind(&lfs, &file) => 0;
+        uint8_t rbuf[SIZE];
+        lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
+        assert(memcmp(rbuf, wbuf, SIZE) == 0);
+    }
+
+    lfsr_file_close(&lfs, &file) => 0;
+    lfsr_unmount(&lfs) => 0;
+'''
+
+[cases.test_traversal_compact_btree_desync]
+defines.CKMETA = [false, true]
+defines.CK = [false, true]
+defines.LOOKAHEAD = [false, true]
+# limit files to very simple btrees
+defines.INLINE_SIZE = 0
+defines.CRYSTAL_THRESH = -1
+defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8'
+defines.SIZE = '2*FRAGMENT_SIZE'
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
+code = '''
+    lfs_t lfs;
+    lfsr_format(&lfs, CFG) => 0;
+    lfsr_mount(&lfs, CFG) => 0;
+
+    uint32_t prng = 42;
+
+    // create a desync file
+    lfsr_file_t file1;
+    lfsr_file_open(&lfs, &file1, "jellyfish",
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0;
+
+    // rewrite our file until btree is >gc_compact_thresh full
+    uint8_t wbuf1[SIZE];
+    while ((file1.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
+        lfsr_file_rewind(&lfs, &file1) => 0;
+        for (lfs_size_t j = 0; j < SIZE; j++) {
+            wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
+        }
+        lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+    }
+
+    // create some overlapping files, these should not get messed with
+    lfsr_file_t file2;
+    lfsr_file_open(&lfs, &file2, "jellyfish",
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
+    uint8_t wbuf2[SIZE];
+    for (lfs_size_t j = 0; j < SIZE; j++) {
+        wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
+    }
+    lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
+    lfsr_file_sync(&lfs, &file2) => 0;
+
+    lfsr_file_t file3;
+    lfsr_file_open(&lfs, &file3, "jellyfish",
+            LFS_O_RDWR) => 0;
+    uint8_t wbuf3[SIZE];
+    for (lfs_size_t j = 0; j < SIZE; j++) {
+        wbuf3[j] = 'a' + (TEST_PRNG(&prng) % 26);
+    }
+    lfsr_file_write(&lfs, &file3, wbuf3, SIZE) => SIZE;
+    lfsr_file_desync(&lfs, &file3) => 0;
+
+    // try traversing and compacting
+    lfsr_traversal_t t;
+    lfsr_traversal_open(&lfs, &t,
+            LFS_T_COMPACT
+                | ((CKMETA) ? LFS_T_CKMETA : 0)
+                | ((CK) ? LFS_T_CK : 0)
+                | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
+    // traverse mroot
+    struct lfs_tinfo tinfo;
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    // traverse btree
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_BTREE);
+    // traverse btree
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_BTREE);
+    // traverse btree
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_BTREE);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
+    lfsr_traversal_close(&lfs, &t) => 0;
+
+    // btree should have been compacted
+    assert((file1.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
+
+    // check we can still read the files
+    lfsr_file_rewind(&lfs, &file1) => 0;
+    uint8_t rbuf[SIZE];
+    lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE;
+    assert(memcmp(rbuf, wbuf1, SIZE) == 0);
+
+    lfsr_file_rewind(&lfs, &file2) => 0;
+    lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE;
+    assert(memcmp(rbuf, wbuf2, SIZE) == 0);
+
+    lfsr_file_rewind(&lfs, &file3) => 0;
+    lfsr_file_read(&lfs, &file3, rbuf, SIZE) => SIZE;
+    assert(memcmp(rbuf, wbuf3, SIZE) == 0);
+
+    // at least try closing/opening our synced file
+    lfsr_file_close(&lfs, &file2) => 0;
+    lfsr_file_open(&lfs, &file2, "jellyfish", LFS_O_RDONLY) => 0;
+    lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE;
+    assert(memcmp(rbuf, wbuf2, SIZE) == 0);
+
+    lfsr_file_close(&lfs, &file1) => 0;
+    lfsr_file_close(&lfs, &file2) => 0;
+    lfsr_file_close(&lfs, &file3) => 0;
+    lfsr_unmount(&lfs) => 0;
+'''
+
+# TODO
+# [cases.test_traversal_compact_bshrub]
+# [cases.test_traversal_compact_bshrub_open]
+# [cases.test_traversal_compact_bshrub_orphan]
+# [cases.test_traversal_compact_bshrub_desync]
+
+
+

 # many/fuzz tests mixed with traversals
 #
@@ -5789,6 +6387,8 @@ defines.CKMETA = [true]
 defines.CK = [true]
 defines.LOOKAHEAD = [false, true]
 defines.COMPACT = [false, true]
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256]
 code = '''
    // test creating directories
@@ -5897,6 +6497,8 @@ defines.CKMETA = [true]
 defines.CK = [true]
 defines.LOOKAHEAD = [false, true]
 defines.COMPACT = [false, true]
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256]
 defines.OPS = '2*N'
 defines.SEED = 42
@@ -6076,6 +6678,8 @@ defines.CKMETA = [true]
 defines.CK = [true]
 defines.LOOKAHEAD = [false, true]
 defines.COMPACT = [false, true]
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
 defines.N = [1, 2, 4, 8, 16, 32, 64]
 defines.SIZE = [
    '0',
@@ -6178,6 +6782,8 @@ defines.CKMETA = [true]
 defines.CK = [true]
 defines.LOOKAHEAD = [false, true]
 defines.COMPACT = [false, true]
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
 defines.N = [1, 2, 4, 8, 16, 32, 64]
 defines.OPS = '2*N'
 defines.SIZE = [
@@ -6418,7 +7024,10 @@ defines.EXCL = [false, true]
 defines.CKMETA = [true]
 defines.CK = [true]
 defines.LOOKAHEAD = [false, true]
-defines.COMPACT = [false, true]
+# TODO !!! Enable this when bshrub compaction is working
+defines.COMPACT = [false]
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
 defines.OPS = 20
 defines.SIZE = [
    'FILE_BUFFER_SIZE/2',
@@ -6580,6 +7189,8 @@ defines.CKMETA = [true]
 defines.CK = [true]
 defines.LOOKAHEAD = [false, true]
 defines.COMPACT = [false, true]
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
 defines.N = [1, 2, 4, 8, 16, 32, 64]
 defines.OPS = '2*N'
 defines.SIZE = [
@@ -6949,6 +7560,8 @@ defines.CKMETA = [true]
 defines.CK = [true]
 defines.LOOKAHEAD = [false, true]
 defines.COMPACT = [false, true]
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
 defines.N = [1, 2, 4, 8, 16, 32, 64]
 defines.OPS = '2*N'
 defines.SIZE = [