t: Implemented gc_compact_thresh over btree nodes

Note, gc_compact_thresh over bshrubs is not yet implemented... That's
_another_ can of worms since we need to be able to commit to non-tracked
bshrubs somehow...

But at least this proves gc_compact_thresh over btrees is possible.

Now, if LFS_T_COMPACT is provided, any btree nodes > gc_compact_thresh
will be compacted during traversal/gc operations.

To make this work required a rather deep modification to the
lfsr_btree_commit/lfsr_bshrub_commit code paths to expose direct-rbyd
commit functions that can commit to arbitrary btree nodes:

- lfsr_btree_commit   - bid, attrs, attr_count
- lfsr_bshrub_commit  - bid, attrs, attr_count
- lfsr_btree_commit_  - bid, rbyd, rid, attrs, attr_count
- lfsr_bshrub_commit_ - bid, rbyb, rid, attrs, attr_count
- lfsr_btree_commit__ - bscratch, bid, rbyd, rid, attrs, attr_count

These are good to have, and will also be useful for implementing
metadata redundancy in the future.

Unfortunately, all of this comes at a significant code/stack cost:

           code          stack
  before: 34652           2640
  after:  35268 (+1.8%)   2776 (+5.2%)

lfs.c

@@ -4291,12 +4291,12 @@ static int lfsr_btree_parent(lfs_t *lfs, const lfsr_btree_t *btree,
 }
 
 
-// extra state needed for non-terminating lfsr_btree_commit_ calls
+// extra state needed for non-terminating lfsr_btree_commit__ calls
-typedef struct lfsr_btree_scratch {
+typedef struct lfsr_bscratch {
     lfsr_attr_t attrs[4];
     lfsr_data_t split_data;
     uint8_t buf[2*LFSR_BRANCH_DSIZE];
-} lfsr_btree_scratch_t;
+} lfsr_bscratch_t;
 
 // core btree algorithm
 //
@@ -4304,53 +4304,31 @@ typedef struct lfsr_btree_scratch {
 // 1. we need a new root
 // 2. we have a shrub root
 //
-static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
+static int lfsr_btree_commit__(lfs_t *lfs, lfsr_btree_t *btree,
-        lfsr_btree_scratch_t *scratch,
+        lfsr_bscratch_t *bscratch,
-        lfsr_bid_t *bid_,
+        lfsr_bid_t bid, lfsr_rbyd_t *rbyd, lfsr_srid_t *rid_,
         const lfsr_attr_t **attrs_, lfs_size_t *attr_count_) {
-    lfsr_bid_t bid = *bid_;
     LFS_ASSERT(bid <= (lfsr_bid_t)btree->weight);
+    lfsr_srid_t rid = *rid_;
     const lfsr_attr_t *attrs = *attrs_;
     lfs_size_t attr_count = *attr_count_;
 
-    // lookup in which leaf our bids resides
-    //
-    // for lfsr_btree_commit operations to work out, we need to
-    // limit our bid to an rid in the tree, which is what this min
-    // is doing
-    lfsr_rbyd_t rbyd = *btree;
-    lfsr_srid_t rid = bid;
-    if (btree->weight > 0) {
-        lfsr_srid_t rid_;
-        int err = lfsr_btree_lookupnext_(lfs, btree,
-                lfs_min(bid, btree->weight-1),
-                &bid, &rbyd, &rid_, NULL, NULL, NULL);
-        if (err) {
-            LFS_ASSERT(err != LFS_ERR_NOENT);
-            return err;
-        }
-
-        // adjust bid to point to the zero-most rid
-        bid -= rid_;
-        rid -= bid;
-    }
-
     // tail-recursively commit to btree
     while (true) {
         // we will always need our parent, so go ahead and find it
         lfsr_rbyd_t parent = {.trunk=0, .weight=0};
         lfsr_srid_t pid = 0;
         // are we root?
-        if (rbyd.blocks[0] == btree->blocks[0]
+        if (rbyd->blocks[0] == btree->blocks[0]
-                || !lfsr_rbyd_trunk(&rbyd)) {
+                || !lfsr_rbyd_trunk(rbyd)) {
             // new root? shrub root? yield the final root commit to
             // higher-level btree/bshrub logic
-            if (!lfsr_rbyd_trunk(&rbyd)
+            if (!lfsr_rbyd_trunk(rbyd)
                     || lfsr_rbyd_isshrub(btree)) {
-                *bid_ = rid;
+                *rid_ = rid;
                 *attrs_ = attrs;
                 *attr_count_ = attr_count;
-                return (!lfsr_rbyd_trunk(&rbyd)) ? LFS_ERR_RANGE : 0;
+                return (!lfsr_rbyd_trunk(rbyd)) ? LFS_ERR_RANGE : 0;
             }
 
             // mark btree as unerased in case of failure, our btree rbyd and
@@ -4359,7 +4337,7 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
             btree->eoff = -1;
 
         } else {
-            int err = lfsr_btree_parent(lfs, btree, bid, &rbyd,
+            int err = lfsr_btree_parent(lfs, btree, bid, rbyd,
                     &parent, &pid);
             if (err) {
                 LFS_ASSERT(err != LFS_ERR_NOENT);
@@ -4374,10 +4352,10 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
         // unfetched
         //
         // a funny benefit is we cache the root of our btree this way
-        if (!lfsr_rbyd_isfetched(&rbyd)) {
+        if (!lfsr_rbyd_isfetched(rbyd)) {
-            int err = lfsr_rbyd_fetchck(lfs, &rbyd,
+            int err = lfsr_rbyd_fetchck(lfs, rbyd,
-                    rbyd.blocks[0], lfsr_rbyd_trunk(&rbyd),
+                    rbyd->blocks[0], lfsr_rbyd_trunk(rbyd),
-                    rbyd.cksum);
+                    rbyd->cksum);
             if (err) {
                 return err;
             }
@@ -4386,7 +4364,7 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
         // is rbyd erased? can we sneak our commit into any remaining
         // erased bytes? note that the btree trunk field prevents this from
         // interacting with other references to the rbyd
-        lfsr_rbyd_t rbyd_ = rbyd;
+        lfsr_rbyd_t rbyd_ = *rbyd;
         int err = lfsr_rbyd_commit(lfs, &rbyd_, rid,
                 attrs, attr_count);
         if (err) {
@@ -4401,7 +4379,7 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
     compact:;
         // estimate our compacted size
         lfsr_srid_t split_rid;
-        lfs_ssize_t estimate = lfsr_rbyd_estimate(lfs, &rbyd, -1, -1,
+        lfs_ssize_t estimate = lfsr_rbyd_estimate(lfs, rbyd, -1, -1,
                 &split_rid);
         if (estimate < 0) {
             return estimate;
@@ -4467,14 +4445,14 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
             }
 
             // try the left sibling
-            if (pid-(lfsr_srid_t)rbyd.weight >= 0) {
+            if (pid-(lfsr_srid_t)rbyd->weight >= 0) {
                 // try looking up the sibling
                 lfsr_srid_t sibling_rid;
                 lfsr_tag_t sibling_tag;
                 lfsr_rid_t sibling_weight;
                 lfsr_data_t sibling_data;
                 err = lfsr_rbyd_lookupnext(lfs, &parent,
-                        pid-rbyd.weight, LFSR_TAG_NAME,
+                        pid-rbyd->weight, LFSR_TAG_NAME,
                         &sibling_rid, &sibling_tag, &sibling_weight,
                         &sibling_data);
                 if (err) {
@@ -4514,11 +4492,11 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
                     // so our sibling is on the right
                     bid -= sibling.weight;
                     rid += sibling.weight;
-                    pid -= rbyd.weight;
+                    pid -= rbyd->weight;
 
                     rbyd_ = sibling;
-                    sibling = rbyd;
+                    sibling = *rbyd;
-                    rbyd = rbyd_;
+                    *rbyd = rbyd_;
 
                     goto merge;
                 }
@@ -4533,7 +4511,7 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
         }
 
         // try to compact
-        err = lfsr_rbyd_compact(lfs, &rbyd_, &rbyd, -1, -1);
+        err = lfsr_rbyd_compact(lfs, &rbyd_, rbyd, -1, -1);
         if (err) {
             LFS_ASSERT(err != LFS_ERR_RANGE);
             // bad prog? try another block
@@ -4561,7 +4539,7 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
     split:;
         // we should have something to split here
         LFS_ASSERT(split_rid > 0
-                && split_rid < (lfsr_srid_t)rbyd.weight);
+                && split_rid < (lfsr_srid_t)rbyd->weight);
 
     split_relocate_l:;
         // allocate a new rbyd
@@ -4571,7 +4549,7 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
         }
 
         // copy over tags < split_rid
-        err = lfsr_rbyd_compact(lfs, &rbyd_, &rbyd, -1, split_rid);
+        err = lfsr_rbyd_compact(lfs, &rbyd_, rbyd, -1, split_rid);
         if (err) {
             LFS_ASSERT(err != LFS_ERR_RANGE);
             // bad prog? try another block
@@ -4615,7 +4593,7 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
         }
 
         // copy over tags >= split_rid
-        err = lfsr_rbyd_compact(lfs, &sibling, &rbyd, split_rid, -1);
+        err = lfsr_rbyd_compact(lfs, &sibling, rbyd, split_rid, -1);
         if (err) {
             LFS_ASSERT(err != LFS_ERR_RANGE);
             // bad prog? try another block
@@ -4666,7 +4644,7 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
         // they introduce a new name!
         lfsr_tag_t split_tag;
         err = lfsr_rbyd_lookupnext(lfs, &sibling, 0, LFSR_TAG_NAME,
-                NULL, &split_tag, NULL, &scratch->split_data);
+                NULL, &split_tag, NULL, &bscratch->split_data);
         if (err) {
             LFS_ASSERT(err != LFS_ERR_NOENT);
             return err;
@@ -4678,48 +4656,48 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
         attr_count = 0;
         // new root?
         if (!lfsr_rbyd_trunk(&parent)) {
-            scratch->attrs[attr_count++] = LFSR_ATTR(
+            bscratch->attrs[attr_count++] = LFSR_ATTR(
                     LFSR_TAG_BRANCH, +rbyd_.weight,
                     LFSR_DATA_BRANCH_(
                         &rbyd_,
-                        &scratch->buf[0*LFSR_BRANCH_DSIZE]));
+                        &bscratch->buf[0*LFSR_BRANCH_DSIZE]));
-            scratch->attrs[attr_count++] = LFSR_ATTR(
+            bscratch->attrs[attr_count++] = LFSR_ATTR(
                     LFSR_TAG_BRANCH, +sibling.weight,
                     LFSR_DATA_BRANCH_(
                         &sibling,
-                        &scratch->buf[1*LFSR_BRANCH_DSIZE]));
+                        &bscratch->buf[1*LFSR_BRANCH_DSIZE]));
             if (lfsr_tag_suptype(split_tag) == LFSR_TAG_NAME) {
-                scratch->attrs[attr_count++] = LFSR_ATTR_CAT_(
+                bscratch->attrs[attr_count++] = LFSR_ATTR_CAT_(
                         LFSR_TAG_NAME, 0,
-                        &scratch->split_data, 1);
+                        &bscratch->split_data, 1);
             }
         // split root?
         } else {
-            bid -= pid - (rbyd.weight-1);
+            bid -= pid - (rbyd->weight-1);
-            scratch->attrs[attr_count++] = LFSR_ATTR(
+            bscratch->attrs[attr_count++] = LFSR_ATTR(
                     LFSR_TAG_BRANCH, 0,
                     LFSR_DATA_BRANCH_(
                         &rbyd_,
-                        &scratch->buf[0*LFSR_BRANCH_DSIZE]));
+                        &bscratch->buf[0*LFSR_BRANCH_DSIZE]));
-            if (rbyd_.weight != rbyd.weight) {
+            if (rbyd_.weight != rbyd->weight) {
-                scratch->attrs[attr_count++] = LFSR_ATTR(
+                bscratch->attrs[attr_count++] = LFSR_ATTR(
-                        LFSR_TAG_GROW, -rbyd.weight + rbyd_.weight,
+                        LFSR_TAG_GROW, -rbyd->weight + rbyd_.weight,
                         LFSR_DATA_NULL());
             }
-            scratch->attrs[attr_count++] = LFSR_ATTR(
+            bscratch->attrs[attr_count++] = LFSR_ATTR(
                     LFSR_TAG_BRANCH, +sibling.weight,
                     LFSR_DATA_BRANCH_(
                         &sibling,
-                        &scratch->buf[1*LFSR_BRANCH_DSIZE]));
+                        &bscratch->buf[1*LFSR_BRANCH_DSIZE]));
             if (lfsr_tag_suptype(split_tag) == LFSR_TAG_NAME) {
-                scratch->attrs[attr_count++] = LFSR_ATTR_CAT_(
+                bscratch->attrs[attr_count++] = LFSR_ATTR_CAT_(
                         LFSR_TAG_NAME, 0,
-                        &scratch->split_data, 1);
+                        &bscratch->split_data, 1);
             }
         }
-        attrs = scratch->attrs;
+        attrs = bscratch->attrs;
 
-        rbyd = parent;
+        *rbyd = parent;
         rid = pid;
         continue;
 
@@ -4732,7 +4710,7 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
         }
 
         // merge the siblings together
-        err = lfsr_rbyd_appendcompactrbyd(lfs, &rbyd_, &rbyd, -1, -1);
+        err = lfsr_rbyd_appendcompactrbyd(lfs, &rbyd_, rbyd, -1, -1);
         if (err) {
             LFS_ASSERT(err != LFS_ERR_RANGE);
             // bad prog? try another block
@@ -4778,7 +4756,7 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
         // we must have a parent at this point, but is our parent the root
         // and is the root degenerate?
         LFS_ASSERT(lfsr_rbyd_trunk(&parent));
-        if (rbyd.weight+sibling.weight == btree->weight) {
+        if (rbyd->weight+sibling.weight == btree->weight) {
             // collapse the root, decreasing the height of the tree
             *btree = rbyd_;
             *attr_count_ = 0;
@@ -4788,20 +4766,20 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
         // prepare commit to parent, tail recursing upwards
         LFS_ASSERT(rbyd_.weight > 0);
         attr_count = 0;
-        bid -= pid - (rbyd.weight-1);
+        bid -= pid - (rbyd->weight-1);
-        scratch->attrs[attr_count++] = LFSR_ATTR(
+        bscratch->attrs[attr_count++] = LFSR_ATTR(
                 LFSR_TAG_RM, -sibling.weight, LFSR_DATA_NULL());
-        scratch->attrs[attr_count++] = LFSR_ATTR(
+        bscratch->attrs[attr_count++] = LFSR_ATTR(
                 LFSR_TAG_BRANCH, 0,
-                LFSR_DATA_BRANCH_(&rbyd_, scratch->buf));
+                LFSR_DATA_BRANCH_(&rbyd_, bscratch->buf));
-        if (rbyd_.weight != rbyd.weight) {
+        if (rbyd_.weight != rbyd->weight) {
-            scratch->attrs[attr_count++] = LFSR_ATTR(
+            bscratch->attrs[attr_count++] = LFSR_ATTR(
-                    LFSR_TAG_GROW, -rbyd.weight + rbyd_.weight,
+                    LFSR_TAG_GROW, -rbyd->weight + rbyd_.weight,
                     LFSR_DATA_NULL());
         }
-        attrs = scratch->attrs;
+        attrs = bscratch->attrs;
 
-        rbyd = parent;
+        *rbyd = parent;
         rid = pid + sibling.weight;
         continue;
 
@@ -4815,7 +4793,7 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
         }
 
         // is our parent the root and is the root degenerate?
-        if (rbyd.weight == btree->weight) {
+        if (rbyd->weight == btree->weight) {
             // collapse the root, decreasing the height of the tree
             *btree = rbyd_;
             *attr_count_ = 0;
@@ -4827,35 +4805,35 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
         // note that since we defer merges to compaction time, we can
         // end up removing an rbyd here
         attr_count = 0;
-        bid -= pid - (rbyd.weight-1);
+        bid -= pid - (rbyd->weight-1);
         if (rbyd_.weight == 0) {
-            scratch->attrs[attr_count++] = LFSR_ATTR(
+            bscratch->attrs[attr_count++] = LFSR_ATTR(
-                    LFSR_TAG_RM, -rbyd.weight, LFSR_DATA_NULL());
+                    LFSR_TAG_RM, -rbyd->weight, LFSR_DATA_NULL());
         } else {
-            scratch->attrs[attr_count++] = LFSR_ATTR(
+            bscratch->attrs[attr_count++] = LFSR_ATTR(
                     LFSR_TAG_BRANCH, 0,
-                    LFSR_DATA_BRANCH_(&rbyd_, scratch->buf));
+                    LFSR_DATA_BRANCH_(&rbyd_, bscratch->buf));
-            if (rbyd_.weight != rbyd.weight) {
+            if (rbyd_.weight != rbyd->weight) {
-                scratch->attrs[attr_count++] = LFSR_ATTR(
+                bscratch->attrs[attr_count++] = LFSR_ATTR(
-                        LFSR_TAG_GROW, -rbyd.weight + rbyd_.weight,
+                        LFSR_TAG_GROW, -rbyd->weight + rbyd_.weight,
                         LFSR_DATA_NULL());
             }
         }
-        attrs = scratch->attrs;
+        attrs = bscratch->attrs;
 
-        rbyd = parent;
+        *rbyd = parent;
         rid = pid;
         continue;
     }
 }
 
-// commit to a btree, this is atomic
+static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree,
-static int lfsr_btree_commit(lfs_t *lfs, lfsr_btree_t *btree,
+        lfsr_bid_t bid, lfsr_rbyd_t *rbyd, lfsr_srid_t rid,
-        lfsr_bid_t bid, const lfsr_attr_t *attrs, lfs_size_t attr_count) {
+        const lfsr_attr_t *attrs, lfs_size_t attr_count) {
     // try to commit to the btree
-    lfsr_btree_scratch_t scratch;
+    lfsr_bscratch_t bscratch;
-    int err = lfsr_btree_commit_(lfs, btree, &scratch,
+    int err = lfsr_btree_commit__(lfs, btree, &bscratch,
-            &bid, &attrs, &attr_count);
+            bid, rbyd, &rid, &attrs, &attr_count);
     if (err && err != LFS_ERR_RANGE) {
         return err;
     }
@@ -4865,13 +4843,12 @@ static int lfsr_btree_commit(lfs_t *lfs, lfsr_btree_t *btree,
         LFS_ASSERT(attr_count > 0);
 
     relocate:;
-        lfsr_rbyd_t rbyd;
+        err = lfsr_rbyd_alloc(lfs, rbyd);
-        err = lfsr_rbyd_alloc(lfs, &rbyd);
         if (err) {
             return err;
         }
 
-        err = lfsr_rbyd_commit(lfs, &rbyd, bid, attrs, attr_count);
+        err = lfsr_rbyd_commit(lfs, rbyd, rid, attrs, attr_count);
         if (err) {
             LFS_ASSERT(err != LFS_ERR_RANGE);
             // bad prog? try another block
@@ -4881,13 +4858,51 @@ static int lfsr_btree_commit(lfs_t *lfs, lfsr_btree_t *btree,
             return err;
         }
 
-        *btree = rbyd;
+        *btree = *rbyd;
     }
 
     LFS_ASSERT(lfsr_rbyd_trunk(btree));
     return 0;
 }
 
+static int lfsr_btree_compact_(lfs_t *lfs, lfsr_btree_t *btree,
+        lfsr_bid_t bid, lfsr_rbyd_t *rbyd) {
+    // the easiest way to do this is to just mark rbyd as unerased
+    // and call lfsr_btree_commit_
+    rbyd->eoff = -1;
+    return lfsr_btree_commit_(lfs, btree, bid, rbyd, 0,
+            NULL, 0);
+}
+
+// commit to a btree, this is atomic
+static int lfsr_btree_commit(lfs_t *lfs, lfsr_btree_t *btree,
+        lfsr_bid_t bid, const lfsr_attr_t *attrs, lfs_size_t attr_count) {
+    // lookup in which leaf our bids resides
+    //
+    // for lfsr_btree_commit operations to work out, we need to
+    // limit our bid to an rid in the tree, which is what this min
+    // is doing
+    lfsr_rbyd_t rbyd = *btree;
+    lfsr_srid_t rid = bid;
+    if (btree->weight > 0) {
+        lfsr_srid_t rid_;
+        int err = lfsr_btree_lookupnext_(lfs, btree,
+                lfs_min(bid, btree->weight-1),
+                &bid, &rbyd, &rid_, NULL, NULL, NULL);
+        if (err) {
+            LFS_ASSERT(err != LFS_ERR_NOENT);
+            return err;
+        }
+
+        // adjust bid to point to the zero-most rid
+        bid -= rid_;
+        rid -= bid;
+    }
+
+    return lfsr_btree_commit_(lfs, btree, bid, &rbyd, rid,
+            attrs, attr_count);
+}
+
 // lookup in a btree by name
 static lfs_scmp_t lfsr_btree_namelookup(lfs_t *lfs, const lfsr_btree_t *btree,
         lfsr_did_t did, const char *name, lfs_size_t name_size,
@@ -5602,14 +5617,14 @@ static int lfsr_bshrub_traverse(lfs_t *lfs,
     }
 }
 
-// needed in lfsr_bshrub_commit
+// needed in lfsr_bshrub_commit_
 static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir,
         const lfsr_attr_t *attrs, lfs_size_t attr_count);
 
-// commit to a bshrub, this is atomic
+static int lfsr_bshrub_commit_(lfs_t *lfs,
-static int lfsr_bshrub_commit(lfs_t *lfs,
         lfsr_mdir_t *mdir, lfsr_bshrub_t *bshrub,
-        lfsr_bid_t bid, const lfsr_attr_t *attrs, lfs_size_t attr_count) {
+        lfsr_bid_t bid, lfsr_rbyd_t *rbyd, lfsr_srid_t rid,
+        const lfsr_attr_t *attrs, lfs_size_t attr_count) {
     // file must be a bshrub/btree here
     LFS_ASSERT(lfsr_bshrub_isbshruborbtree(bshrub));
 
@@ -5631,9 +5646,9 @@ static int lfsr_bshrub_commit(lfs_t *lfs,
     }
 
     // try to commit to the btree
-    lfsr_btree_scratch_t scratch;
+    lfsr_bscratch_t bscratch;
-    int err = lfsr_btree_commit_(lfs, &bshrub->u.btree, &scratch,
+    int err = lfsr_btree_commit__(lfs, &bshrub->u.btree, &bscratch,
-            &bid, &attrs, &attr_count);
+            bid, rbyd, &rid, &attrs, &attr_count);
     if (err && err != LFS_ERR_RANGE) {
         return err;
     }
@@ -5698,7 +5713,7 @@ static int lfsr_bshrub_commit(lfs_t *lfs,
         int err = lfsr_mdir_commit(lfs, mdir, LFSR_ATTRS(
                 LFSR_ATTR_SHRUBCOMMIT(
                     LFSR_TAG_SHRUBCOMMIT, 0,
-                    &bshrub->u.bshrub, bid, attrs, attr_count)));
+                    &bshrub->u.bshrub, rid, attrs, attr_count)));
         if (err) {
             return err;
         }
@@ -5723,15 +5738,14 @@ static int lfsr_bshrub_commit(lfs_t *lfs,
 
 relocate:;
     // convert to btree
-    lfsr_rbyd_t rbyd;
+    err = lfsr_rbyd_alloc(lfs, rbyd);
-    err = lfsr_rbyd_alloc(lfs, &rbyd);
     if (err) {
         return err;
     }
 
     // note this may be a new root
     if (!alloc) {
-        err = lfsr_rbyd_compact(lfs, &rbyd,
+        err = lfsr_rbyd_compact(lfs, rbyd,
                 &bshrub->u.btree, -1, -1);
         if (err) {
             LFS_ASSERT(err != LFS_ERR_RANGE);
@@ -5743,7 +5757,7 @@ relocate:;
         }
     }
 
-    err = lfsr_rbyd_commit(lfs, &rbyd, bid,
+    err = lfsr_rbyd_commit(lfs, rbyd, rid,
             attrs, attr_count);
     if (err) {
         LFS_ASSERT(err != LFS_ERR_RANGE);
@@ -5754,10 +5768,55 @@ relocate:;
         return err;
     }
 
-    bshrub->u.btree = rbyd;
+    bshrub->u.btree = *rbyd;
     return 0;
 }
 
+static int lfsr_bshrub_compact_(lfs_t *lfs,
+        lfsr_mdir_t *mdir, lfsr_bshrub_t *bshrub,
+        lfsr_bid_t bid, lfsr_rbyd_t *rbyd) {
+    // the easiest way to do this is to just mark rbyd as unerased
+    // and call lfsr_btree_commit_
+    rbyd->eoff = -1;
+    return lfsr_bshrub_commit_(lfs, mdir, bshrub, bid, rbyd, 0,
+            NULL, 0);
+}
+
+// commit to a bshrub, this is atomic
+static int lfsr_bshrub_commit(lfs_t *lfs,
+        lfsr_mdir_t *mdir, lfsr_bshrub_t *bshrub,
+        lfsr_bid_t bid, const lfsr_attr_t *attrs, lfs_size_t attr_count) {
+    // file must be a bshrub/btree here
+    LFS_ASSERT(lfsr_bshrub_isbshruborbtree(bshrub));
+
+    // TODO can we dedup the lookup logic?
+    // lookup in which leaf our bids resides
+    //
+    // for lfsr_btree_commit operations to work out, we need to
+    // limit our bid to an rid in the tree, which is what this min
+    // is doing
+    lfsr_rbyd_t rbyd = bshrub->u.btree;
+    lfsr_srid_t rid = bid;
+    if (bshrub->u.btree.weight > 0) {
+        lfsr_srid_t rid_;
+        int err = lfsr_btree_lookupnext_(lfs, &bshrub->u.btree,
+                lfs_min(bid, bshrub->u.btree.weight-1),
+                &bid, &rbyd, &rid_, NULL, NULL, NULL);
+        if (err) {
+            LFS_ASSERT(err != LFS_ERR_NOENT);
+            return err;
+        }
+
+        // TODO
+        // adjust bid to point to the zero-most rid
+        bid -= rid_;
+        rid -= bid;
+    }
+
+    return lfsr_bshrub_commit_(lfs, mdir, bshrub, bid, &rbyd, rid,
+            attrs, attr_count);
+}
+
 
 
 
@@ -8514,7 +8573,10 @@ static int lfsr_mtree_traverse(lfs_t *lfs,
     }
 
     // validate btree nodes? note mdirs are already validated
-    if ((lfsr_t_isckmeta(mt->flags) || lfsr_t_isck(mt->flags))
+    if ((lfsr_t_isckmeta(mt->flags)
+                || lfsr_t_isck(mt->flags)
+                // we also need to fetch to know if we need to compact
+                || lfsr_t_iscompact(mt->flags))
             && mtinfo->tag == LFSR_TAG_BRANCH) {
         err = lfsr_rbyd_fetchck(lfs, &mtinfo->u.rbyd,
                 mtinfo->u.rbyd.blocks[0], mtinfo->u.rbyd.trunk,
@@ -8590,21 +8652,86 @@ static int lfsr_mtree_gc(lfs_t *lfs,
         }
         mtinfo->u.mdir = *mdir;
 
+        // mark as dirty
         mt->flags |= LFS_F_DIRTY;
     }
 
-//    // TODO
+    // compacting btree nodes?
-//    // compacting btree nodes?
+    if (lfsr_t_iscompact(mt->flags)
-//    if (lfsr_t_iscompact(mt->flags)
+            && mtinfo->tag == LFSR_TAG_BRANCH
-//            && mtinfo->tag == LFSR_TAG_BTREE
+            // exceed compaction threshold?
-//            // exceed compaction threshold?
+            && lfsr_rbyd_eoff(&mtinfo->u.rbyd)
-//            && lfsr_rbyd_eoff(&mtinfo->u.rbyd)
+                > ((lfs->cfg->gc_compact_thresh)
-//                > ((lfs->cfg->gc_compact_thresh)
+                    ? lfs->cfg->gc_compact_thresh
-//                    ? lfs->cfg->gc_compact_thresh
+                    : lfs->cfg->block_size - lfs->cfg->block_size/8)) {
-//                    : lfs->cfg->block_size - lfs->cfg->block_size/8)) {
+        LFS_DEBUG("Compacting rbyd 0x%"PRIx32".%"PRIx32" "
-//
+                "(%"PRId32" > %"PRId32")",
-//        // TODO clobber?
+                mtinfo->u.rbyd.blocks[0],
-//    }
+                lfsr_rbyd_trunk(&mtinfo->u.rbyd),
+                lfsr_rbyd_eoff(&mtinfo->u.rbyd),
+                (lfs->cfg->gc_compact_thresh)
+                    ? lfs->cfg->gc_compact_thresh
+                    : lfs->cfg->block_size - lfs->cfg->block_size/8);
+
+        // TODO should we really have two btree copies flying around?
+        LFS_ASSERT(lfsr_rbyd_cmp(&mt->u.bt.branch, &mtinfo->u.rbyd) == 0);
+        if (mt->state == LFSR_MTRAVERSAL_MTREE) {
+            int err = lfsr_btree_compact_(lfs, &mt->bshrub.u.btree,
+                    mt->u.bt.bid, &mt->u.bt.branch);
+            if (err) {
+                return err;
+            }
+        } else {
+            int err = lfsr_bshrub_compact_(lfs, mdir, &mt->bshrub,
+                    mt->u.bt.bid, &mt->u.bt.branch);
+            if (err) {
+                return err;
+            }
+        }
+
+        if (mt->state == LFSR_MTRAVERSAL_OBTREE) {
+            // just update our opened file
+            lfsr_file_t *file_ = (lfsr_file_t*)mt->o;
+            file_->o.flags |= LFS_F_UNSYNC;
+            file_->bshrub = mt->bshrub;
+
+        } else {
+            // commit to mdir
+            uint8_t buf[LFSR_BTREE_DSIZE];
+            int err = lfsr_mdir_commit(lfs, mdir, LFSR_ATTRS(
+                    (mt->state == LFSR_MTRAVERSAL_MTREE)
+                        ? LFSR_ATTR(
+                            LFSR_TAG_SUB | LFSR_TAG_MTREE, 0,
+                            LFSR_DATA_BTREE_(&mt->bshrub.u.btree, buf))
+                    : (lfsr_bshrub_isbshrub(mdir, &mt->bshrub))
+                        ? LFSR_ATTR_SHRUBTRUNK(
+                            LFSR_TAG_SUB | LFSR_TAG_SHRUBTRUNK, 0,
+                            &mt->bshrub.u.bshrub)
+                        : LFSR_ATTR(
+                            LFSR_TAG_BTREE, 0,
+                            LFSR_DATA_BTREE_(&mt->bshrub.u.btree, buf))));
+            if (err) {
+                return err;
+            }
+
+            // update any open files
+            for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) {
+                if (o->type == LFS_TYPE_REG
+                        && o->mdir.mid == mdir->mid
+                        && !lfsr_f_isunsync(o->flags)) {
+                    lfsr_file_t *file_ = (lfsr_file_t*)o;
+                    file_->bshrub = mt->bshrub;
+                }
+            }
+        }
+
+        // reset to btree root
+        mt->u.bt.rid = mt->u.bt.bid;
+        mt->u.bt.branch = mt->bshrub.u.btree;
+
+        // mark as dirty
+        mt->flags |= LFS_F_DIRTY;
+    }
 
     return 0;
 }

tests/test_traversal.toml

@@ -5092,6 +5092,7 @@ code = '''
     lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
     assert(tinfo.btype == LFS_BTYPE_MDIR);
     assert(tinfo.block == 0 || tinfo.block == 1);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
     lfsr_traversal_close(&lfs, &t) => 0;
 
     // mdir should have been compacted
@@ -5256,6 +5257,7 @@ code = '''
     assert(tinfo.btype == LFS_BTYPE_MDIR);
     lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
     assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
     lfsr_traversal_close(&lfs, &t) => 0;
 
     // mrootanchor should have been compacted
@@ -5284,6 +5286,82 @@ code = '''
     lfsr_unmount(&lfs) => 0;
 '''
 
+[cases.test_traversal_compact_mroot_extend]
+defines.CKMETA = [false, true]
+defines.CK = [false, true]
+defines.LOOKAHEAD = [false, true]
+defines.SIZE = 'FILE_BUFFER_SIZE/2'
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
+# force early relocations
+defines.BLOCK_RECYCLES = 0
+in = 'lfs.c'
+code = '''
+    lfs_t lfs;
+    lfsr_format(&lfs, CFG) => 0;
+    lfsr_mount(&lfs, CFG) => 0;
+
+    uint32_t prng = 42;
+
+    // write to our mdir until >gc_compact_thresh full
+    lfsr_file_t file;
+    lfsr_file_open(&lfs, &file, "jellyfish",
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
+
+    // hack, don't use the internals like this
+    uint8_t wbuf[SIZE];
+    while ((file.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
+        lfsr_file_rewind(&lfs, &file) => 0;
+        for (lfs_size_t j = 0; j < SIZE; j++) {
+            wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
+        }
+        lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
+        lfsr_file_sync(&lfs, &file) => 0;
+    }
+
+    // try traversing and compacting
+    lfsr_traversal_t t;
+    lfsr_traversal_open(&lfs, &t,
+            LFS_T_COMPACT
+                | ((CKMETA) ? LFS_T_CKMETA : 0)
+                | ((CK) ? LFS_T_CK : 0)
+                | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
+    // it's a bit unclear if we should follow the mroot or stay on the
+    // mroot anchor during extends, so if this breaks in the future
+    // I wouldn't worry too much about it
+    //
+    // traverse mroot
+    struct lfs_tinfo tinfo;
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
+    lfsr_traversal_close(&lfs, &t) => 0;
+
+    // mdir should have been compacted
+    assert((file.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
+
+    // check we can still read the file
+    for (int remount = 0; remount < 2; remount++) {
+        // remount?
+        if (remount) {
+            lfsr_file_close(&lfs, &file) => 0;
+            lfsr_unmount(&lfs) => 0;
+            lfsr_mount(&lfs, CFG) => 0;
+            lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
+        }
+
+        lfsr_file_rewind(&lfs, &file) => 0;
+        uint8_t rbuf[SIZE];
+        lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
+        assert(memcmp(rbuf, wbuf, SIZE) == 0);
+    }
+
+    lfsr_file_close(&lfs, &file) => 0;
+    lfsr_unmount(&lfs) => 0;
+'''
+
 [cases.test_traversal_compact_mroot_split]
 defines.CKMETA = [false, true]
 defines.CK = [false, true]
@@ -5370,6 +5448,7 @@ code = '''
     assert(tinfo.btype == LFS_BTYPE_MDIR);
     lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
     assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
     lfsr_traversal_close(&lfs, &t) => 0;
 
     // mdirs should have been compacted
@@ -5543,6 +5622,7 @@ code = '''
     assert(tinfo.btype == LFS_BTYPE_MDIR);
     lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
     assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
     lfsr_traversal_close(&lfs, &t) => 0;
 
     // mdirs should have been compacted
@@ -5724,6 +5804,7 @@ code = '''
     assert(tinfo.btype == LFS_BTYPE_MDIR);
     lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
     assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
     lfsr_traversal_close(&lfs, &t) => 0;
 
     // mdirs should have been compacted
@@ -5775,6 +5856,523 @@ code = '''
 
 
 
+# btree/bshrub compactions are quite a bit more difficult
+
+[cases.test_traversal_compact_mtree_btree]
+defines.CKMETA = [false, true]
+defines.CK = [false, true]
+defines.LOOKAHEAD = [false, true]
+defines.SIZE = 'FILE_BUFFER_SIZE/2'
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
+in = 'lfs.c'
+code = '''
+    lfs_t lfs;
+    lfsr_format(&lfs, CFG) => 0;
+    lfsr_mount(&lfs, CFG) => 0;
+
+    uint32_t prng = 42;
+
+    // create two files
+    lfsr_file_t file1;
+    lfsr_file_open(&lfs, &file1, "jellyfish",
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
+    uint8_t wbuf1[SIZE];
+    for (lfs_size_t j = 0; j < SIZE; j++) {
+        wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
+    }
+    lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+    lfsr_file_sync(&lfs, &file1) => 0;
+
+    lfsr_file_t file2;
+    lfsr_file_open(&lfs, &file2, "octopus",
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
+    uint8_t wbuf2[SIZE];
+    for (lfs_size_t j = 0; j < SIZE; j++) {
+        wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
+    }
+    lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
+    lfsr_file_sync(&lfs, &file2) => 0;
+
+    // create enough files for mroot to split
+    lfs_size_t i = 0;
+    while (lfs.mtree.u.weight == 0x80000000) {
+        char name[256];
+        sprintf(name, "medusa%03x", i);
+        lfsr_file_t file;
+        lfsr_file_open(&lfs, &file, name,
+                LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+        lfsr_file_close(&lfs, &file) => 0;
+        i += 1;
+    }
+
+    // switch to early relocations after splitting
+    lfsr_file_close(&lfs, &file1) => 0;
+    lfsr_file_close(&lfs, &file2) => 0;
+    lfsr_unmount(&lfs) => 0;
+    struct lfs_config cfg = *CFG;
+    cfg.block_recycles = 0;
+    lfsr_mount(&lfs, &cfg) => 0;
+    lfsr_file_open(&lfs, &file1, "jellyfish", LFS_O_RDWR) => 0;
+    lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDWR) => 0;
+
+    // rewrite a file until btree is >gc_compact_thresh full
+    while (true) {
+        // we need internals to check this
+        lfsr_mdir_t mdir;
+        // ckmeta needed for eoff
+        lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA);
+        lfsr_mtinfo_t mtinfo;
+        lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
+        assert(mtinfo.tag == LFSR_TAG_MDIR);
+        lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
+        assert(mtinfo.tag == LFSR_TAG_BRANCH);
+        if (lfsr_rbyd_eoff(&mtinfo.u.rbyd) > GC_COMPACT_THRESH) {
+            break;
+        }
+
+        lfsr_file_rewind(&lfs, &file1) => 0;
+        for (lfs_size_t j = 0; j < SIZE; j++) {
+            wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
+        }
+        lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+        lfsr_file_sync(&lfs, &file1) => 0;
+    }
+
+    // switch back to default relocations to avoid mroot extension issues
+    lfsr_file_close(&lfs, &file1) => 0;
+    lfsr_file_close(&lfs, &file2) => 0;
+    lfsr_unmount(&lfs) => 0;
+    lfsr_mount(&lfs, CFG) => 0;
+    lfsr_file_open(&lfs, &file1, "jellyfish", LFS_O_RDWR) => 0;
+    lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDWR) => 0;
+
+    // try traversing and compacting
+    lfsr_traversal_t t;
+    lfsr_traversal_open(&lfs, &t,
+            LFS_T_COMPACT
+                | ((CKMETA) ? LFS_T_CKMETA : 0)
+                | ((CK) ? LFS_T_CK : 0)
+                | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
+    // traverse mroot
+    struct lfs_tinfo tinfo;
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    // traverse btree
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_BTREE);
+    // traverse mdir
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    // traverse mdir
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
+    lfsr_traversal_close(&lfs, &t) => 0;
+
+    // mtree should have been compacted
+    lfsr_mdir_t mdir;
+    lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA);
+    lfsr_mtinfo_t mtinfo;
+    lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
+    assert(mtinfo.tag == LFSR_TAG_MDIR);
+    lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
+    assert(mtinfo.tag == LFSR_TAG_BRANCH);
+    assert(lfsr_rbyd_eoff(&mtinfo.u.rbyd) <= GC_COMPACT_THRESH);
+
+    // check we can still read the files
+    for (int remount = 0; remount < 2; remount++) {
+        // remount?
+        if (remount) {
+            lfsr_file_close(&lfs, &file1) => 0;
+            lfsr_file_close(&lfs, &file2) => 0;
+            lfsr_unmount(&lfs) => 0;
+            lfsr_mount(&lfs, CFG) => 0;
+            lfsr_file_open(&lfs, &file1, "jellyfish", LFS_O_RDONLY) => 0;
+            lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDONLY) => 0;
+        }
+
+        lfsr_file_rewind(&lfs, &file1) => 0;
+        uint8_t rbuf[SIZE];
+        lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE;
+        assert(memcmp(rbuf, wbuf1, SIZE) == 0);
+
+        lfsr_file_rewind(&lfs, &file2) => 0;
+        lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE;
+        assert(memcmp(rbuf, wbuf2, SIZE) == 0);
+    }
+
+    lfsr_file_close(&lfs, &file1) => 0;
+    lfsr_file_close(&lfs, &file2) => 0;
+    lfsr_unmount(&lfs) => 0;
+'''
+
+[cases.test_traversal_compact_btree]
+defines.CKMETA = [false, true]
+defines.CK = [false, true]
+defines.LOOKAHEAD = [false, true]
+# limit files to very simple btrees
+defines.INLINE_SIZE = 0
+defines.CRYSTAL_THRESH = -1
+defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8'
+defines.SIZE = '2*FRAGMENT_SIZE'
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
+code = '''
+    lfs_t lfs;
+    lfsr_format(&lfs, CFG) => 0;
+    lfsr_mount(&lfs, CFG) => 0;
+
+    uint32_t prng = 42;
+
+    // create a file
+    lfsr_file_t file;
+    lfsr_file_open(&lfs, &file, "jellyfish",
+            LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+
+    // rewrite our file until btree is >gc_compact_thresh full
+    uint8_t wbuf[SIZE];
+    while ((file.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
+        lfsr_file_rewind(&lfs, &file) => 0;
+        for (lfs_size_t j = 0; j < SIZE; j++) {
+            wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
+        }
+        lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
+    }
+
+    lfsr_file_close(&lfs, &file) => 0;
+
+    // try traversing and compacting
+    lfsr_traversal_t t;
+    lfsr_traversal_open(&lfs, &t,
+            LFS_T_COMPACT
+                | ((CKMETA) ? LFS_T_CKMETA : 0)
+                | ((CK) ? LFS_T_CK : 0)
+                | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
+    // traverse mroot
+    struct lfs_tinfo tinfo;
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    // traverse btree
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_BTREE);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
+    lfsr_traversal_close(&lfs, &t) => 0;
+
+    // btree should have been compacted
+    lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
+    assert((file.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
+
+    // check we can still read the file
+    for (int remount = 0; remount < 2; remount++) {
+        // remount?
+        if (remount) {
+            lfsr_file_close(&lfs, &file) => 0;
+            lfsr_unmount(&lfs) => 0;
+            lfsr_mount(&lfs, CFG) => 0;
+            lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
+        }
+
+        lfsr_file_rewind(&lfs, &file) => 0;
+        uint8_t rbuf[SIZE];
+        lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
+        assert(memcmp(rbuf, wbuf, SIZE) == 0);
+    }
+
+    lfsr_file_close(&lfs, &file) => 0;
+    lfsr_unmount(&lfs) => 0;
+'''
+
+[cases.test_traversal_compact_btree_open]
+defines.CKMETA = [false, true]
+defines.CK = [false, true]
+defines.LOOKAHEAD = [false, true]
+# limit files to very simple btrees
+defines.INLINE_SIZE = 0
+defines.CRYSTAL_THRESH = -1
+defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8'
+defines.SIZE = '2*FRAGMENT_SIZE'
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
+code = '''
+    lfs_t lfs;
+    lfsr_format(&lfs, CFG) => 0;
+    lfsr_mount(&lfs, CFG) => 0;
+
+    uint32_t prng = 42;
+
+    // create a file
+    lfsr_file_t file;
+    lfsr_file_open(&lfs, &file, "jellyfish",
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
+
+    // rewrite our file until btree is >gc_compact_thresh full
+    uint8_t wbuf[SIZE];
+    while ((file.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
+        lfsr_file_rewind(&lfs, &file) => 0;
+        for (lfs_size_t j = 0; j < SIZE; j++) {
+            wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
+        }
+        lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
+    }
+
+    lfsr_file_sync(&lfs, &file) => 0;
+
+    // try traversing and compacting
+    lfsr_traversal_t t;
+    lfsr_traversal_open(&lfs, &t,
+            LFS_T_COMPACT
+                | ((CKMETA) ? LFS_T_CKMETA : 0)
+                | ((CK) ? LFS_T_CK : 0)
+                | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
+    // traverse mroot
+    struct lfs_tinfo tinfo;
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    // traverse btree
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_BTREE);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
+    lfsr_traversal_close(&lfs, &t) => 0;
+
+    // btree should have been compacted
+    assert((file.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
+
+    // check we can still read the file
+    for (int remount = 0; remount < 2; remount++) {
+        // remount?
+        if (remount) {
+            lfsr_file_close(&lfs, &file) => 0;
+            lfsr_unmount(&lfs) => 0;
+            lfsr_mount(&lfs, CFG) => 0;
+            lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
+        }
+
+        lfsr_file_rewind(&lfs, &file) => 0;
+        uint8_t rbuf[SIZE];
+        lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
+        assert(memcmp(rbuf, wbuf, SIZE) == 0);
+    }
+
+    lfsr_file_close(&lfs, &file) => 0;
+    lfsr_unmount(&lfs) => 0;
+'''
+
+[cases.test_traversal_compact_btree_orphan]
+defines.CKMETA = [false, true]
+defines.CK = [false, true]
+defines.LOOKAHEAD = [false, true]
+# limit files to very simple btrees
+defines.INLINE_SIZE = 0
+defines.CRYSTAL_THRESH = -1
+defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8'
+defines.SIZE = '2*FRAGMENT_SIZE'
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
+code = '''
+    lfs_t lfs;
+    lfsr_format(&lfs, CFG) => 0;
+    lfsr_mount(&lfs, CFG) => 0;
+
+    uint32_t prng = 42;
+
+    // create a file
+    lfsr_file_t file;
+    lfsr_file_open(&lfs, &file, "jellyfish",
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
+
+    // rewrite our file until btree is >gc_compact_thresh full
+    uint8_t wbuf[SIZE];
+    while ((file.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
+        lfsr_file_rewind(&lfs, &file) => 0;
+        for (lfs_size_t j = 0; j < SIZE; j++) {
+            wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
+        }
+        lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
+    }
+
+    // try traversing and compacting
+    lfsr_traversal_t t;
+    lfsr_traversal_open(&lfs, &t,
+            LFS_T_COMPACT
+                | ((CKMETA) ? LFS_T_CKMETA : 0)
+                | ((CK) ? LFS_T_CK : 0)
+                | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
+    // traverse mroot
+    struct lfs_tinfo tinfo;
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    // traverse btree
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_BTREE);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
+    lfsr_traversal_close(&lfs, &t) => 0;
+
+    // btree should have been compacted
+    assert((file.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
+
+    // file should not have accidentally been created or anything
+    struct lfs_info info;
+    lfsr_stat(&lfs, "jellyfish", &info) => LFS_ERR_NOENT;
+
+    // check we can still read the file
+    for (int remount = 0; remount < 2; remount++) {
+        // remount?
+        if (remount) {
+            lfsr_file_close(&lfs, &file) => 0;
+            lfsr_unmount(&lfs) => 0;
+            lfsr_mount(&lfs, CFG) => 0;
+            lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
+        }
+
+        lfsr_file_rewind(&lfs, &file) => 0;
+        uint8_t rbuf[SIZE];
+        lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
+        assert(memcmp(rbuf, wbuf, SIZE) == 0);
+    }
+
+    lfsr_file_close(&lfs, &file) => 0;
+    lfsr_unmount(&lfs) => 0;
+'''
+
+[cases.test_traversal_compact_btree_desync]
+defines.CKMETA = [false, true]
+defines.CK = [false, true]
+defines.LOOKAHEAD = [false, true]
+# limit files to very simple btrees
+defines.INLINE_SIZE = 0
+defines.CRYSTAL_THRESH = -1
+defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8'
+defines.SIZE = '2*FRAGMENT_SIZE'
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
+code = '''
+    lfs_t lfs;
+    lfsr_format(&lfs, CFG) => 0;
+    lfsr_mount(&lfs, CFG) => 0;
+
+    uint32_t prng = 42;
+
+    // create a desync file
+    lfsr_file_t file1;
+    lfsr_file_open(&lfs, &file1, "jellyfish",
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0;
+
+    // rewrite our file until btree is >gc_compact_thresh full
+    uint8_t wbuf1[SIZE];
+    while ((file1.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
+        lfsr_file_rewind(&lfs, &file1) => 0;
+        for (lfs_size_t j = 0; j < SIZE; j++) {
+            wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
+        }
+        lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
+    }
+
+    // create some overlapping files, these should not get messed with
+    lfsr_file_t file2;
+    lfsr_file_open(&lfs, &file2, "jellyfish",
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
+    uint8_t wbuf2[SIZE];
+    for (lfs_size_t j = 0; j < SIZE; j++) {
+        wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
+    }
+    lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
+    lfsr_file_sync(&lfs, &file2) => 0;
+
+    lfsr_file_t file3;
+    lfsr_file_open(&lfs, &file3, "jellyfish",
+            LFS_O_RDWR) => 0;
+    uint8_t wbuf3[SIZE];
+    for (lfs_size_t j = 0; j < SIZE; j++) {
+        wbuf3[j] = 'a' + (TEST_PRNG(&prng) % 26);
+    }
+    lfsr_file_write(&lfs, &file3, wbuf3, SIZE) => SIZE;
+    lfsr_file_desync(&lfs, &file3) => 0;
+
+    // try traversing and compacting
+    lfsr_traversal_t t;
+    lfsr_traversal_open(&lfs, &t,
+            LFS_T_COMPACT
+                | ((CKMETA) ? LFS_T_CKMETA : 0)
+                | ((CK) ? LFS_T_CK : 0)
+                | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
+    // traverse mroot
+    struct lfs_tinfo tinfo;
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_MDIR);
+    assert(tinfo.block == 0 || tinfo.block == 1);
+    // traverse btree
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_BTREE);
+    // traverse btree
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_BTREE);
+    // traverse btree
+    lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
+    assert(tinfo.btype == LFS_BTYPE_BTREE);
+    lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
+    lfsr_traversal_close(&lfs, &t) => 0;
+
+    // btree should have been compacted
+    assert((file1.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
+
+    // check we can still read the files
+    lfsr_file_rewind(&lfs, &file1) => 0;
+    uint8_t rbuf[SIZE];
+    lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE;
+    assert(memcmp(rbuf, wbuf1, SIZE) == 0);
+
+    lfsr_file_rewind(&lfs, &file2) => 0;
+    lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE;
+    assert(memcmp(rbuf, wbuf2, SIZE) == 0);
+
+    lfsr_file_rewind(&lfs, &file3) => 0;
+    lfsr_file_read(&lfs, &file3, rbuf, SIZE) => SIZE;
+    assert(memcmp(rbuf, wbuf3, SIZE) == 0);
+
+    // at least try closing/opening our synced file
+    lfsr_file_close(&lfs, &file2) => 0;
+    lfsr_file_open(&lfs, &file2, "jellyfish", LFS_O_RDONLY) => 0;
+    lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE;
+    assert(memcmp(rbuf, wbuf2, SIZE) == 0);
+
+    lfsr_file_close(&lfs, &file1) => 0;
+    lfsr_file_close(&lfs, &file2) => 0;
+    lfsr_file_close(&lfs, &file3) => 0;
+    lfsr_unmount(&lfs) => 0;
+'''
+
+# TODO
+# [cases.test_traversal_compact_bshrub]
+# [cases.test_traversal_compact_bshrub_open]
+# [cases.test_traversal_compact_bshrub_orphan]
+# [cases.test_traversal_compact_bshrub_desync]
+
+
+
 
 # many/fuzz tests mixed with traversals
 #
@@ -5789,6 +6387,8 @@ defines.CKMETA = [true]
 defines.CK = [true]
 defines.LOOKAHEAD = [false, true]
 defines.COMPACT = [false, true]
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256]
 code = '''
     // test creating directories
@@ -5897,6 +6497,8 @@ defines.CKMETA = [true]
 defines.CK = [true]
 defines.LOOKAHEAD = [false, true]
 defines.COMPACT = [false, true]
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
 defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256]
 defines.OPS = '2*N'
 defines.SEED = 42
@@ -6076,6 +6678,8 @@ defines.CKMETA = [true]
 defines.CK = [true]
 defines.LOOKAHEAD = [false, true]
 defines.COMPACT = [false, true]
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
 defines.N = [1, 2, 4, 8, 16, 32, 64]
 defines.SIZE = [
     '0',
@@ -6178,6 +6782,8 @@ defines.CKMETA = [true]
 defines.CK = [true]
 defines.LOOKAHEAD = [false, true]
 defines.COMPACT = [false, true]
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
 defines.N = [1, 2, 4, 8, 16, 32, 64]
 defines.OPS = '2*N'
 defines.SIZE = [
@@ -6418,7 +7024,10 @@ defines.EXCL = [false, true]
 defines.CKMETA = [true]
 defines.CK = [true]
 defines.LOOKAHEAD = [false, true]
-defines.COMPACT = [false, true]
+# TODO !!! Enable this when bshrub compaction is working
+defines.COMPACT = [false]
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
 defines.OPS = 20
 defines.SIZE = [
     'FILE_BUFFER_SIZE/2',
@@ -6580,6 +7189,8 @@ defines.CKMETA = [true]
 defines.CK = [true]
 defines.LOOKAHEAD = [false, true]
 defines.COMPACT = [false, true]
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
 defines.N = [1, 2, 4, 8, 16, 32, 64]
 defines.OPS = '2*N'
 defines.SIZE = [
@@ -6949,6 +7560,8 @@ defines.CKMETA = [true]
 defines.CK = [true]
 defines.LOOKAHEAD = [false, true]
 defines.COMPACT = [false, true]
+# set compact thresh to minimum
+defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
 defines.N = [1, 2, 4, 8, 16, 32, 64]
 defines.OPS = '2*N'
 defines.SIZE = [