From f3446abfa7b0979c2ab5439fae1156062cfd2c72 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 1 Jul 2024 13:22:35 -0500 Subject: [PATCH] t: Implemented gc_compact_thresh over bshrub nodes These aren't really different than btree nodes, except bshrubs need to be enrolled in our opened list for commits to work. Fortunately this is already true for explicit traversals, which are currently the only traversals where we need to simultaneously mutate the filesystem. This mainly just required adding additional checks for LFS_TYPE_TRAVERSAL bshrubs, tests, and making sure traversal.bshrub is never in an invalid state. This continues to add code/stack cost for what is ultimately a relatively niche feature: code stack before: 35268 2776 after: 35448 (+0.5%) 2800 (+0.9%) Maybe btree/bshrub compactions should be disabled by default? --- lfs.c | 54 +++-- lfs.h | 8 +- tests/test_traversal.toml | 493 +++++++++++++++++++++++++++++++++++++- 3 files changed, 525 insertions(+), 30 deletions(-) diff --git a/lfs.c b/lfs.c index df8f11d8..3a81494d 100644 --- a/lfs.c +++ b/lfs.c @@ -5174,7 +5174,8 @@ static lfs_ssize_t lfsr_sprout_estimate(lfs_t *lfs, const lfsr_sprout_t *last = NULL; for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file_ = (lfsr_file_t*)o; - if (file_->o.type == LFS_TYPE_REG + if ((file_->o.type == LFS_TYPE_REG + || file_->o.type == LFS_TYPE_TRAVERSAL) && lfsr_bshrub_isbsprout(&file_->o.mdir, &file_->bshrub) && lfsr_sprout_cmp(&file_->bshrub.u.bsprout, sprout) == 0) { last = &file_->bshrub.u.bsprout; @@ -5203,7 +5204,8 @@ static int lfsr_sprout_compact(lfs_t *lfs, const lfsr_rbyd_t *rbyd_, // can update these later if our commit is a success for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file_ = (lfsr_file_t*)o; - if (file_->o.type == LFS_TYPE_REG + if ((file_->o.type == LFS_TYPE_REG + || file_->o.type == LFS_TYPE_TRAVERSAL) && lfsr_bshrub_isbsprout(&file_->o.mdir, &file_->bshrub) && lfsr_sprout_cmp( &file_->bshrub.u.bsprout, @@ -5320,7 +5322,8 @@ static lfs_ssize_t lfsr_shrub_estimate(lfs_t *lfs, const lfsr_shrub_t *last = NULL; for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file_ = (lfsr_file_t*)o; - if (file_->o.type == LFS_TYPE_REG + if ((file_->o.type == LFS_TYPE_REG + || file_->o.type == LFS_TYPE_TRAVERSAL) && lfsr_bshrub_isbshrub(&file_->o.mdir, &file_->bshrub) && lfsr_shrub_cmp(&file_->bshrub.u.bshrub, shrub) == 0) { last = &file_->bshrub.u.bshrub; @@ -5352,7 +5355,8 @@ static int lfsr_shrub_compact(lfs_t *lfs, lfsr_rbyd_t *rbyd_, // this should include our current bshrub for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file_ = (lfsr_file_t*)o; - if (file_->o.type == LFS_TYPE_REG + if ((file_->o.type == LFS_TYPE_REG + || file_->o.type == LFS_TYPE_TRAVERSAL) && lfsr_bshrub_isbshrub(&file_->o.mdir, &file_->bshrub) && lfsr_shrub_cmp(&file_->bshrub.u.bshrub, shrub) == 0) { file_->bshrub_.u.bshrub.blocks[0] = rbyd_->blocks[0]; @@ -5456,7 +5460,8 @@ static lfs_ssize_t lfsr_bshrub_estimate(lfs_t *lfs, // this includes our current shrub for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file_ = (lfsr_file_t*)o; - if (file_->o.type == LFS_TYPE_REG + if ((file_->o.type == LFS_TYPE_REG + || file_->o.type == LFS_TYPE_TRAVERSAL) && file_->o.mdir.mid == mdir->mid) { if (lfsr_bshrub_isbsprout(&file_->o.mdir, &file_->bshrub)) { lfs_ssize_t dsize = lfsr_sprout_estimate(lfs, @@ -5633,7 +5638,8 @@ static int lfsr_bshrub_commit_(lfs_t *lfs, if (lfsr_bshrub_isbtree(mdir, bshrub)) { for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file_ = (lfsr_file_t*)o; - if (file_->o.type == LFS_TYPE_REG + if ((file_->o.type == LFS_TYPE_REG + || file_->o.type == LFS_TYPE_TRAVERSAL) && &file_->bshrub != bshrub && lfsr_bshrub_isbshruborbtree(&file_->bshrub) && lfsr_btree_cmp( @@ -5722,7 +5728,8 @@ static int lfsr_bshrub_commit_(lfs_t *lfs, // update _all_ shrubs with the new estimate for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file_ = (lfsr_file_t*)o; - if (file_->o.type == LFS_TYPE_REG + if ((file_->o.type == LFS_TYPE_REG + || file_->o.type == LFS_TYPE_TRAVERSAL) && file_->o.mdir.mid == mdir->mid && lfsr_bshrub_isbshrub(&file_->o.mdir, &file_->bshrub)) { file_->bshrub.u.bshrub.estimate = estimate; @@ -6751,8 +6758,9 @@ static int lfsr_mdir_commit__(lfs_t *lfs, lfsr_mdir_t *mdir, for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file = (lfsr_file_t*)o; // belongs to our mid? - if (file->o.type != LFS_TYPE_REG - || file->o.mdir.mid != mdir__->mid) { + if (!((file->o.type == LFS_TYPE_REG + || file->o.type == LFS_TYPE_TRAVERSAL) + && file->o.mdir.mid == mdir__->mid)) { continue; } @@ -6939,9 +6947,10 @@ static lfs_ssize_t lfsr_mdir_estimate__(lfs_t *lfs, const lfsr_mdir_t *mdir, for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file = (lfsr_file_t*)o; // belongs to our mdir + rid? - if (file->o.type != LFS_TYPE_REG - || lfsr_mdir_cmp(&file->o.mdir, mdir) != 0 - || lfsr_mid_rid(lfs, file->o.mdir.mid) != a_rid) { + if (!((file->o.type == LFS_TYPE_REG + || file->o.type == LFS_TYPE_TRAVERSAL) + && lfsr_mdir_cmp(&file->o.mdir, mdir) == 0 + && lfsr_mid_rid(lfs, file->o.mdir.mid) == a_rid)) { continue; } @@ -7085,11 +7094,12 @@ static int lfsr_mdir_compact__(lfs_t *lfs, lfsr_mdir_t *mdir_, for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file = (lfsr_file_t*)o; // belongs to our mdir? - if (file->o.type != LFS_TYPE_REG - || lfsr_mdir_cmp(&file->o.mdir, mdir) != 0 - || lfsr_mid_rid(lfs, file->o.mdir.mid) < start_rid - || (lfsr_rid_t)lfsr_mid_rid(lfs, file->o.mdir.mid) - >= (lfsr_rid_t)end_rid) { + if (!((file->o.type == LFS_TYPE_REG + || file->o.type == LFS_TYPE_TRAVERSAL) + && lfsr_mdir_cmp(&file->o.mdir, mdir) == 0 + && lfsr_mid_rid(lfs, file->o.mdir.mid) >= start_rid + && (lfsr_rid_t)lfsr_mid_rid(lfs, file->o.mdir.mid) + < (lfsr_rid_t)end_rid)) { continue; } @@ -7346,7 +7356,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, } // stage any bsprouts/bshrubs - if (o->type == LFS_TYPE_REG) { + if (o->type == LFS_TYPE_REG || o->type == LFS_TYPE_TRAVERSAL) { lfsr_file_t *file = (lfsr_file_t*)o; file->bshrub_ = file->bshrub; } @@ -7841,7 +7851,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, // update any staged bsprouts/bshrubs for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { - if (o->type == LFS_TYPE_REG) { + if (o->type == LFS_TYPE_REG || o->type == LFS_TYPE_TRAVERSAL) { lfsr_file_t *file = (lfsr_file_t*)o; file->bshrub = file->bshrub_; } @@ -8231,6 +8241,7 @@ enum { .state=LFSR_MTRAVERSAL_MROOTANCHOR, \ .flags=_flags, \ .o=NULL, \ + .bshrub.u.bshrub.blocks={-1}, \ .u.mtortoise.mptr={{0, 0}}, \ .u.mtortoise.step=0, \ .u.mtortoise.power=0}) @@ -8620,6 +8631,9 @@ static int lfsr_mtree_traverse(lfs_t *lfs, static int lfsr_mtree_gc(lfs_t *lfs, lfsr_mdir_t *mdir, lfsr_mtraversal_t *mt, lfsr_mtinfo_t *mtinfo) { + // TODO traversals need to be enrolled in our opened list for + // lfsr_mtree_gc to work correctly, can we assert this somehow? + int err = lfsr_mtree_traverse(lfs, mdir, mt, mtinfo); if (err) { return err; @@ -8708,7 +8722,7 @@ static int lfsr_mtree_gc(lfs_t *lfs, LFSR_TAG_SUB | LFSR_TAG_SHRUBTRUNK, 0, &mt->bshrub.u.bshrub) : LFSR_ATTR( - LFSR_TAG_BTREE, 0, + LFSR_TAG_SUB | LFSR_TAG_BTREE, 0, LFSR_DATA_BTREE_(&mt->bshrub.u.btree, buf)))); if (err) { return err; diff --git a/lfs.h b/lfs.h index 3a7b3ce2..116f3aa9 100644 --- a/lfs.h +++ b/lfs.h @@ -613,13 +613,15 @@ typedef struct lfsr_btraversal { } lfsr_btraversal_t; typedef struct lfsr_mtraversal { - // core state machine - uint8_t state; - uint16_t flags; // opened file state lfsr_omdir_t *o; // bshrub/btree state + // this lines up with bshrub/btree in lfsr_file_t lfsr_bshrub_t bshrub; + lfsr_bshrub_t bshrub_; + // core state machine + uint8_t state; + uint16_t flags; union { // cycle detection state, only valid when traversing the mroot chain struct { diff --git a/tests/test_traversal.toml b/tests/test_traversal.toml index 5863589b..d335b963 100644 --- a/tests/test_traversal.toml +++ b/tests/test_traversal.toml @@ -6365,11 +6365,491 @@ code = ''' lfsr_unmount(&lfs) => 0; ''' -# TODO -# [cases.test_traversal_compact_bshrub] -# [cases.test_traversal_compact_bshrub_open] -# [cases.test_traversal_compact_bshrub_orphan] -# [cases.test_traversal_compact_bshrub_desync] +[cases.test_traversal_compact_bshrub] +defines.CKMETA = [false, true] +defines.CK = [false, true] +defines.LOOKAHEAD = [false, true] +# this configuration should create a 2-layer bshrub, which may be +# a bit delicate +defines.INLINE_SIZE = 'BLOCK_SIZE/4' +defines.CRYSTAL_THRESH = -1 +defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' +defines.SIZE = 'BLOCK_SIZE' +# set compact thresh to minimum +defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + + // create a file + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jellyfish", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + + // rewrite part of our file until bshrub is >gc_compact_thresh full + while (true) { + // we need internals to check this + lfsr_mdir_t mdir; + // ckmeta needed for eoff + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_BRANCH); + if (lfsr_rbyd_eoff(&mtinfo.u.rbyd) > GC_COMPACT_THRESH) { + break; + } + + lfsr_file_rewind(&lfs, &file) => 0; + for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, FRAGMENT_SIZE) => FRAGMENT_SIZE; + } + + lfsr_file_close(&lfs, &file) => 0; + + // try traversing and compacting + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, + LFS_T_COMPACT + | ((CKMETA) ? LFS_T_CKMETA : 0) + | ((CK) ? LFS_T_CK : 0) + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + // traverse mroot + struct lfs_tinfo tinfo; + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + // compacting our bshrub nodes may cause them to split, so we may + // need to traverse more nodes than we started with + for (lfs_size_t i = 0; i < 5; i++) { + // traverse bshrub + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_BTREE); + } + lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; + lfsr_traversal_close(&lfs, &t) => 0; + + // bshrub should have been compacted + lfsr_mdir_t mdir; + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + while (true) { + int err = lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + break; + } + assert(mtinfo.tag == LFSR_TAG_BRANCH); + assert(lfsr_rbyd_eoff(&mtinfo.u.rbyd) <= GC_COMPACT_THRESH); + } + + // check we can still read the file + for (int remount = 0; remount < 2; remount++) { + // remount? + if (remount) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + } + + lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; + lfsr_file_rewind(&lfs, &file) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf, SIZE) == 0); + lfsr_file_close(&lfs, &file) => 0; + } + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_traversal_compact_bshrub_open] +defines.CKMETA = [false, true] +defines.CK = [false, true] +defines.LOOKAHEAD = [false, true] +# this configuration should create a 2-layer bshrub, which may be +# a bit delicate +defines.INLINE_SIZE = 'BLOCK_SIZE/4' +defines.CRYSTAL_THRESH = -1 +defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' +defines.SIZE = 'BLOCK_SIZE' +# set compact thresh to minimum +defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + + // create a file + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jellyfish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + + // rewrite part of our file until bshrub is >gc_compact_thresh full + while (true) { + // we need internals to check this + lfsr_mdir_t mdir; + // ckmeta needed for eoff + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_BRANCH); + if (lfsr_rbyd_eoff(&mtinfo.u.rbyd) > GC_COMPACT_THRESH) { + break; + } + + lfsr_file_rewind(&lfs, &file) => 0; + for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, FRAGMENT_SIZE) => FRAGMENT_SIZE; + } + + lfsr_file_sync(&lfs, &file) => 0; + + // try traversing and compacting + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, + LFS_T_COMPACT + | ((CKMETA) ? LFS_T_CKMETA : 0) + | ((CK) ? LFS_T_CK : 0) + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + // traverse mroot + struct lfs_tinfo tinfo; + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + // compacting our bshrub nodes may cause them to split, so we may + // need to traverse more nodes than we started with + for (lfs_size_t i = 0; i < 5; i++) { + // traverse bshrub + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_BTREE); + } + lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; + lfsr_traversal_close(&lfs, &t) => 0; + + // bshrub should have been compacted + lfsr_mdir_t mdir; + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + while (true) { + int err = lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + break; + } + assert(mtinfo.tag == LFSR_TAG_BRANCH); + assert(lfsr_rbyd_eoff(&mtinfo.u.rbyd) <= GC_COMPACT_THRESH); + } + + // check we can still read the file + for (int remount = 0; remount < 2; remount++) { + // remount? + if (remount) { + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; + } + + lfsr_file_rewind(&lfs, &file) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf, SIZE) == 0); + } + + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_traversal_compact_bshrub_orphan] +defines.CKMETA = [false, true] +defines.CK = [false, true] +defines.LOOKAHEAD = [false, true] +# this configuration should create a 2-layer bshrub, which may be +# a bit delicate +defines.INLINE_SIZE = 'BLOCK_SIZE/4' +defines.CRYSTAL_THRESH = -1 +defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' +defines.SIZE = 'BLOCK_SIZE' +# set compact thresh to minimum +defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + + // create a file + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jellyfish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + + // rewrite part of our file until bshrub is >gc_compact_thresh full + while (true) { + // we need internals to check this + lfsr_mdir_t mdir; + // ckmeta needed for eoff + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_BRANCH); + if (lfsr_rbyd_eoff(&mtinfo.u.rbyd) > GC_COMPACT_THRESH) { + break; + } + + lfsr_file_rewind(&lfs, &file) => 0; + for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, FRAGMENT_SIZE) => FRAGMENT_SIZE; + } + + // try traversing and compacting + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, + LFS_T_COMPACT + | ((CKMETA) ? LFS_T_CKMETA : 0) + | ((CK) ? LFS_T_CK : 0) + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + // traverse mroot + struct lfs_tinfo tinfo; + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + // compacting our bshrub nodes may cause them to split, so we may + // need to traverse more nodes than we started with + for (lfs_size_t i = 0; i < 5; i++) { + // traverse bshrub + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_BTREE); + } + lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; + lfsr_traversal_close(&lfs, &t) => 0; + + // bshrub should have been compacted + lfsr_mdir_t mdir; + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + while (true) { + int err = lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + break; + } + assert(mtinfo.tag == LFSR_TAG_BRANCH); + assert(lfsr_rbyd_eoff(&mtinfo.u.rbyd) <= GC_COMPACT_THRESH); + } + + // check we can still read the file + for (int remount = 0; remount < 2; remount++) { + // remount? + if (remount) { + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; + } + + lfsr_file_rewind(&lfs, &file) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf, SIZE) == 0); + } + + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_traversal_compact_bshrub_desync] +defines.CKMETA = [false, true] +defines.CK = [false, true] +defines.LOOKAHEAD = [false, true] +# this configuration should create a 2-layer bshrub, which may be +# a bit delicate +defines.INLINE_SIZE = 'BLOCK_SIZE/4' +defines.CRYSTAL_THRESH = -1 +defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' +defines.SIZE = 'BLOCK_SIZE' +# set compact thresh to minimum +defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + + // create a desync file + lfsr_file_t file1; + lfsr_file_open(&lfs, &file1, "jellyfish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0; + + uint8_t wbuf1[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; + + // rewrite part of our file until bshrub is >gc_compact_thresh full + while (true) { + // we need internals to check this + lfsr_mdir_t mdir; + // ckmeta needed for eoff + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_BRANCH); + if (lfsr_rbyd_eoff(&mtinfo.u.rbyd) > GC_COMPACT_THRESH) { + break; + } + + lfsr_file_rewind(&lfs, &file1) => 0; + for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { + wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file1, wbuf1, FRAGMENT_SIZE) => FRAGMENT_SIZE; + } + + // create some overlapping files, these should not get messed with + lfsr_file_t file2; + lfsr_file_open(&lfs, &file2, "jellyfish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + uint8_t wbuf2[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file2) => 0; + + lfsr_file_t file3; + lfsr_file_open(&lfs, &file3, "jellyfish", + LFS_O_RDWR) => 0; + uint8_t wbuf3[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf3[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file3, wbuf3, SIZE) => SIZE; + lfsr_file_desync(&lfs, &file3) => 0; + + // try traversing and compacting + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, + LFS_T_COMPACT + | ((CKMETA) ? LFS_T_CKMETA : 0) + | ((CK) ? LFS_T_CK : 0) + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + // traverse mroot + struct lfs_tinfo tinfo; + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + // compacting our bshrub nodes may cause them to split, so we may + // need to traverse more nodes than we started with + for (lfs_size_t i = 0; i < 3*5; i++) { + // traverse bshrub + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_BTREE); + } + lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; + lfsr_traversal_close(&lfs, &t) => 0; + + // bshrub should have been compacted + lfsr_mdir_t mdir; + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + while (true) { + int err = lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + break; + } + assert(mtinfo.tag == LFSR_TAG_BRANCH); + assert(lfsr_rbyd_eoff(&mtinfo.u.rbyd) <= GC_COMPACT_THRESH); + } + + // check we can still read the files + lfsr_file_rewind(&lfs, &file1) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf1, SIZE) == 0); + + lfsr_file_rewind(&lfs, &file2) => 0; + lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf2, SIZE) == 0); + + lfsr_file_rewind(&lfs, &file3) => 0; + lfsr_file_read(&lfs, &file3, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf3, SIZE) == 0); + + // at least try closing/opening our synced file + lfsr_file_close(&lfs, &file2) => 0; + lfsr_file_open(&lfs, &file2, "jellyfish", LFS_O_RDONLY) => 0; + lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf2, SIZE) == 0); + + lfsr_file_close(&lfs, &file1) => 0; + lfsr_file_close(&lfs, &file2) => 0; + lfsr_file_close(&lfs, &file3) => 0; + lfsr_unmount(&lfs) => 0; +''' @@ -7024,8 +7504,7 @@ defines.EXCL = [false, true] defines.CKMETA = [true] defines.CK = [true] defines.LOOKAHEAD = [false, true] -# TODO !!! Enable this when bshrub compaction is working -defines.COMPACT = [false] +defines.COMPACT = [false, true] # set compact thresh to minimum defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' defines.OPS = 20