diff --git a/lfs.c b/lfs.c index df8f11d8..3a81494d 100644 --- a/lfs.c +++ b/lfs.c @@ -5174,7 +5174,8 @@ static lfs_ssize_t lfsr_sprout_estimate(lfs_t *lfs, const lfsr_sprout_t *last = NULL; for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file_ = (lfsr_file_t*)o; - if (file_->o.type == LFS_TYPE_REG + if ((file_->o.type == LFS_TYPE_REG + || file_->o.type == LFS_TYPE_TRAVERSAL) && lfsr_bshrub_isbsprout(&file_->o.mdir, &file_->bshrub) && lfsr_sprout_cmp(&file_->bshrub.u.bsprout, sprout) == 0) { last = &file_->bshrub.u.bsprout; @@ -5203,7 +5204,8 @@ static int lfsr_sprout_compact(lfs_t *lfs, const lfsr_rbyd_t *rbyd_, // can update these later if our commit is a success for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file_ = (lfsr_file_t*)o; - if (file_->o.type == LFS_TYPE_REG + if ((file_->o.type == LFS_TYPE_REG + || file_->o.type == LFS_TYPE_TRAVERSAL) && lfsr_bshrub_isbsprout(&file_->o.mdir, &file_->bshrub) && lfsr_sprout_cmp( &file_->bshrub.u.bsprout, @@ -5320,7 +5322,8 @@ static lfs_ssize_t lfsr_shrub_estimate(lfs_t *lfs, const lfsr_shrub_t *last = NULL; for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file_ = (lfsr_file_t*)o; - if (file_->o.type == LFS_TYPE_REG + if ((file_->o.type == LFS_TYPE_REG + || file_->o.type == LFS_TYPE_TRAVERSAL) && lfsr_bshrub_isbshrub(&file_->o.mdir, &file_->bshrub) && lfsr_shrub_cmp(&file_->bshrub.u.bshrub, shrub) == 0) { last = &file_->bshrub.u.bshrub; @@ -5352,7 +5355,8 @@ static int lfsr_shrub_compact(lfs_t *lfs, lfsr_rbyd_t *rbyd_, // this should include our current bshrub for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file_ = (lfsr_file_t*)o; - if (file_->o.type == LFS_TYPE_REG + if ((file_->o.type == LFS_TYPE_REG + || file_->o.type == LFS_TYPE_TRAVERSAL) && lfsr_bshrub_isbshrub(&file_->o.mdir, &file_->bshrub) && lfsr_shrub_cmp(&file_->bshrub.u.bshrub, shrub) == 0) { file_->bshrub_.u.bshrub.blocks[0] = rbyd_->blocks[0]; @@ -5456,7 +5460,8 @@ static lfs_ssize_t lfsr_bshrub_estimate(lfs_t *lfs, // this includes our current shrub for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file_ = (lfsr_file_t*)o; - if (file_->o.type == LFS_TYPE_REG + if ((file_->o.type == LFS_TYPE_REG + || file_->o.type == LFS_TYPE_TRAVERSAL) && file_->o.mdir.mid == mdir->mid) { if (lfsr_bshrub_isbsprout(&file_->o.mdir, &file_->bshrub)) { lfs_ssize_t dsize = lfsr_sprout_estimate(lfs, @@ -5633,7 +5638,8 @@ static int lfsr_bshrub_commit_(lfs_t *lfs, if (lfsr_bshrub_isbtree(mdir, bshrub)) { for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file_ = (lfsr_file_t*)o; - if (file_->o.type == LFS_TYPE_REG + if ((file_->o.type == LFS_TYPE_REG + || file_->o.type == LFS_TYPE_TRAVERSAL) && &file_->bshrub != bshrub && lfsr_bshrub_isbshruborbtree(&file_->bshrub) && lfsr_btree_cmp( @@ -5722,7 +5728,8 @@ static int lfsr_bshrub_commit_(lfs_t *lfs, // update _all_ shrubs with the new estimate for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file_ = (lfsr_file_t*)o; - if (file_->o.type == LFS_TYPE_REG + if ((file_->o.type == LFS_TYPE_REG + || file_->o.type == LFS_TYPE_TRAVERSAL) && file_->o.mdir.mid == mdir->mid && lfsr_bshrub_isbshrub(&file_->o.mdir, &file_->bshrub)) { file_->bshrub.u.bshrub.estimate = estimate; @@ -6751,8 +6758,9 @@ static int lfsr_mdir_commit__(lfs_t *lfs, lfsr_mdir_t *mdir, for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file = (lfsr_file_t*)o; // belongs to our mid? - if (file->o.type != LFS_TYPE_REG - || file->o.mdir.mid != mdir__->mid) { + if (!((file->o.type == LFS_TYPE_REG + || file->o.type == LFS_TYPE_TRAVERSAL) + && file->o.mdir.mid == mdir__->mid)) { continue; } @@ -6939,9 +6947,10 @@ static lfs_ssize_t lfsr_mdir_estimate__(lfs_t *lfs, const lfsr_mdir_t *mdir, for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file = (lfsr_file_t*)o; // belongs to our mdir + rid? - if (file->o.type != LFS_TYPE_REG - || lfsr_mdir_cmp(&file->o.mdir, mdir) != 0 - || lfsr_mid_rid(lfs, file->o.mdir.mid) != a_rid) { + if (!((file->o.type == LFS_TYPE_REG + || file->o.type == LFS_TYPE_TRAVERSAL) + && lfsr_mdir_cmp(&file->o.mdir, mdir) == 0 + && lfsr_mid_rid(lfs, file->o.mdir.mid) == a_rid)) { continue; } @@ -7085,11 +7094,12 @@ static int lfsr_mdir_compact__(lfs_t *lfs, lfsr_mdir_t *mdir_, for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { lfsr_file_t *file = (lfsr_file_t*)o; // belongs to our mdir? - if (file->o.type != LFS_TYPE_REG - || lfsr_mdir_cmp(&file->o.mdir, mdir) != 0 - || lfsr_mid_rid(lfs, file->o.mdir.mid) < start_rid - || (lfsr_rid_t)lfsr_mid_rid(lfs, file->o.mdir.mid) - >= (lfsr_rid_t)end_rid) { + if (!((file->o.type == LFS_TYPE_REG + || file->o.type == LFS_TYPE_TRAVERSAL) + && lfsr_mdir_cmp(&file->o.mdir, mdir) == 0 + && lfsr_mid_rid(lfs, file->o.mdir.mid) >= start_rid + && (lfsr_rid_t)lfsr_mid_rid(lfs, file->o.mdir.mid) + < (lfsr_rid_t)end_rid)) { continue; } @@ -7346,7 +7356,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, } // stage any bsprouts/bshrubs - if (o->type == LFS_TYPE_REG) { + if (o->type == LFS_TYPE_REG || o->type == LFS_TYPE_TRAVERSAL) { lfsr_file_t *file = (lfsr_file_t*)o; file->bshrub_ = file->bshrub; } @@ -7841,7 +7851,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, // update any staged bsprouts/bshrubs for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { - if (o->type == LFS_TYPE_REG) { + if (o->type == LFS_TYPE_REG || o->type == LFS_TYPE_TRAVERSAL) { lfsr_file_t *file = (lfsr_file_t*)o; file->bshrub = file->bshrub_; } @@ -8231,6 +8241,7 @@ enum { .state=LFSR_MTRAVERSAL_MROOTANCHOR, \ .flags=_flags, \ .o=NULL, \ + .bshrub.u.bshrub.blocks={-1}, \ .u.mtortoise.mptr={{0, 0}}, \ .u.mtortoise.step=0, \ .u.mtortoise.power=0}) @@ -8620,6 +8631,9 @@ static int lfsr_mtree_traverse(lfs_t *lfs, static int lfsr_mtree_gc(lfs_t *lfs, lfsr_mdir_t *mdir, lfsr_mtraversal_t *mt, lfsr_mtinfo_t *mtinfo) { + // TODO traversals need to be enrolled in our opened list for + // lfsr_mtree_gc to work correctly, can we assert this somehow? + int err = lfsr_mtree_traverse(lfs, mdir, mt, mtinfo); if (err) { return err; @@ -8708,7 +8722,7 @@ static int lfsr_mtree_gc(lfs_t *lfs, LFSR_TAG_SUB | LFSR_TAG_SHRUBTRUNK, 0, &mt->bshrub.u.bshrub) : LFSR_ATTR( - LFSR_TAG_BTREE, 0, + LFSR_TAG_SUB | LFSR_TAG_BTREE, 0, LFSR_DATA_BTREE_(&mt->bshrub.u.btree, buf)))); if (err) { return err; diff --git a/lfs.h b/lfs.h index 3a7b3ce2..116f3aa9 100644 --- a/lfs.h +++ b/lfs.h @@ -613,13 +613,15 @@ typedef struct lfsr_btraversal { } lfsr_btraversal_t; typedef struct lfsr_mtraversal { - // core state machine - uint8_t state; - uint16_t flags; // opened file state lfsr_omdir_t *o; // bshrub/btree state + // this lines up with bshrub/btree in lfsr_file_t lfsr_bshrub_t bshrub; + lfsr_bshrub_t bshrub_; + // core state machine + uint8_t state; + uint16_t flags; union { // cycle detection state, only valid when traversing the mroot chain struct { diff --git a/tests/test_traversal.toml b/tests/test_traversal.toml index 5863589b..d335b963 100644 --- a/tests/test_traversal.toml +++ b/tests/test_traversal.toml @@ -6365,11 +6365,491 @@ code = ''' lfsr_unmount(&lfs) => 0; ''' -# TODO -# [cases.test_traversal_compact_bshrub] -# [cases.test_traversal_compact_bshrub_open] -# [cases.test_traversal_compact_bshrub_orphan] -# [cases.test_traversal_compact_bshrub_desync] +[cases.test_traversal_compact_bshrub] +defines.CKMETA = [false, true] +defines.CK = [false, true] +defines.LOOKAHEAD = [false, true] +# this configuration should create a 2-layer bshrub, which may be +# a bit delicate +defines.INLINE_SIZE = 'BLOCK_SIZE/4' +defines.CRYSTAL_THRESH = -1 +defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' +defines.SIZE = 'BLOCK_SIZE' +# set compact thresh to minimum +defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + + // create a file + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jellyfish", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + + // rewrite part of our file until bshrub is >gc_compact_thresh full + while (true) { + // we need internals to check this + lfsr_mdir_t mdir; + // ckmeta needed for eoff + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_BRANCH); + if (lfsr_rbyd_eoff(&mtinfo.u.rbyd) > GC_COMPACT_THRESH) { + break; + } + + lfsr_file_rewind(&lfs, &file) => 0; + for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, FRAGMENT_SIZE) => FRAGMENT_SIZE; + } + + lfsr_file_close(&lfs, &file) => 0; + + // try traversing and compacting + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, + LFS_T_COMPACT + | ((CKMETA) ? LFS_T_CKMETA : 0) + | ((CK) ? LFS_T_CK : 0) + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + // traverse mroot + struct lfs_tinfo tinfo; + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + // compacting our bshrub nodes may cause them to split, so we may + // need to traverse more nodes than we started with + for (lfs_size_t i = 0; i < 5; i++) { + // traverse bshrub + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_BTREE); + } + lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; + lfsr_traversal_close(&lfs, &t) => 0; + + // bshrub should have been compacted + lfsr_mdir_t mdir; + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + while (true) { + int err = lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + break; + } + assert(mtinfo.tag == LFSR_TAG_BRANCH); + assert(lfsr_rbyd_eoff(&mtinfo.u.rbyd) <= GC_COMPACT_THRESH); + } + + // check we can still read the file + for (int remount = 0; remount < 2; remount++) { + // remount? + if (remount) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + } + + lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; + lfsr_file_rewind(&lfs, &file) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf, SIZE) == 0); + lfsr_file_close(&lfs, &file) => 0; + } + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_traversal_compact_bshrub_open] +defines.CKMETA = [false, true] +defines.CK = [false, true] +defines.LOOKAHEAD = [false, true] +# this configuration should create a 2-layer bshrub, which may be +# a bit delicate +defines.INLINE_SIZE = 'BLOCK_SIZE/4' +defines.CRYSTAL_THRESH = -1 +defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' +defines.SIZE = 'BLOCK_SIZE' +# set compact thresh to minimum +defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + + // create a file + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jellyfish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + + // rewrite part of our file until bshrub is >gc_compact_thresh full + while (true) { + // we need internals to check this + lfsr_mdir_t mdir; + // ckmeta needed for eoff + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_BRANCH); + if (lfsr_rbyd_eoff(&mtinfo.u.rbyd) > GC_COMPACT_THRESH) { + break; + } + + lfsr_file_rewind(&lfs, &file) => 0; + for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, FRAGMENT_SIZE) => FRAGMENT_SIZE; + } + + lfsr_file_sync(&lfs, &file) => 0; + + // try traversing and compacting + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, + LFS_T_COMPACT + | ((CKMETA) ? LFS_T_CKMETA : 0) + | ((CK) ? LFS_T_CK : 0) + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + // traverse mroot + struct lfs_tinfo tinfo; + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + // compacting our bshrub nodes may cause them to split, so we may + // need to traverse more nodes than we started with + for (lfs_size_t i = 0; i < 5; i++) { + // traverse bshrub + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_BTREE); + } + lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; + lfsr_traversal_close(&lfs, &t) => 0; + + // bshrub should have been compacted + lfsr_mdir_t mdir; + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + while (true) { + int err = lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + break; + } + assert(mtinfo.tag == LFSR_TAG_BRANCH); + assert(lfsr_rbyd_eoff(&mtinfo.u.rbyd) <= GC_COMPACT_THRESH); + } + + // check we can still read the file + for (int remount = 0; remount < 2; remount++) { + // remount? + if (remount) { + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; + } + + lfsr_file_rewind(&lfs, &file) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf, SIZE) == 0); + } + + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_traversal_compact_bshrub_orphan] +defines.CKMETA = [false, true] +defines.CK = [false, true] +defines.LOOKAHEAD = [false, true] +# this configuration should create a 2-layer bshrub, which may be +# a bit delicate +defines.INLINE_SIZE = 'BLOCK_SIZE/4' +defines.CRYSTAL_THRESH = -1 +defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' +defines.SIZE = 'BLOCK_SIZE' +# set compact thresh to minimum +defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + + // create a file + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jellyfish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + + // rewrite part of our file until bshrub is >gc_compact_thresh full + while (true) { + // we need internals to check this + lfsr_mdir_t mdir; + // ckmeta needed for eoff + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_BRANCH); + if (lfsr_rbyd_eoff(&mtinfo.u.rbyd) > GC_COMPACT_THRESH) { + break; + } + + lfsr_file_rewind(&lfs, &file) => 0; + for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, FRAGMENT_SIZE) => FRAGMENT_SIZE; + } + + // try traversing and compacting + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, + LFS_T_COMPACT + | ((CKMETA) ? LFS_T_CKMETA : 0) + | ((CK) ? LFS_T_CK : 0) + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + // traverse mroot + struct lfs_tinfo tinfo; + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + // compacting our bshrub nodes may cause them to split, so we may + // need to traverse more nodes than we started with + for (lfs_size_t i = 0; i < 5; i++) { + // traverse bshrub + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_BTREE); + } + lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; + lfsr_traversal_close(&lfs, &t) => 0; + + // bshrub should have been compacted + lfsr_mdir_t mdir; + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + while (true) { + int err = lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + break; + } + assert(mtinfo.tag == LFSR_TAG_BRANCH); + assert(lfsr_rbyd_eoff(&mtinfo.u.rbyd) <= GC_COMPACT_THRESH); + } + + // check we can still read the file + for (int remount = 0; remount < 2; remount++) { + // remount? + if (remount) { + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; + } + + lfsr_file_rewind(&lfs, &file) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf, SIZE) == 0); + } + + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_traversal_compact_bshrub_desync] +defines.CKMETA = [false, true] +defines.CK = [false, true] +defines.LOOKAHEAD = [false, true] +# this configuration should create a 2-layer bshrub, which may be +# a bit delicate +defines.INLINE_SIZE = 'BLOCK_SIZE/4' +defines.CRYSTAL_THRESH = -1 +defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' +defines.SIZE = 'BLOCK_SIZE' +# set compact thresh to minimum +defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + + // create a desync file + lfsr_file_t file1; + lfsr_file_open(&lfs, &file1, "jellyfish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0; + + uint8_t wbuf1[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; + + // rewrite part of our file until bshrub is >gc_compact_thresh full + while (true) { + // we need internals to check this + lfsr_mdir_t mdir; + // ckmeta needed for eoff + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_BRANCH); + if (lfsr_rbyd_eoff(&mtinfo.u.rbyd) > GC_COMPACT_THRESH) { + break; + } + + lfsr_file_rewind(&lfs, &file1) => 0; + for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { + wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file1, wbuf1, FRAGMENT_SIZE) => FRAGMENT_SIZE; + } + + // create some overlapping files, these should not get messed with + lfsr_file_t file2; + lfsr_file_open(&lfs, &file2, "jellyfish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + uint8_t wbuf2[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file2) => 0; + + lfsr_file_t file3; + lfsr_file_open(&lfs, &file3, "jellyfish", + LFS_O_RDWR) => 0; + uint8_t wbuf3[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf3[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file3, wbuf3, SIZE) => SIZE; + lfsr_file_desync(&lfs, &file3) => 0; + + // try traversing and compacting + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, + LFS_T_COMPACT + | ((CKMETA) ? LFS_T_CKMETA : 0) + | ((CK) ? LFS_T_CK : 0) + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + // traverse mroot + struct lfs_tinfo tinfo; + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + // compacting our bshrub nodes may cause them to split, so we may + // need to traverse more nodes than we started with + for (lfs_size_t i = 0; i < 3*5; i++) { + // traverse bshrub + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_BTREE); + } + lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; + lfsr_traversal_close(&lfs, &t) => 0; + + // bshrub should have been compacted + lfsr_mdir_t mdir; + lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA); + lfsr_mtinfo_t mtinfo; + lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0; + assert(mtinfo.tag == LFSR_TAG_MDIR); + while (true) { + int err = lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + break; + } + assert(mtinfo.tag == LFSR_TAG_BRANCH); + assert(lfsr_rbyd_eoff(&mtinfo.u.rbyd) <= GC_COMPACT_THRESH); + } + + // check we can still read the files + lfsr_file_rewind(&lfs, &file1) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf1, SIZE) == 0); + + lfsr_file_rewind(&lfs, &file2) => 0; + lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf2, SIZE) == 0); + + lfsr_file_rewind(&lfs, &file3) => 0; + lfsr_file_read(&lfs, &file3, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf3, SIZE) == 0); + + // at least try closing/opening our synced file + lfsr_file_close(&lfs, &file2) => 0; + lfsr_file_open(&lfs, &file2, "jellyfish", LFS_O_RDONLY) => 0; + lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf2, SIZE) == 0); + + lfsr_file_close(&lfs, &file1) => 0; + lfsr_file_close(&lfs, &file2) => 0; + lfsr_file_close(&lfs, &file3) => 0; + lfsr_unmount(&lfs) => 0; +''' @@ -7024,8 +7504,7 @@ defines.EXCL = [false, true] defines.CKMETA = [true] defines.CK = [true] defines.LOOKAHEAD = [false, true] -# TODO !!! Enable this when bshrub compaction is working -defines.COMPACT = [false] +defines.COMPACT = [false, true] # set compact thresh to minimum defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' defines.OPS = 20