From 35db3bc97ff02ee33a12066688bc3d341c3a4790 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 24 Jul 2024 13:58:26 -0500 Subject: [PATCH] t: Dropped btree node compaction After thinking about this for a while, btree node compaction is subtlety different from mdir compaction, less valuable, and adds more risk: - Unlike mdirs, btree node compaction will always allocate a new block, leading to a higher chance of alloc failure. - Btree node compaction also always requires additional writes to propagate btree changes, whereas mdir compaction is usually self-contained unless it triggers a relocation. If btree nodes are mostly full this risks being counter-productive. - Btree node compaction requires a full tree traversal, whereas mdir compaction requires only traversing the mtree. Though you can always force mtree-only traversal manually with LFS_GC_MTREEONLY. - Btrees/bshrubs are also more likely to be "cold storage", that is it probably won't be uncommon to create long-lived read-only btrees as a part of files. Compacting these btrees can actually be counter- productive as it can encourage splitting. - Btrees/bshrubs are also more likely to be one use, and discarded as a file is truncated and rewritten. Compacting btree nodes in this case is a waste of erase cycles. And since btree node compaction also introduces a lot of complexity/risk of bugs, I'm going to drop this for now and limit LFS_GC_COMPACT to only compacting mdirs. At least this tested implementation will live in the history and can always be reintroduced in the future if it becomes a wanted feature. --- As is usually the case, doing less work ends up with less code: code stack before: 36292 2704 after: 35888 (-1.1%) 2696 (-0.3%) Note this still keeps the rbyd-specific commit logic necessary for committing to specific btree nodes, even though btree node compaction was the only current use case. This should eventually be useful for metadata repair. Hopefully const-propagation can minimize the cost, but realistically this means we're probably leaving some code savings on the table. --- lfs.c | 109 +-- tests/test_traversal.toml | 1795 ------------------------------------- 2 files changed, 1 insertion(+), 1903 deletions(-) diff --git a/lfs.c b/lfs.c index cf6256d4..68305e21 100644 --- a/lfs.c +++ b/lfs.c @@ -4895,15 +4895,6 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree, return 0; } -static int lfsr_btree_compact_(lfs_t *lfs, lfsr_btree_t *btree, - lfsr_bid_t bid, lfsr_rbyd_t *rbyd) { - // the easiest way to do this is to just mark rbyd as unerased - // and call lfsr_btree_commit_ - rbyd->eoff = -1; - return lfsr_btree_commit_(lfs, btree, bid, rbyd, 0, - NULL, 0); -} - // commit to a btree, this is atomic static int lfsr_btree_commit(lfs_t *lfs, lfsr_btree_t *btree, lfsr_bid_t bid, const lfsr_attr_t *attrs, lfs_size_t attr_count) { @@ -5796,16 +5787,6 @@ relocate:; return 0; } -static int lfsr_bshrub_compact_(lfs_t *lfs, - lfsr_mdir_t *mdir, lfsr_bshrub_t *bshrub, - lfsr_bid_t bid, lfsr_rbyd_t *rbyd) { - // the easiest way to do this is to just mark rbyd as unerased - // and call lfsr_btree_commit_ - rbyd->eoff = -1; - return lfsr_bshrub_commit_(lfs, mdir, bshrub, bid, rbyd, 0, - NULL, 0); -} - // commit to a bshrub, this is atomic static int lfsr_bshrub_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfsr_bshrub_t *bshrub, @@ -8640,9 +8621,7 @@ static int lfsr_mtree_traverse(lfs_t *lfs, lfsr_traversal_t *t, // validate btree nodes? note mdirs are already validated if ((lfsr_t_isckmeta(t->o.o.flags) - || lfsr_t_isckdata(t->o.o.flags) - // we also need to fetch to know if we need to compact - || lfsr_t_iscompact(t->o.o.flags)) + || lfsr_t_isckdata(t->o.o.flags)) && tag == LFSR_TAG_BRANCH) { lfsr_rbyd_t *rbyd = (lfsr_rbyd_t*)bptr.data.u.buffer; err = lfsr_rbyd_fetchck(lfs, rbyd, @@ -8754,91 +8733,6 @@ dropped:; } } - // compacting btree nodes? - if (lfsr_t_iscompact(t->o.o.flags) - && tag == LFSR_TAG_BRANCH - // exceed compaction threshold? - && lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - > ((lfs->cfg->gc_compact_thresh) - ? lfs->cfg->gc_compact_thresh - : lfs->cfg->block_size - lfs->cfg->block_size/8)) { - lfsr_rbyd_t *rbyd = (lfsr_rbyd_t*)bptr.data.u.buffer; - LFS_DEBUG("Compacting rbyd 0x%"PRIx32".%"PRIx32" " - "(%"PRId32" > %"PRId32")", - rbyd->blocks[0], - lfsr_rbyd_trunk(rbyd), - lfsr_rbyd_eoff(rbyd), - (lfs->cfg->gc_compact_thresh) - ? lfs->cfg->gc_compact_thresh - : lfs->cfg->block_size - lfs->cfg->block_size/8); - - // traversals need to be enrolled in our opened list for btree - // compactions to work correctly - LFS_ASSERT(lfsr_omdir_isopen(lfs, &t->o.o)); - - // checkpoint the allocator - lfs_alloc_ckpoint(lfs); - - if (t->o.o.state == LFSR_TSTATE_MTREE) { - err = lfsr_btree_compact_(lfs, &t->o.bshrub.u.btree, - // note we may be referencing the btree root here - t->u.bt.bid, rbyd); - if (err) { - goto failed; - } - } else { - err = lfsr_bshrub_compact_(lfs, &t->o.o.mdir, &t->o.bshrub, - // note we may be referencing the btree root here - t->u.bt.bid, rbyd); - if (err) { - goto failed; - } - } - - if (t->o.o.state == LFSR_TSTATE_OBTREE) { - // just update our opened file - lfsr_file_t *file = (lfsr_file_t*)t->ot; - file->o.o.flags |= LFS_F_UNSYNC; - file->o.bshrub = t->o.bshrub; - - } else { - // commit to mdir - uint8_t buf[LFSR_BTREE_DSIZE]; - err = lfsr_mdir_commit(lfs, &t->o.o.mdir, LFSR_ATTRS( - (t->o.o.state == LFSR_TSTATE_MTREE) - ? LFSR_ATTR( - LFSR_TAG_SUB | LFSR_TAG_MTREE, 0, - LFSR_DATA_BTREE_(&t->o.bshrub.u.btree, buf)) - : (lfsr_bshrub_isbshrub(&t->o.o.mdir, &t->o.bshrub)) - ? LFSR_ATTR_SHRUBTRUNK( - LFSR_TAG_SUB | LFSR_TAG_SHRUBTRUNK, 0, - &t->o.bshrub.u.bshrub) - : LFSR_ATTR( - LFSR_TAG_SUB | LFSR_TAG_BTREE, 0, - LFSR_DATA_BTREE_(&t->o.bshrub.u.btree, buf)))); - if (err) { - goto failed; - } - - // update any open files - for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { - if (o->type == LFS_TYPE_REG - && o->mdir.mid == t->o.o.mdir.mid - && !lfsr_f_isunsync(o->flags)) { - lfsr_file_t *file = (lfsr_file_t*)o; - file->o.bshrub = t->o.bshrub; - } - } - } - - // reset to btree root - t->u.bt.branch = &t->o.bshrub.u.btree; - t->u.bt.rid = t->u.bt.bid; - - // mark as dirty, we need to do this manually here - t->o.o.flags |= LFS_F_DIRTY; - } - // swap back dirty/mutated flags t->o.o.flags = lfsr_f_swapdirty(t->o.o.flags); if (tag_) { @@ -13112,7 +13006,6 @@ int lfsr_fs_gc(lfs_t *lfs, lfs_soff_t steps, uint32_t flags) { // do we really need a full traversal? if (!(lfs->gc.o.o.flags & ( LFS_GC_LOOKAHEAD - | LFS_GC_COMPACT | LFS_GC_CKMETA | LFS_GC_CKDATA))) { lfs->gc.o.o.flags |= LFS_T_MTREEONLY; diff --git a/tests/test_traversal.toml b/tests/test_traversal.toml index 231b8c2e..a19cbf4b 100644 --- a/tests/test_traversal.toml +++ b/tests/test_traversal.toml @@ -6226,1343 +6226,6 @@ code = ''' -# btree/bshrub compactions are quite a bit more difficult - -[cases.test_traversal_compact_mtree_btree] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -defines.SIZE = 'FILE_BUFFER_SIZE/2' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -in = 'lfs.c' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create two files - lfsr_file_t file1; - lfsr_file_open(&lfs, &file1, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf1[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file1) => 0; - - lfsr_file_t file2; - lfsr_file_open(&lfs, &file2, "octopus", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf2[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file2) => 0; - - // create enough files for mroot to split - lfs_size_t i = 0; - while (lfs.mtree.u.weight == 0x80000000) { - char name[256]; - sprintf(name, "medusa%03x", i); - lfsr_file_t file; - lfsr_file_open(&lfs, &file, name, - LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; - lfsr_file_close(&lfs, &file) => 0; - i += 1; - } - - // switch to early relocations after splitting - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; - struct lfs_config cfg = *CFG; - cfg.block_recycles = 0; - lfsr_mount(&lfs, LFS_M_RDWR, &cfg) => 0; - lfsr_file_open(&lfs, &file1, "jellyfish", LFS_O_RDWR) => 0; - lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDWR) => 0; - - // rewrite a file until btree is >gc_compact_thresh full - while (true) { - // we need internals to check this - // ckmeta needed for eoff - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_BRANCH); - if (lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - > GC_COMPACT_THRESH) { - break; - } - - lfsr_file_rewind(&lfs, &file1) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file1) => 0; - } - - // switch back to default relocations to avoid mroot extension issues - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file1, "jellyfish", LFS_O_RDWR) => 0; - lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDWR) => 0; - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse mdir - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - // traverse mdir - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // mtree should have been compacted - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // mtree should have been compacted - t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the files - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file1, "jellyfish", LFS_O_RDONLY) => 0; - lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file1) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf1, SIZE) == 0); - - lfsr_file_rewind(&lfs, &file2) => 0; - lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf2, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_btree] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# limit files to very simple btrees -defines.INLINE_SIZE = 0 -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = '2*FRAGMENT_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a file - lfsr_file_t file; - lfsr_file_open(&lfs, &file, "jellyfish", - LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; - - // rewrite our file until btree is >gc_compact_thresh full - uint8_t wbuf[SIZE]; - while ((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; - } - - lfsr_file_close(&lfs, &file) => 0; - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // btree should have been compacted - lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; - assert((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // btree should have been compacted - assert((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the file - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_btree_opened] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# limit files to very simple btrees -defines.INLINE_SIZE = 0 -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = '2*FRAGMENT_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a file - lfsr_file_t file; - lfsr_file_open(&lfs, &file, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - - // rewrite our file until btree is >gc_compact_thresh full - uint8_t wbuf[SIZE]; - while ((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; - } - - lfsr_file_sync(&lfs, &file) => 0; - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // btree should have been compacted - assert((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // btree should have been compacted - assert((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the file - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_btree_orphaned] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# limit files to very simple btrees -defines.INLINE_SIZE = 0 -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = '2*FRAGMENT_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a file - lfsr_file_t file; - lfsr_file_open(&lfs, &file, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - - // rewrite our file until btree is >gc_compact_thresh full - uint8_t wbuf[SIZE]; - while ((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; - } - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // btree should have been compacted - assert((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // btree should have been compacted - assert((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // file should not have accidentally been created or anything - struct lfs_info info; - lfsr_stat(&lfs, "jellyfish", &info) => LFS_ERR_NOENT; - - // check we can still read the file - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_btree_desynced] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# limit files to very simple btrees -defines.INLINE_SIZE = 0 -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = '2*FRAGMENT_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a desync file - lfsr_file_t file1; - lfsr_file_open(&lfs, &file1, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0; - - // rewrite our file until btree is >gc_compact_thresh full - uint8_t wbuf1[SIZE]; - while ((file1.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file1) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - } - - // create some overlapping files, these should not get messed with - lfsr_file_t file2; - lfsr_file_open(&lfs, &file2, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf2[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file2) => 0; - - lfsr_file_t file3; - lfsr_file_open(&lfs, &file3, "jellyfish", - LFS_O_RDWR) => 0; - uint8_t wbuf3[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf3[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file3, wbuf3, SIZE) => SIZE; - lfsr_file_desync(&lfs, &file3) => 0; - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // btree should have been compacted - assert((file1.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // btree should have been compacted - assert((file1.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the files - lfsr_file_rewind(&lfs, &file1) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf1, SIZE) == 0); - - lfsr_file_rewind(&lfs, &file2) => 0; - lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf2, SIZE) == 0); - - lfsr_file_rewind(&lfs, &file3) => 0; - lfsr_file_read(&lfs, &file3, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf3, SIZE) == 0); - - // at least try closing/opening our synced file - lfsr_file_close(&lfs, &file2) => 0; - lfsr_file_open(&lfs, &file2, "jellyfish", LFS_O_RDONLY) => 0; - lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf2, SIZE) == 0); - - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_file_close(&lfs, &file3) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_bshrub] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# this configuration should create a 2-layer bshrub, which may be -# a bit delicate -defines.INLINE_SIZE = 'BLOCK_SIZE/4' -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = 'BLOCK_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -in = 'lfs.c' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a file - lfsr_file_t file; - lfsr_file_open(&lfs, &file, "jellyfish", - LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; - - uint8_t wbuf[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; - - // rewrite part of our file until bshrub is >gc_compact_thresh full - while (true) { - // we need internals to check this - // ckmeta needed for eoff - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_BRANCH); - if (lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - > GC_COMPACT_THRESH) { - break; - } - - lfsr_file_rewind(&lfs, &file) => 0; - for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, FRAGMENT_SIZE) => FRAGMENT_SIZE; - } - - lfsr_file_close(&lfs, &file) => 0; - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // compacting our bshrub nodes may cause them to split, so we may - // need to traverse more nodes than we started with - for (lfs_size_t i = 0; i < 5; i++) { - // traverse bshrub - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // bshrub should have been compacted - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // bshrub should have been compacted - t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the file - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - } - - lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; - lfsr_file_rewind(&lfs, &file) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf, SIZE) == 0); - lfsr_file_close(&lfs, &file) => 0; - } - - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_bshrub_opened] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# this configuration should create a 2-layer bshrub, which may be -# a bit delicate -defines.INLINE_SIZE = 'BLOCK_SIZE/4' -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = 'BLOCK_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -in = 'lfs.c' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a file - lfsr_file_t file; - lfsr_file_open(&lfs, &file, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - - uint8_t wbuf[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; - - // rewrite part of our file until bshrub is >gc_compact_thresh full - while (true) { - // we need internals to check this - // ckmeta needed for eoff - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_BRANCH); - if (lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - > GC_COMPACT_THRESH) { - break; - } - - lfsr_file_rewind(&lfs, &file) => 0; - for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, FRAGMENT_SIZE) => FRAGMENT_SIZE; - } - - lfsr_file_sync(&lfs, &file) => 0; - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // compacting our bshrub nodes may cause them to split, so we may - // need to traverse more nodes than we started with - for (lfs_size_t i = 0; i < 5; i++) { - // traverse bshrub - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // bshrub should have been compacted - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // bshrub should have been compacted - t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the file - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_bshrub_orphaned] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# this configuration should create a 2-layer bshrub, which may be -# a bit delicate -defines.INLINE_SIZE = 'BLOCK_SIZE/4' -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = 'BLOCK_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -in = 'lfs.c' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a file - lfsr_file_t file; - lfsr_file_open(&lfs, &file, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - - uint8_t wbuf[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; - - // rewrite part of our file until bshrub is >gc_compact_thresh full - while (true) { - // we need internals to check this - // ckmeta needed for eoff - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_BRANCH); - if (lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - > GC_COMPACT_THRESH) { - break; - } - - lfsr_file_rewind(&lfs, &file) => 0; - for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, FRAGMENT_SIZE) => FRAGMENT_SIZE; - } - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // compacting our bshrub nodes may cause them to split, so we may - // need to traverse more nodes than we started with - for (lfs_size_t i = 0; i < 5; i++) { - // traverse bshrub - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // bshrub should have been compacted - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // bshrub should have been compacted - t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the file - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_bshrub_desynced] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# this configuration should create a 2-layer bshrub, which may be -# a bit delicate -defines.INLINE_SIZE = 'BLOCK_SIZE/4' -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = 'BLOCK_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -in = 'lfs.c' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a desync file - lfsr_file_t file1; - lfsr_file_open(&lfs, &file1, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0; - - uint8_t wbuf1[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - - // rewrite part of our file until bshrub is >gc_compact_thresh full - while (true) { - // we need internals to check this - // ckmeta needed for eoff - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_BRANCH); - if (lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - > GC_COMPACT_THRESH) { - break; - } - - lfsr_file_rewind(&lfs, &file1) => 0; - for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, FRAGMENT_SIZE) => FRAGMENT_SIZE; - } - - // create some overlapping files, these should not get messed with - lfsr_file_t file2; - lfsr_file_open(&lfs, &file2, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf2[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file2) => 0; - - lfsr_file_t file3; - lfsr_file_open(&lfs, &file3, "jellyfish", - LFS_O_RDWR) => 0; - uint8_t wbuf3[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf3[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file3, wbuf3, SIZE) => SIZE; - lfsr_file_desync(&lfs, &file3) => 0; - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // compacting our bshrub nodes may cause them to split, so we may - // need to traverse more nodes than we started with - for (lfs_size_t i = 0; i < 3*5; i++) { - // traverse bshrub - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // bshrub should have been compacted - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // bshrub should have been compacted - t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the files - lfsr_file_rewind(&lfs, &file1) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf1, SIZE) == 0); - - lfsr_file_rewind(&lfs, &file2) => 0; - lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf2, SIZE) == 0); - - lfsr_file_rewind(&lfs, &file3) => 0; - lfsr_file_read(&lfs, &file3, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf3, SIZE) == 0); - - // at least try closing/opening our synced file - lfsr_file_close(&lfs, &file2) => 0; - lfsr_file_open(&lfs, &file2, "jellyfish", LFS_O_RDONLY) => 0; - lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf2, SIZE) == 0); - - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_file_close(&lfs, &file3) => 0; - lfsr_unmount(&lfs) => 0; -''' # test traversals with mkconsistent @@ -8823,464 +7486,6 @@ code = ''' lfsr_unmount(&lfs) => 0; ''' -[cases.test_traversal_mkconsistent_compact_btree] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# limit files to very simple btrees -defines.INLINE_SIZE = 0 -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = '2*FRAGMENT_SIZE' -# <=2 => grm-able -# >2 => requires orphans -defines.ORPHANS = [0, 1, 2, 3, 100] -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create two files - lfsr_file_t file1; - lfsr_file_open(&lfs, &file1, "cuttlefish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf1[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file1) => 0; - - lfsr_file_t file2; - lfsr_file_open(&lfs, &file2, "octopus", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf2[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file2) => 0; - - // create this many orphaned files - // - // anytime we close a not-yet-created desync file, we create an - // orphan, but note we need these to be different files, and we need - // to close them after all open calls, otherwise we just end up with - // one orphan (littlefs is eager to clean up orphans) - // - lfsr_file_t orphans[ORPHANS]; - for (lfs_size_t i = 0; i < ORPHANS; i++) { - char name[256]; - sprintf(name, "jellyfish%03x", i); - lfsr_file_open(&lfs, &orphans[i], name, - LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0; - } - for (lfs_size_t i = 0; i < ORPHANS; i++) { - lfsr_file_close(&lfs, &orphans[i]) => 0; - } - - // write to our mdirs until >gc_compact_thresh full - // - // hack, don't use the internals like this - while ((file1.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file1) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file1) => 0; - } - - while ((file2.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file2) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file2) => 0; - } - - // we should be marked as inconsistent and uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((ORPHANS > 0) ? LFS_I_INCONSISTENT : 0) - | LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing with mkconsistent - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_MKCONSISTENT - | LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - if (ORPHANS <= 3) { - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } else { - // traverse mtree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse mdirs - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse mdirs - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // we should have cleaned up all grms/orphans - assert(lfs.grm.mids[0] == -1); - assert(lfs.grm.mids[1] == -1); - assert(!(lfs.flags & LFS_F_ORPHANS)); - - // which means there shouldn't be that many files left - assert((lfs.mtree.u.weight & 0x7fffffff) <= (2 << lfs.mdir_bits)); - assert(file1.o.o.mdir.rbyd.weight <= 3); - assert(file2.o.o.mdir.rbyd.weight <= 3); - - // mdirs should have been compacted - assert((file1.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - assert((file2.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // we should be marked as consistent, but because we mutated, we're - // still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // mdirs should have been compacted - assert((file1.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - assert((file2.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the files - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file1, "cuttlefish", LFS_O_RDONLY) => 0; - lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file1) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf1, SIZE) == 0); - - lfsr_file_rewind(&lfs, &file2) => 0; - lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf2, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_mkconsistent_compact_bshrub] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# this configuration should create a 2-layer bshrub, which may be -# a bit delicate -defines.INLINE_SIZE = 'BLOCK_SIZE/4' -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = 'BLOCK_SIZE' -# <=2 => grm-able -# >2 => requires orphans -defines.ORPHANS = [0, 1, 2, 3, 100] -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create two files - lfsr_file_t file1; - lfsr_file_open(&lfs, &file1, "cuttlefish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf1[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file1) => 0; - - lfsr_file_t file2; - lfsr_file_open(&lfs, &file2, "octopus", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf2[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file2) => 0; - - // create this many orphaned files - // - // anytime we close a not-yet-created desync file, we create an - // orphan, but note we need these to be different files, and we need - // to close them after all open calls, otherwise we just end up with - // one orphan (littlefs is eager to clean up orphans) - // - lfsr_file_t orphans[ORPHANS]; - for (lfs_size_t i = 0; i < ORPHANS; i++) { - char name[256]; - sprintf(name, "jellyfish%03x", i); - lfsr_file_open(&lfs, &orphans[i], name, - LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0; - } - for (lfs_size_t i = 0; i < ORPHANS; i++) { - lfsr_file_close(&lfs, &orphans[i]) => 0; - } - - // write to our mdirs until >gc_compact_thresh full - // - // hack, don't use the internals like this - while ((file1.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file1) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file1) => 0; - } - - while ((file2.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file2) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file2) => 0; - } - - // we should be marked as inconsistent and uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((ORPHANS > 0) ? LFS_I_INCONSISTENT : 0) - | LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing with mkconsistent - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_MKCONSISTENT - | LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // needs two passes to compact both bshrubs and mdirs - lfsr_traversal_rewind(&lfs, &t) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - if (ORPHANS <= 3) { - // compacting our bshrub nodes may cause them to split, so we may - // need to traverse more nodes than we started with - for (lfs_size_t i = 0; i < 9; i++) { - // traverse bshrub - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - } else { - // traverse mtree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse mdirs - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - // traverse bshrub - for (lfs_size_t i = 0; i < 5; i++) { - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - // traverse mdirs - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - // traverse bshrub - for (lfs_size_t i = 0; i < 5; i++) { - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - } - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // needs two passes to compact both bshrubs and mdirs - lfsr_traversal_rewind(&lfs, &t) => 0; - // traverse mroot - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - if (ORPHANS <= 3) { - // compacting our bshrub nodes may cause them to split, so we may - // need to traverse more nodes than we started with - for (lfs_size_t i = 0; i < 6; i++) { - // traverse bshrub - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - } else { - // traverse mtree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse mdirs - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - // traverse bshrub - for (lfs_size_t i = 0; i < 3; i++) { - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - // traverse mdirs - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - // traverse bshrub - for (lfs_size_t i = 0; i < 3; i++) { - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - } - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // we should have cleaned up all grms/orphans - assert(lfs.grm.mids[0] == -1); - assert(lfs.grm.mids[1] == -1); - assert(!(lfs.flags & LFS_F_ORPHANS)); - - // which means there shouldn't be that many files left - assert((lfs.mtree.u.weight & 0x7fffffff) <= (2 << lfs.mdir_bits)); - assert(file1.o.o.mdir.rbyd.weight <= 3); - assert(file2.o.o.mdir.rbyd.weight <= 3); - - // mdirs should have been compacted - assert((file1.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - assert((file2.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // we should be marked as consistent, but because we mutated, we're - // still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((ORPHANS > 3) ? LFS_I_UNCOMPACTED : 0) - | ((!LOOKAHEAD || ORPHANS > 3) ? LFS_I_CANLOOKAHEAD : 0))); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // mdirs should have been compacted - assert((file1.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - assert((file2.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the files - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file1, "cuttlefish", LFS_O_RDONLY) => 0; - lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file1) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf1, SIZE) == 0); - - lfsr_file_rewind(&lfs, &file2) => 0; - lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf2, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; -''' -