diff --git a/lfs.c b/lfs.c index cf6256d4..68305e21 100644 --- a/lfs.c +++ b/lfs.c @@ -4895,15 +4895,6 @@ static int lfsr_btree_commit_(lfs_t *lfs, lfsr_btree_t *btree, return 0; } -static int lfsr_btree_compact_(lfs_t *lfs, lfsr_btree_t *btree, - lfsr_bid_t bid, lfsr_rbyd_t *rbyd) { - // the easiest way to do this is to just mark rbyd as unerased - // and call lfsr_btree_commit_ - rbyd->eoff = -1; - return lfsr_btree_commit_(lfs, btree, bid, rbyd, 0, - NULL, 0); -} - // commit to a btree, this is atomic static int lfsr_btree_commit(lfs_t *lfs, lfsr_btree_t *btree, lfsr_bid_t bid, const lfsr_attr_t *attrs, lfs_size_t attr_count) { @@ -5796,16 +5787,6 @@ relocate:; return 0; } -static int lfsr_bshrub_compact_(lfs_t *lfs, - lfsr_mdir_t *mdir, lfsr_bshrub_t *bshrub, - lfsr_bid_t bid, lfsr_rbyd_t *rbyd) { - // the easiest way to do this is to just mark rbyd as unerased - // and call lfsr_btree_commit_ - rbyd->eoff = -1; - return lfsr_bshrub_commit_(lfs, mdir, bshrub, bid, rbyd, 0, - NULL, 0); -} - // commit to a bshrub, this is atomic static int lfsr_bshrub_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfsr_bshrub_t *bshrub, @@ -8640,9 +8621,7 @@ static int lfsr_mtree_traverse(lfs_t *lfs, lfsr_traversal_t *t, // validate btree nodes? note mdirs are already validated if ((lfsr_t_isckmeta(t->o.o.flags) - || lfsr_t_isckdata(t->o.o.flags) - // we also need to fetch to know if we need to compact - || lfsr_t_iscompact(t->o.o.flags)) + || lfsr_t_isckdata(t->o.o.flags)) && tag == LFSR_TAG_BRANCH) { lfsr_rbyd_t *rbyd = (lfsr_rbyd_t*)bptr.data.u.buffer; err = lfsr_rbyd_fetchck(lfs, rbyd, @@ -8754,91 +8733,6 @@ dropped:; } } - // compacting btree nodes? - if (lfsr_t_iscompact(t->o.o.flags) - && tag == LFSR_TAG_BRANCH - // exceed compaction threshold? - && lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - > ((lfs->cfg->gc_compact_thresh) - ? lfs->cfg->gc_compact_thresh - : lfs->cfg->block_size - lfs->cfg->block_size/8)) { - lfsr_rbyd_t *rbyd = (lfsr_rbyd_t*)bptr.data.u.buffer; - LFS_DEBUG("Compacting rbyd 0x%"PRIx32".%"PRIx32" " - "(%"PRId32" > %"PRId32")", - rbyd->blocks[0], - lfsr_rbyd_trunk(rbyd), - lfsr_rbyd_eoff(rbyd), - (lfs->cfg->gc_compact_thresh) - ? lfs->cfg->gc_compact_thresh - : lfs->cfg->block_size - lfs->cfg->block_size/8); - - // traversals need to be enrolled in our opened list for btree - // compactions to work correctly - LFS_ASSERT(lfsr_omdir_isopen(lfs, &t->o.o)); - - // checkpoint the allocator - lfs_alloc_ckpoint(lfs); - - if (t->o.o.state == LFSR_TSTATE_MTREE) { - err = lfsr_btree_compact_(lfs, &t->o.bshrub.u.btree, - // note we may be referencing the btree root here - t->u.bt.bid, rbyd); - if (err) { - goto failed; - } - } else { - err = lfsr_bshrub_compact_(lfs, &t->o.o.mdir, &t->o.bshrub, - // note we may be referencing the btree root here - t->u.bt.bid, rbyd); - if (err) { - goto failed; - } - } - - if (t->o.o.state == LFSR_TSTATE_OBTREE) { - // just update our opened file - lfsr_file_t *file = (lfsr_file_t*)t->ot; - file->o.o.flags |= LFS_F_UNSYNC; - file->o.bshrub = t->o.bshrub; - - } else { - // commit to mdir - uint8_t buf[LFSR_BTREE_DSIZE]; - err = lfsr_mdir_commit(lfs, &t->o.o.mdir, LFSR_ATTRS( - (t->o.o.state == LFSR_TSTATE_MTREE) - ? LFSR_ATTR( - LFSR_TAG_SUB | LFSR_TAG_MTREE, 0, - LFSR_DATA_BTREE_(&t->o.bshrub.u.btree, buf)) - : (lfsr_bshrub_isbshrub(&t->o.o.mdir, &t->o.bshrub)) - ? LFSR_ATTR_SHRUBTRUNK( - LFSR_TAG_SUB | LFSR_TAG_SHRUBTRUNK, 0, - &t->o.bshrub.u.bshrub) - : LFSR_ATTR( - LFSR_TAG_SUB | LFSR_TAG_BTREE, 0, - LFSR_DATA_BTREE_(&t->o.bshrub.u.btree, buf)))); - if (err) { - goto failed; - } - - // update any open files - for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { - if (o->type == LFS_TYPE_REG - && o->mdir.mid == t->o.o.mdir.mid - && !lfsr_f_isunsync(o->flags)) { - lfsr_file_t *file = (lfsr_file_t*)o; - file->o.bshrub = t->o.bshrub; - } - } - } - - // reset to btree root - t->u.bt.branch = &t->o.bshrub.u.btree; - t->u.bt.rid = t->u.bt.bid; - - // mark as dirty, we need to do this manually here - t->o.o.flags |= LFS_F_DIRTY; - } - // swap back dirty/mutated flags t->o.o.flags = lfsr_f_swapdirty(t->o.o.flags); if (tag_) { @@ -13112,7 +13006,6 @@ int lfsr_fs_gc(lfs_t *lfs, lfs_soff_t steps, uint32_t flags) { // do we really need a full traversal? if (!(lfs->gc.o.o.flags & ( LFS_GC_LOOKAHEAD - | LFS_GC_COMPACT | LFS_GC_CKMETA | LFS_GC_CKDATA))) { lfs->gc.o.o.flags |= LFS_T_MTREEONLY; diff --git a/tests/test_traversal.toml b/tests/test_traversal.toml index 231b8c2e..a19cbf4b 100644 --- a/tests/test_traversal.toml +++ b/tests/test_traversal.toml @@ -6226,1343 +6226,6 @@ code = ''' -# btree/bshrub compactions are quite a bit more difficult - -[cases.test_traversal_compact_mtree_btree] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -defines.SIZE = 'FILE_BUFFER_SIZE/2' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -in = 'lfs.c' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create two files - lfsr_file_t file1; - lfsr_file_open(&lfs, &file1, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf1[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file1) => 0; - - lfsr_file_t file2; - lfsr_file_open(&lfs, &file2, "octopus", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf2[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file2) => 0; - - // create enough files for mroot to split - lfs_size_t i = 0; - while (lfs.mtree.u.weight == 0x80000000) { - char name[256]; - sprintf(name, "medusa%03x", i); - lfsr_file_t file; - lfsr_file_open(&lfs, &file, name, - LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; - lfsr_file_close(&lfs, &file) => 0; - i += 1; - } - - // switch to early relocations after splitting - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; - struct lfs_config cfg = *CFG; - cfg.block_recycles = 0; - lfsr_mount(&lfs, LFS_M_RDWR, &cfg) => 0; - lfsr_file_open(&lfs, &file1, "jellyfish", LFS_O_RDWR) => 0; - lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDWR) => 0; - - // rewrite a file until btree is >gc_compact_thresh full - while (true) { - // we need internals to check this - // ckmeta needed for eoff - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_BRANCH); - if (lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - > GC_COMPACT_THRESH) { - break; - } - - lfsr_file_rewind(&lfs, &file1) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file1) => 0; - } - - // switch back to default relocations to avoid mroot extension issues - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file1, "jellyfish", LFS_O_RDWR) => 0; - lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDWR) => 0; - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse mdir - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - // traverse mdir - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // mtree should have been compacted - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // mtree should have been compacted - t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the files - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file1, "jellyfish", LFS_O_RDONLY) => 0; - lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file1) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf1, SIZE) == 0); - - lfsr_file_rewind(&lfs, &file2) => 0; - lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf2, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_btree] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# limit files to very simple btrees -defines.INLINE_SIZE = 0 -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = '2*FRAGMENT_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a file - lfsr_file_t file; - lfsr_file_open(&lfs, &file, "jellyfish", - LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; - - // rewrite our file until btree is >gc_compact_thresh full - uint8_t wbuf[SIZE]; - while ((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; - } - - lfsr_file_close(&lfs, &file) => 0; - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // btree should have been compacted - lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; - assert((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // btree should have been compacted - assert((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the file - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_btree_opened] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# limit files to very simple btrees -defines.INLINE_SIZE = 0 -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = '2*FRAGMENT_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a file - lfsr_file_t file; - lfsr_file_open(&lfs, &file, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - - // rewrite our file until btree is >gc_compact_thresh full - uint8_t wbuf[SIZE]; - while ((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; - } - - lfsr_file_sync(&lfs, &file) => 0; - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // btree should have been compacted - assert((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // btree should have been compacted - assert((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the file - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_btree_orphaned] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# limit files to very simple btrees -defines.INLINE_SIZE = 0 -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = '2*FRAGMENT_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a file - lfsr_file_t file; - lfsr_file_open(&lfs, &file, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - - // rewrite our file until btree is >gc_compact_thresh full - uint8_t wbuf[SIZE]; - while ((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; - } - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // btree should have been compacted - assert((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // btree should have been compacted - assert((file.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // file should not have accidentally been created or anything - struct lfs_info info; - lfsr_stat(&lfs, "jellyfish", &info) => LFS_ERR_NOENT; - - // check we can still read the file - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_btree_desynced] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# limit files to very simple btrees -defines.INLINE_SIZE = 0 -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = '2*FRAGMENT_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a desync file - lfsr_file_t file1; - lfsr_file_open(&lfs, &file1, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0; - - // rewrite our file until btree is >gc_compact_thresh full - uint8_t wbuf1[SIZE]; - while ((file1.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file1) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - } - - // create some overlapping files, these should not get messed with - lfsr_file_t file2; - lfsr_file_open(&lfs, &file2, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf2[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file2) => 0; - - lfsr_file_t file3; - lfsr_file_open(&lfs, &file3, "jellyfish", - LFS_O_RDWR) => 0; - uint8_t wbuf3[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf3[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file3, wbuf3, SIZE) => SIZE; - lfsr_file_desync(&lfs, &file3) => 0; - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // btree should have been compacted - assert((file1.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // btree should have been compacted - assert((file1.o.bshrub.u.btree.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the files - lfsr_file_rewind(&lfs, &file1) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf1, SIZE) == 0); - - lfsr_file_rewind(&lfs, &file2) => 0; - lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf2, SIZE) == 0); - - lfsr_file_rewind(&lfs, &file3) => 0; - lfsr_file_read(&lfs, &file3, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf3, SIZE) == 0); - - // at least try closing/opening our synced file - lfsr_file_close(&lfs, &file2) => 0; - lfsr_file_open(&lfs, &file2, "jellyfish", LFS_O_RDONLY) => 0; - lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf2, SIZE) == 0); - - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_file_close(&lfs, &file3) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_bshrub] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# this configuration should create a 2-layer bshrub, which may be -# a bit delicate -defines.INLINE_SIZE = 'BLOCK_SIZE/4' -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = 'BLOCK_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -in = 'lfs.c' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a file - lfsr_file_t file; - lfsr_file_open(&lfs, &file, "jellyfish", - LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; - - uint8_t wbuf[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; - - // rewrite part of our file until bshrub is >gc_compact_thresh full - while (true) { - // we need internals to check this - // ckmeta needed for eoff - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_BRANCH); - if (lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - > GC_COMPACT_THRESH) { - break; - } - - lfsr_file_rewind(&lfs, &file) => 0; - for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, FRAGMENT_SIZE) => FRAGMENT_SIZE; - } - - lfsr_file_close(&lfs, &file) => 0; - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // compacting our bshrub nodes may cause them to split, so we may - // need to traverse more nodes than we started with - for (lfs_size_t i = 0; i < 5; i++) { - // traverse bshrub - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // bshrub should have been compacted - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // bshrub should have been compacted - t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the file - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - } - - lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; - lfsr_file_rewind(&lfs, &file) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf, SIZE) == 0); - lfsr_file_close(&lfs, &file) => 0; - } - - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_bshrub_opened] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# this configuration should create a 2-layer bshrub, which may be -# a bit delicate -defines.INLINE_SIZE = 'BLOCK_SIZE/4' -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = 'BLOCK_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -in = 'lfs.c' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a file - lfsr_file_t file; - lfsr_file_open(&lfs, &file, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - - uint8_t wbuf[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; - - // rewrite part of our file until bshrub is >gc_compact_thresh full - while (true) { - // we need internals to check this - // ckmeta needed for eoff - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_BRANCH); - if (lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - > GC_COMPACT_THRESH) { - break; - } - - lfsr_file_rewind(&lfs, &file) => 0; - for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, FRAGMENT_SIZE) => FRAGMENT_SIZE; - } - - lfsr_file_sync(&lfs, &file) => 0; - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // compacting our bshrub nodes may cause them to split, so we may - // need to traverse more nodes than we started with - for (lfs_size_t i = 0; i < 5; i++) { - // traverse bshrub - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // bshrub should have been compacted - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // bshrub should have been compacted - t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the file - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_bshrub_orphaned] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# this configuration should create a 2-layer bshrub, which may be -# a bit delicate -defines.INLINE_SIZE = 'BLOCK_SIZE/4' -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = 'BLOCK_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -in = 'lfs.c' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a file - lfsr_file_t file; - lfsr_file_open(&lfs, &file, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - - uint8_t wbuf[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; - - // rewrite part of our file until bshrub is >gc_compact_thresh full - while (true) { - // we need internals to check this - // ckmeta needed for eoff - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_BRANCH); - if (lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - > GC_COMPACT_THRESH) { - break; - } - - lfsr_file_rewind(&lfs, &file) => 0; - for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { - wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file, wbuf, FRAGMENT_SIZE) => FRAGMENT_SIZE; - } - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // compacting our bshrub nodes may cause them to split, so we may - // need to traverse more nodes than we started with - for (lfs_size_t i = 0; i < 5; i++) { - // traverse bshrub - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // bshrub should have been compacted - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // bshrub should have been compacted - t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the file - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_compact_bshrub_desynced] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# this configuration should create a 2-layer bshrub, which may be -# a bit delicate -defines.INLINE_SIZE = 'BLOCK_SIZE/4' -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = 'BLOCK_SIZE' -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -in = 'lfs.c' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create a desync file - lfsr_file_t file1; - lfsr_file_open(&lfs, &file1, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0; - - uint8_t wbuf1[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - - // rewrite part of our file until bshrub is >gc_compact_thresh full - while (true) { - // we need internals to check this - // ckmeta needed for eoff - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_BRANCH); - if (lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - > GC_COMPACT_THRESH) { - break; - } - - lfsr_file_rewind(&lfs, &file1) => 0; - for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, FRAGMENT_SIZE) => FRAGMENT_SIZE; - } - - // create some overlapping files, these should not get messed with - lfsr_file_t file2; - lfsr_file_open(&lfs, &file2, "jellyfish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf2[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file2) => 0; - - lfsr_file_t file3; - lfsr_file_open(&lfs, &file3, "jellyfish", - LFS_O_RDWR) => 0; - uint8_t wbuf3[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf3[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file3, wbuf3, SIZE) => SIZE; - lfsr_file_desync(&lfs, &file3) => 0; - - // we should be marked as uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing and compacting - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - // compacting our bshrub nodes may cause them to split, so we may - // need to traverse more nodes than we started with - for (lfs_size_t i = 0; i < 3*5; i++) { - // traverse bshrub - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // bshrub should have been compacted - lfsr_traversal_t t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_tag_t tag; - lfsr_bptr_t bptr; - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // but because we mutated, we're still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // bshrub should have been compacted - t_ = LFSR_TRAVERSAL(LFS_T_CKMETA); - lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr) => 0; - assert(tag == LFSR_TAG_MDIR); - while (true) { - int err = lfsr_mtree_traverse(&lfs, &t_, &tag, &bptr); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - assert(tag == LFSR_TAG_BRANCH); - assert(lfsr_rbyd_eoff((lfsr_rbyd_t*)bptr.data.u.buffer) - <= GC_COMPACT_THRESH); - } - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the files - lfsr_file_rewind(&lfs, &file1) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf1, SIZE) == 0); - - lfsr_file_rewind(&lfs, &file2) => 0; - lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf2, SIZE) == 0); - - lfsr_file_rewind(&lfs, &file3) => 0; - lfsr_file_read(&lfs, &file3, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf3, SIZE) == 0); - - // at least try closing/opening our synced file - lfsr_file_close(&lfs, &file2) => 0; - lfsr_file_open(&lfs, &file2, "jellyfish", LFS_O_RDONLY) => 0; - lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf2, SIZE) == 0); - - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_file_close(&lfs, &file3) => 0; - lfsr_unmount(&lfs) => 0; -''' # test traversals with mkconsistent @@ -8823,464 +7486,6 @@ code = ''' lfsr_unmount(&lfs) => 0; ''' -[cases.test_traversal_mkconsistent_compact_btree] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# limit files to very simple btrees -defines.INLINE_SIZE = 0 -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = '2*FRAGMENT_SIZE' -# <=2 => grm-able -# >2 => requires orphans -defines.ORPHANS = [0, 1, 2, 3, 100] -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create two files - lfsr_file_t file1; - lfsr_file_open(&lfs, &file1, "cuttlefish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf1[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file1) => 0; - - lfsr_file_t file2; - lfsr_file_open(&lfs, &file2, "octopus", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf2[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file2) => 0; - - // create this many orphaned files - // - // anytime we close a not-yet-created desync file, we create an - // orphan, but note we need these to be different files, and we need - // to close them after all open calls, otherwise we just end up with - // one orphan (littlefs is eager to clean up orphans) - // - lfsr_file_t orphans[ORPHANS]; - for (lfs_size_t i = 0; i < ORPHANS; i++) { - char name[256]; - sprintf(name, "jellyfish%03x", i); - lfsr_file_open(&lfs, &orphans[i], name, - LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0; - } - for (lfs_size_t i = 0; i < ORPHANS; i++) { - lfsr_file_close(&lfs, &orphans[i]) => 0; - } - - // write to our mdirs until >gc_compact_thresh full - // - // hack, don't use the internals like this - while ((file1.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file1) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file1) => 0; - } - - while ((file2.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file2) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file2) => 0; - } - - // we should be marked as inconsistent and uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((ORPHANS > 0) ? LFS_I_INCONSISTENT : 0) - | LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing with mkconsistent - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_MKCONSISTENT - | LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - if (ORPHANS <= 3) { - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } else { - // traverse mtree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse mdirs - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse mdirs - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - // traverse btree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // we should have cleaned up all grms/orphans - assert(lfs.grm.mids[0] == -1); - assert(lfs.grm.mids[1] == -1); - assert(!(lfs.flags & LFS_F_ORPHANS)); - - // which means there shouldn't be that many files left - assert((lfs.mtree.u.weight & 0x7fffffff) <= (2 << lfs.mdir_bits)); - assert(file1.o.o.mdir.rbyd.weight <= 3); - assert(file2.o.o.mdir.rbyd.weight <= 3); - - // mdirs should have been compacted - assert((file1.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - assert((file2.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // we should be marked as consistent, but because we mutated, we're - // still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // mdirs should have been compacted - assert((file1.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - assert((file2.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the files - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file1, "cuttlefish", LFS_O_RDONLY) => 0; - lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file1) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf1, SIZE) == 0); - - lfsr_file_rewind(&lfs, &file2) => 0; - lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf2, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; -''' - -[cases.test_traversal_mkconsistent_compact_bshrub] -defines.LOOKAHEAD = [false, true] -defines.CKMETA = [false, true] -defines.CKDATA = [false, true] -# this configuration should create a 2-layer bshrub, which may be -# a bit delicate -defines.INLINE_SIZE = 'BLOCK_SIZE/4' -defines.CRYSTAL_THRESH = -1 -defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8' -defines.SIZE = 'BLOCK_SIZE' -# <=2 => grm-able -# >2 => requires orphans -defines.ORPHANS = [0, 1, 2, 3, 100] -# set compact thresh to minimum -defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' -code = ''' - lfs_t lfs; - lfsr_format(&lfs, CFG) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - - uint32_t prng = 42; - - // create two files - lfsr_file_t file1; - lfsr_file_open(&lfs, &file1, "cuttlefish", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf1[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file1) => 0; - - lfsr_file_t file2; - lfsr_file_open(&lfs, &file2, "octopus", - LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; - uint8_t wbuf2[SIZE]; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file2) => 0; - - // create this many orphaned files - // - // anytime we close a not-yet-created desync file, we create an - // orphan, but note we need these to be different files, and we need - // to close them after all open calls, otherwise we just end up with - // one orphan (littlefs is eager to clean up orphans) - // - lfsr_file_t orphans[ORPHANS]; - for (lfs_size_t i = 0; i < ORPHANS; i++) { - char name[256]; - sprintf(name, "jellyfish%03x", i); - lfsr_file_open(&lfs, &orphans[i], name, - LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0; - } - for (lfs_size_t i = 0; i < ORPHANS; i++) { - lfsr_file_close(&lfs, &orphans[i]) => 0; - } - - // write to our mdirs until >gc_compact_thresh full - // - // hack, don't use the internals like this - while ((file1.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file1) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file1) => 0; - } - - while ((file2.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { - lfsr_file_rewind(&lfs, &file2) => 0; - for (lfs_size_t j = 0; j < SIZE; j++) { - wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); - } - lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; - lfsr_file_sync(&lfs, &file2) => 0; - } - - // we should be marked as inconsistent and uncompacted - struct lfs_fsinfo fsinfo; - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((ORPHANS > 0) ? LFS_I_INCONSISTENT : 0) - | LFS_I_CANLOOKAHEAD - | LFS_I_UNCOMPACTED)); - - // try traversing with mkconsistent - lfsr_traversal_t t; - lfsr_traversal_open(&lfs, &t, - LFS_T_MKCONSISTENT - | LFS_T_COMPACT - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) - | ((CKMETA) ? LFS_T_CKMETA : 0) - | ((CKDATA) ? LFS_T_CKDATA : 0)) => 0; - // needs two passes to compact both bshrubs and mdirs - lfsr_traversal_rewind(&lfs, &t) => 0; - // traverse mroot - struct lfs_tinfo tinfo; - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - if (ORPHANS <= 3) { - // compacting our bshrub nodes may cause them to split, so we may - // need to traverse more nodes than we started with - for (lfs_size_t i = 0; i < 9; i++) { - // traverse bshrub - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - } else { - // traverse mtree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse mdirs - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - // traverse bshrub - for (lfs_size_t i = 0; i < 5; i++) { - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - // traverse mdirs - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - // traverse bshrub - for (lfs_size_t i = 0; i < 5; i++) { - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - } - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // needs two passes to compact both bshrubs and mdirs - lfsr_traversal_rewind(&lfs, &t) => 0; - // traverse mroot - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - assert(tinfo.block == 0 || tinfo.block == 1); - if (ORPHANS <= 3) { - // compacting our bshrub nodes may cause them to split, so we may - // need to traverse more nodes than we started with - for (lfs_size_t i = 0; i < 6; i++) { - // traverse bshrub - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - } else { - // traverse mtree - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - // traverse mdirs - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - // traverse bshrub - for (lfs_size_t i = 0; i < 3; i++) { - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - // traverse mdirs - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_MDIR); - // traverse bshrub - for (lfs_size_t i = 0; i < 3; i++) { - lfsr_traversal_read(&lfs, &t, &tinfo) => 0; - assert(tinfo.btype == LFS_BTYPE_BTREE); - } - } - lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT; - - // we should have cleaned up all grms/orphans - assert(lfs.grm.mids[0] == -1); - assert(lfs.grm.mids[1] == -1); - assert(!(lfs.flags & LFS_F_ORPHANS)); - - // which means there shouldn't be that many files left - assert((lfs.mtree.u.weight & 0x7fffffff) <= (2 << lfs.mdir_bits)); - assert(file1.o.o.mdir.rbyd.weight <= 3); - assert(file2.o.o.mdir.rbyd.weight <= 3); - - // mdirs should have been compacted - assert((file1.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - assert((file2.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // we should be marked as consistent, but because we mutated, we're - // still marked as uncompacted - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((ORPHANS > 3) ? LFS_I_UNCOMPACTED : 0) - | ((!LOOKAHEAD || ORPHANS > 3) ? LFS_I_CANLOOKAHEAD : 0))); - - // running another traversal should clear the uncompacted flag - lfsr_traversal_rewind(&lfs, &t) => 0; - while (true) { - int err = lfsr_traversal_read(&lfs, &t, &tinfo); - assert(!err || err == LFS_ERR_NOENT); - if (err == LFS_ERR_NOENT) { - break; - } - } - lfsr_traversal_close(&lfs, &t) => 0; - - // mdirs should have been compacted - assert((file1.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - assert((file2.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); - - // uncompacted flag should have been cleared - lfsr_fs_stat(&lfs, &fsinfo) => 0; - assert(fsinfo.flags == ( - ((!LOOKAHEAD) ? LFS_I_CANLOOKAHEAD : 0))); - - // check we can still read the files - for (int remount = 0; remount < 2; remount++) { - // remount? - if (remount) { - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; - lfsr_file_open(&lfs, &file1, "cuttlefish", LFS_O_RDONLY) => 0; - lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDONLY) => 0; - } - - lfsr_file_rewind(&lfs, &file1) => 0; - uint8_t rbuf[SIZE]; - lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf1, SIZE) == 0); - - lfsr_file_rewind(&lfs, &file2) => 0; - lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; - assert(memcmp(rbuf, wbuf2, SIZE) == 0); - } - - lfsr_file_close(&lfs, &file1) => 0; - lfsr_file_close(&lfs, &file2) => 0; - lfsr_unmount(&lfs) => 0; -''' -