From 4d06fc2e0e8b7326429802a52a678b9180096936 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 24 Jun 2024 16:21:38 -0500 Subject: [PATCH] t: (Re)implemented gc_compact_thresh, at least over mdirs lfs_fs_gc is still not reimplemented, but this is accessible through the traversal API with LFS_T_COMPACT. This is also the first traversal operation that can mutate the filesystem, which brings its own set of problems: - We need to set LFS_F_DIRTY in lfsr_mtree_gc now, which really highlights how much of a mess having two flag fields is... We do _not_ clobber in this case, since we assume lfsr_mtree_gc knows what it's doing. - We can now commit to an mroot in the mroot chain outside of the normal mroot chain update logic. This is a bit scary, but should just work. The only issue so far is that we need to allow mdirs to follow the mroot during mroot splits if mid=-1, even if they aren't lfs_t's mroot mdir. This should now be decently tested with the new test_traversal_compact_* tests. - It's easy for mtraversal's mdir and mtinfo's mdir to fall out of sync when mutating... Why do we have two of these? The actual compaction itself is pretty straightforward: just mark as unerased, eoff=-1, and call lfsr_mdir_commit with an empty commit. This is now wrapped up in lfsr_mdir_compact. Code changes: code stack before: 34528 2640 after: 34652 (+0.4%) 2640 (+0.0%) Though the real hard part will be implementing gc_compact_thresh over btree nodes... --- lfs.c | 120 ++++-- lfs.h | 11 + runners/bench_runner.h | 2 + runners/test_runner.h | 2 + tests/test_traversal.toml | 771 +++++++++++++++++++++++++++++++++++++- 5 files changed, 875 insertions(+), 31 deletions(-) diff --git a/lfs.c b/lfs.c index 54350e35..02c9202e 100644 --- a/lfs.c +++ b/lfs.c @@ -5903,12 +5903,12 @@ static void lfsr_traversal_clobber(lfs_t *lfs, lfsr_traversal_t *t, static void lfsr_omdir_clobber(lfs_t *lfs, lfsr_omdir_t *o, bool dirty) { for (lfsr_omdir_t *o_ = lfs->omdirs; o_; o_ = o_->next) { if (o_->type == LFS_TYPE_TRAVERSAL) { + lfsr_traversal_t *t = (lfsr_traversal_t*)o_; // mark _all_ traversals as dirty if we're mutating the // filesystem at all - o_->flags |= (dirty) ? LFS_F_DIRTY : 0; + t->mt.flags |= (dirty) ? LFS_F_DIRTY : 0; // clobber any traversals referencing our mdir - lfsr_traversal_t *t = (lfsr_traversal_t*)o_; if (t->mt.o == o) { lfsr_traversal_clobber(lfs, t, -1); } @@ -7229,7 +7229,9 @@ static void lfs_alloc_ckpoint(lfs_t *lfs); static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, const lfsr_attr_t *attrs, lfs_size_t attr_count) { // non-mroot mdirs must have weight - LFS_ASSERT(lfsr_mdir_cmp(mdir, &lfs->mroot) == 0 + LFS_ASSERT(mdir->mid == -1 + // note inlined mdirs are mroots with mid != -1 + || lfsr_mdir_cmp(mdir, &lfs->mroot) == 0 || mdir->rbyd.weight > 0); // rid in-bounds? LFS_ASSERT(lfsr_mid_rid(lfs, mdir->mid) @@ -7789,8 +7791,14 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, // clobber any related traversals for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) { if (o->type == LFS_TYPE_TRAVERSAL) { + // don't clobber the current mdir, we assume upper layers know + // what they're doing + if (&o->mdir == mdir) { + continue; + } + // mark all traversals as dirty - o->flags |= LFS_F_DIRTY; + ((lfsr_traversal_t*)o)->mt.flags |= LFS_F_DIRTY; // clobber any mdir related traversals if (lfsr_mdir_cmp(&o->mdir, mdir) == 0) { @@ -7831,8 +7839,10 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, } // update mdir to follow requested rid - LFS_ASSERT(mdir->mid != -1 || mdir == &lfs->mroot); if (mdelta > 0 + && mdir->mid == -1) { + mdir->rbyd = mroot_.rbyd; + } else if (mdelta > 0 && lfsr_mid_rid(lfs, mdir->mid) >= (lfsr_srid_t)mdir_[0].rbyd.weight) { mdir->mid += (1 << lfs->mdir_bits) - mdir_[0].rbyd.weight; @@ -7853,6 +7863,13 @@ failed:; return err; } +static int lfsr_mdir_compact(lfs_t *lfs, lfsr_mdir_t *mdir) { + // the easiest way to do this is to just mark mdir as unerased + // and call lfsr_mdir_commit + mdir->rbyd.eoff = -1; + return lfsr_mdir_commit(lfs, mdir, NULL, 0); +} + /// Mtree path/name lookup /// @@ -8538,11 +8555,58 @@ static int lfsr_mtree_traverse(lfs_t *lfs, // high-level mutating traversal, handle extra features that require // mutation here, upper layers should call lfs_alloc_ckpoint as needed -static int lfsr_mtree_traversemut(lfs_t *lfs, +static int lfsr_mtree_gc(lfs_t *lfs, lfsr_mdir_t *mdir, lfsr_mtraversal_t *mt, lfsr_mtinfo_t *mtinfo) { - // TODO - return lfsr_mtree_traverse(lfs, mdir, mt, mtinfo); + int err = lfsr_mtree_traverse(lfs, mdir, mt, mtinfo); + if (err) { + return err; + } + + // compacting mdirs? + if (lfsr_t_iscompact(mt->flags) + && mtinfo->tag == LFSR_TAG_MDIR + // exceed compaction threshold? + && lfsr_rbyd_eoff(&mtinfo->u.mdir.rbyd) + > ((lfs->cfg->gc_compact_thresh) + ? lfs->cfg->gc_compact_thresh + : lfs->cfg->block_size - lfs->cfg->block_size/8)) { + LFS_DEBUG("Compacting mdir %"PRId32" " + "0x{%"PRIx32",%"PRIx32"} " + "(%"PRId32" > %"PRId32")", + mtinfo->u.mdir.mid >> lfs->mdir_bits, + mtinfo->u.mdir.rbyd.blocks[0], + mtinfo->u.mdir.rbyd.blocks[1], + lfsr_rbyd_eoff(&mtinfo->u.mdir.rbyd), + (lfs->cfg->gc_compact_thresh) + ? lfs->cfg->gc_compact_thresh + : lfs->cfg->block_size - lfs->cfg->block_size/8); + + // TODO should we really have two mdir copies flying around? + LFS_ASSERT(lfsr_mdir_cmp(mdir, &mtinfo->u.mdir) == 0); + int err = lfsr_mdir_compact(lfs, mdir); + if (err) { + return err; + } + mtinfo->u.mdir = *mdir; + + mt->flags |= LFS_F_DIRTY; + } + +// // TODO +// // compacting btree nodes? +// if (lfsr_t_iscompact(mt->flags) +// && mtinfo->tag == LFSR_TAG_BTREE +// // exceed compaction threshold? +// && lfsr_rbyd_eoff(&mtinfo->u.rbyd) +// > ((lfs->cfg->gc_compact_thresh) +// ? lfs->cfg->gc_compact_thresh +// : lfs->cfg->block_size - lfs->cfg->block_size/8)) { +// +// // TODO clobber? +// } + + return 0; } @@ -11465,6 +11529,15 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { // // wear-leveling. // LFS_ASSERT(lfs->cfg->block_cycles != 0); + // check that gc_compact_thresh makes sense + // + // metadata can't be compacted below block_size/2, and metadata can't + // exceed a block + LFS_ASSERT(lfs->cfg->gc_compact_thresh == 0 + || lfs->cfg->gc_compact_thresh >= lfs->cfg->block_size/2); + LFS_ASSERT(lfs->cfg->gc_compact_thresh == (lfs_size_t)-1 + || lfs->cfg->gc_compact_thresh <= lfs->cfg->block_size); + // inline_size must be <= block_size/4 LFS_ASSERT(lfs->cfg->inline_size <= lfs->cfg->block_size/4); // shrub_size must be <= block_size/4 @@ -12618,8 +12691,8 @@ int lfsr_traversal_read(lfs_t *lfs, lfsr_traversal_t *t, LFS_ASSERT(lfsr_omdir_isopen(lfs, &t->o)); // traversal dirty and excl? terminate early - if (lfsr_t_isexcl(t->o.flags) - && lfsr_f_isdirty(t->o.flags)) { + if (lfsr_t_isexcl(t->mt.flags) + && lfsr_f_isdirty(t->mt.flags)) { return LFS_ERR_BUSY; } @@ -12627,7 +12700,7 @@ int lfsr_traversal_read(lfs_t *lfs, lfsr_traversal_t *t, // some redund blocks left over? if (t->blocks[0] != -1) { // write our traversal info - tinfo->btype = lfsr_t_btype(t->o.flags); + tinfo->btype = lfsr_t_btype(t->mt.flags); tinfo->block = t->blocks[0]; t->blocks[0] = t->blocks[1]; @@ -12637,7 +12710,7 @@ int lfsr_traversal_read(lfs_t *lfs, lfsr_traversal_t *t, // find next block lfsr_mtinfo_t mtinfo; - int err = lfsr_mtree_traversemut(lfs, &t->o.mdir, &t->mt, &mtinfo); + int err = lfsr_mtree_gc(lfs, &t->o.mdir, &t->mt, &mtinfo); if (err) { // end of traversal? if (err == LFS_ERR_NOENT) { @@ -12646,19 +12719,26 @@ int lfsr_traversal_read(lfs_t *lfs, lfsr_traversal_t *t, return err; } + // traversal may itself set the dirty flag if it required + // mutation to make progress + if (lfsr_t_isexcl(t->mt.flags) + && lfsr_f_isdirty(t->mt.flags)) { + return LFS_ERR_BUSY; + } + // figure out type/blocks if (mtinfo.tag == LFSR_TAG_MDIR) { - t->o.flags = (t->o.flags & ~0x7) | LFS_BTYPE_MDIR; + t->mt.flags = (t->mt.flags & ~0x7) | LFS_BTYPE_MDIR; t->blocks[0] = mtinfo.u.mdir.rbyd.blocks[0]; t->blocks[1] = mtinfo.u.mdir.rbyd.blocks[1]; } else if (mtinfo.tag == LFSR_TAG_BRANCH) { - t->o.flags = (t->o.flags & ~0x7) | LFS_BTYPE_BTREE; + t->mt.flags = (t->mt.flags & ~0x7) | LFS_BTYPE_BTREE; t->blocks[0] = mtinfo.u.rbyd.blocks[0]; t->blocks[1] = -1; } else if (mtinfo.tag == LFSR_TAG_BLOCK) { - t->o.flags = (t->o.flags & ~0x7) | LFS_BTYPE_DATA; + t->mt.flags = (t->mt.flags & ~0x7) | LFS_BTYPE_DATA; t->blocks[0] = mtinfo.u.bptr.data.u.disk.block; t->blocks[1] = -1; @@ -12669,8 +12749,8 @@ int lfsr_traversal_read(lfs_t *lfs, lfsr_traversal_t *t, done:; // was a lookahead scan successful? - if (lfsr_t_islookahead(t->o.flags) - && !lfsr_f_isdirty(t->o.flags)) { + if (lfsr_t_islookahead(t->mt.flags) + && !lfsr_f_isdirty(t->mt.flags)) { lfs_alloc_markfree(lfs); } @@ -12703,9 +12783,7 @@ static void lfsr_traversal_clobber(lfs_t *lfs, lfsr_traversal_t *t, static int lfsr_traversal_rewind_(lfs_t *lfs, lfsr_traversal_t *t) { (void)lfs; - // clear sticky flags - t->o.flags &= ~LFS_F_DIRTY; - // reset traversal + // reset traversal, note this clears any sticky bits t->o.mdir = LFSR_MDIR_NULL(); t->mt = LFSR_MTRAVERSAL(t->o.flags); @@ -12714,7 +12792,7 @@ static int lfsr_traversal_rewind_(lfs_t *lfs, lfsr_traversal_t *t) { t->blocks[1] = -1; // shift the lookahead buffer if requested - if (lfsr_t_islookahead(t->o.flags)) { + if (lfsr_t_islookahead(t->mt.flags)) { lfs_alloc_shift(lfs); } diff --git a/lfs.h b/lfs.h index 042a6ebd..3a7b3ce2 100644 --- a/lfs.h +++ b/lfs.h @@ -268,6 +268,17 @@ struct lfs_config { // can track 8 blocks. lfs_size_t lookahead_size; + // Threshold for metadata compaction during gc in bytes. Metadata logs + // that exceed this threshold will be compacted during gc operations. + // Defaults to ~88% block_size when zero, though this default may change + // in the future. + // + // Note this only affects explicit gc operations. Otherwise metadata is + // only compacted when full. + // + // Set to -1 to disable metadata compaction during gc. + lfs_size_t gc_compact_thresh; + // Optional statically allocated read buffer. Must be rcache_size. By // default lfs_malloc is used to allocate this buffer. void *rcache_buffer; diff --git a/runners/bench_runner.h b/runners/bench_runner.h index 5c7a30ac..f5c3ef60 100644 --- a/runners/bench_runner.h +++ b/runners/bench_runner.h @@ -120,6 +120,7 @@ void bench_permutation(size_t i, uint32_t *buffer, size_t size); BENCH_DEFINE(PCACHE_SIZE, LFS_MAX(16, PROG_SIZE) ) \ BENCH_DEFINE(FILE_BUFFER_SIZE, 16 ) \ BENCH_DEFINE(LOOKAHEAD_SIZE, 16 ) \ + BENCH_DEFINE(GC_COMPACT_THRESH, 0 ) \ BENCH_DEFINE(INLINE_SIZE, BLOCK_SIZE/4 ) \ BENCH_DEFINE(SHRUB_SIZE, INLINE_SIZE ) \ BENCH_DEFINE(FRAGMENT_SIZE, BLOCK_SIZE/8 ) \ @@ -149,6 +150,7 @@ void bench_permutation(size_t i, uint32_t *buffer, size_t size); .pcache_size = PCACHE_SIZE, \ .file_buffer_size = FILE_BUFFER_SIZE, \ .lookahead_size = LOOKAHEAD_SIZE, \ + .gc_compact_thresh = GC_COMPACT_THRESH, \ .inline_size = INLINE_SIZE, \ .shrub_size = SHRUB_SIZE, \ .fragment_size = FRAGMENT_SIZE, \ diff --git a/runners/test_runner.h b/runners/test_runner.h index b24abdd3..2220b960 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -105,6 +105,7 @@ void test_permutation(size_t i, uint32_t *buffer, size_t size); TEST_DEFINE(PCACHE_SIZE, LFS_MAX(16, PROG_SIZE) ) \ TEST_DEFINE(FILE_BUFFER_SIZE, 16 ) \ TEST_DEFINE(LOOKAHEAD_SIZE, 16 ) \ + TEST_DEFINE(GC_COMPACT_THRESH, 0 ) \ TEST_DEFINE(INLINE_SIZE, BLOCK_SIZE/4 ) \ TEST_DEFINE(SHRUB_SIZE, INLINE_SIZE ) \ TEST_DEFINE(FRAGMENT_SIZE, BLOCK_SIZE/8 ) \ @@ -134,6 +135,7 @@ void test_permutation(size_t i, uint32_t *buffer, size_t size); .pcache_size = PCACHE_SIZE, \ .file_buffer_size = FILE_BUFFER_SIZE, \ .lookahead_size = LOOKAHEAD_SIZE, \ + .gc_compact_thresh = GC_COMPACT_THRESH, \ .inline_size = INLINE_SIZE, \ .shrub_size = SHRUB_SIZE, \ .fragment_size = FRAGMENT_SIZE, \ diff --git a/tests/test_traversal.toml b/tests/test_traversal.toml index c0f0efc1..2cc54b1a 100644 --- a/tests/test_traversal.toml +++ b/tests/test_traversal.toml @@ -12,6 +12,7 @@ after = [ defines.CKMETA = [false, true] defines.CK = [false, true] defines.LOOKAHEAD = [false, true] +defines.COMPACT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, CFG) => 0; @@ -22,7 +23,8 @@ code = ''' lfsr_traversal_open(&lfs, &t, ((CKMETA) ? LFS_T_CKMETA : 0) | ((CK) ? LFS_T_CK : 0) - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) + | ((COMPACT) ? LFS_T_COMPACT : 0)) => 0; struct lfs_tinfo tinfo; lfsr_traversal_read(&lfs, &t, &tinfo) => 0; assert(tinfo.btype == LFS_BTYPE_MDIR); @@ -41,6 +43,7 @@ code = ''' defines.CKMETA = [false, true] defines.CK = [false, true] defines.LOOKAHEAD = [false, true] +defines.COMPACT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, CFG) => 0; @@ -51,7 +54,8 @@ code = ''' lfsr_traversal_open(&lfs, &t, ((CKMETA) ? LFS_T_CKMETA : 0) | ((CK) ? LFS_T_CK : 0) - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) + | ((COMPACT) ? LFS_T_COMPACT : 0)) => 0; struct lfs_tinfo tinfo; lfsr_traversal_read(&lfs, &t, &tinfo) => 0; assert(tinfo.btype == LFS_BTYPE_MDIR); @@ -79,6 +83,7 @@ code = ''' defines.CKMETA = [false, true] defines.CK = [false, true] defines.LOOKAHEAD = [false, true] +defines.COMPACT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, CFG) => 0; @@ -89,7 +94,8 @@ code = ''' lfsr_traversal_open(&lfs, &t, ((CKMETA) ? LFS_T_CKMETA : 0) | ((CK) ? LFS_T_CK : 0) - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) + | ((COMPACT) ? LFS_T_COMPACT : 0)) => 0; struct lfs_tinfo tinfo; lfsr_traversal_read(&lfs, &t, &tinfo) => 0; assert(tinfo.btype == LFS_BTYPE_MDIR); @@ -5039,6 +5045,737 @@ code = ''' +# test traversals with mdir compaction + +[cases.test_traversal_compact] +defines.CKMETA = [false, true] +defines.CK = [false, true] +defines.LOOKAHEAD = [false, true] +defines.SIZE = 'FILE_BUFFER_SIZE/2' +# set compact thresh to minimum +defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + + // write to our mdir until >gc_compact_thresh full + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jellyfish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + + // hack, don't use the internals like this + uint8_t wbuf[SIZE]; + while ((file.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { + lfsr_file_rewind(&lfs, &file) => 0; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file) => 0; + } + + // try traversing and compacting + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, + LFS_T_COMPACT + | ((CKMETA) ? LFS_T_CKMETA : 0) + | ((CK) ? LFS_T_CK : 0) + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + // traverse mroot + struct lfs_tinfo tinfo; + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + lfsr_traversal_close(&lfs, &t) => 0; + + // mdir should have been compacted + assert((file.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); + + // check we can still read the file + for (int remount = 0; remount < 2; remount++) { + // remount? + if (remount) { + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; + } + + lfsr_file_rewind(&lfs, &file) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf, SIZE) == 0); + } + + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; +''' + +# yes, compactions should return busy if excl +[cases.test_traversal_compact_excl] +defines.CKMETA = [false, true] +defines.CK = [false, true] +defines.LOOKAHEAD = [false, true] +defines.SIZE = 'FILE_BUFFER_SIZE/2' +# set compact thresh to minimum +defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + + // write to our mdir until >gc_compact_thresh full + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jellyfish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + + // hack, don't use the internals like this + uint8_t wbuf[SIZE]; + while ((file.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { + lfsr_file_rewind(&lfs, &file) => 0; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file) => 0; + } + + // try traversing and compacting + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, + LFS_T_COMPACT + | LFS_T_EXCL + | ((CKMETA) ? LFS_T_CKMETA : 0) + | ((CK) ? LFS_T_CK : 0) + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + // traverse mroot + struct lfs_tinfo tinfo; + // compact, return bust + lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_BUSY; + lfsr_traversal_close(&lfs, &t) => 0; + + // mdir should have been compacted + assert((file.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); + + // check we can still read the file + for (int remount = 0; remount < 2; remount++) { + // remount? + if (remount) { + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; + } + + lfsr_file_rewind(&lfs, &file) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf, SIZE) == 0); + } + + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_traversal_compact_mrootchain] +defines.CKMETA = [false, true] +defines.CK = [false, true] +defines.LOOKAHEAD = [false, true] +defines.SIZE = 'FILE_BUFFER_SIZE/2' +# set compact thresh to minimum +defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' +# force early relocations +defines.BLOCK_RECYCLES = 0 +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + + // write to our mdir until mroot extends + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jellyfish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint8_t wbuf[SIZE]; + while (lfs.mroot.rbyd.blocks[0] == 0 + || lfs.mroot.rbyd.blocks[0] == 1) { + lfsr_file_rewind(&lfs, &file) => 0; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file) => 0; + } + + // now write to our mdir until mrootanchor >gc_compact_thresh full + while (true) { + // we need internals to check this + lfsr_mdir_t mrootanchor; + lfsr_mdir_fetch(&lfs, &mrootanchor, + -1, &LFSR_MPTR_MROOTANCHOR()) => 0; + if (lfsr_rbyd_eoff(&mrootanchor.rbyd) > GC_COMPACT_THRESH) { + break; + } + + lfsr_file_rewind(&lfs, &file) => 0; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file) => 0; + } + + // try traversing and compacting + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, + LFS_T_COMPACT + | ((CKMETA) ? LFS_T_CKMETA : 0) + | ((CK) ? LFS_T_CK : 0) + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + // traverse mrootanchor + struct lfs_tinfo tinfo; + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + // traverse mroot + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + lfsr_traversal_close(&lfs, &t) => 0; + + // mrootanchor should have been compacted + lfsr_mdir_t mrootanchor; + lfsr_mdir_fetch(&lfs, &mrootanchor, + -1, &LFSR_MPTR_MROOTANCHOR()) => 0; + assert(lfsr_rbyd_eoff(&mrootanchor.rbyd) <= GC_COMPACT_THRESH); + + // check we can still read the file + for (int remount = 0; remount < 2; remount++) { + // remount? + if (remount) { + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0; + } + + lfsr_file_rewind(&lfs, &file) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf, SIZE) == 0); + } + + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_traversal_compact_mroot_split] +defines.CKMETA = [false, true] +defines.CK = [false, true] +defines.LOOKAHEAD = [false, true] +defines.SIZE = 'FILE_BUFFER_SIZE/2' +# set compact thresh to minimum +defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + + // create two files + lfsr_file_t file1; + lfsr_file_open(&lfs, &file1, "jellyfish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + uint8_t wbuf1[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file1) => 0; + + lfsr_file_t file2; + lfsr_file_open(&lfs, &file2, "octopus", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + uint8_t wbuf2[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file2) => 0; + + // create enough files to both compact and split + lfs_size_t i = 0; + while (true) { + // we need internals to check this + lfs_ssize_t estimate = lfsr_mdir_estimate__(&lfs, + &file1.o.mdir, -1, -1, + NULL); + assert(estimate >= 0); + if ((file1.o.mdir.rbyd.eoff & 0x7fffffff) > GC_COMPACT_THRESH + && estimate > BLOCK_SIZE/2) { + break; + } + + char name[256]; + sprintf(name, "medusaaaaaaaaaaaaaaaaaaaaaaaaaaaa%03x", i); + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_close(&lfs, &file) => 0; + i += 1; + } + + // try traversing and compacting + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, + LFS_T_COMPACT + | ((CKMETA) ? LFS_T_CKMETA : 0) + | ((CK) ? LFS_T_CK : 0) + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + // traverse mroot + struct lfs_tinfo tinfo; + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + // should have split, traverse mtree + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_BTREE); + // traverse mdir + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + // traverse mdir + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + lfsr_traversal_close(&lfs, &t) => 0; + + // mdirs should have been compacted + assert((file1.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); + assert((file2.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); + + // check we can still read the files + for (int remount = 0; remount < 2; remount++) { + // remount? + if (remount) { + lfsr_file_close(&lfs, &file1) => 0; + lfsr_file_close(&lfs, &file2) => 0; + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + lfsr_file_open(&lfs, &file1, "jellyfish", LFS_O_RDONLY) => 0; + lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDONLY) => 0; + } + + lfsr_file_rewind(&lfs, &file1) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf1, SIZE) == 0); + + lfsr_file_rewind(&lfs, &file2) => 0; + lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf2, SIZE) == 0); + } + + lfsr_file_close(&lfs, &file1) => 0; + lfsr_file_close(&lfs, &file2) => 0; + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_traversal_compact_mtree] +defines.CKMETA = [false, true] +defines.CK = [false, true] +defines.LOOKAHEAD = [false, true] +defines.SIZE = 'FILE_BUFFER_SIZE/2' +# set compact thresh to minimum +defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' +defines.COMPACTSET = 'range(0x7)' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + + // create three files + lfsr_file_t file1; + lfsr_file_open(&lfs, &file1, "cuttlefish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + uint8_t wbuf1[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file1) => 0; + + lfsr_file_t file2; + lfsr_file_open(&lfs, &file2, "jellyfish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + uint8_t wbuf2[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file2) => 0; + + lfsr_file_t file3; + lfsr_file_open(&lfs, &file3, "octopus", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + uint8_t wbuf3[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf3[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file3, wbuf3, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file3) => 0; + + // create enough files for mroot to split twice + lfs_size_t i = 0; + while (lfs.mtree.u.weight == 0x80000000) { + char name[256]; + sprintf(name, "hydroid%03x", i); + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_close(&lfs, &file) => 0; + i += 1; + } + + i = 0; + lfs_size_t orig = lfs.mtree.u.weight; + while (lfs.mtree.u.weight == orig) { + char name[256]; + sprintf(name, "medusa%03x", i); + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_close(&lfs, &file) => 0; + i += 1; + } + + // write to each file until mdir >gc_compact_thresh full + if (COMPACTSET & 0x1) { + // hack, don't use the internals like this + while ((file1.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { + lfsr_file_rewind(&lfs, &file1) => 0; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file1) => 0; + } + } + + if (COMPACTSET & 0x2) { + // hack, don't use the internals like this + while ((file2.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { + lfsr_file_rewind(&lfs, &file2) => 0; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file2) => 0; + } + } + + if (COMPACTSET & 0x4) { + // hack, don't use the internals like this + while ((file3.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) { + lfsr_file_rewind(&lfs, &file3) => 0; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf3[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file3, wbuf3, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file3) => 0; + } + } + + // try traversing and compacting + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, + LFS_T_COMPACT + | ((CKMETA) ? LFS_T_CKMETA : 0) + | ((CK) ? LFS_T_CK : 0) + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + // traverse mroot + struct lfs_tinfo tinfo; + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + // traverse mtree + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_BTREE); + // traverse mdir + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + // traverse mdir + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + // traverse mdir + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + lfsr_traversal_close(&lfs, &t) => 0; + + // mdirs should have been compacted + assert((file1.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); + assert((file2.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); + assert((file3.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); + + // check we can still read the files + for (int remount = 0; remount < 2; remount++) { + // remount? + if (remount) { + lfsr_file_close(&lfs, &file1) => 0; + lfsr_file_close(&lfs, &file2) => 0; + lfsr_file_close(&lfs, &file3) => 0; + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + lfsr_file_open(&lfs, &file1, "cuttlefish", LFS_O_RDONLY) => 0; + lfsr_file_open(&lfs, &file2, "jellyfish", LFS_O_RDONLY) => 0; + lfsr_file_open(&lfs, &file3, "octopus", LFS_O_RDONLY) => 0; + } + + lfsr_file_rewind(&lfs, &file1) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf1, SIZE) == 0); + + lfsr_file_rewind(&lfs, &file2) => 0; + lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf2, SIZE) == 0); + + lfsr_file_rewind(&lfs, &file3) => 0; + lfsr_file_read(&lfs, &file3, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf3, SIZE) == 0); + } + + lfsr_file_close(&lfs, &file1) => 0; + lfsr_file_close(&lfs, &file2) => 0; + lfsr_file_close(&lfs, &file3) => 0; + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_traversal_compact_mtree_split] +defines.CKMETA = [false, true] +defines.CK = [false, true] +defines.LOOKAHEAD = [false, true] +defines.SIZE = 'FILE_BUFFER_SIZE/2' +# set compact thresh to minimum +defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2' +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + + // create four files + lfsr_file_t file1; + lfsr_file_open(&lfs, &file1, "cuttlefish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + uint8_t wbuf1[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file1) => 0; + + lfsr_file_t file2; + lfsr_file_open(&lfs, &file2, "jellyfish", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + uint8_t wbuf2[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file2) => 0; + + lfsr_file_t file3; + lfsr_file_open(&lfs, &file3, "octopus", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + uint8_t wbuf3[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf3[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file3, wbuf3, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file3) => 0; + + lfsr_file_t file4; + lfsr_file_open(&lfs, &file4, "squid", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + uint8_t wbuf4[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf4[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file4, wbuf4, SIZE) => SIZE; + lfsr_file_sync(&lfs, &file4) => 0; + + // create enough files for mroot to split twice + lfs_size_t i = 0; + while (lfs.mtree.u.weight == 0x80000000) { + char name[256]; + sprintf(name, "hydroid%03x", i); + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_close(&lfs, &file) => 0; + i += 1; + } + + i = 0; + lfs_size_t orig = lfs.mtree.u.weight; + while (lfs.mtree.u.weight == orig) { + char name[256]; + sprintf(name, "polyp%03x", i); + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_close(&lfs, &file) => 0; + i += 1; + } + + // create enough files to both compact and split + i = 0; + while (true) { + // we need internals to check this + lfs_ssize_t estimate = lfsr_mdir_estimate__(&lfs, + &file2.o.mdir, -1, -1, + NULL); + assert(estimate >= 0); + if ((file2.o.mdir.rbyd.eoff & 0x7fffffff) > GC_COMPACT_THRESH + && estimate > BLOCK_SIZE/2) { + break; + } + + char name[256]; + sprintf(name, "medusaaaaaaaaaaaaaaaaaaaaaaaaaaaa%03x", i); + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_close(&lfs, &file) => 0; + i += 1; + } + + // try traversing and compacting + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, + LFS_T_COMPACT + | ((CKMETA) ? LFS_T_CKMETA : 0) + | ((CK) ? LFS_T_CK : 0) + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + // traverse mroot + struct lfs_tinfo tinfo; + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + assert(tinfo.block == 0 || tinfo.block == 1); + // traverse mtree + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_BTREE); + // traverse mdir + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + // traverse mdir + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + // should have split, traverse mdir + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + // traverse mdir + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + lfsr_traversal_read(&lfs, &t, &tinfo) => 0; + assert(tinfo.btype == LFS_BTYPE_MDIR); + lfsr_traversal_close(&lfs, &t) => 0; + + // mdirs should have been compacted + assert((file1.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); + assert((file2.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); + assert((file3.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); + assert((file4.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH); + + // check we can still read the files + for (int remount = 0; remount < 2; remount++) { + // remount? + if (remount) { + lfsr_file_close(&lfs, &file1) => 0; + lfsr_file_close(&lfs, &file2) => 0; + lfsr_file_close(&lfs, &file3) => 0; + lfsr_file_close(&lfs, &file4) => 0; + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + lfsr_file_open(&lfs, &file1, "cuttlefish", LFS_O_RDONLY) => 0; + lfsr_file_open(&lfs, &file2, "jellyfish", LFS_O_RDONLY) => 0; + lfsr_file_open(&lfs, &file3, "octopus", LFS_O_RDONLY) => 0; + lfsr_file_open(&lfs, &file4, "squid", LFS_O_RDONLY) => 0; + } + + lfsr_file_rewind(&lfs, &file1) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf1, SIZE) == 0); + + lfsr_file_rewind(&lfs, &file2) => 0; + lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf2, SIZE) == 0); + + lfsr_file_rewind(&lfs, &file3) => 0; + lfsr_file_read(&lfs, &file3, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf3, SIZE) == 0); + + lfsr_file_rewind(&lfs, &file4) => 0; + lfsr_file_read(&lfs, &file4, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf4, SIZE) == 0); + } + + lfsr_file_close(&lfs, &file1) => 0; + lfsr_file_close(&lfs, &file2) => 0; + lfsr_file_close(&lfs, &file3) => 0; + lfsr_file_close(&lfs, &file4) => 0; + lfsr_unmount(&lfs) => 0; +''' + + + + # many/fuzz tests mixed with traversals # # these should hopefully test a bunch of messy traversal state @@ -5051,6 +5788,7 @@ defines.EXCL = [false, true] defines.CKMETA = [true] defines.CK = [true] defines.LOOKAHEAD = [false, true] +defines.COMPACT = [false, true] defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256] code = ''' // test creating directories @@ -5064,7 +5802,8 @@ code = ''' ((EXCL) ? LFS_T_EXCL : 0) | ((CKMETA) ? LFS_T_CKMETA : 0) | ((CK) ? LFS_T_CK : 0) - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) + | ((COMPACT) ? LFS_T_COMPACT : 0)) => 0; // make this many directories for (lfs_size_t i = 0; i < N; i++) { @@ -5157,6 +5896,7 @@ defines.EXCL = [false, true] defines.CKMETA = [true] defines.CK = [true] defines.LOOKAHEAD = [false, true] +defines.COMPACT = [false, true] defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256] defines.OPS = '2*N' defines.SEED = 42 @@ -5177,7 +5917,8 @@ code = ''' ((EXCL) ? LFS_T_EXCL : 0) | ((CKMETA) ? LFS_T_CKMETA : 0) | ((CK) ? LFS_T_CK : 0) - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) + | ((COMPACT) ? LFS_T_COMPACT : 0)) => 0; uint32_t prng = SEED; for (lfs_size_t i = 0; i < OPS; i++) { @@ -5334,6 +6075,7 @@ defines.EXCL = [false, true] defines.CKMETA = [true] defines.CK = [true] defines.LOOKAHEAD = [false, true] +defines.COMPACT = [false, true] defines.N = [1, 2, 4, 8, 16, 32, 64] defines.SIZE = [ '0', @@ -5357,7 +6099,8 @@ code = ''' ((EXCL) ? LFS_T_EXCL : 0) | ((CKMETA) ? LFS_T_CKMETA : 0) | ((CK) ? LFS_T_CK : 0) - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) + | ((COMPACT) ? LFS_T_COMPACT : 0)) => 0; // create this many files uint32_t prng = 42; @@ -5434,6 +6177,7 @@ defines.EXCL = [false, true] defines.CKMETA = [true] defines.CK = [true] defines.LOOKAHEAD = [false, true] +defines.COMPACT = [false, true] defines.N = [1, 2, 4, 8, 16, 32, 64] defines.OPS = '2*N' defines.SIZE = [ @@ -5465,7 +6209,8 @@ code = ''' ((EXCL) ? LFS_T_EXCL : 0) | ((CKMETA) ? LFS_T_CKMETA : 0) | ((CK) ? LFS_T_CK : 0) - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) + | ((COMPACT) ? LFS_T_COMPACT : 0)) => 0; uint32_t prng = SEED; for (lfs_size_t i = 0; i < OPS; i++) { @@ -5673,6 +6418,7 @@ defines.EXCL = [false, true] defines.CKMETA = [true] defines.CK = [true] defines.LOOKAHEAD = [false, true] +defines.COMPACT = [false, true] defines.OPS = 20 defines.SIZE = [ 'FILE_BUFFER_SIZE/2', @@ -5736,7 +6482,8 @@ code = ''' ((EXCL) ? LFS_T_EXCL : 0) | ((CKMETA) ? LFS_T_CKMETA : 0) | ((CK) ? LFS_T_CK : 0) - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) + | ((COMPACT) ? LFS_T_COMPACT : 0)) => 0; for (lfs_size_t i = 0; i < OPS; i++) { // choose a random location @@ -5832,6 +6579,7 @@ defines.EXCL = [false, true] defines.CKMETA = [true] defines.CK = [true] defines.LOOKAHEAD = [false, true] +defines.COMPACT = [false, true] defines.N = [1, 2, 4, 8, 16, 32, 64] defines.OPS = '2*N' defines.SIZE = [ @@ -5873,7 +6621,8 @@ code = ''' ((EXCL) ? LFS_T_EXCL : 0) | ((CKMETA) ? LFS_T_CKMETA : 0) | ((CK) ? LFS_T_CK : 0) - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) + | ((COMPACT) ? LFS_T_COMPACT : 0)) => 0; uint32_t prng = SEED; for (lfs_size_t i = 0; i < OPS; i++) { @@ -6199,6 +6948,7 @@ defines.EXCL = [false, true] defines.CKMETA = [true] defines.CK = [true] defines.LOOKAHEAD = [false, true] +defines.COMPACT = [false, true] defines.N = [1, 2, 4, 8, 16, 32, 64] defines.OPS = '2*N' defines.SIZE = [ @@ -6241,7 +6991,8 @@ code = ''' ((EXCL) ? LFS_T_EXCL : 0) | ((CKMETA) ? LFS_T_CKMETA : 0) | ((CK) ? LFS_T_CK : 0) - | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0; + | ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0) + | ((COMPACT) ? LFS_T_COMPACT : 0)) => 0; uint32_t prng = SEED; for (lfs_size_t i = 0; i < OPS; i++) {