From ef4fb9d3d3832bc44ff5175e2ff34051510bdf3e Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 12 May 2023 01:56:55 -0500 Subject: [PATCH] Added specific tests to cover complex mdir split/drop corner cases Dropped the high-level "large entry" tests in exchange for these low-level tests. The high-level tests accomplished the same thing, but worse and less reliably. Added some rough fixes (this whole code path needs to be rewritten). Also made lfsr_rbyd_bisect a bit better behaved when dealing with a small number of large entries. This was necessary for the split/drop corner case tests since these rely on precise control of when mdirs split. --- lfs.c | 136 ++++---- tests/test_mtree.toml | 703 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 741 insertions(+), 98 deletions(-) diff --git a/lfs.c b/lfs.c index a827530c..73aa728f 100644 --- a/lfs.c +++ b/lfs.c @@ -2786,7 +2786,7 @@ static int lfsr_rbyd_estimate(lfs_t *lfs, const lfsr_rbyd_t *rbyd, if (real_dsize > threshold) { // TODO do these need to be conditional? if (lower_id_) { - *lower_id_ = id; + *lower_id_ = id+1; } if (lower_dsize_) { *lower_dsize_ = dsize; @@ -2847,14 +2847,8 @@ static lfs_ssize_t lfsr_rbyd_bisect(lfs_t *lfs, const lfsr_rbyd_t *rbyd, } upper_dsize += dsize; - // TODO need this still? - // // done when upper/lower dsizes are close to balanced - // - // but we also make sure at least one id is removed, in case our - // compact did not terminate on a clean id boundary - // - if (upper_dsize >= lower_dsize && lower_id_ < lower_id) { + if (upper_dsize >= lower_dsize) { break; } @@ -4596,24 +4590,38 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid, // append our mtree? if (mdir->mid == -1 && dirty_mtree) { - lfsr_tag_t tag; - uint8_t buf[LFSR_BTREE_DSIZE]; - lfs_ssize_t d = lfsr_btree_todisk(lfs, &lfs->mtree, &tag, buf); - if (d < 0) { - return d; - } + // TODO hm, this is messy + if (lfsr_btree_isnull(&lfs->mtree)) { + err = lfsr_rbyd_appendall(lfs, &mdir_.rbyd, -1, -1, LFSR_ATTRS( + LFSR_ATTR(-1, RMMDIR, 0, NULL, 0), + LFSR_ATTR(-1, RMBTREE, 0, NULL, 0))); + if (err && err != LFS_ERR_RANGE) { + //TODO should we also move if there is corruption here? + return err; + } + if (err) { + goto compact; + } + } else { + lfsr_tag_t tag; + uint8_t buf[LFSR_BTREE_DSIZE]; + lfs_ssize_t d = lfsr_btree_todisk(lfs, &lfs->mtree, &tag, buf); + if (d < 0) { + return d; + } - // TODO yeah we're going to need a wide-rm - err = lfsr_rbyd_appendall(lfs, &mdir_.rbyd, -1, -1, LFSR_ATTRS( - LFSR_ATTR(-1, RMMDIR, 0, NULL, 0), - LFSR_ATTR(-1, RMBTREE, 0, NULL, 0), - LFSR_ATTR_(-1, tag, 0, buf, d))); - if (err && err != LFS_ERR_RANGE) { - //TODO should we also move if there is corruption here? - return err; - } - if (err) { - goto compact; + // TODO yeah we're going to need a wide-rm + err = lfsr_rbyd_appendall(lfs, &mdir_.rbyd, -1, -1, LFSR_ATTRS( + LFSR_ATTR(-1, RMMDIR, 0, NULL, 0), + LFSR_ATTR(-1, RMBTREE, 0, NULL, 0), + LFSR_ATTR_(-1, tag, 0, buf, d))); + if (err && err != LFS_ERR_RANGE) { + //TODO should we also move if there is corruption here? + return err; + } + if (err) { + goto compact; + } } } @@ -4685,7 +4693,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid, // mroots without inlined mdirs must fit, skip the check for // compaction threshold in this case, we'll error in lfsr_rbyd_append // if we don't fit - if (!(mdir->mid < 0 && !lfsr_mtree_isinlined(lfs))) { + if (!(mdir->mid < 0 && (!lfsr_mtree_isinlined(lfs) || uninlined))) { // check if we're within our compaction threshold, otherwise we // need to split int fits = lfsr_rbyd_estimate(lfs, &mdir->rbyd, -1, @@ -4792,30 +4800,44 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid, // append our mtree? if (mdir->mid == -1 && dirty_mtree) { - lfsr_tag_t tag; - uint8_t buf[LFSR_BTREE_DSIZE]; - lfs_ssize_t d = lfsr_btree_todisk(lfs, &lfs->mtree, &tag, buf); - if (d < 0) { - return d; - } + // TODO hm, this is messy + if (lfsr_btree_isnull(&lfs->mtree)) { + err = lfsr_rbyd_appendall(lfs, &mdir_.rbyd, -1, -1, LFSR_ATTRS( + LFSR_ATTR(-1, RMMDIR, 0, NULL, 0), + LFSR_ATTR(-1, RMBTREE, 0, NULL, 0))); + if (err && err != LFS_ERR_RANGE) { + //TODO should we also move if there is corruption here? + return err; + } + if (err) { + goto compact; + } + } else { + lfsr_tag_t tag; + uint8_t buf[LFSR_BTREE_DSIZE]; + lfs_ssize_t d = lfsr_btree_todisk(lfs, &lfs->mtree, &tag, buf); + if (d < 0) { + return d; + } - // TODO yeah we're going to need a wide-rm - err = lfsr_rbyd_appendall(lfs, &mdir_.rbyd, -1, -1, LFSR_ATTRS( - LFSR_ATTR(-1, RMMDIR, 0, NULL, 0), - LFSR_ATTR(-1, RMBTREE, 0, NULL, 0), - LFSR_ATTR_(-1, tag, 0, buf, d))); - if (err && err != LFS_ERR_RANGE) { - //TODO should we also move if there is corruption here? - return err; - } - if (err) { - goto compact; + // TODO yeah we're going to need a wide-rm + err = lfsr_rbyd_appendall(lfs, &mdir_.rbyd, -1, -1, LFSR_ATTRS( + LFSR_ATTR(-1, RMMDIR, 0, NULL, 0), + LFSR_ATTR(-1, RMBTREE, 0, NULL, 0), + LFSR_ATTR_(-1, tag, 0, buf, d))); + if (err && err != LFS_ERR_RANGE) { + //TODO should we also move if there is corruption here? + return err; + } + if (err) { + goto compact; + } } } // TODO the number of conditions here feels like a mess, it would be // nice if this could be cleaned up - if (mdir->mid >= 0 && mdir_.rbyd.weight == 0) { + if (mdir_.mid >= 0 && mdir_.rbyd.weight == 0) { // if our weight goes to zero, drop our mdir lfs_cache_zero(lfs, &lfs->pcache); LFS_DEBUG("Dropping mdir 0x{%"PRIx32",%"PRIx32"}", @@ -4826,21 +4848,23 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid, // we need to commit our superattrs to the mroot, but that's it if (!uninlining) { // remove from mtree - err = lfsr_btree_pop(lfs, &lfs->mtree, mdir->mid); + err = lfsr_btree_pop(lfs, &lfs->mtree, mdir_.mid); if (err) { return err; } + + // update our mdir, prepare mroot + mdir_.mid = -3; + *mdir = mdir_; + mdir = &lfs->mroot; } - - // update our mdir, prepare mroot - mdir_.mid = -3; - *mdir = mdir_; - mdir = &lfs->mroot; // TODO synchronize open mdirs? // TODO wait where do we synchronize open mdirs that makes sense // if we fail after this point? + // TODO do we need to mark mtree as dirty if we were uninlining? + // // mark mtree as dirty and tail recurse to write it and any // pending superattrs to the mroot dirty_mtree = true; @@ -5227,17 +5251,19 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid, if (err) { return err; } + + // update our mdir, prepare mroot + mdir_.mid = -3; + *mdir = mdir_; + mdir = &lfs->mroot; } - - // update our mdir, prepare mroot - mdir_.mid = -3; - *mdir = mdir_; - mdir = &lfs->mroot; // TODO synchronize open mdirs? // TODO wait where do we synchronize open mdirs that makes sense // if we fail after this point? + // TODO do we need to mark mtree as dirty if we were uninlining? + // // mark mtree as dirty and tail recurse to write it and any // pending superattrs to the mroot dirty_mtree = true; diff --git a/tests/test_mtree.toml b/tests/test_mtree.toml index 02fef374..cd00a546 100644 --- a/tests/test_mtree.toml +++ b/tests/test_mtree.toml @@ -58,13 +58,176 @@ code = ''' lfsr_unmount(&lfs) => 0; ''' +## Splitting operations ## + +# specific split corner cases +[cases.test_mtree_uninline] +# this should be set so only one entry can fit in a metadata block +defines.SIZE = 'BLOCK_SIZE / 4' +in = 'lfs.c' +code = ''' + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + lfs_t lfs; + lfsr_format(&lfs, cfg) => 0; + lfsr_mount(&lfs, cfg) => 0; + + // prepare mroot with a large attr so the next entry can not fit + uint8_t buffer[SIZE]; + memset(buffer, alphas[0 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(-1, UATTR(1), 0, buffer, SIZE))) => 0; + + // create a large entry that needs to be uninlined (but not split!) + memset(buffer, alphas[1 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(0, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mroot to compact + lfs.mroot.rbyd.off = BLOCK_SIZE; + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, NULL, 0) => 0; + + // assert mdir was unininlined correctly + assert(lfsr_mtree_weight(&lfs) == 1); + // assert mroot now has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // assert that our attr is still in the mroot + lfsr_mdir_get(&lfs, &lfs.mroot, -1, LFSR_TAG_UATTR(1), + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[0 % 26], 1) == 0); + + // assert that our entry is still in the mtree + lfsr_mdir_t mdir; + lfsr_mtree_lookup(&lfs, 0, &mdir) => 0; + assert(mdir.rbyd.weight == 1); + + lfsr_mdir_get(&lfs, &mdir, 0, LFSR_TAG_INLINED, + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[1 % 26], 1) == 0); + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_mtree_uninline_split] +# this should be set so only one entry can fit in a metadata block +defines.SIZE = 'BLOCK_SIZE / 4' +in = 'lfs.c' +code = ''' + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + lfs_t lfs; + lfsr_format(&lfs, cfg) => 0; + lfsr_mount(&lfs, cfg) => 0; + + // create 2 large entries that needs to be uninlined and split + uint8_t buffer[SIZE]; + memset(buffer, alphas[0 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(0, MKINLINED, +1, buffer, SIZE))) => 0; + + memset(buffer, alphas[1 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(1, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mroot to compact + lfs.mroot.rbyd.off = BLOCK_SIZE; + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, NULL, 0) => 0; + + // assert mdirs were unininlined and split + assert(lfsr_mtree_weight(&lfs) == 2); + // assert mroot now has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // assert that our entries are still in the mtree + lfsr_mdir_t mdir; + lfsr_mtree_lookup(&lfs, 0, &mdir) => 0; + assert(mdir.rbyd.weight == 1); + lfsr_mdir_get(&lfs, &mdir, 0, LFSR_TAG_INLINED, + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[0 % 26], 1) == 0); + + lfsr_mtree_lookup(&lfs, 1, &mdir) => 0; + assert(mdir.rbyd.weight == 1); + lfsr_mdir_get(&lfs, &mdir, 0, LFSR_TAG_INLINED, + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[1 % 26], 1) == 0); + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_mtree_split] +# this should be set so only one entry can fit in a metadata block +defines.SIZE = 'BLOCK_SIZE / 4' +in = 'lfs.c' +code = ''' + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + lfs_t lfs; + lfsr_format(&lfs, cfg) => 0; + lfsr_mount(&lfs, cfg) => 0; + + // create an uninlined mdir + uint8_t buffer[SIZE]; + memset(buffer, alphas[0 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(-1, UATTR(1), 0, buffer, SIZE))) => 0; + + memset(buffer, alphas[1 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(0, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mroot to compact + lfs.mroot.rbyd.off = BLOCK_SIZE; + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, NULL, 0) => 0; + + // assert mdir was unininlined correctly + assert(lfsr_mtree_weight(&lfs) == 1); + // assert mroot now has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // now add another large entry to the mdir, forcing a split + lfsr_mdir_t mdir; + lfsr_mtree_lookup(&lfs, 0, &mdir) => 0; + assert(mdir.rbyd.weight == 1); + + memset(buffer, alphas[2 % 26], SIZE); + lfsr_mdir_commit(&lfs, &mdir, &(lfs_ssize_t){1}, LFSR_ATTRS( + LFSR_ATTR(1, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mdir to compact + mdir.rbyd.off = BLOCK_SIZE; + lfsr_mdir_commit(&lfs, &mdir, &(lfs_ssize_t){1}, NULL, 0) => 0; + + // assert mdir was split correctly + assert(lfsr_mtree_weight(&lfs) == 2); + // assert mroot still has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // assert that our attr is still in the mroot + lfsr_mdir_get(&lfs, &lfs.mroot, -1, LFSR_TAG_UATTR(1), + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[0 % 26], 1) == 0); + + // assert that our entries are still in the mtree + lfsr_mtree_lookup(&lfs, 0, &mdir) => 0; + assert(mdir.rbyd.weight == 1); + lfsr_mdir_get(&lfs, &mdir, 0, LFSR_TAG_INLINED, + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[1 % 26], 1) == 0); + + lfsr_mtree_lookup(&lfs, 1, &mdir) => 0; + assert(mdir.rbyd.weight == 1); + lfsr_mdir_get(&lfs, &mdir, 0, LFSR_TAG_INLINED, + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[2 % 26], 1) == 0); + + lfsr_unmount(&lfs) => 0; +''' + + # TODO test many mroots # try creating a range of entries that may or may not split our mtree -[cases.test_mtree_split] +[cases.test_mtree_split_many] defines.N = [5, 10, 20, 40, 80, 160, 320] -defines.SIZE = ['1', 'BLOCK_SIZE/8'] -if = 'SIZE == 1 || N <= 20' defines.FORCE_COMPACTION = [false, true] in = 'lfs.c' code = ''' @@ -85,13 +248,12 @@ code = ''' lfs.mroot.rbyd.off = cfg->block_size; } - uint8_t buffer[SIZE]; - memset(buffer, alphas[i % 26], SIZE); lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS( - LFSR_ATTR(rid, MKINLINED, +1, buffer, SIZE))) => 0; + LFSR_ATTR(rid, MKINLINED, +1, &alphas[i % 26], 1))) => 0; + uint8_t buffer[4]; lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, - buffer, SIZE) => SIZE; + buffer, 4) => 1; assert(memcmp(buffer, &alphas[i % 26], 1) == 0); rid += 1; @@ -107,9 +269,9 @@ code = ''' for (lfs_ssize_t rid = 0; rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir); rid++) { - uint8_t buffer[SIZE]; + uint8_t buffer[4]; lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, - buffer, SIZE) => SIZE; + buffer, 4) => 1; assert(memcmp(buffer, &alphas[i % 26], 1) == 0); i += 1; } @@ -122,8 +284,6 @@ code = ''' # create random entries [cases.test_mtree_split_fuzz] defines.N = [5, 10, 20, 40, 80, 160] -defines.SIZE = ['1', 'BLOCK_SIZE/8'] -if = 'SIZE == 1 || N <= 20' defines.FORCE_COMPACTION = [false, true] defines.SAMPLES = 100 # -1 => all pseudo-random seeds @@ -166,14 +326,13 @@ code = ''' } // add to rbyd, potentially splitting the mdir - uint8_t buffer[SIZE]; - memset(buffer, alphas[i % 26], SIZE); lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS( - LFSR_ATTR(rid, MKINLINED, +1, buffer, SIZE))) => 0; + LFSR_ATTR(rid, MKINLINED, +1, &alphas[i % 26], 1))) => 0; // make sure we can look up the new entry + uint8_t buffer[4]; lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, - buffer, SIZE) => SIZE; + buffer, 4) => 1; assert(memcmp(buffer, &alphas[i % 26], 1) == 0); count += 1; @@ -190,9 +349,9 @@ code = ''' for (lfs_ssize_t rid = 0; rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir); rid++) { - uint8_t buffer[SIZE]; + uint8_t buffer[4]; lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, - buffer, SIZE) => SIZE; + buffer, 4) => 1; count_ += 1; } @@ -206,12 +365,476 @@ code = ''' } ''' -# try creating an mtree and then dropping mdirs +## Dropping operations ## + +# specific drop corner cases [cases.test_mtree_drop] +# this should be set so only one entry can fit in a metadata block +defines.SIZE = 'BLOCK_SIZE / 4' +in = 'lfs.c' +code = ''' + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + lfs_t lfs; + lfsr_format(&lfs, cfg) => 0; + lfsr_mount(&lfs, cfg) => 0; + + // create an uninlined mdir + uint8_t buffer[SIZE]; + memset(buffer, alphas[0 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(-1, UATTR(1), 0, buffer, SIZE))) => 0; + + memset(buffer, alphas[1 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(0, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mroot to compact + lfs.mroot.rbyd.off = BLOCK_SIZE; + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, NULL, 0) => 0; + + // assert mdir was unininlined correctly + assert(lfsr_mtree_weight(&lfs) == 1); + // assert mroot now has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // remove the entry, forcing the mdir to be dropped + lfsr_mdir_t mdir; + lfsr_mtree_lookup(&lfs, 0, &mdir) => 0; + assert(mdir.rbyd.weight == 1); + + lfsr_mdir_commit(&lfs, &mdir, &(lfs_ssize_t){1}, LFSR_ATTRS( + LFSR_ATTR(0, MKUNR, -1, NULL, 0))) => 0; + + // assert mdir was dropped + assert(lfsr_mtree_weight(&lfs) == 0); + // assert mroot still has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // assert that our attr is still in the mroot + lfsr_mdir_get(&lfs, &lfs.mroot, -1, LFSR_TAG_UATTR(1), + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[0 % 26], 1) == 0); + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_mtree_drop_compact] +# this should be set so only one entry can fit in a metadata block +defines.SIZE = 'BLOCK_SIZE / 4' +in = 'lfs.c' +code = ''' + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + lfs_t lfs; + lfsr_format(&lfs, cfg) => 0; + lfsr_mount(&lfs, cfg) => 0; + + // create an uninlined mdir + uint8_t buffer[SIZE]; + memset(buffer, alphas[0 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(-1, UATTR(1), 0, buffer, SIZE))) => 0; + + memset(buffer, alphas[1 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(0, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mroot to compact + lfs.mroot.rbyd.off = BLOCK_SIZE; + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, NULL, 0) => 0; + + // assert mdir was unininlined correctly + assert(lfsr_mtree_weight(&lfs) == 1); + // assert mroot now has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // remove the entry, forcing the mdir to be dropped + lfsr_mdir_t mdir; + lfsr_mtree_lookup(&lfs, 0, &mdir) => 0; + assert(mdir.rbyd.weight == 1); + + // force mdir to compact while we're removing + mdir.rbyd.off = BLOCK_SIZE; + + lfsr_mdir_commit(&lfs, &mdir, &(lfs_ssize_t){1}, LFSR_ATTRS( + LFSR_ATTR(0, MKUNR, -1, NULL, 0))) => 0; + + // assert mdir was dropped + assert(lfsr_mtree_weight(&lfs) == 0); + // assert mroot still has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // assert that our attr is still in the mroot + lfsr_mdir_get(&lfs, &lfs.mroot, -1, LFSR_TAG_UATTR(1), + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[0 % 26], 1) == 0); + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_mtree_drop_uninline] +# this should be set so only one entry can fit in a metadata block +defines.SIZE = 'BLOCK_SIZE / 4' +in = 'lfs.c' +code = ''' + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + lfs_t lfs; + lfsr_format(&lfs, cfg) => 0; + lfsr_mount(&lfs, cfg) => 0; + + // create an uninlined mdir + uint8_t buffer[SIZE]; + memset(buffer, alphas[0 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(-1, UATTR(1), 0, buffer, SIZE))) => 0; + + memset(buffer, alphas[1 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(0, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mroot to compact + lfs.mroot.rbyd.off = BLOCK_SIZE; + + // remove the entry as we compact, forcing the mdir to be dropped + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(0, MKUNR, -1, NULL, 0))) => 0; + + // assert mdir was dropped + assert(lfsr_mtree_weight(&lfs) == 0); + // assert mroot has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // assert that our attr is still in the mroot + lfsr_mdir_get(&lfs, &lfs.mroot, -1, LFSR_TAG_UATTR(1), + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[0 % 26], 1) == 0); + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_mtree_drop_uninline_split_l] +# this should be set so only one entry can fit in a metadata block +defines.SIZE = 'BLOCK_SIZE / 4' +in = 'lfs.c' +code = ''' + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + lfs_t lfs; + lfsr_format(&lfs, cfg) => 0; + lfsr_mount(&lfs, cfg) => 0; + + // create an mdir that needs to be split + uint8_t buffer[SIZE]; + memset(buffer, alphas[0 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(0, MKINLINED, +1, buffer, SIZE))) => 0; + + memset(buffer, alphas[1 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(1, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mroot to compact + lfs.mroot.rbyd.off = BLOCK_SIZE; + + // remove the left entry as we compact, forcing the left + // mdir to be dropped + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(0, MKUNR, -1, NULL, 0))) => 0; + + // assert mdir was dropped + assert(lfsr_mtree_weight(&lfs) == 1); + // assert mroot has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // assert that one entry is still in the mtree + lfsr_mdir_t mdir; + lfsr_mtree_lookup(&lfs, 0, &mdir) => 0; + assert(mdir.rbyd.weight == 1); + + lfsr_mdir_get(&lfs, &mdir, 0, LFSR_TAG_INLINED, + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[1 % 26], 1) == 0); + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_mtree_drop_uninline_split_r] +# this should be set so only one entry can fit in a metadata block +defines.SIZE = 'BLOCK_SIZE / 4' +in = 'lfs.c' +code = ''' + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + lfs_t lfs; + lfsr_format(&lfs, cfg) => 0; + lfsr_mount(&lfs, cfg) => 0; + + // create an mdir that needs to be split + uint8_t buffer[SIZE]; + memset(buffer, alphas[0 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(0, MKINLINED, +1, buffer, SIZE))) => 0; + + memset(buffer, alphas[1 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(1, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mroot to compact + lfs.mroot.rbyd.off = BLOCK_SIZE; + + // remove the right entry as we compact, forcing the right mdir + // to be dropped + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(1, MKUNR, -1, NULL, 0))) => 0; + + // assert mdir was dropped + assert(lfsr_mtree_weight(&lfs) == 1); + // assert mroot has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // assert that one entry is still in the mtree + lfsr_mdir_t mdir; + lfsr_mtree_lookup(&lfs, 0, &mdir) => 0; + assert(mdir.rbyd.weight == 1); + + lfsr_mdir_get(&lfs, &mdir, 0, LFSR_TAG_INLINED, + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[0 % 26], 1) == 0); + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_mtree_drop_uninline_split_both] +# this should be set so only one entry can fit in a metadata block +defines.SIZE = 'BLOCK_SIZE / 4' +in = 'lfs.c' +code = ''' + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + lfs_t lfs; + lfsr_format(&lfs, cfg) => 0; + lfsr_mount(&lfs, cfg) => 0; + + // create an mdir that needs to be split + uint8_t buffer[SIZE]; + memset(buffer, alphas[0 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(0, MKINLINED, +1, buffer, SIZE))) => 0; + + memset(buffer, alphas[1 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(1, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mroot to compact + lfs.mroot.rbyd.off = BLOCK_SIZE; + + // remove both entries as we compact, forcing both mdirs to be dropped + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(0, MKUNR, -1, NULL, 0), + LFSR_ATTR(0, MKUNR, -1, NULL, 0))) => 0; + + // assert mdir was dropped + assert(lfsr_mtree_weight(&lfs) == 0); + // assert mroot has no entries + assert(lfs.mroot.rbyd.weight == 0); + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_mtree_drop_split_l] +# this should be set so only one entry can fit in a metadata block +defines.SIZE = 'BLOCK_SIZE / 4' +in = 'lfs.c' +code = ''' + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + lfs_t lfs; + lfsr_format(&lfs, cfg) => 0; + lfsr_mount(&lfs, cfg) => 0; + + // create an uninlined mdir + uint8_t buffer[SIZE]; + memset(buffer, alphas[0 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(-1, UATTR(1), 0, buffer, SIZE))) => 0; + + memset(buffer, alphas[1 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(0, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mroot to compact + lfs.mroot.rbyd.off = BLOCK_SIZE; + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, NULL, 0) => 0; + + // assert mdir was unininlined correctly + assert(lfsr_mtree_weight(&lfs) == 1); + // assert mroot now has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // now add another large entry to the mdir, forcing a split + lfsr_mdir_t mdir; + lfsr_mtree_lookup(&lfs, 0, &mdir) => 0; + assert(mdir.rbyd.weight == 1); + + memset(buffer, alphas[2 % 26], SIZE); + lfsr_mdir_commit(&lfs, &mdir, &(lfs_ssize_t){1}, LFSR_ATTRS( + LFSR_ATTR(1, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mdir to compact + mdir.rbyd.off = BLOCK_SIZE; + + // remove the left entry as we compact, forcing the left + // mdir to be dropped + lfsr_mdir_commit(&lfs, &mdir, &(lfs_ssize_t){0}, LFSR_ATTRS( + LFSR_ATTR(0, MKUNR, -1, NULL, 0))) => 0; + + // assert mdir was dropped + assert(lfsr_mtree_weight(&lfs) == 1); + // assert mroot has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // assert that our attr is still in the mroot + lfsr_mdir_get(&lfs, &lfs.mroot, -1, LFSR_TAG_UATTR(1), + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[0 % 26], 1) == 0); + + // assert that one entry is still in the mtree + lfsr_mtree_lookup(&lfs, 0, &mdir) => 0; + assert(mdir.rbyd.weight == 1); + + lfsr_mdir_get(&lfs, &mdir, 0, LFSR_TAG_INLINED, + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[2 % 26], 1) == 0); + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_mtree_drop_split_r] +# this should be set so only one entry can fit in a metadata block +defines.SIZE = 'BLOCK_SIZE / 4' +in = 'lfs.c' +code = ''' + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + lfs_t lfs; + lfsr_format(&lfs, cfg) => 0; + lfsr_mount(&lfs, cfg) => 0; + + // create an uninlined mdir + uint8_t buffer[SIZE]; + memset(buffer, alphas[0 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(-1, UATTR(1), 0, buffer, SIZE))) => 0; + + memset(buffer, alphas[1 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(0, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mroot to compact + lfs.mroot.rbyd.off = BLOCK_SIZE; + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, NULL, 0) => 0; + + // assert mdir was unininlined correctly + assert(lfsr_mtree_weight(&lfs) == 1); + // assert mroot now has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // now add another large entry to the mdir, forcing a split + lfsr_mdir_t mdir; + lfsr_mtree_lookup(&lfs, 0, &mdir) => 0; + assert(mdir.rbyd.weight == 1); + + memset(buffer, alphas[2 % 26], SIZE); + lfsr_mdir_commit(&lfs, &mdir, &(lfs_ssize_t){1}, LFSR_ATTRS( + LFSR_ATTR(1, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mdir to compact + mdir.rbyd.off = BLOCK_SIZE; + + // remove the right entry as we compact, forcing the right + // mdir to be dropped + lfsr_mdir_commit(&lfs, &mdir, &(lfs_ssize_t){1}, LFSR_ATTRS( + LFSR_ATTR(1, MKUNR, -1, NULL, 0))) => 0; + + // assert mdir was dropped + assert(lfsr_mtree_weight(&lfs) == 1); + // assert mroot has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // assert that our attr is still in the mroot + lfsr_mdir_get(&lfs, &lfs.mroot, -1, LFSR_TAG_UATTR(1), + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[0 % 26], 1) == 0); + + // assert that one entry is still in the mtree + lfsr_mtree_lookup(&lfs, 0, &mdir) => 0; + assert(mdir.rbyd.weight == 1); + + lfsr_mdir_get(&lfs, &mdir, 0, LFSR_TAG_INLINED, + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[1 % 26], 1) == 0); + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_mtree_drop_split_both] +# this should be set so only one entry can fit in a metadata block +defines.SIZE = 'BLOCK_SIZE / 4' +in = 'lfs.c' +code = ''' + const char *alphas = "abcdefghijklmnopqrstuvwxyz"; + lfs_t lfs; + lfsr_format(&lfs, cfg) => 0; + lfsr_mount(&lfs, cfg) => 0; + + // create an uninlined mdir + uint8_t buffer[SIZE]; + memset(buffer, alphas[0 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(-1, UATTR(1), 0, buffer, SIZE))) => 0; + + memset(buffer, alphas[1 % 26], SIZE); + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS( + LFSR_ATTR(0, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mroot to compact + lfs.mroot.rbyd.off = BLOCK_SIZE; + lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, NULL, 0) => 0; + + // assert mdir was unininlined correctly + assert(lfsr_mtree_weight(&lfs) == 1); + // assert mroot now has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // now add another large entry to the mdir, forcing a split + lfsr_mdir_t mdir; + lfsr_mtree_lookup(&lfs, 0, &mdir) => 0; + assert(mdir.rbyd.weight == 1); + + memset(buffer, alphas[2 % 26], SIZE); + lfsr_mdir_commit(&lfs, &mdir, &(lfs_ssize_t){1}, LFSR_ATTRS( + LFSR_ATTR(1, MKINLINED, +1, buffer, SIZE))) => 0; + + // force mdir to compact + mdir.rbyd.off = BLOCK_SIZE; + + // remove both entries as we compact, forcing both mdirs to be dropped + lfsr_mdir_commit(&lfs, &mdir, &(lfs_ssize_t){1}, LFSR_ATTRS( + LFSR_ATTR(0, MKUNR, -1, NULL, 0), + LFSR_ATTR(0, MKUNR, -1, NULL, 0))) => 0; + + // assert mdir was dropped + assert(lfsr_mtree_weight(&lfs) == 0); + // assert mroot has no entries + assert(lfs.mroot.rbyd.weight == 0); + + // assert that our attr is still in the mroot + lfsr_mdir_get(&lfs, &lfs.mroot, -1, LFSR_TAG_UATTR(1), + buffer, SIZE) => SIZE; + assert(memcmp(buffer, &alphas[0 % 26], 1) == 0); + + lfsr_unmount(&lfs) => 0; +''' + +# try creating an mtree and then dropping mdirs +[cases.test_mtree_drop_many] defines.N = [5, 10, 20, 40, 80, 160, 320] defines.REMAINING = [20, 5, 1, 0] -defines.SIZE = ['1', 'BLOCK_SIZE/8'] -if = 'N > REMAINING && (SIZE == 1 || N <= 20)' +if = 'N > REMAINING' defines.FORCE_COMPACTION = [false, true] in = 'lfs.c' code = ''' @@ -226,13 +849,12 @@ code = ''' lfs_ssize_t rid = 0; for (lfs_size_t i = 0; i < N; i++) { - uint8_t buffer[SIZE]; - memset(buffer, alphas[i % 26], SIZE); lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS( - LFSR_ATTR(rid, MKINLINED, +1, buffer, SIZE))) => 0; + LFSR_ATTR(rid, MKINLINED, +1, &alphas[i % 26], 1))) => 0; + uint8_t buffer[4]; lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, - buffer, SIZE) => SIZE; + buffer, 4) => 1; assert(memcmp(buffer, &alphas[i % 26], 1) == 0); rid += 1; @@ -267,9 +889,9 @@ code = ''' for (lfs_ssize_t rid = 0; rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir); rid++) { - uint8_t buffer[SIZE]; + uint8_t buffer[4]; lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, - buffer, SIZE) => SIZE; + buffer, 4) => 1; assert(memcmp(buffer, &alphas[i % 26], 1) == 0); i += 1; } @@ -281,9 +903,7 @@ code = ''' # this one has some pretty nasty corner cases [cases.test_mtree_repeated_drop] -defines.N = [5, 10, 20, 40, 80] -defines.SIZE = ['1', 'BLOCK_SIZE/8'] -if = 'SIZE == 1 || N <= 5' +defines.N = [5, 10, 20, 40] defines.FORCE_COMPACTION = [false, true] defines.CYCLES = 10 in = 'lfs.c' @@ -300,13 +920,12 @@ code = ''' lfs_ssize_t rid = 0; for (lfs_size_t i = 0; i < N; i++) { - uint8_t buffer[SIZE]; - memset(buffer, alphas[i % 26], SIZE); lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS( - LFSR_ATTR(rid, MKINLINED, +1, buffer, SIZE))) => 0; + LFSR_ATTR(rid, MKINLINED, +1, &alphas[i % 26], 1))) => 0; + uint8_t buffer[4]; lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, - buffer, SIZE) => SIZE; + buffer, 4) => 1; assert(memcmp(buffer, &alphas[i % 26], 1) == 0); rid += 1; @@ -322,9 +941,9 @@ code = ''' for (lfs_ssize_t rid = 0; rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir); rid++) { - uint8_t buffer[SIZE]; + uint8_t buffer[4]; lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, - buffer, SIZE) => SIZE; + buffer, 4) => 1; assert(memcmp(buffer, &alphas[i % 26], 1) == 0); i += 1; } @@ -359,8 +978,6 @@ code = ''' [cases.test_mtree_drop_fuzz] defines.N = [5, 10, 20, 40, 80, 160] -defines.SIZE = ['1', 'BLOCK_SIZE/8'] -if = 'SIZE == 1 || N <= 20' defines.FORCE_COMPACTION = [false, true] defines.SAMPLES = 100 # -1 => all pseudo-random seeds @@ -409,14 +1026,14 @@ code = ''' // create if (op == 0) { // add to rbyd, potentially splitting the mdir - uint8_t buffer[SIZE]; - memset(buffer, alphas[i % 26], SIZE); lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS( - LFSR_ATTR(rid, MKINLINED, +1, buffer, SIZE))) => 0; + LFSR_ATTR(rid, MKINLINED, +1, + &alphas[i % 26], 1))) => 0; // make sure we can look up the new entry + uint8_t buffer[4]; lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, - buffer, SIZE) => SIZE; + buffer, 4) => 1; assert(memcmp(buffer, &alphas[i % 26], 1) == 0); count += 1; @@ -445,9 +1062,9 @@ code = ''' for (lfs_ssize_t rid = 0; rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir); rid++) { - uint8_t buffer[SIZE]; + uint8_t buffer[4]; lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, - buffer, SIZE) => SIZE; + buffer, 4) => 1; count_ += 1; }