Github/littlefs
1
Fork 0
Code Actions Activity

Added a very rough implementation of mdir drops

Browse Source 
mdirs behave a bit differently than btree nodes here. When an mdir's
weight drops to zero, we eagerly drop the mdir. Unfortunately this
introduce a large number of conditions into lfsr_mdir_commit. Maybe
there's some different way to structure to code to avoid this...

Also expanded mtree tests to cover more corner cases, these are
desperately for any confidence that mdir drops work.
This commit is contained in:
Christopher Haster
2023-05-11 16:22:11 -05:00
parent ea28413eb2
commit 6bc85375ea
3 changed files with 657 additions and 273 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+222 -42
View File
@@ -4407,10 +4407,14 @@ static lfs_ssize_t lfsr_mpair_fromdisk(lfs_t *lfs, lfsr_mpair_t *mpair,
return d;
}
static lfsr_mpair_t lfsr_mdir_mpair(const lfsr_mdir_t *mdir) {
static inline lfsr_mpair_t lfsr_mdir_mpair(const lfsr_mdir_t *mdir) {
return LFSR_MPAIR(mdir->rbyd.block, mdir->other_block);
}
static inline lfs_size_t lfsr_mdir_weight(const lfsr_mdir_t *mdir) {
return mdir->rbyd.weight;
}
static int lfsr_mdir_alloc(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t mid) {
// allocate two blocks
lfs_block_t blocks[2];
@@ -4519,7 +4523,7 @@ static lfs_ssize_t lfsr_mdir_get(lfs_t *lfs, const lfsr_mdir_t *mdir,
// mtree is the core tree of mdirs in littlefs
static inline int lfsr_mtree_isinlined(lfs_t *lfs) {
return lfsr_btree_isnull(&lfs->mtree);
return lfsr_btree_weight(&lfs->mtree) == 0;
}
static inline lfs_ssize_t lfsr_mtree_weight(lfs_t *lfs) {
@@ -4574,13 +4578,13 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
// TODO need both dirty_mtree and uninlining?
bool dirty_mtree = false;
// TODO this can be done better, we shouldn't need all these different flags
bool uninlined = false;
while (true) {
// try to commit
// TODO do we need a backup rbyd here?
// TODO this was a quick hack, should rbyd_ be the same as mdir_?
lfsr_rbyd_t rbyd_ = mdir->rbyd;
int err = lfsr_rbyd_appendall(lfs, &rbyd_, -1, -1,
lfsr_mdir_t mdir_ = *mdir;
int err = lfsr_rbyd_appendall(lfs, &mdir_.rbyd, -1, -1,
attrs, attr_count);
if (err && err != LFS_ERR_RANGE) {
//TODO should we also move if there is corruption here?
@@ -4600,7 +4604,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
}
// TODO yeah we're going to need a wide-rm
err = lfsr_rbyd_appendall(lfs, &rbyd_, -1, -1, LFSR_ATTRS(
err = lfsr_rbyd_appendall(lfs, &mdir_.rbyd, -1, -1, LFSR_ATTRS(
LFSR_ATTR(-1, RMMDIR, 0, NULL, 0),
LFSR_ATTR(-1, RMBTREE, 0, NULL, 0),
LFSR_ATTR_(-1, tag, 0, buf, d)));
@@ -4613,8 +4617,36 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
}
}
if (mdir->mid >= 0 && mdir_.rbyd.weight == 0) {
// if our weight goes to zero, drop our mdir
lfs_cache_zero(lfs, &lfs->pcache);
LFS_DEBUG("Dropping mdir 0x{%"PRIx32",%"PRIx32"}",
mdir_.rbyd.block, mdir_.other_block);
// remove from mtree
err = lfsr_btree_pop(lfs, &lfs->mtree, mdir->mid);
if (err) {
return err;
}
// update our mdir, prepare mroot
mdir_.mid = -3;
*mdir = mdir_;
mdir = &lfs->mroot;
// TODO synchronize open mdirs?
// TODO wait where do we synchronize open mdirs that makes sense
// if we fail after this point?
// mark mtree as dirty and tail recurse to write it and any pending
// superattrs to the mroot
dirty_mtree = true;
attr_count = 0;
continue;
} else {
// finalize commit
err = lfsr_rbyd_commit(lfs, &rbyd_, NULL, 0);
err = lfsr_rbyd_commit(lfs, &mdir_.rbyd, NULL, 0);
if (err && err != LFS_ERR_RANGE) {
//TODO should we also move if there is corruption here?
return err;
@@ -4622,7 +4654,9 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
if (err) {
goto compact;
}
mdir->rbyd = rbyd_;
// update our mdir
*mdir = mdir_;
// TODO synchronize open mdirs?
// synchronize mroot
@@ -4632,6 +4666,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
// successful commit
break;
}
compact:;
// can't commit, try to compact
@@ -4641,7 +4676,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
// normally the new mdir is just the flipped version of our
// current mdir
lfsr_mdir_t mdir_ = *mdir;
mdir_ = *mdir;
bool uninlining = false;
lfs_size_t lower_id;
@@ -4730,7 +4765,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
//
err = lfsr_rbyd_compact(lfs, &mdir_.rbyd,
(uninlining ? 0 : -1),
(mdir_.mid < 0 && !lfsr_mtree_isinlined(lfs) ? 0 : -1),
(uninlined ? 0 : -1),
false,
&mdir->rbyd);
if (err) {
@@ -4748,7 +4783,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
// mroot
err = lfsr_rbyd_appendall(lfs, &mdir_.rbyd,
(uninlining ? 0 : -1),
(mdir_.mid < 0 && !lfsr_mtree_isinlined(lfs) ? 0 : -1),
(uninlined ? 0 : -1),
attrs, attr_count);
if (err) {
LFS_ASSERT(err != LFS_ERR_RANGE);
@@ -4778,6 +4813,45 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
}
}
// TODO the number of conditions here feels like a mess, it would be
// nice if this could be cleaned up
if (mdir->mid >= 0 && mdir_.rbyd.weight == 0) {
// if our weight goes to zero, drop our mdir
lfs_cache_zero(lfs, &lfs->pcache);
LFS_DEBUG("Dropping mdir 0x{%"PRIx32",%"PRIx32"}",
mdir_.rbyd.block, mdir_.other_block);
// weird case, going to zero while uninlining
//
// we need to commit our superattrs to the mroot, but that's it
if (!uninlining) {
// remove from mtree
err = lfsr_btree_pop(lfs, &lfs->mtree, mdir->mid);
if (err) {
return err;
}
}
// update our mdir, prepare mroot
mdir_.mid = -3;
*mdir = mdir_;
mdir = &lfs->mroot;
// TODO synchronize open mdirs?
// TODO wait where do we synchronize open mdirs that makes sense
// if we fail after this point?
// mark mtree as dirty and tail recurse to write it and any
// pending superattrs to the mroot
dirty_mtree = true;
if (!uninlining) {
attr_count = 0;
} else {
uninlined = true;
}
continue;
} else {
// finalize commit
err = lfsr_rbyd_commit(lfs, &mdir_.rbyd, NULL, 0);
if (err) {
@@ -4826,25 +4900,12 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
return err;
}
// mark mtree as dirty and tail recurse to write it and any pending
// superattrs to the mroot
// mark mtree as dirty and tail recurse to write it and any
// pending superattrs to the mroot
dirty_mtree = true;
uninlined = true;
continue;
// // prepare commit to mroot
// lfsr_tag_t tag;
// d = lfsr_btree_todisk(lfs, &lfs->mtree, &tag, recurse_buf);
// if (d < 0) {
// return d;
// }
//
// // TODO yeah we're going to need a wide-rm
// recurse_attrs[0] = LFSR_ATTR(-1, RMMDIR, 0, NULL, 0);
// recurse_attrs[1] = LFSR_ATTR(-1, RMBTREE, 0, NULL, 0);
// recurse_attrs[2] = LFSR_ATTR_(-1, tag, 0, recurse_buf, d);
// attrs = recurse_attrs;
// attr_count = 3;
// continue;
}
}
split:;
@@ -4951,12 +5012,18 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
return err;
}
if (mdir_.rbyd.weight == 0) {
// if our weight goes to zero, drop our mdir
lfs_cache_zero(lfs, &lfs->pcache);
} else {
// finalize commit
err = lfsr_rbyd_commit(lfs, &mdir_.rbyd, NULL, 0);
if (err) {
LFS_ASSERT(err != LFS_ERR_RANGE);
return err;
}
}
// copy over tags >= split_id
err = lfsr_rbyd_compact(lfs, &sibling.rbyd, split_id, -1, false,
@@ -4977,13 +5044,20 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
return err;
}
if (sibling.rbyd.weight == 0) {
// if our weight goes to zero, drop our mdir
lfs_cache_zero(lfs, &lfs->pcache);
} else {
// finalize commit
err = lfsr_rbyd_commit(lfs, &sibling.rbyd, NULL, 0);
if (err) {
LFS_ASSERT(err != LFS_ERR_RANGE);
return err;
}
}
if (mdir_.rbyd.weight > 0 && sibling.rbyd.weight > 0) {
// lookup first name in sibling to use as the split name
//
// note we need to do this after playing out pending attrs in case
@@ -5052,22 +5126,128 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
// only include superattrs if we're uninlining
if (!uninlining) {
attr_count = 0;
} else {
uninlined = true;
}
continue;
// // prepare commit to mroot
// lfs_ssize_t d = lfsr_btree_todisk(lfs, &lfs->mtree, &tag, recurse_buf);
// if (d < 0) {
// return d;
// }
//
// // TODO yeah we're going to need a wide-rm
// recurse_attrs[0] = LFSR_ATTR(-1, RMMDIR, 0, NULL, 0);
// recurse_attrs[1] = LFSR_ATTR(-1, RMBTREE, 0, NULL, 0);
// recurse_attrs[2] = LFSR_ATTR_(-1, tag, 0, recurse_buf, d);
// attrs = recurse_attrs;
// attr_count = 3;
// continue;
// weird case, one sibling went to zero while splitt
} else if (mdir_.rbyd.weight > 0 || sibling.rbyd.weight > 0) {
// set mdir_ to whichever sibling has weight still
if (mdir_.rbyd.weight == 0) {
LFS_DEBUG("Dropping mdir 0x{%"PRIx32",%"PRIx32"}",
mdir_.rbyd.block, mdir_.other_block);
sibling.mid -= 1;
mdir_ = sibling;
} else {
LFS_DEBUG("Dropping mdir 0x{%"PRIx32",%"PRIx32"}",
sibling.rbyd.block, sibling.other_block);
}
// TODO maybe mdir->mid != mdir_.mid can be used as uninlining?
if (!uninlining) {
// update our mdir
*mdir = mdir_;
// TODO deduplicate mdir synchronization?
// TODO synchronize open mdirs?
// synchronize mroot
if (mdir->mid == -1 && mdir != &lfs->mroot) {
lfs->mroot = *mdir;
}
// update our mtree
uint8_t buf[LFSR_MPAIR_DSIZE];
lfs_ssize_t d = lfsr_mpair_todisk(lfs, lfsr_mdir_mpair(&mdir_),
buf);
if (d < 0) {
return d;
}
err = lfsr_btree_update(lfs, &lfs->mtree,
mdir->mid, LFSR_TAG_MDIR, 1,
LFSR_DATA_BUF(buf, d));
if (err) {
return err;
}
// mark mtree as dirty and tail recurse to write it and any
// pending superattrs to the mroot
dirty_mtree = true;
attr_count = 0;
continue;
} else {
// TODO how the heck does rid interact with mdir splits
// + lazy attrs + mdir drops, this seems hard!
// update our mdir, prepare mroot
if (*rid < 0) {
// wait to update mdir after supdermdir update
} else {
*mdir = mdir_;
mdir = &lfs->mroot;
}
// TODO synchronize open mdirs?
// TODO wait where do we synchronize open mdirs that makes sense
// if we fail after this point?
// update our mtree
uint8_t buf[LFSR_MPAIR_DSIZE];
lfs_ssize_t d = lfsr_mpair_todisk(lfs, lfsr_mdir_mpair(&mdir_),
buf);
if (d < 0) {
return d;
}
err = lfsr_btree_push(lfs, &lfs->mtree, 0, LFSR_TAG_MDIR, 1,
LFSR_DATA_BUF(buf, d));
if (err) {
return err;
}
// mark mtree as dirty and tail recurse to write it and any
// pending superattrs to the mroot
dirty_mtree = true;
uninlined = true;
continue;
}
// weird case, both siblings went to zero, so this split actually ends
// up dropping both mdirs
} else {
LFS_DEBUG("Dropping mdir 0x{%"PRIx32",%"PRIx32"}",
mdir_.rbyd.block, mdir_.other_block);
LFS_DEBUG("Dropping mdir 0x{%"PRIx32",%"PRIx32"}",
sibling.rbyd.block, sibling.other_block);
// even weirder this can happen while uninlining
if (!uninlining) {
// remove from mtree
err = lfsr_btree_pop(lfs, &lfs->mtree, mdir->mid);
if (err) {
return err;
}
}
// update our mdir, prepare mroot
mdir_.mid = -3;
*mdir = mdir_;
mdir = &lfs->mroot;
// TODO synchronize open mdirs?
// TODO wait where do we synchronize open mdirs that makes sense
// if we fail after this point?
// mark mtree as dirty and tail recurse to write it and any
// pending superattrs to the mroot
dirty_mtree = true;
if (!uninlining) {
attr_count = 0;
} else {
uninlined = true;
}
continue;
}
}
// done
lfs.h
+1
View File
@@ -371,6 +371,7 @@ typedef union lfsr_btree {
} lfsr_btree_t;
typedef struct lfsr_mdir {
// -3 => deleted
// -2 => an out-of-tree mdir
// -1 => mroot
// >=0 => bid in the mtree
tests/test_mtree.toml
+309 -106
View File
@@ -13,8 +13,8 @@ in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(1), 0, "ardvark", 7))) => 0;
@@ -22,12 +22,11 @@ code = '''
lfsr_mdir_get(&lfs, &lfs.mroot,
-1, LFSR_TAG_UATTR(1), buffer, 7) => 7;
assert(memcmp(buffer, "ardvark", 7) == 0);
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
lfsr_mdir_get(&lfs, &lfs.mroot,
-1, LFSR_TAG_UATTR(1), buffer, 7) => 7;
assert(memcmp(buffer, "ardvark", 7) == 0);
lfsr_unmount(&lfs) => 0;
'''
@@ -39,8 +38,8 @@ code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
for (lfs_size_t i = 0; i < N; i++) {
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(1), 0, &alphas[i % 26], 1))) => 0;
@@ -50,13 +49,12 @@ code = '''
-1, LFSR_TAG_UATTR(1), buffer, 4) => 1;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
}
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &lfs.mroot,
-1, LFSR_TAG_UATTR(1), buffer, 4) => 1;
assert(memcmp(buffer, &alphas[(N-1) % 26], 1) == 0);
lfsr_unmount(&lfs) => 0;
'''
@@ -65,125 +63,69 @@ code = '''
# try creating a range of entries that may or may not split our mtree
[cases.test_mtree_split]
defines.N = [5, 10, 20, 40, 80, 160, 320]
defines.SIZE = ['1', 'BLOCK_SIZE/8']
if = 'SIZE == 1 || N <= 20'
defines.FORCE_COMPACTION = [false, true]
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
// create entries
lfsr_mount(&lfs, cfg) => 0;
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, lfsr_mtree_weight(&lfs)-1, &mdir) => 0;
lfs_ssize_t rid = 0;
for (lfs_size_t i = 0; i < N; i++) {
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKINLINED, +1, &alphas[i % 26], 1))) => 0;
// force a compaction?
if (FORCE_COMPACTION) {
mdir.rbyd.off = cfg->block_size;
lfs.mroot.rbyd.off = cfg->block_size;
}
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, buffer, 4) => 1;
uint8_t buffer[SIZE];
memset(buffer, alphas[i % 26], SIZE);
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKINLINED, +1, buffer, SIZE))) => 0;
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
rid += 1;
}
lfsr_unmount(&lfs) => 0;
// try looking up each entry
lfsr_mount(&lfs, cfg) => 0;
lfs_ssize_t mid = -1;
rid = 0;
mdir = lfs.mroot;
for (lfs_size_t i = 0; i < N; i++) {
if (rid >= (lfs_ssize_t)mdir.rbyd.weight) {
mid += 1;
rid = 0;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
}
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, buffer, 4) => 1;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
rid += 1;
}
lfsr_unmount(&lfs) => 0;
'''
# create a range of entries, and commit to the mroot several times,
# this makes it more likely to force uninlining without necessarily
# splitting
[cases.test_mtree_uninline]
defines.N = [5, 10, 20, 40, 80, 160, 320]
defines.M = [5, 5000]
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
// create entries
lfsr_mount(&lfs, cfg) => 0;
lfs_size_t i = 0;
for (lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
mid < lfsr_mtree_weight(&lfs);
mid++) {
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, lfsr_mtree_weight(&lfs)-1, &mdir) => 0;
lfs_ssize_t rid = 0;
for (lfs_size_t i = 0; i < N; i++) {
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKINLINED, +1, &alphas[i % 26], 1))) => 0;
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, buffer, 4) => 1;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
rid += 1;
}
// also append a bunch of fs-level attributes, if the mdir should be
// uninlined this will force it to happen
for (lfs_size_t i = 0; i < M; i++) {
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(1), 0, &alphas[i % 26], 1))) => 0;
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &lfs.mroot,
-1, LFSR_TAG_UATTR(1), buffer, 4) => 1;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
}
lfsr_unmount(&lfs) => 0;
// try looking up each entry
lfsr_mount(&lfs, cfg) => 0;
lfs_ssize_t mid = -1;
rid = 0;
mdir = lfs.mroot;
for (lfs_size_t i = 0; i < N; i++) {
if (rid >= (lfs_ssize_t)mdir.rbyd.weight) {
mid += 1;
rid = 0;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
}
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, buffer, 4) => 1;
for (lfs_ssize_t rid = 0;
rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir);
rid++) {
uint8_t buffer[SIZE];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
rid += 1;
i += 1;
}
}
assert(i == N);
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &lfs.mroot,
-1, LFSR_TAG_UATTR(1), buffer, 4) => 1;
assert(memcmp(buffer, &alphas[(M-1) % 26], 1) == 0);
lfsr_unmount(&lfs) => 0;
'''
# create random entries
[cases.test_mtree_split_fuzz]
defines.N = [5, 10, 20, 40, 80, 160, 320]
defines.SAMPLES = 10
defines.N = [5, 10, 20, 40, 80, 160]
defines.SIZE = ['1', 'BLOCK_SIZE/8']
if = 'SIZE == 1 || N <= 20'
defines.FORCE_COMPACTION = [false, true]
defines.SAMPLES = 100
# -1 => all pseudo-random seeds
# n => reproduce a specific seed
defines.SEED = -1
@@ -200,8 +142,8 @@ code = '''
// space constrained and we can't expect gc to work at this point
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
// at least keep track of the number of entries we expect
lfs_size_t count = 0;
@@ -215,22 +157,28 @@ code = '''
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
// choose a pseudo-random rid
lfs_ssize_t rid = TEST_PRNG(&prng) % (mdir.rbyd.weight+1);
lfs_ssize_t rid = TEST_PRNG(&prng) % (lfsr_mdir_weight(&mdir)+1);
// force a compaction?
if (FORCE_COMPACTION) {
mdir.rbyd.off = cfg->block_size;
lfs.mroot.rbyd.off = cfg->block_size;
}
// add to rbyd, potentially splitting the mdir
uint8_t buffer[SIZE];
memset(buffer, alphas[i % 26], SIZE);
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKINLINED, +1, &alphas[i % 26], 1))) => 0;
LFSR_ATTR(rid, MKINLINED, +1, buffer, SIZE))) => 0;
// make sure we can look up the new entry
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, buffer, 4) => 1;
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
count += 1;
}
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// try looking up each entry
lfs_size_t count_ = 0;
@@ -240,11 +188,266 @@ code = '''
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
for (lfs_ssize_t rid = 0;
rid < (lfs_ssize_t)mdir.rbyd.weight;
rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir);
rid++) {
uint8_t buffer[4];
uint8_t buffer[SIZE];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, 4) => 1;
buffer, SIZE) => SIZE;
count_ += 1;
}
}
// the mtree is a bit difficult to simulate, but we can at least test
// we ended up with the right number of entries
assert(count_ == count);
lfsr_unmount(&lfs) => 0;
}
'''
# try creating an mtree and then dropping mdirs
[cases.test_mtree_drop]
defines.N = [5, 10, 20, 40, 80, 160, 320]
defines.REMAINING = [20, 5, 1, 0]
defines.SIZE = ['1', 'BLOCK_SIZE/8']
if = 'N > REMAINING && (SIZE == 1 || N <= 20)'
defines.FORCE_COMPACTION = [false, true]
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
// create entries
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, lfsr_mtree_weight(&lfs)-1, &mdir) => 0;
lfs_ssize_t rid = 0;
for (lfs_size_t i = 0; i < N; i++) {
uint8_t buffer[SIZE];
memset(buffer, alphas[i % 26], SIZE);
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKINLINED, +1, buffer, SIZE))) => 0;
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
rid += 1;
}
// remove entries
for (lfs_size_t i = 0; i < N - REMAINING; i++) {
lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
// drop should make sure we never have empty mdirs
assert(mdir.mid == -1 || mdir.rbyd.weight > 0);
// force a compaction?
if (FORCE_COMPACTION) {
mdir.rbyd.off = cfg->block_size;
lfs.mroot.rbyd.off = cfg->block_size;
}
lfsr_mdir_commit(&lfs, &mdir, &(lfs_ssize_t){0}, LFSR_ATTRS(
LFSR_ATTR(0, MKUNR, -1, NULL, 0))) => 0;
}
// try looking up each entry
lfs_size_t i = N - REMAINING;
for (lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
mid < lfsr_mtree_weight(&lfs);
mid++) {
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
for (lfs_ssize_t rid = 0;
rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir);
rid++) {
uint8_t buffer[SIZE];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
i += 1;
}
}
assert(i == N);
lfsr_unmount(&lfs) => 0;
'''
# this one has some pretty nasty corner cases
[cases.test_mtree_repeated_drop]
defines.N = [5, 10, 20, 40, 80]
defines.SIZE = ['1', 'BLOCK_SIZE/8']
if = 'SIZE == 1 || N <= 5'
defines.FORCE_COMPACTION = [false, true]
defines.CYCLES = 10
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
for (lfs_size_t cycle = 0; cycle < CYCLES; cycle++) {
// create entries
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, lfsr_mtree_weight(&lfs)-1, &mdir) => 0;
lfs_ssize_t rid = 0;
for (lfs_size_t i = 0; i < N; i++) {
uint8_t buffer[SIZE];
memset(buffer, alphas[i % 26], SIZE);
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKINLINED, +1, buffer, SIZE))) => 0;
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
rid += 1;
}
// try looking up each entry
lfs_size_t i = 0;
for (lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
mid < lfsr_mtree_weight(&lfs);
mid++) {
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
for (lfs_ssize_t rid = 0;
rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir);
rid++) {
uint8_t buffer[SIZE];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
i += 1;
}
}
assert(i == N);
// remove entries
for (lfs_size_t i = 0; i < N; i++) {
lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
// drop should make sure we never have empty mdirs
assert(mdir.mid == -1 || mdir.rbyd.weight > 0);
// force a compaction?
if (FORCE_COMPACTION) {
mdir.rbyd.off = cfg->block_size;
lfs.mroot.rbyd.off = cfg->block_size;
}
lfsr_mdir_commit(&lfs, &mdir, &(lfs_ssize_t){0}, LFSR_ATTRS(
LFSR_ATTR(0, MKUNR, -1, NULL, 0))) => 0;
}
assert(lfsr_mtree_weight(&lfs) == 0);
assert(lfsr_mdir_weight(&lfs.mroot) == 0);
}
lfsr_unmount(&lfs) => 0;
'''
[cases.test_mtree_drop_fuzz]
defines.N = [5, 10, 20, 40, 80, 160]
defines.SIZE = ['1', 'BLOCK_SIZE/8']
if = 'SIZE == 1 || N <= 20'
defines.FORCE_COMPACTION = [false, true]
defines.SAMPLES = 100
# -1 => all pseudo-random seeds
# n => reproduce a specific seed
defines.SEED = -1
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
// iterate through severals seeds that we can reproduce easily
for (uint32_t seed = (SEED == -1 ? 1 : SEED);
(SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
seed++) {
printf("--- seed: %d ---\n", seed);
// create lfs here since we need to reset each iteration, we're
// space constrained and we can't expect gc to work at this point
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
// at least keep track of the number of entries we expect
lfs_size_t count = 0;
uint32_t prng = seed;
for (lfs_size_t i = 0; i < N; i++) {
// choose a pseudo-random mid
lfs_ssize_t mid = lfsr_mtree_weight(&lfs) == 0
? -1
: (lfs_ssize_t)(TEST_PRNG(&prng) % lfsr_mtree_weight(&lfs));
// fetch mdir
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
// choose a pseudo-random rid
lfs_ssize_t rid = TEST_PRNG(&prng) % (lfsr_mdir_weight(&mdir)+1);
// choose to create or delete
uint8_t op = (lfs_size_t)rid == lfsr_mdir_weight(&mdir)
? 0
: TEST_PRNG(&prng) % 2;
// force a compaction?
if (FORCE_COMPACTION) {
mdir.rbyd.off = cfg->block_size;
lfs.mroot.rbyd.off = cfg->block_size;
}
// create
if (op == 0) {
// add to rbyd, potentially splitting the mdir
uint8_t buffer[SIZE];
memset(buffer, alphas[i % 26], SIZE);
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKINLINED, +1, buffer, SIZE))) => 0;
// make sure we can look up the new entry
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
count += 1;
// delete
} else {
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKUNR, -1, NULL, 0))) => 0;
count -= 1;
}
}
// try looking up each entry
lfs_size_t count_ = 0;
for (lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
mid < lfsr_mtree_weight(&lfs);
mid++) {
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
// drop should make sure we never have empty mdirs
assert(mdir.mid == -1 || mdir.rbyd.weight > 0);
for (lfs_ssize_t rid = 0;
rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir);
rid++) {
uint8_t buffer[SIZE];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
count_ += 1;
}