Github/littlefs
1
Fork 0
Code Actions Activity

Added a very rough implementation of mdir drops

Browse Source 
mdirs behave a bit differently than btree nodes here. When an mdir's
weight drops to zero, we eagerly drop the mdir. Unfortunately this
introduce a large number of conditions into lfsr_mdir_commit. Maybe
there's some different way to structure to code to avoid this...

Also expanded mtree tests to cover more corner cases, these are
desperately for any confidence that mdir drops work.
This commit is contained in:
Christopher Haster
2023-05-11 16:22:11 -05:00
parent ea28413eb2
commit 6bc85375ea
3 changed files with 657 additions and 273 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+346 -166
View File
@@ -4407,10 +4407,14 @@ static lfs_ssize_t lfsr_mpair_fromdisk(lfs_t *lfs, lfsr_mpair_t *mpair,
return d;
}
static lfsr_mpair_t lfsr_mdir_mpair(const lfsr_mdir_t *mdir) {
static inline lfsr_mpair_t lfsr_mdir_mpair(const lfsr_mdir_t *mdir) {
return LFSR_MPAIR(mdir->rbyd.block, mdir->other_block);
}
static inline lfs_size_t lfsr_mdir_weight(const lfsr_mdir_t *mdir) {
return mdir->rbyd.weight;
}
static int lfsr_mdir_alloc(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t mid) {
// allocate two blocks
lfs_block_t blocks[2];
@@ -4519,7 +4523,7 @@ static lfs_ssize_t lfsr_mdir_get(lfs_t *lfs, const lfsr_mdir_t *mdir,
// mtree is the core tree of mdirs in littlefs
static inline int lfsr_mtree_isinlined(lfs_t *lfs) {
return lfsr_btree_isnull(&lfs->mtree);
return lfsr_btree_weight(&lfs->mtree) == 0;
}
static inline lfs_ssize_t lfsr_mtree_weight(lfs_t *lfs) {
@@ -4574,13 +4578,13 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
// TODO need both dirty_mtree and uninlining?
bool dirty_mtree = false;
// TODO this can be done better, we shouldn't need all these different flags
bool uninlined = false;
while (true) {
// try to commit
// TODO do we need a backup rbyd here?
// TODO this was a quick hack, should rbyd_ be the same as mdir_?
lfsr_rbyd_t rbyd_ = mdir->rbyd;
int err = lfsr_rbyd_appendall(lfs, &rbyd_, -1, -1,
lfsr_mdir_t mdir_ = *mdir;
int err = lfsr_rbyd_appendall(lfs, &mdir_.rbyd, -1, -1,
attrs, attr_count);
if (err && err != LFS_ERR_RANGE) {
//TODO should we also move if there is corruption here?
@@ -4600,7 +4604,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
}
// TODO yeah we're going to need a wide-rm
err = lfsr_rbyd_appendall(lfs, &rbyd_, -1, -1, LFSR_ATTRS(
err = lfsr_rbyd_appendall(lfs, &mdir_.rbyd, -1, -1, LFSR_ATTRS(
LFSR_ATTR(-1, RMMDIR, 0, NULL, 0),
LFSR_ATTR(-1, RMBTREE, 0, NULL, 0),
LFSR_ATTR_(-1, tag, 0, buf, d)));
@@ -4613,25 +4617,56 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
}
}
// finalize commit
err = lfsr_rbyd_commit(lfs, &rbyd_, NULL, 0);
if (err && err != LFS_ERR_RANGE) {
//TODO should we also move if there is corruption here?
return err;
}
if (err) {
goto compact;
}
mdir->rbyd = rbyd_;
if (mdir->mid >= 0 && mdir_.rbyd.weight == 0) {
// if our weight goes to zero, drop our mdir
lfs_cache_zero(lfs, &lfs->pcache);
LFS_DEBUG("Dropping mdir 0x{%"PRIx32",%"PRIx32"}",
mdir_.rbyd.block, mdir_.other_block);
// TODO synchronize open mdirs?
// synchronize mroot
if (mdir->mid == -1 && mdir != &lfs->mroot) {
lfs->mroot = *mdir;
}
// remove from mtree
err = lfsr_btree_pop(lfs, &lfs->mtree, mdir->mid);
if (err) {
return err;
}
// successful commit
break;
// update our mdir, prepare mroot
mdir_.mid = -3;
*mdir = mdir_;
mdir = &lfs->mroot;
// TODO synchronize open mdirs?
// TODO wait where do we synchronize open mdirs that makes sense
// if we fail after this point?
// mark mtree as dirty and tail recurse to write it and any pending
// superattrs to the mroot
dirty_mtree = true;
attr_count = 0;
continue;
} else {
// finalize commit
err = lfsr_rbyd_commit(lfs, &mdir_.rbyd, NULL, 0);
if (err && err != LFS_ERR_RANGE) {
//TODO should we also move if there is corruption here?
return err;
}
if (err) {
goto compact;
}
// update our mdir
*mdir = mdir_;
// TODO synchronize open mdirs?
// synchronize mroot
if (mdir->mid == -1 && mdir != &lfs->mroot) {
lfs->mroot = *mdir;
}
// successful commit
break;
}
compact:;
// can't commit, try to compact
@@ -4641,7 +4676,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
// normally the new mdir is just the flipped version of our
// current mdir
lfsr_mdir_t mdir_ = *mdir;
mdir_ = *mdir;
bool uninlining = false;
lfs_size_t lower_id;
@@ -4730,7 +4765,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
//
err = lfsr_rbyd_compact(lfs, &mdir_.rbyd,
(uninlining ? 0 : -1),
(mdir_.mid < 0 && !lfsr_mtree_isinlined(lfs) ? 0 : -1),
(uninlined ? 0 : -1),
false,
&mdir->rbyd);
if (err) {
@@ -4748,7 +4783,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
// mroot
err = lfsr_rbyd_appendall(lfs, &mdir_.rbyd,
(uninlining ? 0 : -1),
(mdir_.mid < 0 && !lfsr_mtree_isinlined(lfs) ? 0 : -1),
(uninlined ? 0 : -1),
attrs, attr_count);
if (err) {
LFS_ASSERT(err != LFS_ERR_RANGE);
@@ -4778,73 +4813,99 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
}
}
// finalize commit
err = lfsr_rbyd_commit(lfs, &mdir_.rbyd, NULL, 0);
if (err) {
LFS_ASSERT(err != LFS_ERR_RANGE);
return err;
}
// TODO the number of conditions here feels like a mess, it would be
// nice if this could be cleaned up
if (mdir->mid >= 0 && mdir_.rbyd.weight == 0) {
// if our weight goes to zero, drop our mdir
lfs_cache_zero(lfs, &lfs->pcache);
LFS_DEBUG("Dropping mdir 0x{%"PRIx32",%"PRIx32"}",
mdir_.rbyd.block, mdir_.other_block);
// TODO maybe mdir->mid != mdir_.mid can be used as uninlining?
if (!uninlining) {
// update our mdir
*mdir = mdir_;
// TODO deduplicate mdir synchronization?
// TODO synchronize open mdirs?
// synchronize mroot
if (mdir->mid == -1 && mdir != &lfs->mroot) {
lfs->mroot = *mdir;
// weird case, going to zero while uninlining
//
// we need to commit our superattrs to the mroot, but that's it
if (!uninlining) {
// remove from mtree
err = lfsr_btree_pop(lfs, &lfs->mtree, mdir->mid);
if (err) {
return err;
}
}
break;
} else {
// update our mdir, prepare mroot
if (*rid < 0) {
// wait to update mdir after supdermdir update
} else {
*mdir = mdir_;
mdir = &lfs->mroot;
}
mdir_.mid = -3;
*mdir = mdir_;
mdir = &lfs->mroot;
// TODO synchronize open mdirs?
// TODO wait where do we synchronize open mdirs that makes sense
// if we fail after this point?
// update our mtree
uint8_t buf[LFSR_MPAIR_DSIZE];
lfs_ssize_t d = lfsr_mpair_todisk(lfs, lfsr_mdir_mpair(&mdir_),
buf);
if (d < 0) {
return d;
// mark mtree as dirty and tail recurse to write it and any
// pending superattrs to the mroot
dirty_mtree = true;
if (!uninlining) {
attr_count = 0;
} else {
uninlined = true;
}
continue;
err = lfsr_btree_push(lfs, &lfs->mtree, 0, LFSR_TAG_MDIR, 1,
LFSR_DATA_BUF(buf, d));
} else {
// finalize commit
err = lfsr_rbyd_commit(lfs, &mdir_.rbyd, NULL, 0);
if (err) {
LFS_ASSERT(err != LFS_ERR_RANGE);
return err;
}
// mark mtree as dirty and tail recurse to write it and any pending
// superattrs to the mroot
dirty_mtree = true;
continue;
// TODO maybe mdir->mid != mdir_.mid can be used as uninlining?
if (!uninlining) {
// update our mdir
*mdir = mdir_;
// // prepare commit to mroot
// lfsr_tag_t tag;
// d = lfsr_btree_todisk(lfs, &lfs->mtree, &tag, recurse_buf);
// if (d < 0) {
// return d;
// }
//
// // TODO yeah we're going to need a wide-rm
// recurse_attrs[0] = LFSR_ATTR(-1, RMMDIR, 0, NULL, 0);
// recurse_attrs[1] = LFSR_ATTR(-1, RMBTREE, 0, NULL, 0);
// recurse_attrs[2] = LFSR_ATTR_(-1, tag, 0, recurse_buf, d);
// attrs = recurse_attrs;
// attr_count = 3;
// continue;
// TODO deduplicate mdir synchronization?
// TODO synchronize open mdirs?
// synchronize mroot
if (mdir->mid == -1 && mdir != &lfs->mroot) {
lfs->mroot = *mdir;
}
break;
} else {
// update our mdir, prepare mroot
if (*rid < 0) {
// wait to update mdir after supdermdir update
} else {
*mdir = mdir_;
mdir = &lfs->mroot;
}
// TODO synchronize open mdirs?
// TODO wait where do we synchronize open mdirs that makes sense
// if we fail after this point?
// update our mtree
uint8_t buf[LFSR_MPAIR_DSIZE];
lfs_ssize_t d = lfsr_mpair_todisk(lfs, lfsr_mdir_mpair(&mdir_),
buf);
if (d < 0) {
return d;
}
err = lfsr_btree_push(lfs, &lfs->mtree, 0, LFSR_TAG_MDIR, 1,
LFSR_DATA_BUF(buf, d));
if (err) {
return err;
}
// mark mtree as dirty and tail recurse to write it and any
// pending superattrs to the mroot
dirty_mtree = true;
uninlined = true;
continue;
}
}
split:;
@@ -4951,11 +5012,17 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
return err;
}
// finalize commit
err = lfsr_rbyd_commit(lfs, &mdir_.rbyd, NULL, 0);
if (err) {
LFS_ASSERT(err != LFS_ERR_RANGE);
return err;
if (mdir_.rbyd.weight == 0) {
// if our weight goes to zero, drop our mdir
lfs_cache_zero(lfs, &lfs->pcache);
} else {
// finalize commit
err = lfsr_rbyd_commit(lfs, &mdir_.rbyd, NULL, 0);
if (err) {
LFS_ASSERT(err != LFS_ERR_RANGE);
return err;
}
}
// copy over tags >= split_id
@@ -4977,97 +5044,210 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
return err;
}
// finalize commit
err = lfsr_rbyd_commit(lfs, &sibling.rbyd, NULL, 0);
if (err) {
LFS_ASSERT(err != LFS_ERR_RANGE);
return err;
}
if (sibling.rbyd.weight == 0) {
// if our weight goes to zero, drop our mdir
lfs_cache_zero(lfs, &lfs->pcache);
// lookup first name in sibling to use as the split name
//
// note we need to do this after playing out pending attrs in case
// they introduce a new name!
lfsr_tag_t stag;
lfsr_data_t sdata;
err = lfsr_rbyd_lookupnext(lfs, &sibling.rbyd, 0, LFSR_TAG_NAME,
NULL, &stag, NULL, &sdata);
if (err) {
LFS_ASSERT(err != LFS_ERR_NOENT);
return err;
}
// update our mdir, prepare mroot
if (uninlining && *rid < 0) {
// wait to update mdir after supdermdir update
} else if (*rid < split_id) {
*mdir = mdir_;
mdir = &lfs->mroot;
} else if (*rid >= split_id) {
*mdir = sibling;
*rid -= split_id;
mdir = &lfs->mroot;
}
// TODO synchronize open mdirs?
// TODO wait where do we synchronize open mdirs that makes sense
// if we fail after this point?
// update our mtree
if (uninlining) {
// TODO do we really need an explicit push when creating a new,
// 2-sized btree?
err = lfsr_btree_push(lfs, &lfs->mtree, 0, LFSR_TAG_MDIR, 1,
LFSR_DATA_NULL);
} else {
// finalize commit
err = lfsr_rbyd_commit(lfs, &sibling.rbyd, NULL, 0);
if (err) {
LFS_ASSERT(err != LFS_ERR_RANGE);
return err;
}
}
uint8_t buf1[LFSR_MPAIR_DSIZE];
lfs_ssize_t d1 = lfsr_mpair_todisk(lfs, lfsr_mdir_mpair(&mdir_),
buf1);
if (d1 < 0) {
return d1;
}
uint8_t buf2[LFSR_MPAIR_DSIZE];
lfs_ssize_t d2 = lfsr_mpair_todisk(lfs, lfsr_mdir_mpair(&sibling),
buf2);
if (d2 < 0) {
return d2;
}
if (mdir_.rbyd.weight > 0 && sibling.rbyd.weight > 0) {
// lookup first name in sibling to use as the split name
//
// note we need to do this after playing out pending attrs in case
// they introduce a new name!
lfsr_tag_t stag;
lfsr_data_t sdata;
err = lfsr_rbyd_lookupnext(lfs, &sibling.rbyd, 0, LFSR_TAG_NAME,
NULL, &stag, NULL, &sdata);
if (err) {
LFS_ASSERT(err != LFS_ERR_NOENT);
return err;
}
err = lfsr_btree_split(lfs, &lfs->mtree, mdir_.mid,
(lfsr_tag_suptype(stag) == LFSR_TAG_NAME
? sdata
: LFSR_DATA_NULL),
LFSR_TAG_MDIR, 1, LFSR_DATA_BUF(buf1, d1),
LFSR_TAG_MDIR, 1, LFSR_DATA_BUF(buf2, d2));
if (err) {
return err;
}
// update our mdir, prepare mroot
if (uninlining && *rid < 0) {
// wait to update mdir after supdermdir update
} else if (*rid < split_id) {
*mdir = mdir_;
mdir = &lfs->mroot;
} else if (*rid >= split_id) {
*mdir = sibling;
*rid -= split_id;
mdir = &lfs->mroot;
}
// mark mtree as dirty and tail recurse to write it to the mroot
dirty_mtree = true;
// only include superattrs if we're uninlining
if (!uninlining) {
attr_count = 0;
}
continue;
// TODO synchronize open mdirs?
// TODO wait where do we synchronize open mdirs that makes sense
// if we fail after this point?
// // prepare commit to mroot
// lfs_ssize_t d = lfsr_btree_todisk(lfs, &lfs->mtree, &tag, recurse_buf);
// if (d < 0) {
// return d;
// }
//
// // TODO yeah we're going to need a wide-rm
// recurse_attrs[0] = LFSR_ATTR(-1, RMMDIR, 0, NULL, 0);
// recurse_attrs[1] = LFSR_ATTR(-1, RMBTREE, 0, NULL, 0);
// recurse_attrs[2] = LFSR_ATTR_(-1, tag, 0, recurse_buf, d);
// attrs = recurse_attrs;
// attr_count = 3;
// continue;
// update our mtree
if (uninlining) {
// TODO do we really need an explicit push when creating a new,
// 2-sized btree?
err = lfsr_btree_push(lfs, &lfs->mtree, 0, LFSR_TAG_MDIR, 1,
LFSR_DATA_NULL);
if (err) {
return err;
}
}
uint8_t buf1[LFSR_MPAIR_DSIZE];
lfs_ssize_t d1 = lfsr_mpair_todisk(lfs, lfsr_mdir_mpair(&mdir_),
buf1);
if (d1 < 0) {
return d1;
}
uint8_t buf2[LFSR_MPAIR_DSIZE];
lfs_ssize_t d2 = lfsr_mpair_todisk(lfs, lfsr_mdir_mpair(&sibling),
buf2);
if (d2 < 0) {
return d2;
}
err = lfsr_btree_split(lfs, &lfs->mtree, mdir_.mid,
(lfsr_tag_suptype(stag) == LFSR_TAG_NAME
? sdata
: LFSR_DATA_NULL),
LFSR_TAG_MDIR, 1, LFSR_DATA_BUF(buf1, d1),
LFSR_TAG_MDIR, 1, LFSR_DATA_BUF(buf2, d2));
if (err) {
return err;
}
// mark mtree as dirty and tail recurse to write it to the mroot
dirty_mtree = true;
// only include superattrs if we're uninlining
if (!uninlining) {
attr_count = 0;
} else {
uninlined = true;
}
continue;
// weird case, one sibling went to zero while splitt
} else if (mdir_.rbyd.weight > 0 || sibling.rbyd.weight > 0) {
// set mdir_ to whichever sibling has weight still
if (mdir_.rbyd.weight == 0) {
LFS_DEBUG("Dropping mdir 0x{%"PRIx32",%"PRIx32"}",
mdir_.rbyd.block, mdir_.other_block);
sibling.mid -= 1;
mdir_ = sibling;
} else {
LFS_DEBUG("Dropping mdir 0x{%"PRIx32",%"PRIx32"}",
sibling.rbyd.block, sibling.other_block);
}
// TODO maybe mdir->mid != mdir_.mid can be used as uninlining?
if (!uninlining) {
// update our mdir
*mdir = mdir_;
// TODO deduplicate mdir synchronization?
// TODO synchronize open mdirs?
// synchronize mroot
if (mdir->mid == -1 && mdir != &lfs->mroot) {
lfs->mroot = *mdir;
}
// update our mtree
uint8_t buf[LFSR_MPAIR_DSIZE];
lfs_ssize_t d = lfsr_mpair_todisk(lfs, lfsr_mdir_mpair(&mdir_),
buf);
if (d < 0) {
return d;
}
err = lfsr_btree_update(lfs, &lfs->mtree,
mdir->mid, LFSR_TAG_MDIR, 1,
LFSR_DATA_BUF(buf, d));
if (err) {
return err;
}
// mark mtree as dirty and tail recurse to write it and any
// pending superattrs to the mroot
dirty_mtree = true;
attr_count = 0;
continue;
} else {
// TODO how the heck does rid interact with mdir splits
// + lazy attrs + mdir drops, this seems hard!
// update our mdir, prepare mroot
if (*rid < 0) {
// wait to update mdir after supdermdir update
} else {
*mdir = mdir_;
mdir = &lfs->mroot;
}
// TODO synchronize open mdirs?
// TODO wait where do we synchronize open mdirs that makes sense
// if we fail after this point?
// update our mtree
uint8_t buf[LFSR_MPAIR_DSIZE];
lfs_ssize_t d = lfsr_mpair_todisk(lfs, lfsr_mdir_mpair(&mdir_),
buf);
if (d < 0) {
return d;
}
err = lfsr_btree_push(lfs, &lfs->mtree, 0, LFSR_TAG_MDIR, 1,
LFSR_DATA_BUF(buf, d));
if (err) {
return err;
}
// mark mtree as dirty and tail recurse to write it and any
// pending superattrs to the mroot
dirty_mtree = true;
uninlined = true;
continue;
}
// weird case, both siblings went to zero, so this split actually ends
// up dropping both mdirs
} else {
LFS_DEBUG("Dropping mdir 0x{%"PRIx32",%"PRIx32"}",
mdir_.rbyd.block, mdir_.other_block);
LFS_DEBUG("Dropping mdir 0x{%"PRIx32",%"PRIx32"}",
sibling.rbyd.block, sibling.other_block);
// even weirder this can happen while uninlining
if (!uninlining) {
// remove from mtree
err = lfsr_btree_pop(lfs, &lfs->mtree, mdir->mid);
if (err) {
return err;
}
}
// update our mdir, prepare mroot
mdir_.mid = -3;
*mdir = mdir_;
mdir = &lfs->mroot;
// TODO synchronize open mdirs?
// TODO wait where do we synchronize open mdirs that makes sense
// if we fail after this point?
// mark mtree as dirty and tail recurse to write it and any
// pending superattrs to the mroot
dirty_mtree = true;
if (!uninlining) {
attr_count = 0;
} else {
uninlined = true;
}
continue;
}
}
// done
lfs.h
+1
View File
@@ -371,6 +371,7 @@ typedef union lfsr_btree {
} lfsr_btree_t;
typedef struct lfsr_mdir {
// -3 => deleted
// -2 => an out-of-tree mdir
// -1 => mroot
// >=0 => bid in the mtree
tests/test_mtree.toml
+310 -107
View File
@@ -13,8 +13,8 @@ in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(1), 0, "ardvark", 7))) => 0;
@@ -22,12 +22,11 @@ code = '''
lfsr_mdir_get(&lfs, &lfs.mroot,
-1, LFSR_TAG_UATTR(1), buffer, 7) => 7;
assert(memcmp(buffer, "ardvark", 7) == 0);
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
lfsr_mdir_get(&lfs, &lfs.mroot,
-1, LFSR_TAG_UATTR(1), buffer, 7) => 7;
assert(memcmp(buffer, "ardvark", 7) == 0);
lfsr_unmount(&lfs) => 0;
'''
@@ -39,8 +38,8 @@ code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
for (lfs_size_t i = 0; i < N; i++) {
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(1), 0, &alphas[i % 26], 1))) => 0;
@@ -50,13 +49,12 @@ code = '''
-1, LFSR_TAG_UATTR(1), buffer, 4) => 1;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
}
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &lfs.mroot,
-1, LFSR_TAG_UATTR(1), buffer, 4) => 1;
assert(memcmp(buffer, &alphas[(N-1) % 26], 1) == 0);
lfsr_unmount(&lfs) => 0;
'''
@@ -65,125 +63,69 @@ code = '''
# try creating a range of entries that may or may not split our mtree
[cases.test_mtree_split]
defines.N = [5, 10, 20, 40, 80, 160, 320]
defines.SIZE = ['1', 'BLOCK_SIZE/8']
if = 'SIZE == 1 || N <= 20'
defines.FORCE_COMPACTION = [false, true]
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
// create entries
lfsr_mount(&lfs, cfg) => 0;
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, lfsr_mtree_weight(&lfs)-1, &mdir) => 0;
lfs_ssize_t rid = 0;
for (lfs_size_t i = 0; i < N; i++) {
// force a compaction?
if (FORCE_COMPACTION) {
mdir.rbyd.off = cfg->block_size;
lfs.mroot.rbyd.off = cfg->block_size;
}
uint8_t buffer[SIZE];
memset(buffer, alphas[i % 26], SIZE);
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKINLINED, +1, &alphas[i % 26], 1))) => 0;
LFSR_ATTR(rid, MKINLINED, +1, buffer, SIZE))) => 0;
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, buffer, 4) => 1;
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
rid += 1;
}
lfsr_unmount(&lfs) => 0;
// try looking up each entry
lfsr_mount(&lfs, cfg) => 0;
lfs_ssize_t mid = -1;
rid = 0;
mdir = lfs.mroot;
for (lfs_size_t i = 0; i < N; i++) {
if (rid >= (lfs_ssize_t)mdir.rbyd.weight) {
mid += 1;
rid = 0;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
lfs_size_t i = 0;
for (lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
mid < lfsr_mtree_weight(&lfs);
mid++) {
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
for (lfs_ssize_t rid = 0;
rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir);
rid++) {
uint8_t buffer[SIZE];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
i += 1;
}
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, buffer, 4) => 1;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
rid += 1;
}
lfsr_unmount(&lfs) => 0;
'''
assert(i == N);
# create a range of entries, and commit to the mroot several times,
# this makes it more likely to force uninlining without necessarily
# splitting
[cases.test_mtree_uninline]
defines.N = [5, 10, 20, 40, 80, 160, 320]
defines.M = [5, 5000]
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
// create entries
lfsr_mount(&lfs, cfg) => 0;
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, lfsr_mtree_weight(&lfs)-1, &mdir) => 0;
lfs_ssize_t rid = 0;
for (lfs_size_t i = 0; i < N; i++) {
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKINLINED, +1, &alphas[i % 26], 1))) => 0;
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, buffer, 4) => 1;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
rid += 1;
}
// also append a bunch of fs-level attributes, if the mdir should be
// uninlined this will force it to happen
for (lfs_size_t i = 0; i < M; i++) {
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(1), 0, &alphas[i % 26], 1))) => 0;
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &lfs.mroot,
-1, LFSR_TAG_UATTR(1), buffer, 4) => 1;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
}
lfsr_unmount(&lfs) => 0;
// try looking up each entry
lfsr_mount(&lfs, cfg) => 0;
lfs_ssize_t mid = -1;
rid = 0;
mdir = lfs.mroot;
for (lfs_size_t i = 0; i < N; i++) {
if (rid >= (lfs_ssize_t)mdir.rbyd.weight) {
mid += 1;
rid = 0;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
}
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, buffer, 4) => 1;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
rid += 1;
}
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &lfs.mroot,
-1, LFSR_TAG_UATTR(1), buffer, 4) => 1;
assert(memcmp(buffer, &alphas[(M-1) % 26], 1) == 0);
lfsr_unmount(&lfs) => 0;
'''
# create random entries
[cases.test_mtree_split_fuzz]
defines.N = [5, 10, 20, 40, 80, 160, 320]
defines.SAMPLES = 10
defines.N = [5, 10, 20, 40, 80, 160]
defines.SIZE = ['1', 'BLOCK_SIZE/8']
if = 'SIZE == 1 || N <= 20'
defines.FORCE_COMPACTION = [false, true]
defines.SAMPLES = 100
# -1 => all pseudo-random seeds
# n => reproduce a specific seed
defines.SEED = -1
@@ -200,8 +142,8 @@ code = '''
// space constrained and we can't expect gc to work at this point
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
// at least keep track of the number of entries we expect
lfs_size_t count = 0;
@@ -215,22 +157,28 @@ code = '''
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
// choose a pseudo-random rid
lfs_ssize_t rid = TEST_PRNG(&prng) % (mdir.rbyd.weight+1);
lfs_ssize_t rid = TEST_PRNG(&prng) % (lfsr_mdir_weight(&mdir)+1);
// force a compaction?
if (FORCE_COMPACTION) {
mdir.rbyd.off = cfg->block_size;
lfs.mroot.rbyd.off = cfg->block_size;
}
// add to rbyd, potentially splitting the mdir
uint8_t buffer[SIZE];
memset(buffer, alphas[i % 26], SIZE);
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKINLINED, +1, &alphas[i % 26], 1))) => 0;
LFSR_ATTR(rid, MKINLINED, +1, buffer, SIZE))) => 0;
// make sure we can look up the new entry
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED, buffer, 4) => 1;
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
count += 1;
}
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// try looking up each entry
lfs_size_t count_ = 0;
@@ -240,11 +188,266 @@ code = '''
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
for (lfs_ssize_t rid = 0;
rid < (lfs_ssize_t)mdir.rbyd.weight;
rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir);
rid++) {
uint8_t buffer[4];
uint8_t buffer[SIZE];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, 4) => 1;
buffer, SIZE) => SIZE;
count_ += 1;
}
}
// the mtree is a bit difficult to simulate, but we can at least test
// we ended up with the right number of entries
assert(count_ == count);
lfsr_unmount(&lfs) => 0;
}
'''
# try creating an mtree and then dropping mdirs
[cases.test_mtree_drop]
defines.N = [5, 10, 20, 40, 80, 160, 320]
defines.REMAINING = [20, 5, 1, 0]
defines.SIZE = ['1', 'BLOCK_SIZE/8']
if = 'N > REMAINING && (SIZE == 1 || N <= 20)'
defines.FORCE_COMPACTION = [false, true]
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
// create entries
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, lfsr_mtree_weight(&lfs)-1, &mdir) => 0;
lfs_ssize_t rid = 0;
for (lfs_size_t i = 0; i < N; i++) {
uint8_t buffer[SIZE];
memset(buffer, alphas[i % 26], SIZE);
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKINLINED, +1, buffer, SIZE))) => 0;
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
rid += 1;
}
// remove entries
for (lfs_size_t i = 0; i < N - REMAINING; i++) {
lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
// drop should make sure we never have empty mdirs
assert(mdir.mid == -1 || mdir.rbyd.weight > 0);
// force a compaction?
if (FORCE_COMPACTION) {
mdir.rbyd.off = cfg->block_size;
lfs.mroot.rbyd.off = cfg->block_size;
}
lfsr_mdir_commit(&lfs, &mdir, &(lfs_ssize_t){0}, LFSR_ATTRS(
LFSR_ATTR(0, MKUNR, -1, NULL, 0))) => 0;
}
// try looking up each entry
lfs_size_t i = N - REMAINING;
for (lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
mid < lfsr_mtree_weight(&lfs);
mid++) {
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
for (lfs_ssize_t rid = 0;
rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir);
rid++) {
uint8_t buffer[SIZE];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
i += 1;
}
}
assert(i == N);
lfsr_unmount(&lfs) => 0;
'''
# this one has some pretty nasty corner cases
[cases.test_mtree_repeated_drop]
defines.N = [5, 10, 20, 40, 80]
defines.SIZE = ['1', 'BLOCK_SIZE/8']
if = 'SIZE == 1 || N <= 5'
defines.FORCE_COMPACTION = [false, true]
defines.CYCLES = 10
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
for (lfs_size_t cycle = 0; cycle < CYCLES; cycle++) {
// create entries
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, lfsr_mtree_weight(&lfs)-1, &mdir) => 0;
lfs_ssize_t rid = 0;
for (lfs_size_t i = 0; i < N; i++) {
uint8_t buffer[SIZE];
memset(buffer, alphas[i % 26], SIZE);
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKINLINED, +1, buffer, SIZE))) => 0;
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
rid += 1;
}
// try looking up each entry
lfs_size_t i = 0;
for (lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
mid < lfsr_mtree_weight(&lfs);
mid++) {
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
for (lfs_ssize_t rid = 0;
rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir);
rid++) {
uint8_t buffer[SIZE];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
i += 1;
}
}
assert(i == N);
// remove entries
for (lfs_size_t i = 0; i < N; i++) {
lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
// drop should make sure we never have empty mdirs
assert(mdir.mid == -1 || mdir.rbyd.weight > 0);
// force a compaction?
if (FORCE_COMPACTION) {
mdir.rbyd.off = cfg->block_size;
lfs.mroot.rbyd.off = cfg->block_size;
}
lfsr_mdir_commit(&lfs, &mdir, &(lfs_ssize_t){0}, LFSR_ATTRS(
LFSR_ATTR(0, MKUNR, -1, NULL, 0))) => 0;
}
assert(lfsr_mtree_weight(&lfs) == 0);
assert(lfsr_mdir_weight(&lfs.mroot) == 0);
}
lfsr_unmount(&lfs) => 0;
'''
[cases.test_mtree_drop_fuzz]
defines.N = [5, 10, 20, 40, 80, 160]
defines.SIZE = ['1', 'BLOCK_SIZE/8']
if = 'SIZE == 1 || N <= 20'
defines.FORCE_COMPACTION = [false, true]
defines.SAMPLES = 100
# -1 => all pseudo-random seeds
# n => reproduce a specific seed
defines.SEED = -1
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
// iterate through severals seeds that we can reproduce easily
for (uint32_t seed = (SEED == -1 ? 1 : SEED);
(SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
seed++) {
printf("--- seed: %d ---\n", seed);
// create lfs here since we need to reset each iteration, we're
// space constrained and we can't expect gc to work at this point
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
// at least keep track of the number of entries we expect
lfs_size_t count = 0;
uint32_t prng = seed;
for (lfs_size_t i = 0; i < N; i++) {
// choose a pseudo-random mid
lfs_ssize_t mid = lfsr_mtree_weight(&lfs) == 0
? -1
: (lfs_ssize_t)(TEST_PRNG(&prng) % lfsr_mtree_weight(&lfs));
// fetch mdir
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
// choose a pseudo-random rid
lfs_ssize_t rid = TEST_PRNG(&prng) % (lfsr_mdir_weight(&mdir)+1);
// choose to create or delete
uint8_t op = (lfs_size_t)rid == lfsr_mdir_weight(&mdir)
? 0
: TEST_PRNG(&prng) % 2;
// force a compaction?
if (FORCE_COMPACTION) {
mdir.rbyd.off = cfg->block_size;
lfs.mroot.rbyd.off = cfg->block_size;
}
// create
if (op == 0) {
// add to rbyd, potentially splitting the mdir
uint8_t buffer[SIZE];
memset(buffer, alphas[i % 26], SIZE);
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKINLINED, +1, buffer, SIZE))) => 0;
// make sure we can look up the new entry
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
count += 1;
// delete
} else {
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKUNR, -1, NULL, 0))) => 0;
count -= 1;
}
}
// try looking up each entry
lfs_size_t count_ = 0;
for (lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
mid < lfsr_mtree_weight(&lfs);
mid++) {
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
// drop should make sure we never have empty mdirs
assert(mdir.mid == -1 || mdir.rbyd.weight > 0);
for (lfs_ssize_t rid = 0;
rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir);
rid++) {
uint8_t buffer[SIZE];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
count_ += 1;
}