Github/littlefs
1
Fork 0
Code Actions Activity

Implemented incremental mtree traversal

Browse Source 
Just like lfsr_btree_traversal_t, lfsr_mtree_traversal_t provides a
mechanism for traversing the mtree incrementally, including any inner
btree nodes.

This is one level more complex than btree traversal because we also need
to handle the mroot chain and traversal of rids in each mdir.

Again, mtree traversal returns temporary decoded rbyd structs for inner
nodes. Actually, mtree traversal only returns inner nodes... so maybe
using lfsr_data_t here is the wrong choice:

- tag=LFSR_TAG_BTREE => lfsr_rbyd_t
- tag=LFSR_TAG_MDIR  => lfsr_mdir_t
This commit is contained in:
Christopher Haster
2023-05-27 04:15:03 -05:00
parent 93bf68c84b
commit 34bcb62a9e
3 changed files with 1014 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+180 -12
View File
@@ -4363,18 +4363,12 @@ typedef struct lfsr_btree_traversal {
lfsr_rbyd_t branch;
} lfsr_btree_traversal_t;
static int lfsr_btree_traversal_start(lfs_t *lfs,
const lfsr_btree_t *btree,
lfsr_btree_traversal_t *traversal) {
(void)lfs;
(void)btree;
// setup traversal to fetch the root next call
traversal->bid = 0;
traversal->rid = 0;
traversal->branch.trunk = 0;
traversal->branch.weight = 0;
return 0;
}
#define LFSR_BTREE_TRAVERSAL_INIT ((lfsr_btree_traversal_t){ \
.bid = 0, \
.rid = 0, \
.branch.trunk = 0, \
.branch.weight = 0, \
})
static int lfsr_btree_traversal_next(lfs_t *lfs,
const lfsr_btree_t *btree,
@@ -5567,6 +5561,180 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
}
// incremental mtree traversal
typedef struct lfsr_mtree_traversal {
lfsr_mdir_t mdir;
lfsr_btree_traversal_t mtraversal;
} lfsr_mtree_traversal_t;
#define LFSR_MTREE_TRAVERSAL_INIT ((lfsr_mtree_traversal_t){ \
.mdir.rbyd.trunk = 0, \
.mtraversal = LFSR_BTREE_TRAVERSAL_INIT, \
})
static int lfsr_mtree_traversal_next(lfs_t *lfs,
lfsr_mtree_traversal_t *traversal,
lfs_size_t *mid_, lfsr_tag_t *tag_, lfsr_data_t *data_) {
// new traversal? start with 0x{0,1}
//
// note we make sure to include fake mroots!
//
if (traversal->mdir.rbyd.trunk == 0) {
int err = lfsr_mdir_fetch(lfs, &traversal->mdir,
-1, LFSR_MPAIR(0, 1), NULL);
if (err) {
return err;
}
if (mid_) {
*mid_ = -1;
}
if (tag_) {
*tag_ = LFSR_TAG_MDIR;
}
if (data_) {
*data_ = LFSR_DATA_BUF(&traversal->mdir, sizeof(lfsr_mdir_t));
}
return 0;
// check for mroot/mtree/mdir
} else if (traversal->mdir.mid == -1) {
// lookup mroot, if we find one this is a fake mroot
lfsr_data_t data;
int err = lfsr_mdir_lookup(lfs, &traversal->mdir,
-1, LFSR_TAG_MROOT, &data);
if (err && err != LFS_ERR_NOENT) {
return err;
}
// found a new mroot
if (err != LFS_ERR_NOENT) {
lfsr_mpair_t mpair;
lfs_ssize_t d = lfsr_mpair_fromdisk(lfs, &mpair, data);
if (d < 0) {
return d;
}
int err = lfsr_mdir_fetch(lfs, &traversal->mdir,
-1, mpair, NULL);
if (err) {
return err;
}
if (mid_) {
*mid_ = -1;
}
if (tag_) {
*tag_ = LFSR_TAG_MDIR;
}
if (data_) {
*data_ = LFSR_DATA_BUF(&traversal->mdir, sizeof(lfsr_mdir_t));
}
return 0;
// no more mroots, which makes this our real mroot
} else {
// update our mroot
lfs->mroot = traversal->mdir;
// TODO we may be able to combine this lookup with the mroot if the
// mroot is also a struct tag
// do we have an mtree? mdir?
lfs_ssize_t rid;
lfsr_tag_t tag;
int err = lfsr_mdir_lookupnext(lfs, &traversal->mdir,
-1, LFSR_TAG_STRUCT,
&rid, &tag, NULL, &data);
if (err && err != LFS_ERR_NOENT) {
return err;
}
// fetch our mtree
if (err != LFS_ERR_NOENT
&& rid == -1
&& lfsr_tag_suptype(tag) == LFSR_TAG_STRUCT) {
if (tag != LFSR_TAG_MDIR && tag != LFSR_TAG_BTREE) {
LFS_ERROR("Weird mstruct? 0x%"PRIx32, tag);
return LFS_ERR_CORRUPT;
}
lfs_ssize_t d = lfsr_btree_fromdisk(lfs, &lfs->mtree,
tag, 1, data);
if (d < 0) {
return d;
}
// no mtree
} else {
lfs->mtree = LFSR_BTREE_NULL;
}
// initialize our mtree traversal
traversal->mtraversal = LFSR_BTREE_TRAVERSAL_INIT;
}
}
// traverse through the mtree
lfs_size_t mid;
lfsr_tag_t tag;
lfsr_data_t data;
int err = lfsr_btree_traversal_next(
lfs, &lfs->mtree, &traversal->mtraversal,
&mid, &tag, NULL, &data);
if (err) {
return err;
}
// inner btree nodes already decoded
if (tag == LFSR_TAG_BTREE) {
// still update our mdir mid so we don't get stuck in a loop
// traversing mroots
traversal->mdir.mid = mid;
if (mid_) {
*mid_ = mid;
}
if (tag_) {
*tag_ = tag;
}
if (data_) {
*data_ = data;
}
return 0;
// fetch mdir if we're on a leaf
} else if (tag == LFSR_TAG_MDIR) {
lfsr_mpair_t mpair;
lfs_ssize_t d = lfsr_mpair_fromdisk(lfs, &mpair, data);
if (d < 0) {
return d;
}
int err = lfsr_mdir_fetch(lfs, &traversal->mdir,
mid, mpair, NULL);
if (err) {
return err;
}
if (mid_) {
*mid_ = mid;
}
if (tag_) {
*tag_ = tag;
}
if (data_) {
*data_ = LFSR_DATA_BUF(&traversal->mdir, sizeof(lfsr_mdir_t));
}
return 0;
} else {
LFS_ERROR("Weird mtree entry? 0x%"PRIx32, tag);
return LFS_ERR_CORRUPT;
}
}
// TODO how much of this code can we share with btree_commit?
// TODO share commit?
// TODO share split?
tests/test_btree.toml
+8 -15
View File
@@ -4117,11 +4117,10 @@ code = '''
uint8_t *seen = malloc((BLOCK_COUNT+7)/8);
memset(seen, 0, (BLOCK_COUNT+7)/8);
lfsr_btree_traversal_t traversal;
lfsr_btree_traversal_start(&lfs, &btree, &traversal) => 0;
lfsr_btree_traversal_t traversal = LFSR_BTREE_TRAVERSAL_INIT;
for (lfs_block_t i = 0;; i++) {
// a bit hacky, but catch infinite loops
// a bit hacky, but this catches infinite loops
assert(i < 2*N);
lfs_size_t bid_;
@@ -4136,7 +4135,7 @@ code = '''
}
if (tag_ == LFSR_TAG_BTREE) {
const lfsr_rbyd_t *branch = (const lfsr_rbyd_t *)data_.buf.buffer;
lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.buf.buffer;
printf("traversal: %d 0x%x w%d btree 0x%x.%x\n",
bid_,
tag_,
@@ -4163,6 +4162,7 @@ code = '''
cfg->prog(cfg, block, 0, buffer_, BLOCK_SIZE) => 0;
}
}
free(seen);
// and the tree should still work
@@ -4180,9 +4180,6 @@ code = '''
lfsr_btree_get(&lfs, &btree, n,
&tag_, &weight_,
buffer, 4, false) => LFS_ERR_NOENT;
// clean up traversal stuff
free(seen);
'''
[cases.test_btree_traversal_fuzz]
@@ -4282,11 +4279,10 @@ code = '''
uint8_t *seen = malloc((BLOCK_COUNT+7)/8);
memset(seen, 0, (BLOCK_COUNT+7)/8);
lfsr_btree_traversal_t traversal;
lfsr_btree_traversal_start(&lfs, &btree, &traversal) => 0;
lfsr_btree_traversal_t traversal = LFSR_BTREE_TRAVERSAL_INIT;
for (lfs_block_t i = 0;; i++) {
// a bit hacky, but catch infinite loops
// a bit hacky, but this catches infinite loops
assert(i < 2*N);
lfs_size_t bid_;
@@ -4301,8 +4297,7 @@ code = '''
}
if (tag_ == LFSR_TAG_BTREE) {
const lfsr_rbyd_t *branch = (
(const lfsr_rbyd_t *)data_.buf.buffer);
lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.buf.buffer;
printf("traversal: %d 0x%x w%d btree 0x%x.%x\n",
bid_,
tag_,
@@ -4329,6 +4324,7 @@ code = '''
cfg->prog(cfg, block, 0, buffer_, BLOCK_SIZE) => 0;
}
}
free(seen);
// and the tree should still work
@@ -4363,9 +4359,6 @@ code = '''
&tag_, &weight_,
buffer, 4, false) => LFS_ERR_NOENT;
// clean up traversal stuff
free(seen);
// clean up sim
free(sim);
lfs_deinit(&lfs) => 0;
tests/test_mtree.toml
+826
View File
@@ -2817,6 +2817,67 @@ code = '''
lfsr_unmount(&lfs) => 0;
'''
[cases.test_mtree_neighbor_extend]
# this should be set so only one entry can fit in a metadata block
defines.SIZE = 'BLOCK_SIZE / 4'
# make it so blocks relocate every two compacts
defines.BLOCK_CYCLES = 2
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
// setup our neighbors
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(0, MKINLINED, +1, &alphas[0 % 26], 1),
LFSR_ATTR(1, MKINLINED, +1, &alphas[1 % 26], 1))) => 0;
// this test only works if these all fit in the mroot
assert(lfsr_mtree_isinlined(&lfs));
lfsr_openedmdir_t left_neighbor = {.rid=0, .mdir=lfs.mroot};
lfsr_openedmdir_t right_neighbor = {.rid=1, .mdir=lfs.mroot};
lfsr_mdir_addopened(&lfs, &left_neighbor);
lfsr_mdir_addopened(&lfs, &right_neighbor);
// prepare mroot with an attr
uint8_t buffer[SIZE];
memset(buffer, alphas[0 % 26], SIZE);
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(1), 0, buffer, SIZE))) => 0;
// force mroot to compact twice, this should extend the mroot
lfsr_mdir_t old_mroot = lfs.mroot;
lfs.mroot.rbyd.off = BLOCK_SIZE;
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, NULL, 0) => 0;
lfs.mroot.rbyd.off = BLOCK_SIZE;
memset(buffer, alphas[1 % 26], SIZE);
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(1), 0, buffer, SIZE))) => 0;
// assert we relocated
assert(!lfsr_mdir_eq(&old_mroot, &lfs.mroot));
// assert that our attr is still in the mroot
lfsr_mdir_get(&lfs, &lfs.mroot, -1, LFSR_TAG_UATTR(1),
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[1 % 26], 1) == 0);
// assert that our neighbors were updated correctly
assert(left_neighbor.rid == 0);
assert(left_neighbor.mdir.mid == -1);
assert(memcmp(&left_neighbor.mdir, &lfs.mroot, sizeof(lfsr_mdir_t)) == 0);
assert(right_neighbor.rid == 1);
assert(right_neighbor.mdir.mid == -1);
assert(memcmp(&right_neighbor.mdir, &lfs.mroot, sizeof(lfsr_mdir_t)) == 0);
lfsr_mdir_removeopened(&lfs, &left_neighbor);
lfsr_mdir_removeopened(&lfs, &right_neighbor);
lfsr_unmount(&lfs) => 0;
'''
[cases.test_mtree_neighbor_relocate]
# this should be set so only one entry can fit in a metadata block
defines.SIZE = 'BLOCK_SIZE / 4'
@@ -2903,3 +2964,768 @@ code = '''
lfsr_mdir_removeopened(&lfs, &right_neighbor);
lfsr_unmount(&lfs) => 0;
'''
## mtree traversal ##
# test specific corner cases
[cases.test_mtree_traversal]
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
// insert a new entry, this should update our neighbors
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(0, MKINLINED, +1, &alphas[0 % 26], 1))) => 0;
// assert that our entry is still in the mtree
assert(lfs.mroot.rbyd.weight == 1);
uint8_t buffer[1];
lfsr_mdir_get(&lfs, &lfs.mroot, 0, LFSR_TAG_INLINED,
buffer, 1) => 1;
assert(memcmp(buffer, &alphas[0 % 26], 1) == 0);
// test that we can traverse the tree, keeping track of all blocks we see
uint8_t *seen = malloc((BLOCK_COUNT+7)/8);
memset(seen, 0, (BLOCK_COUNT+7)/8);
lfsr_mtree_traversal_t traversal = LFSR_MTREE_TRAVERSAL_INIT;
for (lfs_block_t i = 0;; i++) {
// a bit hacky, but this catches infinite loops
assert(i < 2*1);
lfs_size_t mid_;
lfsr_tag_t tag_;
lfsr_data_t data_;
int err = lfsr_mtree_traversal_next(&lfs, &traversal,
&mid_, &tag_, &data_);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
if (tag_ == LFSR_TAG_BTREE) {
lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.buf.buffer;
printf("traversal: %d 0x%x btree 0x%x.%x\n",
mid_,
tag_,
branch->block, branch->trunk);
// keep track of seen blocks
seen[branch->block / 8] |= 1 << (branch->block % 8);
} else if (tag_ == LFSR_TAG_MDIR) {
lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.buf.buffer;
printf("traversal: %d 0x%x mdir 0x{%x,%x}\n",
mid_,
tag_,
mdir->rbyd.block, mdir->other_block);
// keep track of seen blocks
seen[mdir->rbyd.block / 8] |= 1 << (mdir->rbyd.block % 8);
seen[mdir->other_block / 8] |= 1 << (mdir->other_block % 8);
} else {
// this shouldn't happen
printf("traversal: %d 0x%x %d\n",
mid_,
tag_,
lfsr_data_size(data_));
assert(false);
}
}
// if traversal worked, we should be able to clobber all other blocks
uint8_t buffer_[BLOCK_SIZE];
memset(buffer_, 0xcc, BLOCK_SIZE);
for (lfs_block_t block = 0; block < BLOCK_COUNT; block++) {
if (!(seen[block / 8] & (1 << (block % 8)))) {
cfg->erase(cfg, block) => 0;
cfg->prog(cfg, block, 0, buffer_, BLOCK_SIZE) => 0;
}
}
free(seen);
// and the tree should still work
// assert that our entry is still in the mtree
assert(lfs.mroot.rbyd.weight == 1);
lfsr_mdir_get(&lfs, &lfs.mroot, 0, LFSR_TAG_INLINED,
buffer, 1) => 1;
assert(memcmp(buffer, &alphas[0 % 26], 1) == 0);
lfsr_unmount(&lfs) => 0;
'''
[cases.test_mtree_traversal_uninline]
# this should be set so only one entry can fit in a metadata block
defines.SIZE = 'BLOCK_SIZE / 4'
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
// prepare mroot with a large attr so the next entry can not fit
uint8_t buffer[SIZE];
memset(buffer, alphas[0 % 26], SIZE);
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(1), 0, buffer, SIZE))) => 0;
// create a large entry that needs to be uninlined (but not split!)
memset(buffer, alphas[1 % 26], SIZE);
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(0, MKINLINED, +1, buffer, SIZE))) => 0;
// force mroot to compact
lfs.mroot.rbyd.off = BLOCK_SIZE;
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, NULL, 0) => 0;
// assert mdir was unininlined correctly
assert(lfsr_mtree_weight(&lfs) == 1);
// assert mroot now has no entries
assert(lfs.mroot.rbyd.weight == 0);
// assert that our attr is still in the mroot
lfsr_mdir_get(&lfs, &lfs.mroot, -1, LFSR_TAG_UATTR(1),
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[0 % 26], 1) == 0);
// assert that our entry is still in the mtree
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, 0, &mdir) => 0;
assert(mdir.rbyd.weight == 1);
lfsr_mdir_get(&lfs, &mdir, 0, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[1 % 26], 1) == 0);
// test that we can traverse the tree, keeping track of all blocks we see
uint8_t *seen = malloc((BLOCK_COUNT+7)/8);
memset(seen, 0, (BLOCK_COUNT+7)/8);
lfsr_mtree_traversal_t traversal = LFSR_MTREE_TRAVERSAL_INIT;
for (lfs_block_t i = 0;; i++) {
// a bit hacky, but this catches infinite loops
assert(i < 2*2);
lfs_size_t mid_;
lfsr_tag_t tag_;
lfsr_data_t data_;
int err = lfsr_mtree_traversal_next(&lfs, &traversal,
&mid_, &tag_, &data_);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
if (tag_ == LFSR_TAG_BTREE) {
lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.buf.buffer;
printf("traversal: %d 0x%x btree 0x%x.%x\n",
mid_,
tag_,
branch->block, branch->trunk);
// keep track of seen blocks
seen[branch->block / 8] |= 1 << (branch->block % 8);
} else if (tag_ == LFSR_TAG_MDIR) {
lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.buf.buffer;
printf("traversal: %d 0x%x mdir 0x{%x,%x}\n",
mid_,
tag_,
mdir->rbyd.block, mdir->other_block);
// keep track of seen blocks
seen[mdir->rbyd.block / 8] |= 1 << (mdir->rbyd.block % 8);
seen[mdir->other_block / 8] |= 1 << (mdir->other_block % 8);
} else {
// this shouldn't happen
printf("traversal: %d 0x%x %d\n",
mid_,
tag_,
lfsr_data_size(data_));
assert(false);
}
}
// if traversal worked, we should be able to clobber all other blocks
uint8_t buffer_[BLOCK_SIZE];
memset(buffer_, 0xcc, BLOCK_SIZE);
for (lfs_block_t block = 0; block < BLOCK_COUNT; block++) {
if (!(seen[block / 8] & (1 << (block % 8)))) {
cfg->erase(cfg, block) => 0;
cfg->prog(cfg, block, 0, buffer_, BLOCK_SIZE) => 0;
}
}
free(seen);
// and the tree should still work
// assert mdir was unininlined correctly
assert(lfsr_mtree_weight(&lfs) == 1);
// assert mroot now has no entries
assert(lfs.mroot.rbyd.weight == 0);
// assert that our attr is still in the mroot
lfsr_mdir_get(&lfs, &lfs.mroot, -1, LFSR_TAG_UATTR(1),
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[0 % 26], 1) == 0);
// assert that our entry is still in the mtree
lfsr_mtree_lookup(&lfs, 0, &mdir) => 0;
assert(mdir.rbyd.weight == 1);
lfsr_mdir_get(&lfs, &mdir, 0, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[1 % 26], 1) == 0);
lfsr_unmount(&lfs) => 0;
'''
[cases.test_mtree_traversal_split]
# this should be set so only one entry can fit in a metadata block
defines.SIZE = 'BLOCK_SIZE / 4'
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
// create 2 large entries that needs to be uninlined and split
uint8_t buffer[SIZE];
memset(buffer, alphas[0 % 26], SIZE);
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(0, MKINLINED, +1, buffer, SIZE))) => 0;
memset(buffer, alphas[1 % 26], SIZE);
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(1, MKINLINED, +1, buffer, SIZE))) => 0;
// force mroot to compact
lfs.mroot.rbyd.off = BLOCK_SIZE;
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, NULL, 0) => 0;
// assert mdirs were unininlined and split
assert(lfsr_mtree_weight(&lfs) == 2);
// assert mroot now has no entries
assert(lfs.mroot.rbyd.weight == 0);
// assert that our entries are still in the mtree
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, 0, &mdir) => 0;
assert(mdir.rbyd.weight == 1);
lfsr_mdir_get(&lfs, &mdir, 0, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[0 % 26], 1) == 0);
lfsr_mdir_t msibling;
lfsr_mtree_lookup(&lfs, 1, &msibling) => 0;
assert(msibling.rbyd.weight == 1);
lfsr_mdir_get(&lfs, &msibling, 0, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[1 % 26], 1) == 0);
// test that we can traverse the tree, keeping track of all blocks we see
uint8_t *seen = malloc((BLOCK_COUNT+7)/8);
memset(seen, 0, (BLOCK_COUNT+7)/8);
lfsr_mtree_traversal_t traversal = LFSR_MTREE_TRAVERSAL_INIT;
for (lfs_block_t i = 0;; i++) {
// a bit hacky, but this catches infinite loops
assert(i < 2*3);
lfs_size_t mid_;
lfsr_tag_t tag_;
lfsr_data_t data_;
int err = lfsr_mtree_traversal_next(&lfs, &traversal,
&mid_, &tag_, &data_);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
if (tag_ == LFSR_TAG_BTREE) {
lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.buf.buffer;
printf("traversal: %d 0x%x btree 0x%x.%x\n",
mid_,
tag_,
branch->block, branch->trunk);
// keep track of seen blocks
seen[branch->block / 8] |= 1 << (branch->block % 8);
} else if (tag_ == LFSR_TAG_MDIR) {
lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.buf.buffer;
printf("traversal: %d 0x%x mdir 0x{%x,%x}\n",
mid_,
tag_,
mdir->rbyd.block, mdir->other_block);
// keep track of seen blocks
seen[mdir->rbyd.block / 8] |= 1 << (mdir->rbyd.block % 8);
seen[mdir->other_block / 8] |= 1 << (mdir->other_block % 8);
} else {
// this shouldn't happen
printf("traversal: %d 0x%x %d\n",
mid_,
tag_,
lfsr_data_size(data_));
assert(false);
}
}
// if traversal worked, we should be able to clobber all other blocks
uint8_t buffer_[BLOCK_SIZE];
memset(buffer_, 0xcc, BLOCK_SIZE);
for (lfs_block_t block = 0; block < BLOCK_COUNT; block++) {
if (!(seen[block / 8] & (1 << (block % 8)))) {
cfg->erase(cfg, block) => 0;
cfg->prog(cfg, block, 0, buffer_, BLOCK_SIZE) => 0;
}
}
free(seen);
// and the tree should still work
// assert mdirs were unininlined and split
assert(lfsr_mtree_weight(&lfs) == 2);
// assert mroot now has no entries
assert(lfs.mroot.rbyd.weight == 0);
// assert that our entries are still in the mtree
lfsr_mtree_lookup(&lfs, 0, &mdir) => 0;
assert(mdir.rbyd.weight == 1);
lfsr_mdir_get(&lfs, &mdir, 0, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[0 % 26], 1) == 0);
lfsr_mtree_lookup(&lfs, 1, &msibling) => 0;
assert(msibling.rbyd.weight == 1);
lfsr_mdir_get(&lfs, &msibling, 0, LFSR_TAG_INLINED,
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[1 % 26], 1) == 0);
lfsr_unmount(&lfs) => 0;
'''
[cases.test_mtree_traversal_extend]
# this should be set so only one entry can fit in a metadata block
defines.SIZE = 'BLOCK_SIZE / 4'
# make it so blocks relocate every two compacts
defines.BLOCK_CYCLES = 2
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
// prepare mroot with an attr
uint8_t buffer[SIZE];
memset(buffer, alphas[0 % 26], SIZE);
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(1), 0, buffer, SIZE))) => 0;
// force mroot to compact twice, this should extend the mroot
lfsr_mdir_t old_mroot = lfs.mroot;
lfs.mroot.rbyd.off = BLOCK_SIZE;
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, NULL, 0) => 0;
lfs.mroot.rbyd.off = BLOCK_SIZE;
memset(buffer, alphas[1 % 26], SIZE);
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(1), 0, buffer, SIZE))) => 0;
// assert we relocated
assert(!lfsr_mdir_eq(&old_mroot, &lfs.mroot));
// assert that our attr is still in the mroot
lfsr_mdir_get(&lfs, &lfs.mroot, -1, LFSR_TAG_UATTR(1),
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[1 % 26], 1) == 0);
// test that we can traverse the tree, keeping track of all blocks we see
uint8_t *seen = malloc((BLOCK_COUNT+7)/8);
memset(seen, 0, (BLOCK_COUNT+7)/8);
lfsr_mtree_traversal_t traversal = LFSR_MTREE_TRAVERSAL_INIT;
for (lfs_block_t i = 0;; i++) {
// a bit hacky, but this catches infinite loops
assert(i < 2*3);
lfs_size_t mid_;
lfsr_tag_t tag_;
lfsr_data_t data_;
int err = lfsr_mtree_traversal_next(&lfs, &traversal,
&mid_, &tag_, &data_);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
if (tag_ == LFSR_TAG_BTREE) {
lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.buf.buffer;
printf("traversal: %d 0x%x btree 0x%x.%x\n",
mid_,
tag_,
branch->block, branch->trunk);
// keep track of seen blocks
seen[branch->block / 8] |= 1 << (branch->block % 8);
} else if (tag_ == LFSR_TAG_MDIR) {
lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.buf.buffer;
printf("traversal: %d 0x%x mdir 0x{%x,%x}\n",
mid_,
tag_,
mdir->rbyd.block, mdir->other_block);
// keep track of seen blocks
seen[mdir->rbyd.block / 8] |= 1 << (mdir->rbyd.block % 8);
seen[mdir->other_block / 8] |= 1 << (mdir->other_block % 8);
} else {
// this shouldn't happen
printf("traversal: %d 0x%x %d\n",
mid_,
tag_,
lfsr_data_size(data_));
assert(false);
}
}
// if traversal worked, we should be able to clobber all other blocks
uint8_t buffer_[BLOCK_SIZE];
memset(buffer_, 0xcc, BLOCK_SIZE);
for (lfs_block_t block = 0; block < BLOCK_COUNT; block++) {
if (!(seen[block / 8] & (1 << (block % 8)))) {
cfg->erase(cfg, block) => 0;
cfg->prog(cfg, block, 0, buffer_, BLOCK_SIZE) => 0;
}
}
free(seen);
// and the tree should still work
// assert we relocated
assert(!lfsr_mdir_eq(&old_mroot, &lfs.mroot));
// assert that our attr is still in the mroot
lfsr_mdir_get(&lfs, &lfs.mroot, -1, LFSR_TAG_UATTR(1),
buffer, SIZE) => SIZE;
assert(memcmp(buffer, &alphas[1 % 26], 1) == 0);
lfsr_unmount(&lfs) => 0;
'''
# larger traversal tests
[cases.test_mtree_traversal_many]
defines.N = [5, 10, 20, 40, 80, 160, 320]
defines.FORCE_COMPACTION = [false, true]
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
// create entries
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, lfsr_mtree_weight(&lfs)-1, &mdir) => 0;
lfs_ssize_t rid = 0;
for (lfs_size_t i = 0; i < N; i++) {
// force a compaction?
if (FORCE_COMPACTION) {
mdir.rbyd.off = cfg->block_size;
lfs.mroot.rbyd.off = cfg->block_size;
}
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKINLINED, +1, &alphas[i % 26], 1))) => 0;
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, 4) => 1;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
rid += 1;
}
// try looking up each entry
lfs_size_t i = 0;
for (lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
mid < lfsr_mtree_weight(&lfs);
mid++) {
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
for (lfs_ssize_t rid = 0;
rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir);
rid++) {
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, 4) => 1;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
i += 1;
}
}
assert(i == N);
// test that we can traverse the tree, keeping track of all blocks we see
uint8_t *seen = malloc((BLOCK_COUNT+7)/8);
memset(seen, 0, (BLOCK_COUNT+7)/8);
lfsr_mtree_traversal_t traversal = LFSR_MTREE_TRAVERSAL_INIT;
for (lfs_block_t i = 0;; i++) {
// a bit hacky, but this catches infinite loops
assert(i < 2*(1+N));
lfs_size_t mid_;
lfsr_tag_t tag_;
lfsr_data_t data_;
int err = lfsr_mtree_traversal_next(&lfs, &traversal,
&mid_, &tag_, &data_);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
if (tag_ == LFSR_TAG_BTREE) {
lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.buf.buffer;
printf("traversal: %d 0x%x btree 0x%x.%x\n",
mid_,
tag_,
branch->block, branch->trunk);
// keep track of seen blocks
seen[branch->block / 8] |= 1 << (branch->block % 8);
} else if (tag_ == LFSR_TAG_MDIR) {
lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.buf.buffer;
printf("traversal: %d 0x%x mdir 0x{%x,%x}\n",
mid_,
tag_,
mdir->rbyd.block, mdir->other_block);
// keep track of seen blocks
seen[mdir->rbyd.block / 8] |= 1 << (mdir->rbyd.block % 8);
seen[mdir->other_block / 8] |= 1 << (mdir->other_block % 8);
} else {
// this shouldn't happen
printf("traversal: %d 0x%x %d\n",
mid_,
tag_,
lfsr_data_size(data_));
assert(false);
}
}
// if traversal worked, we should be able to clobber all other blocks
uint8_t buffer_[BLOCK_SIZE];
memset(buffer_, 0xcc, BLOCK_SIZE);
for (lfs_block_t block = 0; block < BLOCK_COUNT; block++) {
if (!(seen[block / 8] & (1 << (block % 8)))) {
cfg->erase(cfg, block) => 0;
cfg->prog(cfg, block, 0, buffer_, BLOCK_SIZE) => 0;
}
}
free(seen);
// and the tree should still work
// try looking up each entry
i = 0;
for (lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
mid < lfsr_mtree_weight(&lfs);
mid++) {
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
for (lfs_ssize_t rid = 0;
rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir);
rid++) {
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, 4) => 1;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
i += 1;
}
}
assert(i == N);
lfsr_unmount(&lfs) => 0;
'''
[cases.test_mtree_traversal_fuzz]
defines.N = [5, 10, 20, 40, 80, 160]
defines.FORCE_COMPACTION = [false, true]
defines.SAMPLES = 100
# -1 => all pseudo-random seeds
# n => reproduce a specific seed
defines.SEED = -1
in = 'lfs.c'
code = '''
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
// iterate through severals seeds that we can reproduce easily
for (uint32_t seed = (SEED == -1 ? 1 : SEED);
(SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
seed++) {
printf("--- seed: %d ---\n", seed);
// create lfs here since we need to reset each iteration, we're
// space constrained and we can't expect gc to work at this point
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
// at least keep track of the number of entries we expect
lfs_size_t count = 0;
uint32_t prng = seed;
for (lfs_size_t i = 0; i < N; i++) {
// choose a pseudo-random mid
lfs_ssize_t mid = lfsr_mtree_weight(&lfs) == 0
? -1
: (lfs_ssize_t)(TEST_PRNG(&prng) % lfsr_mtree_weight(&lfs));
// fetch mdir
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
// choose a pseudo-random rid
lfs_ssize_t rid = TEST_PRNG(&prng) % (lfsr_mdir_weight(&mdir)+1);
// force a compaction?
if (FORCE_COMPACTION) {
mdir.rbyd.off = cfg->block_size;
lfs.mroot.rbyd.off = cfg->block_size;
}
// add to rbyd, potentially splitting the mdir
lfsr_mdir_commit(&lfs, &mdir, &rid, LFSR_ATTRS(
LFSR_ATTR(rid, MKINLINED, +1, &alphas[i % 26], 1))) => 0;
// make sure we can look up the new entry
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, 4) => 1;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
count += 1;
}
// try looking up each entry
lfs_size_t count_ = 0;
for (lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
mid < lfsr_mtree_weight(&lfs);
mid++) {
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
for (lfs_ssize_t rid = 0;
rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir);
rid++) {
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, 4) => 1;
count_ += 1;
}
}
// the mtree is a bit difficult to simulate, but we can at least test
// we ended up with the right number of entries
assert(count_ == count);
// test that we can traverse the tree, keeping track of all blocks we see
uint8_t *seen = malloc((BLOCK_COUNT+7)/8);
memset(seen, 0, (BLOCK_COUNT+7)/8);
lfsr_mtree_traversal_t traversal = LFSR_MTREE_TRAVERSAL_INIT;
for (lfs_block_t i = 0;; i++) {
// a bit hacky, but this catches infinite loops
assert(i < 2*(1+N));
lfs_size_t mid_;
lfsr_tag_t tag_;
lfsr_data_t data_;
int err = lfsr_mtree_traversal_next(&lfs, &traversal,
&mid_, &tag_, &data_);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
if (tag_ == LFSR_TAG_BTREE) {
lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.buf.buffer;
printf("traversal: %d 0x%x btree 0x%x.%x\n",
mid_,
tag_,
branch->block, branch->trunk);
// keep track of seen blocks
seen[branch->block / 8] |= 1 << (branch->block % 8);
} else if (tag_ == LFSR_TAG_MDIR) {
lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.buf.buffer;
printf("traversal: %d 0x%x mdir 0x{%x,%x}\n",
mid_,
tag_,
mdir->rbyd.block, mdir->other_block);
// keep track of seen blocks
seen[mdir->rbyd.block / 8] |= 1 << (mdir->rbyd.block % 8);
seen[mdir->other_block / 8] |= 1 << (mdir->other_block % 8);
} else {
// this shouldn't happen
printf("traversal: %d 0x%x %d\n",
mid_,
tag_,
lfsr_data_size(data_));
assert(false);
}
}
// if traversal worked, we should be able to clobber all other blocks
uint8_t buffer_[BLOCK_SIZE];
memset(buffer_, 0xcc, BLOCK_SIZE);
for (lfs_block_t block = 0; block < BLOCK_COUNT; block++) {
if (!(seen[block / 8] & (1 << (block % 8)))) {
cfg->erase(cfg, block) => 0;
cfg->prog(cfg, block, 0, buffer_, BLOCK_SIZE) => 0;
}
}
free(seen);
// and the tree should still work
// try looking up each entry
count_ = 0;
for (lfs_ssize_t mid = (lfsr_mtree_isinlined(&lfs) ? -1 : 0);
mid < lfsr_mtree_weight(&lfs);
mid++) {
lfsr_mdir_t mdir;
lfsr_mtree_lookup(&lfs, mid, &mdir) => 0;
for (lfs_ssize_t rid = 0;
rid < (lfs_ssize_t)lfsr_mdir_weight(&mdir);
rid++) {
uint8_t buffer[4];
lfsr_mdir_get(&lfs, &mdir, rid, LFSR_TAG_INLINED,
buffer, 4) => 1;
count_ += 1;
}
}
// the mtree is a bit difficult to simulate, but we can at least test
// we ended up with the right number of entries
assert(count_ == count);
lfsr_unmount(&lfs) => 0;
}
'''