Github/littlefs
1
Fork 0
Code Actions Activity

t: Implemented gc_compact_thresh over bshrub nodes

Browse Source 
These aren't really different than btree nodes, except bshrubs need to
be enrolled in our opened list for commits to work.

Fortunately this is already true for explicit traversals, which are
currently the only traversals where we need to simultaneously mutate the
filesystem. This mainly just required adding additional checks for
LFS_TYPE_TRAVERSAL bshrubs, tests, and making sure traversal.bshrub is
never in an invalid state.

This continues to add code/stack cost for what is ultimately a
relatively niche feature:

           code          stack
  before: 35268           2776
  after:  35448 (+0.5%)   2800 (+0.9%)

Maybe btree/bshrub compactions should be disabled by default?
This commit is contained in:
Christopher Haster
2024-07-01 13:22:35 -05:00
parent 227b804c7a
commit f3446abfa7
3 changed files with 525 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+34 -20
View File
@@ -5174,7 +5174,8 @@ static lfs_ssize_t lfsr_sprout_estimate(lfs_t *lfs,
const lfsr_sprout_t *last = NULL;
for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) {
lfsr_file_t *file_ = (lfsr_file_t*)o;
if (file_->o.type == LFS_TYPE_REG
if ((file_->o.type == LFS_TYPE_REG
|| file_->o.type == LFS_TYPE_TRAVERSAL)
&& lfsr_bshrub_isbsprout(&file_->o.mdir, &file_->bshrub)
&& lfsr_sprout_cmp(&file_->bshrub.u.bsprout, sprout) == 0) {
last = &file_->bshrub.u.bsprout;
@@ -5203,7 +5204,8 @@ static int lfsr_sprout_compact(lfs_t *lfs, const lfsr_rbyd_t *rbyd_,
// can update these later if our commit is a success
for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) {
lfsr_file_t *file_ = (lfsr_file_t*)o;
if (file_->o.type == LFS_TYPE_REG
if ((file_->o.type == LFS_TYPE_REG
|| file_->o.type == LFS_TYPE_TRAVERSAL)
&& lfsr_bshrub_isbsprout(&file_->o.mdir, &file_->bshrub)
&& lfsr_sprout_cmp(
&file_->bshrub.u.bsprout,
@@ -5320,7 +5322,8 @@ static lfs_ssize_t lfsr_shrub_estimate(lfs_t *lfs,
const lfsr_shrub_t *last = NULL;
for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) {
lfsr_file_t *file_ = (lfsr_file_t*)o;
if (file_->o.type == LFS_TYPE_REG
if ((file_->o.type == LFS_TYPE_REG
|| file_->o.type == LFS_TYPE_TRAVERSAL)
&& lfsr_bshrub_isbshrub(&file_->o.mdir, &file_->bshrub)
&& lfsr_shrub_cmp(&file_->bshrub.u.bshrub, shrub) == 0) {
last = &file_->bshrub.u.bshrub;
@@ -5352,7 +5355,8 @@ static int lfsr_shrub_compact(lfs_t *lfs, lfsr_rbyd_t *rbyd_,
// this should include our current bshrub
for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) {
lfsr_file_t *file_ = (lfsr_file_t*)o;
if (file_->o.type == LFS_TYPE_REG
if ((file_->o.type == LFS_TYPE_REG
|| file_->o.type == LFS_TYPE_TRAVERSAL)
&& lfsr_bshrub_isbshrub(&file_->o.mdir, &file_->bshrub)
&& lfsr_shrub_cmp(&file_->bshrub.u.bshrub, shrub) == 0) {
file_->bshrub_.u.bshrub.blocks[0] = rbyd_->blocks[0];
@@ -5456,7 +5460,8 @@ static lfs_ssize_t lfsr_bshrub_estimate(lfs_t *lfs,
// this includes our current shrub
for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) {
lfsr_file_t *file_ = (lfsr_file_t*)o;
if (file_->o.type == LFS_TYPE_REG
if ((file_->o.type == LFS_TYPE_REG
|| file_->o.type == LFS_TYPE_TRAVERSAL)
&& file_->o.mdir.mid == mdir->mid) {
if (lfsr_bshrub_isbsprout(&file_->o.mdir, &file_->bshrub)) {
lfs_ssize_t dsize = lfsr_sprout_estimate(lfs,
@@ -5633,7 +5638,8 @@ static int lfsr_bshrub_commit_(lfs_t *lfs,
if (lfsr_bshrub_isbtree(mdir, bshrub)) {
for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) {
lfsr_file_t *file_ = (lfsr_file_t*)o;
if (file_->o.type == LFS_TYPE_REG
if ((file_->o.type == LFS_TYPE_REG
|| file_->o.type == LFS_TYPE_TRAVERSAL)
&& &file_->bshrub != bshrub
&& lfsr_bshrub_isbshruborbtree(&file_->bshrub)
&& lfsr_btree_cmp(
@@ -5722,7 +5728,8 @@ static int lfsr_bshrub_commit_(lfs_t *lfs,
// update _all_ shrubs with the new estimate
for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) {
lfsr_file_t *file_ = (lfsr_file_t*)o;
if (file_->o.type == LFS_TYPE_REG
if ((file_->o.type == LFS_TYPE_REG
|| file_->o.type == LFS_TYPE_TRAVERSAL)
&& file_->o.mdir.mid == mdir->mid
&& lfsr_bshrub_isbshrub(&file_->o.mdir, &file_->bshrub)) {
file_->bshrub.u.bshrub.estimate = estimate;
@@ -6751,8 +6758,9 @@ static int lfsr_mdir_commit__(lfs_t *lfs, lfsr_mdir_t *mdir,
for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) {
lfsr_file_t *file = (lfsr_file_t*)o;
// belongs to our mid?
if (file->o.type != LFS_TYPE_REG
|| file->o.mdir.mid != mdir__->mid) {
if (!((file->o.type == LFS_TYPE_REG
|| file->o.type == LFS_TYPE_TRAVERSAL)
&& file->o.mdir.mid == mdir__->mid)) {
continue;
}
@@ -6939,9 +6947,10 @@ static lfs_ssize_t lfsr_mdir_estimate__(lfs_t *lfs, const lfsr_mdir_t *mdir,
for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) {
lfsr_file_t *file = (lfsr_file_t*)o;
// belongs to our mdir + rid?
if (file->o.type != LFS_TYPE_REG
|| lfsr_mdir_cmp(&file->o.mdir, mdir) != 0
|| lfsr_mid_rid(lfs, file->o.mdir.mid) != a_rid) {
if (!((file->o.type == LFS_TYPE_REG
|| file->o.type == LFS_TYPE_TRAVERSAL)
&& lfsr_mdir_cmp(&file->o.mdir, mdir) == 0
&& lfsr_mid_rid(lfs, file->o.mdir.mid) == a_rid)) {
continue;
}
@@ -7085,11 +7094,12 @@ static int lfsr_mdir_compact__(lfs_t *lfs, lfsr_mdir_t *mdir_,
for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) {
lfsr_file_t *file = (lfsr_file_t*)o;
// belongs to our mdir?
if (file->o.type != LFS_TYPE_REG
|| lfsr_mdir_cmp(&file->o.mdir, mdir) != 0
|| lfsr_mid_rid(lfs, file->o.mdir.mid) < start_rid
|| (lfsr_rid_t)lfsr_mid_rid(lfs, file->o.mdir.mid)
>= (lfsr_rid_t)end_rid) {
if (!((file->o.type == LFS_TYPE_REG
|| file->o.type == LFS_TYPE_TRAVERSAL)
&& lfsr_mdir_cmp(&file->o.mdir, mdir) == 0
&& lfsr_mid_rid(lfs, file->o.mdir.mid) >= start_rid
&& (lfsr_rid_t)lfsr_mid_rid(lfs, file->o.mdir.mid)
< (lfsr_rid_t)end_rid)) {
continue;
}
@@ -7346,7 +7356,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir,
}
// stage any bsprouts/bshrubs
if (o->type == LFS_TYPE_REG) {
if (o->type == LFS_TYPE_REG || o->type == LFS_TYPE_TRAVERSAL) {
lfsr_file_t *file = (lfsr_file_t*)o;
file->bshrub_ = file->bshrub;
}
@@ -7841,7 +7851,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir,
// update any staged bsprouts/bshrubs
for (lfsr_omdir_t *o = lfs->omdirs; o; o = o->next) {
if (o->type == LFS_TYPE_REG) {
if (o->type == LFS_TYPE_REG || o->type == LFS_TYPE_TRAVERSAL) {
lfsr_file_t *file = (lfsr_file_t*)o;
file->bshrub = file->bshrub_;
}
@@ -8231,6 +8241,7 @@ enum {
.state=LFSR_MTRAVERSAL_MROOTANCHOR, \
.flags=_flags, \
.o=NULL, \
.bshrub.u.bshrub.blocks={-1}, \
.u.mtortoise.mptr={{0, 0}}, \
.u.mtortoise.step=0, \
.u.mtortoise.power=0})
@@ -8620,6 +8631,9 @@ static int lfsr_mtree_traverse(lfs_t *lfs,
static int lfsr_mtree_gc(lfs_t *lfs,
lfsr_mdir_t *mdir, lfsr_mtraversal_t *mt,
lfsr_mtinfo_t *mtinfo) {
// TODO traversals need to be enrolled in our opened list for
// lfsr_mtree_gc to work correctly, can we assert this somehow?
int err = lfsr_mtree_traverse(lfs, mdir, mt, mtinfo);
if (err) {
return err;
@@ -8708,7 +8722,7 @@ static int lfsr_mtree_gc(lfs_t *lfs,
LFSR_TAG_SUB | LFSR_TAG_SHRUBTRUNK, 0,
&mt->bshrub.u.bshrub)
: LFSR_ATTR(
LFSR_TAG_BTREE, 0,
LFSR_TAG_SUB | LFSR_TAG_BTREE, 0,
LFSR_DATA_BTREE_(&mt->bshrub.u.btree, buf))));
if (err) {
return err;
lfs.h
+5 -3
View File
@@ -613,13 +613,15 @@ typedef struct lfsr_btraversal {
} lfsr_btraversal_t;
typedef struct lfsr_mtraversal {
// core state machine
uint8_t state;
uint16_t flags;
// opened file state
lfsr_omdir_t *o;
// bshrub/btree state
// this lines up with bshrub/btree in lfsr_file_t
lfsr_bshrub_t bshrub;
lfsr_bshrub_t bshrub_;
// core state machine
uint8_t state;
uint16_t flags;
union {
// cycle detection state, only valid when traversing the mroot chain
struct {
tests/test_traversal.toml
+486 -7
View File
@@ -6365,11 +6365,491 @@ code = '''
lfsr_unmount(&lfs) => 0;
'''
# TODO
# [cases.test_traversal_compact_bshrub]
# [cases.test_traversal_compact_bshrub_open]
# [cases.test_traversal_compact_bshrub_orphan]
# [cases.test_traversal_compact_bshrub_desync]
[cases.test_traversal_compact_bshrub]
defines.CKMETA = [false, true]
defines.CK = [false, true]
defines.LOOKAHEAD = [false, true]
# this configuration should create a 2-layer bshrub, which may be
# a bit delicate
defines.INLINE_SIZE = 'BLOCK_SIZE/4'
defines.CRYSTAL_THRESH = -1
defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8'
defines.SIZE = 'BLOCK_SIZE'
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create a file
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "jellyfish",
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
// rewrite part of our file until bshrub is >gc_compact_thresh full
while (true) {
// we need internals to check this
lfsr_mdir_t mdir;
// ckmeta needed for eoff
lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA);
lfsr_mtinfo_t mtinfo;
lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
assert(mtinfo.tag == LFSR_TAG_MDIR);
lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
assert(mtinfo.tag == LFSR_TAG_BRANCH);
if (lfsr_rbyd_eoff(&mtinfo.u.rbyd) > GC_COMPACT_THRESH) {
break;
}
lfsr_file_rewind(&lfs, &file) => 0;
for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, FRAGMENT_SIZE) => FRAGMENT_SIZE;
}
lfsr_file_close(&lfs, &file) => 0;
// try traversing and compacting
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t,
LFS_T_COMPACT
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
// traverse mroot
struct lfs_tinfo tinfo;
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
// compacting our bshrub nodes may cause them to split, so we may
// need to traverse more nodes than we started with
for (lfs_size_t i = 0; i < 5; i++) {
// traverse bshrub
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_BTREE);
}
lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
lfsr_traversal_close(&lfs, &t) => 0;
// bshrub should have been compacted
lfsr_mdir_t mdir;
lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA);
lfsr_mtinfo_t mtinfo;
lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
assert(mtinfo.tag == LFSR_TAG_MDIR);
while (true) {
int err = lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
assert(mtinfo.tag == LFSR_TAG_BRANCH);
assert(lfsr_rbyd_eoff(&mtinfo.u.rbyd) <= GC_COMPACT_THRESH);
}
// check we can still read the file
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
}
lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
lfsr_file_rewind(&lfs, &file) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
lfsr_file_close(&lfs, &file) => 0;
}
lfsr_unmount(&lfs) => 0;
'''
[cases.test_traversal_compact_bshrub_open]
defines.CKMETA = [false, true]
defines.CK = [false, true]
defines.LOOKAHEAD = [false, true]
# this configuration should create a 2-layer bshrub, which may be
# a bit delicate
defines.INLINE_SIZE = 'BLOCK_SIZE/4'
defines.CRYSTAL_THRESH = -1
defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8'
defines.SIZE = 'BLOCK_SIZE'
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create a file
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "jellyfish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
// rewrite part of our file until bshrub is >gc_compact_thresh full
while (true) {
// we need internals to check this
lfsr_mdir_t mdir;
// ckmeta needed for eoff
lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA);
lfsr_mtinfo_t mtinfo;
lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
assert(mtinfo.tag == LFSR_TAG_MDIR);
lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
assert(mtinfo.tag == LFSR_TAG_BRANCH);
if (lfsr_rbyd_eoff(&mtinfo.u.rbyd) > GC_COMPACT_THRESH) {
break;
}
lfsr_file_rewind(&lfs, &file) => 0;
for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, FRAGMENT_SIZE) => FRAGMENT_SIZE;
}
lfsr_file_sync(&lfs, &file) => 0;
// try traversing and compacting
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t,
LFS_T_COMPACT
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
// traverse mroot
struct lfs_tinfo tinfo;
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
// compacting our bshrub nodes may cause them to split, so we may
// need to traverse more nodes than we started with
for (lfs_size_t i = 0; i < 5; i++) {
// traverse bshrub
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_BTREE);
}
lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
lfsr_traversal_close(&lfs, &t) => 0;
// bshrub should have been compacted
lfsr_mdir_t mdir;
lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA);
lfsr_mtinfo_t mtinfo;
lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
assert(mtinfo.tag == LFSR_TAG_MDIR);
while (true) {
int err = lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
assert(mtinfo.tag == LFSR_TAG_BRANCH);
assert(lfsr_rbyd_eoff(&mtinfo.u.rbyd) <= GC_COMPACT_THRESH);
}
// check we can still read the file
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
}
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
[cases.test_traversal_compact_bshrub_orphan]
defines.CKMETA = [false, true]
defines.CK = [false, true]
defines.LOOKAHEAD = [false, true]
# this configuration should create a 2-layer bshrub, which may be
# a bit delicate
defines.INLINE_SIZE = 'BLOCK_SIZE/4'
defines.CRYSTAL_THRESH = -1
defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8'
defines.SIZE = 'BLOCK_SIZE'
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create a file
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "jellyfish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
// rewrite part of our file until bshrub is >gc_compact_thresh full
while (true) {
// we need internals to check this
lfsr_mdir_t mdir;
// ckmeta needed for eoff
lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA);
lfsr_mtinfo_t mtinfo;
lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
assert(mtinfo.tag == LFSR_TAG_MDIR);
lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
assert(mtinfo.tag == LFSR_TAG_BRANCH);
if (lfsr_rbyd_eoff(&mtinfo.u.rbyd) > GC_COMPACT_THRESH) {
break;
}
lfsr_file_rewind(&lfs, &file) => 0;
for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, FRAGMENT_SIZE) => FRAGMENT_SIZE;
}
// try traversing and compacting
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t,
LFS_T_COMPACT
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
// traverse mroot
struct lfs_tinfo tinfo;
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
// compacting our bshrub nodes may cause them to split, so we may
// need to traverse more nodes than we started with
for (lfs_size_t i = 0; i < 5; i++) {
// traverse bshrub
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_BTREE);
}
lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
lfsr_traversal_close(&lfs, &t) => 0;
// bshrub should have been compacted
lfsr_mdir_t mdir;
lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA);
lfsr_mtinfo_t mtinfo;
lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
assert(mtinfo.tag == LFSR_TAG_MDIR);
while (true) {
int err = lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
assert(mtinfo.tag == LFSR_TAG_BRANCH);
assert(lfsr_rbyd_eoff(&mtinfo.u.rbyd) <= GC_COMPACT_THRESH);
}
// check we can still read the file
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
}
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
[cases.test_traversal_compact_bshrub_desync]
defines.CKMETA = [false, true]
defines.CK = [false, true]
defines.LOOKAHEAD = [false, true]
# this configuration should create a 2-layer bshrub, which may be
# a bit delicate
defines.INLINE_SIZE = 'BLOCK_SIZE/4'
defines.CRYSTAL_THRESH = -1
defines.FRAGMENT_SIZE = 'BLOCK_SIZE/8'
defines.SIZE = 'BLOCK_SIZE'
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create a desync file
lfsr_file_t file1;
lfsr_file_open(&lfs, &file1, "jellyfish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0;
uint8_t wbuf1[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
// rewrite part of our file until bshrub is >gc_compact_thresh full
while (true) {
// we need internals to check this
lfsr_mdir_t mdir;
// ckmeta needed for eoff
lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA);
lfsr_mtinfo_t mtinfo;
lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
assert(mtinfo.tag == LFSR_TAG_MDIR);
lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
assert(mtinfo.tag == LFSR_TAG_BRANCH);
if (lfsr_rbyd_eoff(&mtinfo.u.rbyd) > GC_COMPACT_THRESH) {
break;
}
lfsr_file_rewind(&lfs, &file1) => 0;
for (lfs_size_t j = 0; j < FRAGMENT_SIZE; j++) {
wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file1, wbuf1, FRAGMENT_SIZE) => FRAGMENT_SIZE;
}
// create some overlapping files, these should not get messed with
lfsr_file_t file2;
lfsr_file_open(&lfs, &file2, "jellyfish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf2[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file2) => 0;
lfsr_file_t file3;
lfsr_file_open(&lfs, &file3, "jellyfish",
LFS_O_RDWR) => 0;
uint8_t wbuf3[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf3[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file3, wbuf3, SIZE) => SIZE;
lfsr_file_desync(&lfs, &file3) => 0;
// try traversing and compacting
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t,
LFS_T_COMPACT
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
// traverse mroot
struct lfs_tinfo tinfo;
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
// compacting our bshrub nodes may cause them to split, so we may
// need to traverse more nodes than we started with
for (lfs_size_t i = 0; i < 3*5; i++) {
// traverse bshrub
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_BTREE);
}
lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
lfsr_traversal_close(&lfs, &t) => 0;
// bshrub should have been compacted
lfsr_mdir_t mdir;
lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(LFS_T_CKMETA);
lfsr_mtinfo_t mtinfo;
lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo) => 0;
assert(mtinfo.tag == LFSR_TAG_MDIR);
while (true) {
int err = lfsr_mtree_traverse(&lfs, &mdir, &mt, &mtinfo);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
assert(mtinfo.tag == LFSR_TAG_BRANCH);
assert(lfsr_rbyd_eoff(&mtinfo.u.rbyd) <= GC_COMPACT_THRESH);
}
// check we can still read the files
lfsr_file_rewind(&lfs, &file1) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf1, SIZE) == 0);
lfsr_file_rewind(&lfs, &file2) => 0;
lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf2, SIZE) == 0);
lfsr_file_rewind(&lfs, &file3) => 0;
lfsr_file_read(&lfs, &file3, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf3, SIZE) == 0);
// at least try closing/opening our synced file
lfsr_file_close(&lfs, &file2) => 0;
lfsr_file_open(&lfs, &file2, "jellyfish", LFS_O_RDONLY) => 0;
lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf2, SIZE) == 0);
lfsr_file_close(&lfs, &file1) => 0;
lfsr_file_close(&lfs, &file2) => 0;
lfsr_file_close(&lfs, &file3) => 0;
lfsr_unmount(&lfs) => 0;
'''
@@ -7024,8 +7504,7 @@ defines.EXCL = [false, true]
defines.CKMETA = [true]
defines.CK = [true]
defines.LOOKAHEAD = [false, true]
# TODO !!! Enable this when bshrub compaction is working
defines.COMPACT = [false]
defines.COMPACT = [false, true]
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
defines.OPS = 20