Github/littlefs
1
Fork 0
Code Actions Activity

Added filesystem-level info flags to lfsr_fs_stat

Browse Source 
Thinking again of use cases, lfsr_fs_gc provides the perfect API to call
in the background to perform any pending filesystem work. But what if
there's no work to be done? Sure we could just spin forever, but that's
a waste. Especially on devices that can turn on sleep modes to save
power.

To help with this, this commit adds a set of flags to struct lfs_fsinfo
that signals when lfsr_fs_gc can accomplish work:

  LFS_I_INCONSISTENT     = 0x01, // Filesystem needs mkconsistent to write
  LFS_I_NEEDSUPGRADE*    = 0x02, // Filesystem needs an upgrade to write
  LFS_I_CANLOOKAHEAD     = 0x04, // Lookahead buffer is not full
  LFS_I_CANPREERASE+     = 0x08, // Pre-erase buffer is not full
  LFS_I_UNCOMPACTED      = 0x10, // Filesystem may have uncompacted metadata
  LFS_I_NEEDSREPAIRMETA+ = 0x20, // Filesystem contains damaged metadata
  LFS_I_NEEDSREPAIRDATA+ = 0x40, // Filesystem contains damaged data

  *Hypothetical
  +Planned

This flags field also provides a useful place internally to store other
filesystem-related flags, currently LFS_F_ORPHANS, though this may be
expanded in the future.

These flags allow users to know exactly what work can/needs to be done
for the filesystem to make progress:

- LFS_I_INCONSISTENT => LFS_GC_MKCONSISTENT or lfsr_fs_mkconsistent
- LFS_I_CANLOOKAHEAD => LFS_GC_LOOKAHEAD

- LFS_I_UNCOMPACTED => LFS_GC_COMPACT

  The one is new!

  If we complete a compaction-traversal without any mutation, we know
  all mdirs/btree nodes have been compacted and future traversals won't
  accomplish anything. Of course, we need to clear this bit on
  filesystem mutation.

  Right now we just pessimistically assume the filesystem is uncompacted
  during mount, but in theory we can also figure this out during our
  initial mount traversal.

- LFS_GC_CKMETA/CKDATA?

  LFS_GC_CKMETA and LFS_GC_CKDATA are a bit trickier. In theory,
  LFS_GC_CKMETA/CKDATA will always accomplish something, since time is
  the only ingredient necessary to introduce bit errors.

  So there isn't really a reasonable flag here. It's entirely up to the
  user to decide when to do an LFS_GC_CKMETA/CKDATA traversal.

Code changes:

           code          stack
  before: 35740           2672
  after:  35880 (+0.4%)   2672 (+0.0%)
This commit is contained in:
Christopher Haster
2024-07-14 19:27:27 -05:00
parent d18633e4e8
commit 0a3cb2dd3a
5 changed files with 1594 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+60 -13
View File
@@ -5905,8 +5905,20 @@ static int lfsr_data_readmptr(lfs_t *lfs, lfsr_data_t *data,
}
// opened mdir things
//
/// various flag things ///
static inline bool lfsr_i_isuncompacted(uint32_t flags) {
return flags & LFS_I_UNCOMPACTED;
}
static inline bool lfsr_f_hasorphans(uint32_t flags) {
return flags & LFS_F_ORPHANS;
}
/// opened mdir things ///
// we maintain a linked-list of all opened mdirs, in order to keep
// metadata state in-sync, these may be casted to specific file types
@@ -7831,6 +7843,10 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir,
lfs->seed ^= mdir_[1].rbyd.cksum;
}
// we may have touched any number of mdirs, so assume uncompacted
// until lfsr_fs_gc can prove otherwise
lfs->flags |= LFS_I_UNCOMPACTED;
// update any gstate changes
lfsr_fs_commitgdelta(lfs);
@@ -8682,7 +8698,7 @@ dropped:;
// mkconsistencing mdirs?
if (lfsr_t_ismkconsistent(t->o.o.flags)
&& tag == LFSR_TAG_MDIR
&& lfs->hasorphans) {
&& lfsr_f_hasorphans(lfs->flags)) {
lfsr_mdir_t *mdir = (lfsr_mdir_t*)bptr.data.u.buffer;
err = lfsr_mdir_fixorphans(lfs, mdir);
if (err) {
@@ -8802,6 +8818,9 @@ dropped:;
// reset to btree root
t->u.bt.branch = &t->o.bshrub.u.btree;
t->u.bt.rid = t->u.bt.bid;
// mark as dirty, we need to do this manually here
t->o.o.flags |= LFS_F_DIRTY;
}
// swap back dirty/mutated flags
@@ -8822,7 +8841,7 @@ eot:;
// was mkconsistent successful?
if (lfsr_t_ismkconsistent(t->o.o.flags)
&& !lfsr_f_isdirty(t->o.o.flags)) {
lfs->hasorphans = false;
lfs->flags &= ~LFS_F_ORPHANS;
}
// was lookahead scan successful?
@@ -8832,6 +8851,14 @@ eot:;
lfs_alloc_markfree(lfs);
}
// was compaction successful? note we may need multiple passes if
// we want to be sure everything is compacted
if (lfsr_t_iscompact(t->o.o.flags)
&& !lfsr_f_isdirty(t->o.o.flags)
&& !lfsr_f_ismutated(t->o.o.flags)) {
lfs->flags &= ~LFS_I_UNCOMPACTED;
}
return LFS_ERR_NOENT;
failed:;
@@ -10118,7 +10145,7 @@ int lfsr_file_close(lfs_t *lfs, lfsr_file_t *file) {
// fallback to just marking the filesystem as orphaned
} else {
lfs->hasorphans = true;
lfs->flags |= LFS_F_ORPHANS;
}
}
@@ -11796,6 +11823,9 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
// fragment_size must be <= block_size/4
LFS_ASSERT(lfs->cfg->fragment_size <= lfs->cfg->block_size/4);
// zero flags
lfs->flags = 0;
// setup block_count so we can mutate it
lfs->block_count = lfs->cfg->block_count;
@@ -11870,10 +11900,6 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
// lfs->lfs1 = NULL;
//#endif
// TODO maybe reorganize this function?
lfs->hasorphans = false;
// TODO do we need to recalculate these after mount?
// find the number of bits to use for recycle counters
@@ -12460,7 +12486,7 @@ static int lfsr_mountinited(lfs_t *lfs) {
"%"PRId32".%"PRId32,
lfsr_mid_bid(lfs, mdir->mid) >> lfs->mdir_bits,
rid);
lfs->hasorphans = true;
lfs->flags |= LFS_F_ORPHANS;
// found an unknown file type?
} else if (lfsr_tag_isunknown(tag)) {
@@ -12521,6 +12547,10 @@ static int lfsr_mountinited(lfs_t *lfs) {
lfsr_mid_rid(lfs, lfs->grm.mids[0]));
}
// default to assuming we need a compaction somewhere, worst case this
// just makes lfsr_fs_gc read more than is strictly needed
lfs->flags |= LFS_I_UNCOMPACTED;
return 0;
}
@@ -12671,10 +12701,24 @@ int lfsr_format(lfs_t *lfs, const struct lfs_config *cfg) {
/// Other filesystem things ///
int lfsr_fs_stat(lfs_t *lfs, struct lfs_fsinfo *fsinfo) {
// return various filesystem flags
fsinfo->flags = lfs->flags & LFS_I_UNCOMPACTED;
// some flags we calculate on demand
if (lfsr_grm_count(lfs) > 0 || lfsr_f_hasorphans(lfs->flags)) {
fsinfo->flags |= LFS_I_INCONSISTENT;
}
if (lfs->lookahead.next > 0 || lfs->lookahead.size == 0) {
fsinfo->flags |= LFS_I_CANLOOKAHEAD;
}
// return filesystem config, this may come from disk
fsinfo->block_size = lfs->cfg->block_size;
fsinfo->block_count = lfs->block_count;
fsinfo->name_limit = lfs->name_limit;
fsinfo->file_limit = lfs->file_limit;
return 0;
}
@@ -12839,7 +12883,7 @@ int lfsr_fs_mkconsistent(lfs_t *lfs) {
// this must happen after fixgrm, since removing orphaned files risks
// outdating the grm
//
if (lfs->hasorphans) {
if (lfsr_f_hasorphans(lfs->flags)) {
LFS_DEBUG("Fixing orphans...");
int err = lfsr_fs_fixorphans(lfs);
@@ -12886,10 +12930,11 @@ int lfsr_fs_gc(lfs_t *lfs, uint32_t flags) {
}
// do we need to do anything?
if (!((lfsr_t_ismkconsistent(flags) && lfs->hasorphans)
if (!((lfsr_t_ismkconsistent(flags) && lfsr_f_hasorphans(lfs->flags))
|| (lfsr_t_islookahead(flags)
&& (lfs->lookahead.next > 0 || lfs->lookahead.size == 0))
|| lfsr_t_iscompact(flags)
|| (lfsr_t_iscompact(flags)
&& lfsr_i_isuncompacted(lfs->flags))
|| lfsr_t_isckmeta(flags)
|| lfsr_t_isckdata(flags))) {
return 0;
@@ -12923,6 +12968,7 @@ int lfsr_fs_gc(lfs_t *lfs, uint32_t flags) {
// shift the lookahead buffer if requested
if (lfsr_t_islookahead(lfs->gc.o.o.flags)) {
lfs_alloc_shift(lfs);
lfs_alloc_ckpoint(lfs);
}
}
@@ -13173,6 +13219,7 @@ static int lfsr_traversal_rewind_(lfs_t *lfs, lfsr_traversal_t *t) {
// shift the lookahead buffer if requested
if (lfsr_t_islookahead(t->o.o.flags)) {
lfs_alloc_shift(lfs);
lfs_alloc_ckpoint(lfs);
}
return 0;
lfs.h
+22 -13
View File
@@ -125,6 +125,16 @@ enum lfs_type {
LFS_TYPE_TRAVERSAL = 9,
};
// Block types
enum lfs_btype {
LFS_BTYPE_MDIR = 1,
LFS_BTYPE_BTREE = 2,
LFS_BTYPE_DATA = 3,
// TODO
// LFS_BTYPE_PARITY = 4,
// LFS_BTYPE_BAD = 5,
};
// File open flags
enum lfs_open_flags {
// open flags
@@ -155,14 +165,15 @@ enum lfs_whence_flags {
LFS_SEEK_END = 2, // Seek relative to the end of the file
};
// Block types
enum lfs_btype {
LFS_BTYPE_MDIR = 1,
LFS_BTYPE_BTREE = 2,
LFS_BTYPE_DATA = 3,
// TODO
// LFS_BTYPE_PARITY = 4,
// LFS_BTYPE_BAD = 5,
// Filesystem info flags
enum lfs_fsinfo_flags {
// state flags
LFS_I_INCONSISTENT = 0x01, // Filesystem needs mkconsistent to write
LFS_I_CANLOOKAHEAD = 0x04, // Lookahead buffer is not full
LFS_I_UNCOMPACTED = 0x10, // Filesystem may have uncompacted metadata
// internally used flags
LFS_F_ORPHANS = 0x80, // Filesystem may have untracked orphans
};
// Traversal flags
@@ -391,7 +402,8 @@ struct lfs_info {
// Filesystem info structure
struct lfs_fsinfo {
// TODO should we add rcompat/wcompat flags here?
// Filesystem flags
uint8_t flags;
// Size of a logical block in bytes.
lfs_size_t block_size;
@@ -713,10 +725,7 @@ typedef struct lfs {
lfs_size_t name_limit;
lfs_off_t file_limit;
// TODO we should put this flag somewhere, should lfs_t have a general
// purpose flags field? this has been useful for lfsr_file_t
bool hasorphans;
uint8_t flags;
int8_t recycle_bits;
uint8_t attr_estimate;
uint8_t mdir_bits;
tests/test_forphans.toml
+15 -3
View File
@@ -4701,7 +4701,11 @@ code = '''
// we should have cleaned up all grms/orphans
assert(lfs.grm.mids[0] == -1);
assert(lfs.grm.mids[1] == -1);
assert(lfs.hasorphans == false);
assert(!(lfs.flags & LFS_F_ORPHANS));
struct lfs_fsinfo fsinfo;
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(!(fsinfo.flags & LFS_I_INCONSISTENT));
// double check the actual disk state, it's easy for littlefs to
// lie here
@@ -4831,7 +4835,11 @@ code = '''
// we should have cleaned up all grms/orphans
assert(lfs.grm.mids[0] == -1);
assert(lfs.grm.mids[1] == -1);
assert(lfs.hasorphans == false);
assert(!(lfs.flags & LFS_F_ORPHANS));
struct lfs_fsinfo fsinfo;
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(!(fsinfo.flags & LFS_I_INCONSISTENT));
// double check the actual disk state, it's easy for littlefs to
// lie here
@@ -4975,7 +4983,11 @@ code = '''
// we should have cleaned up all grms/orphans
assert(lfs.grm.mids[0] == -1);
assert(lfs.grm.mids[1] == -1);
assert(lfs.hasorphans == false);
assert(!(lfs.flags & LFS_F_ORPHANS));
struct lfs_fsinfo fsinfo;
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(!(fsinfo.flags & LFS_I_INCONSISTENT));
// double check the actual disk state, it's easy for littlefs to
// lie here
tests/test_gc.toml
+459 -50
View File
@@ -17,7 +17,6 @@ defines.SIZE = [
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
@@ -37,8 +36,10 @@ code = '''
lfsr_file_close(&lfs, &file) => 0;
// expect dirty initial state or else our test doesn't work
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfs.lookahead.next > 0 || lfs.lookahead.size == 0);
struct lfs_fsinfo fsinfo;
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_CANLOOKAHEAD);
assert(lfs.omdirs != &lfs.gc.o.o);
// run GC until our traversal is done
while (true) {
@@ -48,13 +49,14 @@ code = '''
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
// internal traversal done?
if (!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
if (lfs.omdirs != &lfs.gc.o.o) {
break;
}
}
// we should have made progress
assert(!(lfs.lookahead.next > 0 || lfs.lookahead.size == 0));
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(!(fsinfo.flags & LFS_I_CANLOOKAHEAD));
// check the file contents
lfsr_file_open(&lfs, &file, "spider", LFS_O_RDONLY) => 0;
@@ -66,7 +68,7 @@ code = '''
lfsr_unmount(&lfs) => 0;
'''
# test that lookahead clobbering still works with the GC API
# test that lookahead dirtying still works with the GC API
[cases.test_gc_lookahead_mutation]
defines.GC_STEPS = 1
defines.CKMETA = [false, true]
@@ -79,7 +81,6 @@ defines.SIZE = [
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
@@ -99,15 +100,17 @@ code = '''
lfsr_file_close(&lfs, &file) => 0;
// expect dirty initial state or else our test doesn't work
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfs.lookahead.next > 0 || lfs.lookahead.size == 0);
struct lfs_fsinfo fsinfo;
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_CANLOOKAHEAD);
assert(lfs.omdirs != &lfs.gc.o.o);
// run GC one step
lfsr_fs_gc(&lfs,
LFS_GC_LOOKAHEAD
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
assert(lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfs.omdirs == &lfs.gc.o.o);
// mutate the filesystem
lfsr_file_open(&lfs, &file, "spider",
@@ -119,7 +122,7 @@ code = '''
lfsr_file_close(&lfs, &file) => 0;
// run GC until our traversal is done
while (lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
while (lfs.omdirs == &lfs.gc.o.o) {
lfsr_fs_gc(&lfs,
LFS_GC_LOOKAHEAD
| ((CKMETA) ? LFS_GC_CKMETA : 0)
@@ -127,7 +130,8 @@ code = '''
}
// we should _not_ make progress
assert(lfs.lookahead.next > 0 || lfs.lookahead.size == 0);
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_CANLOOKAHEAD);
// check the file contents
lfsr_file_open(&lfs, &file, "spider", LFS_O_RDONLY) => 0;
@@ -154,7 +158,6 @@ defines.SIZE = [
]
# we need something to keep the traversal running
if = 'CKMETA || CKDATA'
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
@@ -174,17 +177,19 @@ code = '''
lfsr_file_close(&lfs, &file) => 0;
// expect dirty initial state or else our test doesn't work
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfs.lookahead.next > 0 || lfs.lookahead.size == 0);
struct lfs_fsinfo fsinfo;
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_CANLOOKAHEAD);
assert(lfs.omdirs != &lfs.gc.o.o);
// run GC one step
lfsr_fs_gc(&lfs,
((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
assert(lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfs.omdirs == &lfs.gc.o.o);
// change flags and run GC until our traversal is done
while (lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
while (lfs.omdirs == &lfs.gc.o.o) {
lfsr_fs_gc(&lfs,
LFS_GC_LOOKAHEAD
| ((CKMETA) ? LFS_GC_CKMETA : 0)
@@ -192,7 +197,8 @@ code = '''
}
// we should _not_ make progress
assert(lfs.lookahead.next > 0 || lfs.lookahead.size == 0);
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_CANLOOKAHEAD);
// check the file contents
lfsr_file_open(&lfs, &file, "spider", LFS_O_RDONLY) => 0;
@@ -219,7 +225,6 @@ defines.SIZE = [
]
# we need something to keep the traversal running
if = 'CKMETA || CKDATA'
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
@@ -239,25 +244,28 @@ code = '''
lfsr_file_close(&lfs, &file) => 0;
// expect dirty initial state or else our test doesn't work
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfs.lookahead.next > 0 || lfs.lookahead.size == 0);
struct lfs_fsinfo fsinfo;
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_CANLOOKAHEAD);
assert(lfs.omdirs != &lfs.gc.o.o);
// run GC one step
lfsr_fs_gc(&lfs,
LFS_GC_LOOKAHEAD
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
assert(lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfs.omdirs == &lfs.gc.o.o);
// change flags and run GC until our traversal is done
while (lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
while (lfs.omdirs == &lfs.gc.o.o) {
lfsr_fs_gc(&lfs,
((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
}
// we should _not_ make progress
assert(lfs.lookahead.next > 0 || lfs.lookahead.size == 0);
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_CANLOOKAHEAD);
// check the file contents
lfsr_file_open(&lfs, &file, "spider", LFS_O_RDONLY) => 0;
@@ -270,6 +278,398 @@ code = '''
'''
# test that compact can make progress in isolation
[cases.test_gc_compact_progress]
defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000]
defines.LOOKAHEAD = [false, true]
defines.CKMETA = [false, true]
defines.CKDATA = [false, true]
defines.SIZE = [
'FILE_BUFFER_SIZE/2',
'2*FILE_BUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// write to our mdir until >gc_compact_thresh full
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "jellyfish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
// hack, don't use the internals like this
uint8_t wbuf[SIZE];
while ((file.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
lfsr_file_rewind(&lfs, &file) => 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file) => 0;
}
// expect dirty initial state or else our test doesn't work
struct lfs_fsinfo fsinfo;
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_UNCOMPACTED);
assert(lfs.omdirs != &lfs.gc.o.o);
// run GC until our traversal is done (twice for compact)
for (int i = 0; i < 2; i++) {
while (true) {
lfsr_fs_gc(&lfs,
LFS_GC_COMPACT
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
// internal traversal done?
if (lfs.omdirs != &lfs.gc.o.o) {
break;
}
}
}
// mdir should have been compacted
assert((file.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
// we should have made progress
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(!(fsinfo.flags & LFS_I_UNCOMPACTED));
// check we can still read the file
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
}
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
# test that compact dirtying still works with the GC API
[cases.test_gc_compact_mutation]
defines.GC_STEPS = 1
defines.LOOKAHEAD = [false, true]
defines.CKMETA = [false, true]
defines.CKDATA = [false, true]
defines.SIZE = [
'FILE_BUFFER_SIZE/2',
'2*FILE_BUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// write to our mdir until >gc_compact_thresh full
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "jellyfish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
// hack, don't use the internals like this
uint8_t wbuf[SIZE];
while ((file.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
lfsr_file_rewind(&lfs, &file) => 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file) => 0;
}
// expect dirty initial state or else our test doesn't work
struct lfs_fsinfo fsinfo;
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_UNCOMPACTED);
assert(lfs.omdirs != &lfs.gc.o.o);
// run GC one traversal + one step
while (true) {
lfsr_fs_gc(&lfs,
LFS_GC_COMPACT
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
// internal traversal done?
if (lfs.omdirs != &lfs.gc.o.o) {
break;
}
}
lfsr_fs_gc(&lfs,
LFS_GC_COMPACT
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
assert(lfs.omdirs == &lfs.gc.o.o);
// mutate the filesystem
lfsr_file_rewind(&lfs, &file) => 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file) => 0;
// run GC until our traversal is done (twice for compact)
while (lfs.omdirs == &lfs.gc.o.o) {
lfsr_fs_gc(&lfs,
LFS_GC_COMPACT
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
}
// we should _not_ make progress
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_UNCOMPACTED);
// check we can still read the file
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
}
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
# test that adding flags doesn't break compact
[cases.test_gc_compact_add_flags]
defines.GC_STEPS = 1
defines.LOOKAHEAD = [false, true]
defines.CKMETA = [false, true]
defines.CKDATA = [false, true]
defines.SIZE = [
'FILE_BUFFER_SIZE/2',
'2*FILE_BUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
# we need something to keep the traversal running
if = 'CKMETA || CKDATA'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// write to our mdir until >gc_compact_thresh full
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "jellyfish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
// hack, don't use the internals like this
uint8_t wbuf[SIZE];
while ((file.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
lfsr_file_rewind(&lfs, &file) => 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file) => 0;
}
// expect dirty initial state or else our test doesn't work
struct lfs_fsinfo fsinfo;
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_UNCOMPACTED);
assert(lfs.omdirs != &lfs.gc.o.o);
// run GC one traversal + one step
while (true) {
lfsr_fs_gc(&lfs,
((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
// internal traversal done?
if (lfs.omdirs != &lfs.gc.o.o) {
break;
}
}
lfsr_fs_gc(&lfs,
((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
assert(lfs.omdirs == &lfs.gc.o.o);
// change flags and run GC until our traversal is done (twice for compact)
while (lfs.omdirs == &lfs.gc.o.o) {
lfsr_fs_gc(&lfs,
LFS_GC_COMPACT
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
}
// we should _not_ make progress
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_UNCOMPACTED);
// check we can still read the file
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
}
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
# test that removing flags invalidates compact
[cases.test_gc_compact_remove_flags]
defines.GC_STEPS = 1
defines.LOOKAHEAD = [false, true]
defines.CKMETA = [false, true]
defines.CKDATA = [false, true]
defines.SIZE = [
'FILE_BUFFER_SIZE/2',
'2*FILE_BUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
# we need something to keep the traversal running
if = 'CKMETA || CKDATA'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// write to our mdir until >gc_compact_thresh full
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "jellyfish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
// hack, don't use the internals like this
uint8_t wbuf[SIZE];
while ((file.o.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
lfsr_file_rewind(&lfs, &file) => 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file) => 0;
}
// expect dirty initial state or else our test doesn't work
struct lfs_fsinfo fsinfo;
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_UNCOMPACTED);
assert(lfs.omdirs != &lfs.gc.o.o);
// run GC one traversal + one step
while (true) {
lfsr_fs_gc(&lfs,
LFS_GC_COMPACT
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
// internal traversal done?
if (lfs.omdirs != &lfs.gc.o.o) {
break;
}
}
lfsr_fs_gc(&lfs,
LFS_GC_COMPACT
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
assert(lfs.omdirs == &lfs.gc.o.o);
// change flags and run GC until our traversal is done (twice for compact)
while (lfs.omdirs == &lfs.gc.o.o) {
lfsr_fs_gc(&lfs,
((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
}
// we should _not_ make progress
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_UNCOMPACTED);
// check we can still read the file
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
}
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
# test that mkconsistent can make progress in isolation
[cases.test_gc_mkconsistent_progress]
defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000]
@@ -281,7 +681,6 @@ defines.SIZE = 'FILE_BUFFER_SIZE/2'
# <=2 => grm-able
# >2 => requires orphans
defines.ORPHANS = [1, 2, 3, 100]
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
@@ -329,8 +728,10 @@ code = '''
}
// expect dirty initial state or else our test doesn't work
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans);
struct lfs_fsinfo fsinfo;
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_INCONSISTENT);
assert(lfs.omdirs != &lfs.gc.o.o);
// run GC until our traversal is done
while (true) {
@@ -342,13 +743,14 @@ code = '''
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
// internal traversal done?
if (!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
if (lfs.omdirs != &lfs.gc.o.o) {
break;
}
}
// we should have made progress
assert(!(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans));
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(!(fsinfo.flags & LFS_I_INCONSISTENT));
// check we can still read the files
for (int remount = 0; remount < 2; remount++) {
@@ -377,7 +779,7 @@ code = '''
lfsr_unmount(&lfs) => 0;
'''
# test that mkconsistent clobbering still works with the GC API
# test that mkconsistent dirtying still works with the GC API
[cases.test_gc_mkconsistent_mutation]
defines.GC_STEPS = 1
defines.LOOKAHEAD = [false, true]
@@ -388,7 +790,6 @@ defines.SIZE = 'FILE_BUFFER_SIZE/2'
# <=2 => grm-able
# >2 => requires orphans
defines.ORPHANS = [3, 100]
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
@@ -430,14 +831,14 @@ code = '''
}
// run GC one step
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfs.omdirs != &lfs.gc.o.o);
lfsr_fs_gc(&lfs,
LFS_GC_MKCONSISTENT
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_GC_COMPACT : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
assert(lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfs.omdirs == &lfs.gc.o.o);
// create the rest of the orphans after GC has started
for (lfs_size_t i = 0; i < ORPHANS; i++) {
@@ -451,10 +852,12 @@ code = '''
}
// we should now have dirty state
assert(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans);
struct lfs_fsinfo fsinfo;
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_INCONSISTENT);
// run GC until our traversal is done
while (lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
while (lfs.omdirs == &lfs.gc.o.o) {
lfsr_fs_gc(&lfs,
LFS_GC_MKCONSISTENT
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
@@ -464,7 +867,8 @@ code = '''
}
// we should _not_ make progress
assert(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans);
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_INCONSISTENT);
// check we can still read the files
for (int remount = 0; remount < 2; remount++) {
@@ -505,7 +909,7 @@ defines.SIZE = 'FILE_BUFFER_SIZE/2'
# >2 => requires orphans
defines.ORPHANS = [3, 100]
# we need something to keep the traversal running
if = 'COMPACT || CKMETA || CKDATA'
if = 'CKMETA || CKDATA'
in = 'lfs.c'
code = '''
lfs_t lfs;
@@ -554,8 +958,10 @@ code = '''
}
// expect dirty initial state or else our test doesn't work
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans);
struct lfs_fsinfo fsinfo;
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_INCONSISTENT);
assert(lfs.omdirs != &lfs.gc.o.o);
// run GC one step
lfsr_fs_gc(&lfs,
@@ -563,10 +969,10 @@ code = '''
| ((COMPACT) ? LFS_GC_COMPACT : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
assert(lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfs.omdirs == &lfs.gc.o.o);
// change flags and run GC until our traversal is done
while (lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
while (lfs.omdirs == &lfs.gc.o.o) {
lfsr_fs_gc(&lfs,
LFS_GC_MKCONSISTENT
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
@@ -576,7 +982,8 @@ code = '''
}
// we should _not_ make progress
assert(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans);
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_INCONSISTENT);
// check we can still read the files
for (int remount = 0; remount < 2; remount++) {
@@ -617,8 +1024,7 @@ defines.SIZE = 'FILE_BUFFER_SIZE/2'
# >2 => requires orphans
defines.ORPHANS = [3, 100]
# we need something to keep the traversal running
if = 'COMPACT || CKMETA || CKDATA'
in = 'lfs.c'
if = 'CKMETA || CKDATA'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
@@ -666,8 +1072,10 @@ code = '''
}
// expect dirty initial state or else our test doesn't work
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans);
struct lfs_fsinfo fsinfo;
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_INCONSISTENT);
assert(lfs.omdirs != &lfs.gc.o.o);
// run GC one step
lfsr_fs_gc(&lfs,
@@ -676,10 +1084,10 @@ code = '''
| ((COMPACT) ? LFS_GC_COMPACT : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
assert(lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfs.omdirs == &lfs.gc.o.o);
// change flags and run GC until our traversal is done
while (lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
while (lfs.omdirs == &lfs.gc.o.o) {
lfsr_fs_gc(&lfs,
((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_GC_COMPACT : 0)
@@ -688,7 +1096,8 @@ code = '''
}
// we should _not_ make progress
assert(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans);
lfsr_fs_stat(&lfs, &fsinfo) => 0;
assert(fsinfo.flags & LFS_I_INCONSISTENT);
// check we can still read the files
for (int remount = 0; remount < 2; remount++) {
@@ -718,7 +1127,7 @@ code = '''
'''
# pseudo-fuzz test that clobbering still works with the GC API
# pseudo-fuzz test that dirtying still works with the GC API
[cases.test_gc_mutation]
defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000]
defines.STEPS = 100
tests/test_traversal.toml
+1038 -22
View File
File diff suppressed because it is too large Load Diff