Github/littlefs
1
Fork 0
Code Actions Activity

Reworked lfsr_fs_gc to be incremental

Browse Source 
Thinking about use case a bit, most lfsr_fs_gc will be to perform
background work, and can benefit from being incremental.

We already support incremental gc and all the mess associated with
traversal invalidation via the traversal API, so we might as well expose
this through lfsr_fs_gc.

The main downside is that we need to store an lfsr_traversal_t object
somewhere, which is not exactly a cheap struct. I was originally
considering limiting incremental gc to the traversal API for this
reason, but I think the value add of an incremental lfsr_fs_gc is too
compelling... Though we really should add a compile-time option
(LFS_NO_GC? LFS_NO_INCRGC?) to allow users to opt-out of this RAM cost
if they're never going to call this function.

Oh, and lfs_t also becomes self-referential, which might become a
problem for higher-level language users...

---

The incremental behavior of lfsr_fs_gc can be controlled by the new
gc_steps config option. This allows more than one step to be performed
at a time, which may allow for more progress when intermixed with
write-heavy filesystem operations. Setting gc_steps=-1 performs a full
traversal every call, which guarantees always making some amount of
progress.

This adds a bit of code, since we now need to check for/resume existing
traversals. But the real cost is the added RAM to lfs_t, which is
unfortunately wasted if you never call lfsr_fs_gc:

          code           stack          lfs_t
  before: 35708           2672            164
  after:  35756 (+0.1%)   2672 (+0.0%)    296 (+80.5%)
This commit is contained in:
Christopher Haster
2024-07-12 17:02:16 -05:00
parent a0e0ea2081
commit fc486ca4f7
5 changed files with 940 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+58 -50
View File
@@ -12532,6 +12532,11 @@ int lfsr_mount(lfs_t *lfs, const struct lfs_config *cfg) {
}
int lfsr_unmount(lfs_t *lfs) {
// close any ongoing gc traversals
if (lfsr_omdir_isopen(lfs, &lfs->gc.o.o)) {
lfsr_omdir_close(lfs, &lfs->gc.o.o);
}
// all files/dirs should be closed before lfsr_unmount
LFS_ASSERT(lfs->omdirs == NULL);
@@ -12858,64 +12863,67 @@ int lfsr_fs_gc(lfs_t *lfs, uint32_t flags) {
}
}
// we need multiple passes because of potential mutation issues
for (int i = 0;; i++) {
// do we need to do anything?
if (!((lfsr_t_ismkconsistent(flags) && lfs->hasorphans)
|| (lfsr_t_islookahead(flags)
&& (lfs->lookahead.next > 0 || lfs->lookahead.size == 0))
|| (lfsr_t_iscompact(flags) && i == 0)
|| (lfsr_t_isckmeta(flags) && i == 0)
|| (lfsr_t_isckdata(flags) && i == 0))) {
break;
}
// do we need to do anything?
if (!((lfsr_t_ismkconsistent(flags) && lfs->hasorphans)
|| (lfsr_t_islookahead(flags)
&& (lfs->lookahead.next > 0 || lfs->lookahead.size == 0))
|| lfsr_t_iscompact(flags)
|| lfsr_t_isckmeta(flags)
|| lfsr_t_isckdata(flags))) {
return 0;
}
if (lfsr_t_islookahead(flags)) {
// existing traversal?
if (lfsr_omdir_isopen(lfs, &lfs->gc.o.o)) {
// note that we mask out flags! if you change flags mid-traversal,
// the result is equivalent to the worst-case set of flags
lfs->gc.o.o.flags &= (
~LFS_GC_MTREEONLY
& ~LFS_GC_MKCONSISTENT
& ~LFS_GC_LOOKAHEAD
& ~LFS_GC_COMPACT
& ~LFS_GC_CKMETA
& ~LFS_GC_CKDATA
) | flags;
// start a new traversal
} else {
lfs->gc = LFSR_TRAVERSAL(flags);
lfsr_omdir_open(lfs, &lfs->gc.o.o);
// shift the lookahead buffer if requested
if (lfsr_t_islookahead(lfs->gc.o.o.flags)) {
lfs_alloc_shift(lfs);
}
}
lfsr_traversal_t t = LFSR_TRAVERSAL(flags);
// note we need to be tracked for bshrub commits to work
lfsr_omdir_open(lfs, &t.o.o);
while (true) {
// let lfsr_mtree_gc do most of the work
int err = lfsr_mtree_gc(lfs, &t,
NULL, NULL);
if (err) {
if (err == LFS_ERR_NOENT) {
break;
for (uint32_t i = 0;
i < (lfs->cfg->gc_steps ? (uint32_t)lfs->cfg->gc_steps : 1);
i++) {
int err = lfsr_mtree_gc(lfs, &lfs->gc,
NULL, NULL);
if (err) {
if (err == LFS_ERR_NOENT) {
// was mkconsistent successful?
if (lfsr_t_ismkconsistent(lfs->gc.o.o.flags)
&& !lfsr_f_isdirty(lfs->gc.o.o.flags)) {
lfs->hasorphans = false;
}
lfsr_omdir_close(lfs, &t.o.o);
return err;
// was lookahead scan successful?
if (lfsr_t_islookahead(lfs->gc.o.o.flags)
&& !lfsr_f_isdirty(lfs->gc.o.o.flags)
&& !lfsr_f_ismutated(lfs->gc.o.o.flags)) {
lfs_alloc_markfree(lfs);
}
lfsr_omdir_close(lfs, &lfs->gc.o.o);
break;
}
lfsr_omdir_close(lfs, &lfs->gc.o.o);
return err;
}
lfsr_omdir_close(lfs, &t.o.o);
// no more orphans?
if (lfsr_t_ismkconsistent(t.o.o.flags)) {
LFS_ASSERT(!lfsr_f_isdirty(t.o.o.flags));
lfs->hasorphans = false;
}
// was lookahead scan successful?
if (lfsr_t_islookahead(t.o.o.flags)
&& !lfsr_f_ismutated(t.o.o.flags)) {
LFS_ASSERT(!lfsr_f_isdirty(t.o.o.flags));
lfs_alloc_markfree(lfs);
}
// update flags, clear mutated/dirty
flags = t.o.o.flags & ~LFS_F_DIRTY & ~LFS_F_MUTATED;
}
if (lfsr_t_ismkconsistent(flags)) {
LFS_ASSERT(lfsr_grm_count(lfs) == 0);
LFS_ASSERT(lfs->hasorphans == false);
}
if (lfsr_t_islookahead(flags)) {
LFS_ASSERT(lfs->lookahead.next == 0);
LFS_ASSERT(lfs->lookahead.size > 0);
}
return 0;
}
@@ -13137,7 +13145,7 @@ static void lfsr_traversal_clobber(lfs_t *lfs, lfsr_traversal_t *t) {
t->o.o.state = LFSR_TSTATE_OMDIRS;
t->o.bshrub.u.bshrub.weight = 0;
t->o.bshrub.u.bshrub.blocks[0] = -1;
t->ot = t->ot->next;
t->ot = (t->ot) ? t->ot->next : NULL;
// done traversals should never need clobbering
} else {
LFS_UNREACHABLE();
lfs.h
+16 -3
View File
@@ -281,6 +281,17 @@ struct lfs_config {
// can track 8 blocks.
lfs_size_t lookahead_size;
// How many gc steps to perform on each lfsr_fs_gc call.
//
// Each gc step progresses janitorial work by ~1 block (this is equivalent
// to lfsr_traversal_read). More steps per call may make more progress if
// interleaving with other work.
//
// 0 defaults to 1 step, and -1 will perform a full traversal every call,
// though multiple traversals may still be needed to complete all
// janitorial work.
int32_t gc_steps;
// Threshold for metadata compaction during gc in bytes. Metadata logs
// that exceed this threshold will be compacted during gc operations.
// Defaults to ~88% block_size when zero, though this default may change
@@ -741,6 +752,9 @@ typedef struct lfs {
lfsr_grm_t grm;
uint8_t grm_p[LFSR_GRM_DSIZE];
uint8_t grm_d[LFSR_GRM_DSIZE];
// TODO allow compile time opt-out to reclaim RAM
lfsr_traversal_t gc;
} lfs_t;
@@ -1130,9 +1144,8 @@ int lfsr_fs_mkconsistent(lfs_t *lfs);
#ifndef LFS_READONLY
// Attempt any janitorial work that may be pending.
//
// The exact janitorial work depends on the provided flags. Note that most
// of this work can also be accomplished incrementally via
// lfsr_traversal_read.
// The exact janitorial work depends on the provided flags. Note multiple
// calls may be required to complete all janitorial work.
//
// Calling this function is not required, but may allow the offloading of
// expensive janitorial work to a less time-critical code path.
runners/bench_runner.h
+2
View File
@@ -120,6 +120,7 @@ void bench_permutation(size_t i, uint32_t *buffer, size_t size);
BENCH_DEFINE(PCACHE_SIZE, LFS_MAX(16, PROG_SIZE) ) \
BENCH_DEFINE(FILE_BUFFER_SIZE, 16 ) \
BENCH_DEFINE(LOOKAHEAD_SIZE, 16 ) \
BENCH_DEFINE(GC_STEPS, 0 ) \
BENCH_DEFINE(GC_COMPACT_THRESH, 0 ) \
BENCH_DEFINE(INLINE_SIZE, BLOCK_SIZE/4 ) \
BENCH_DEFINE(SHRUB_SIZE, INLINE_SIZE ) \
@@ -150,6 +151,7 @@ void bench_permutation(size_t i, uint32_t *buffer, size_t size);
.pcache_size = PCACHE_SIZE, \
.file_buffer_size = FILE_BUFFER_SIZE, \
.lookahead_size = LOOKAHEAD_SIZE, \
.gc_steps = GC_STEPS, \
.gc_compact_thresh = GC_COMPACT_THRESH, \
.inline_size = INLINE_SIZE, \
.shrub_size = SHRUB_SIZE, \
runners/test_runner.h
+2
View File
@@ -105,6 +105,7 @@ void test_permutation(size_t i, uint32_t *buffer, size_t size);
TEST_DEFINE(PCACHE_SIZE, LFS_MAX(16, PROG_SIZE) ) \
TEST_DEFINE(FILE_BUFFER_SIZE, 16 ) \
TEST_DEFINE(LOOKAHEAD_SIZE, 16 ) \
TEST_DEFINE(GC_STEPS, 0 ) \
TEST_DEFINE(GC_COMPACT_THRESH, 0 ) \
TEST_DEFINE(INLINE_SIZE, BLOCK_SIZE/4 ) \
TEST_DEFINE(SHRUB_SIZE, INLINE_SIZE ) \
@@ -135,6 +136,7 @@ void test_permutation(size_t i, uint32_t *buffer, size_t size);
.pcache_size = PCACHE_SIZE, \
.file_buffer_size = FILE_BUFFER_SIZE, \
.lookahead_size = LOOKAHEAD_SIZE, \
.gc_steps = GC_STEPS, \
.gc_compact_thresh = GC_COMPACT_THRESH, \
.inline_size = INLINE_SIZE, \
.shrub_size = SHRUB_SIZE, \
tests/test_gc.toml
+862
View File
@@ -1,10 +1,866 @@
# Test GC things
# most of the GC logic is tested in test_traversal, we just test a few
# GC-specific things here
after = ['test_traversal']
# test that lookahead can make progress in isolation
[cases.test_gc_lookahead_progress]
defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000]
defines.CKMETA = [false, true]
defines.CKDATA = [false, true]
defines.SIZE = [
'FILE_BUFFER_SIZE/2',
'2*FILE_BUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create a file
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "spider",
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_close(&lfs, &file) => 0;
// expect dirty initial state or else our test doesn't work
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfs.lookahead.next > 0 || lfs.lookahead.size == 0);
// run GC until our traversal is done
while (true) {
lfsr_fs_gc(&lfs,
LFS_GC_LOOKAHEAD
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
// internal traversal done?
if (!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
break;
}
}
// we should have made progress
assert(!(lfs.lookahead.next > 0 || lfs.lookahead.size == 0));
// check the file contents
lfsr_file_open(&lfs, &file, "spider", LFS_O_RDONLY) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
# test that lookahead clobbering still works with the GC API
[cases.test_gc_lookahead_mutation]
defines.GC_STEPS = 1
defines.CKMETA = [false, true]
defines.CKDATA = [false, true]
defines.SIZE = [
'FILE_BUFFER_SIZE/2',
'2*FILE_BUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create a file
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "spider",
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_close(&lfs, &file) => 0;
// expect dirty initial state or else our test doesn't work
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfs.lookahead.next > 0 || lfs.lookahead.size == 0);
// run GC one step
lfsr_fs_gc(&lfs,
LFS_GC_LOOKAHEAD
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
assert(lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
// mutate the filesystem
lfsr_file_open(&lfs, &file, "spider",
LFS_O_WRONLY | LFS_O_TRUNC) => 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_close(&lfs, &file) => 0;
// run GC until our traversal is done
while (lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
lfsr_fs_gc(&lfs,
LFS_GC_LOOKAHEAD
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
}
// we should _not_ make progress
assert(lfs.lookahead.next > 0 || lfs.lookahead.size == 0);
// check the file contents
lfsr_file_open(&lfs, &file, "spider", LFS_O_RDONLY) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
# test that adding flags doesn't break lookahead
[cases.test_gc_lookahead_add_flags]
defines.GC_STEPS = 1
defines.CKMETA = [false, true]
defines.CKDATA = [false, true]
defines.SIZE = [
'FILE_BUFFER_SIZE/2',
'2*FILE_BUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
# we need something to keep the traversal running
if = 'CKMETA || CKDATA'
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create a file
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "spider",
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_close(&lfs, &file) => 0;
// expect dirty initial state or else our test doesn't work
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfs.lookahead.next > 0 || lfs.lookahead.size == 0);
// run GC one step
lfsr_fs_gc(&lfs,
((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
assert(lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
// change flags and run GC until our traversal is done
while (lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
lfsr_fs_gc(&lfs,
LFS_GC_LOOKAHEAD
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
}
// we should _not_ make progress
assert(lfs.lookahead.next > 0 || lfs.lookahead.size == 0);
// check the file contents
lfsr_file_open(&lfs, &file, "spider", LFS_O_RDONLY) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
# test that removing flags invalidates lookahead
[cases.test_gc_lookahead_remove_flags]
defines.GC_STEPS = 1
defines.CKMETA = [false, true]
defines.CKDATA = [false, true]
defines.SIZE = [
'FILE_BUFFER_SIZE/2',
'2*FILE_BUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
# we need something to keep the traversal running
if = 'CKMETA || CKDATA'
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create a file
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "spider",
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_close(&lfs, &file) => 0;
// expect dirty initial state or else our test doesn't work
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfs.lookahead.next > 0 || lfs.lookahead.size == 0);
// run GC one step
lfsr_fs_gc(&lfs,
LFS_GC_LOOKAHEAD
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
assert(lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
// change flags and run GC until our traversal is done
while (lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
lfsr_fs_gc(&lfs,
((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
}
// we should _not_ make progress
assert(lfs.lookahead.next > 0 || lfs.lookahead.size == 0);
// check the file contents
lfsr_file_open(&lfs, &file, "spider", LFS_O_RDONLY) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
# test that mkconsistent can make progress in isolation
[cases.test_gc_mkconsistent_progress]
defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
defines.CKMETA = [false, true]
defines.CKDATA = [false, true]
defines.SIZE = 'FILE_BUFFER_SIZE/2'
# <=2 => grm-able
# >2 => requires orphans
defines.ORPHANS = [1, 2, 3, 100]
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create two files
lfsr_file_t file1;
lfsr_file_open(&lfs, &file1, "cuttlefish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf1[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file1) => 0;
lfsr_file_t file2;
lfsr_file_open(&lfs, &file2, "octopus",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf2[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file2) => 0;
// create this many orphaned files
//
// anytime we close a not-yet-created desync file, we create an
// orphan, but note we need these to be different files, and we need
// to close them after all open calls, otherwise we just end up with
// one orphan (littlefs is eager to clean up orphans)
//
lfsr_file_t orphans[ORPHANS];
for (lfs_size_t i = 0; i < ORPHANS; i++) {
char name[256];
sprintf(name, "jellyfish%03x", i);
lfsr_file_open(&lfs, &orphans[i], name,
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0;
}
for (lfs_size_t i = 0; i < ORPHANS; i++) {
lfsr_file_close(&lfs, &orphans[i]) => 0;
}
// expect dirty initial state or else our test doesn't work
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans);
// run GC until our traversal is done
while (true) {
lfsr_fs_gc(&lfs,
LFS_GC_MKCONSISTENT
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_GC_COMPACT : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
// internal traversal done?
if (!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
break;
}
}
// we should have made progress
assert(!(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans));
// check we can still read the files
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file1) => 0;
lfsr_file_close(&lfs, &file2) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file1, "cuttlefish", LFS_O_RDONLY) => 0;
lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file1) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf1, SIZE) == 0);
lfsr_file_rewind(&lfs, &file2) => 0;
lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf2, SIZE) == 0);
}
lfsr_file_close(&lfs, &file1) => 0;
lfsr_file_close(&lfs, &file2) => 0;
lfsr_unmount(&lfs) => 0;
'''
# test that mkconsistent clobbering still works with the GC API
[cases.test_gc_mkconsistent_mutation]
defines.GC_STEPS = 1
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
defines.CKMETA = [false, true]
defines.CKDATA = [false, true]
defines.SIZE = 'FILE_BUFFER_SIZE/2'
# <=2 => grm-able
# >2 => requires orphans
defines.ORPHANS = [3, 100]
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create two files
lfsr_file_t file1;
lfsr_file_open(&lfs, &file1, "cuttlefish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf1[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file1) => 0;
lfsr_file_t file2;
lfsr_file_open(&lfs, &file2, "octopus",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf2[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file2) => 0;
// create at least 3 orphans so GC will start
lfsr_file_t orphans[ORPHANS];
for (lfs_size_t i = 0; i < 3; i++) {
char name[256];
sprintf(name, "jellyfish%03x", i);
lfsr_file_open(&lfs, &orphans[i], name,
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0;
}
for (lfs_size_t i = 0; i < 3; i++) {
lfsr_file_close(&lfs, &orphans[i]) => 0;
}
// run GC one step
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
lfsr_fs_gc(&lfs,
LFS_GC_MKCONSISTENT
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_GC_COMPACT : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
assert(lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
// create the rest of the orphans after GC has started
for (lfs_size_t i = 0; i < ORPHANS; i++) {
char name[256];
sprintf(name, "jellyfish%03x", i);
lfsr_file_open(&lfs, &orphans[i], name,
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0;
}
for (lfs_size_t i = 0; i < ORPHANS; i++) {
lfsr_file_close(&lfs, &orphans[i]) => 0;
}
// we should now have dirty state
assert(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans);
// run GC until our traversal is done
while (lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
lfsr_fs_gc(&lfs,
LFS_GC_MKCONSISTENT
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_GC_COMPACT : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
}
// we should _not_ make progress
assert(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans);
// check we can still read the files
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file1) => 0;
lfsr_file_close(&lfs, &file2) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file1, "cuttlefish", LFS_O_RDONLY) => 0;
lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file1) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf1, SIZE) == 0);
lfsr_file_rewind(&lfs, &file2) => 0;
lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf2, SIZE) == 0);
}
lfsr_file_close(&lfs, &file1) => 0;
lfsr_file_close(&lfs, &file2) => 0;
lfsr_unmount(&lfs) => 0;
'''
# test that adding flags doesn't break mkconsistent
[cases.test_gc_mkconsistent_add_flags]
defines.GC_STEPS = 1
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
defines.CKMETA = [false, true]
defines.CKDATA = [false, true]
defines.SIZE = 'FILE_BUFFER_SIZE/2'
# <=2 => grm-able
# >2 => requires orphans
defines.ORPHANS = [3, 100]
# we need something to keep the traversal running
if = 'COMPACT || CKMETA || CKDATA'
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create two files
lfsr_file_t file1;
lfsr_file_open(&lfs, &file1, "cuttlefish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf1[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file1) => 0;
lfsr_file_t file2;
lfsr_file_open(&lfs, &file2, "octopus",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf2[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file2) => 0;
// create this many orphaned files
//
// anytime we close a not-yet-created desync file, we create an
// orphan, but note we need these to be different files, and we need
// to close them after all open calls, otherwise we just end up with
// one orphan (littlefs is eager to clean up orphans)
//
lfsr_file_t orphans[ORPHANS];
for (lfs_size_t i = 0; i < ORPHANS; i++) {
char name[256];
sprintf(name, "jellyfish%03x", i);
lfsr_file_open(&lfs, &orphans[i], name,
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0;
}
for (lfs_size_t i = 0; i < ORPHANS; i++) {
lfsr_file_close(&lfs, &orphans[i]) => 0;
}
// expect dirty initial state or else our test doesn't work
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans);
// run GC one step
lfsr_fs_gc(&lfs,
((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_GC_COMPACT : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
assert(lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
// change flags and run GC until our traversal is done
while (lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
lfsr_fs_gc(&lfs,
LFS_GC_MKCONSISTENT
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_GC_COMPACT : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
}
// we should _not_ make progress
assert(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans);
// check we can still read the files
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file1) => 0;
lfsr_file_close(&lfs, &file2) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file1, "cuttlefish", LFS_O_RDONLY) => 0;
lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file1) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf1, SIZE) == 0);
lfsr_file_rewind(&lfs, &file2) => 0;
lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf2, SIZE) == 0);
}
lfsr_file_close(&lfs, &file1) => 0;
lfsr_file_close(&lfs, &file2) => 0;
lfsr_unmount(&lfs) => 0;
'''
# test that removing flags invalidates mkconsistent
[cases.test_gc_mkconsistent_remove_flags]
defines.GC_STEPS = 1
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
defines.CKMETA = [false, true]
defines.CKDATA = [false, true]
defines.SIZE = 'FILE_BUFFER_SIZE/2'
# <=2 => grm-able
# >2 => requires orphans
defines.ORPHANS = [3, 100]
# we need something to keep the traversal running
if = 'COMPACT || CKMETA || CKDATA'
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create two files
lfsr_file_t file1;
lfsr_file_open(&lfs, &file1, "cuttlefish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf1[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file1) => 0;
lfsr_file_t file2;
lfsr_file_open(&lfs, &file2, "octopus",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf2[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file2) => 0;
// create this many orphaned files
//
// anytime we close a not-yet-created desync file, we create an
// orphan, but note we need these to be different files, and we need
// to close them after all open calls, otherwise we just end up with
// one orphan (littlefs is eager to clean up orphans)
//
lfsr_file_t orphans[ORPHANS];
for (lfs_size_t i = 0; i < ORPHANS; i++) {
char name[256];
sprintf(name, "jellyfish%03x", i);
lfsr_file_open(&lfs, &orphans[i], name,
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL | LFS_O_DESYNC) => 0;
}
for (lfs_size_t i = 0; i < ORPHANS; i++) {
lfsr_file_close(&lfs, &orphans[i]) => 0;
}
// expect dirty initial state or else our test doesn't work
assert(!lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
assert(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans);
// run GC one step
lfsr_fs_gc(&lfs,
LFS_GC_MKCONSISTENT
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_GC_COMPACT : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
assert(lfsr_omdir_isopen(&lfs, &lfs.gc.o.o));
// change flags and run GC until our traversal is done
while (lfsr_omdir_isopen(&lfs, &lfs.gc.o.o)) {
lfsr_fs_gc(&lfs,
((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_GC_COMPACT : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
}
// we should _not_ make progress
assert(lfsr_grm_count(&lfs) > 0 || lfs.hasorphans);
// check we can still read the files
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file1) => 0;
lfsr_file_close(&lfs, &file2) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file1, "cuttlefish", LFS_O_RDONLY) => 0;
lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file1) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf1, SIZE) == 0);
lfsr_file_rewind(&lfs, &file2) => 0;
lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf2, SIZE) == 0);
}
lfsr_file_close(&lfs, &file1) => 0;
lfsr_file_close(&lfs, &file2) => 0;
lfsr_unmount(&lfs) => 0;
'''
# pseudo-fuzz test that clobbering still works with the GC API
[cases.test_gc_mutation]
defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000]
defines.STEPS = 100
defines.MKCONSISTENT = [false, true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
defines.CKMETA = [false, true]
defines.CKDATA = [false, true]
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
defines.SIZE = [
'FILE_BUFFER_SIZE/2',
'2*FILE_BUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create a file
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "spider",
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_close(&lfs, &file) => 0;
for (uint32_t i = 0; i < STEPS; i++) {
// rewrite the file every gc cycle
lfsr_file_open(&lfs, &file, "spider",
LFS_O_WRONLY | LFS_O_TRUNC) => 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_close(&lfs, &file) => 0;
// gc!
lfsr_fs_gc(&lfs,
((MKCONSISTENT) ? LFS_GC_MKCONSISTENT : 0)
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_GC_COMPACT : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)) => 0;
}
// check the file contents
lfsr_file_open(&lfs, &file, "spider", LFS_O_RDONLY) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
# pseudo-fuzz test that adding/removing flags doesn't break anything
[cases.test_gc_changing_flags]
defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000]
defines.STEPS = 100
defines.MKCONSISTENT = [false, true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
defines.CKMETA = [false, true]
defines.CKDATA = [false, true]
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
defines.SIZE = [
'FILE_BUFFER_SIZE/2',
'2*FILE_BUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create a file
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "spider",
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_close(&lfs, &file) => 0;
for (uint32_t i = 0; i < STEPS; i++) {
// rewrite the file every gc cycle
lfsr_file_open(&lfs, &file, "spider",
LFS_O_WRONLY | LFS_O_TRUNC) => 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_close(&lfs, &file) => 0;
// choose a new subset of flags every cycle
uint32_t flags = (
((MKCONSISTENT) ? LFS_GC_MKCONSISTENT : 0)
| ((LOOKAHEAD) ? LFS_GC_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_GC_COMPACT : 0)
| ((CKMETA) ? LFS_GC_CKMETA : 0)
| ((CKDATA) ? LFS_GC_CKDATA : 0)
) & TEST_PRNG(&prng);
// gc!
lfsr_fs_gc(&lfs, flags) => 0;
}
// check the file contents
lfsr_file_open(&lfs, &file, "spider", LFS_O_RDONLY) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
# many/fuzz tests mixed with GC
#
[cases.test_gc_spam_dir_many]
defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000]
defines.MKCONSISTENT = [false, true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
@@ -99,6 +955,7 @@ code = '''
'''
[cases.test_gc_spam_dir_fuzz]
defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000]
defines.MKCONSISTENT = [false, true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
@@ -264,6 +1121,7 @@ code = '''
'''
[cases.test_gc_spam_file_many]
defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000]
defines.MKCONSISTENT = [false, true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
@@ -352,6 +1210,7 @@ code = '''
'''
[cases.test_gc_spam_file_fuzz]
defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000]
defines.MKCONSISTENT = [false, true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
@@ -579,6 +1438,7 @@ code = '''
'''
[cases.test_gc_spam_fwrite_fuzz]
defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000]
defines.MKCONSISTENT = [false, true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
@@ -726,6 +1586,7 @@ code = '''
'''
[cases.test_gc_spam_orphanzombie_fuzz]
defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000]
defines.MKCONSISTENT = [false, true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
@@ -1081,6 +1942,7 @@ code = '''
'''
[cases.test_gc_spam_orphanzombiedir_fuzz]
defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000]
defines.MKCONSISTENT = [false, true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]