t: (Re)implemented gc_compact_thresh, at least over mdirs

lfs_fs_gc is still not reimplemented, but this is accessible through the
traversal API with LFS_T_COMPACT.

This is also the first traversal operation that can mutate the
filesystem, which brings its own set of problems:

- We need to set LFS_F_DIRTY in lfsr_mtree_gc now, which really
  highlights how much of a mess having two flag fields is...

  We do _not_ clobber in this case, since we assume lfsr_mtree_gc knows
  what it's doing.

- We can now commit to an mroot in the mroot chain outside of the normal
  mroot chain update logic.

  This is a bit scary, but should just work.

  The only issue so far is that we need to allow mdirs to follow the
  mroot during mroot splits if mid=-1, even if they aren't lfs_t's mroot
  mdir.

  This should now be decently tested with the new
  test_traversal_compact_* tests.

- It's easy for mtraversal's mdir and mtinfo's mdir to fall out of sync
  when mutating... Why do we have two of these?

The actual compaction itself is pretty straightforward: just mark as
unerased, eoff=-1, and call lfsr_mdir_commit with an empty commit. This
is now wrapped up in lfsr_mdir_compact.

Code changes:

           code          stack
  before: 34528           2640
  after:  34652 (+0.4%)   2640 (+0.0%)

Though the real hard part will be implementing gc_compact_thresh over
btree nodes...
This commit is contained in:
Christopher Haster
2024-06-24 16:21:38 -05:00
parent ff0271ecbe
commit 4d06fc2e0e
5 changed files with 875 additions and 31 deletions
+761 -10
View File
@@ -12,6 +12,7 @@ after = [
defines.CKMETA = [false, true]
defines.CK = [false, true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
@@ -22,7 +23,8 @@ code = '''
lfsr_traversal_open(&lfs, &t,
((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_T_COMPACT : 0)) => 0;
struct lfs_tinfo tinfo;
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
@@ -41,6 +43,7 @@ code = '''
defines.CKMETA = [false, true]
defines.CK = [false, true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
@@ -51,7 +54,8 @@ code = '''
lfsr_traversal_open(&lfs, &t,
((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_T_COMPACT : 0)) => 0;
struct lfs_tinfo tinfo;
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
@@ -79,6 +83,7 @@ code = '''
defines.CKMETA = [false, true]
defines.CK = [false, true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
@@ -89,7 +94,8 @@ code = '''
lfsr_traversal_open(&lfs, &t,
((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_T_COMPACT : 0)) => 0;
struct lfs_tinfo tinfo;
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
@@ -5039,6 +5045,737 @@ code = '''
# test traversals with mdir compaction
[cases.test_traversal_compact]
defines.CKMETA = [false, true]
defines.CK = [false, true]
defines.LOOKAHEAD = [false, true]
defines.SIZE = 'FILE_BUFFER_SIZE/2'
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// write to our mdir until >gc_compact_thresh full
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "jellyfish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
// hack, don't use the internals like this
uint8_t wbuf[SIZE];
while ((file.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
lfsr_file_rewind(&lfs, &file) => 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file) => 0;
}
// try traversing and compacting
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t,
LFS_T_COMPACT
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
// traverse mroot
struct lfs_tinfo tinfo;
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
lfsr_traversal_close(&lfs, &t) => 0;
// mdir should have been compacted
assert((file.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
// check we can still read the file
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
}
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
# yes, compactions should return busy if excl
[cases.test_traversal_compact_excl]
defines.CKMETA = [false, true]
defines.CK = [false, true]
defines.LOOKAHEAD = [false, true]
defines.SIZE = 'FILE_BUFFER_SIZE/2'
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// write to our mdir until >gc_compact_thresh full
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "jellyfish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
// hack, don't use the internals like this
uint8_t wbuf[SIZE];
while ((file.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
lfsr_file_rewind(&lfs, &file) => 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file) => 0;
}
// try traversing and compacting
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t,
LFS_T_COMPACT
| LFS_T_EXCL
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
// traverse mroot
struct lfs_tinfo tinfo;
// compact, return bust
lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_BUSY;
lfsr_traversal_close(&lfs, &t) => 0;
// mdir should have been compacted
assert((file.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
// check we can still read the file
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
}
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
[cases.test_traversal_compact_mrootchain]
defines.CKMETA = [false, true]
defines.CK = [false, true]
defines.LOOKAHEAD = [false, true]
defines.SIZE = 'FILE_BUFFER_SIZE/2'
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
# force early relocations
defines.BLOCK_RECYCLES = 0
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// write to our mdir until mroot extends
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "jellyfish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf[SIZE];
while (lfs.mroot.rbyd.blocks[0] == 0
|| lfs.mroot.rbyd.blocks[0] == 1) {
lfsr_file_rewind(&lfs, &file) => 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file) => 0;
}
// now write to our mdir until mrootanchor >gc_compact_thresh full
while (true) {
// we need internals to check this
lfsr_mdir_t mrootanchor;
lfsr_mdir_fetch(&lfs, &mrootanchor,
-1, &LFSR_MPTR_MROOTANCHOR()) => 0;
if (lfsr_rbyd_eoff(&mrootanchor.rbyd) > GC_COMPACT_THRESH) {
break;
}
lfsr_file_rewind(&lfs, &file) => 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file) => 0;
}
// try traversing and compacting
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t,
LFS_T_COMPACT
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
// traverse mrootanchor
struct lfs_tinfo tinfo;
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
// traverse mroot
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
lfsr_traversal_close(&lfs, &t) => 0;
// mrootanchor should have been compacted
lfsr_mdir_t mrootanchor;
lfsr_mdir_fetch(&lfs, &mrootanchor,
-1, &LFSR_MPTR_MROOTANCHOR()) => 0;
assert(lfsr_rbyd_eoff(&mrootanchor.rbyd) <= GC_COMPACT_THRESH);
// check we can still read the file
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "jellyfish", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf, SIZE) == 0);
}
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
[cases.test_traversal_compact_mroot_split]
defines.CKMETA = [false, true]
defines.CK = [false, true]
defines.LOOKAHEAD = [false, true]
defines.SIZE = 'FILE_BUFFER_SIZE/2'
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create two files
lfsr_file_t file1;
lfsr_file_open(&lfs, &file1, "jellyfish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf1[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file1) => 0;
lfsr_file_t file2;
lfsr_file_open(&lfs, &file2, "octopus",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf2[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file2) => 0;
// create enough files to both compact and split
lfs_size_t i = 0;
while (true) {
// we need internals to check this
lfs_ssize_t estimate = lfsr_mdir_estimate__(&lfs,
&file1.o.mdir, -1, -1,
NULL);
assert(estimate >= 0);
if ((file1.o.mdir.rbyd.eoff & 0x7fffffff) > GC_COMPACT_THRESH
&& estimate > BLOCK_SIZE/2) {
break;
}
char name[256];
sprintf(name, "medusaaaaaaaaaaaaaaaaaaaaaaaaaaaa%03x", i);
lfsr_file_t file;
lfsr_file_open(&lfs, &file, name,
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
lfsr_file_close(&lfs, &file) => 0;
i += 1;
}
// try traversing and compacting
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t,
LFS_T_COMPACT
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
// traverse mroot
struct lfs_tinfo tinfo;
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
// should have split, traverse mtree
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_BTREE);
// traverse mdir
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
// traverse mdir
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
lfsr_traversal_close(&lfs, &t) => 0;
// mdirs should have been compacted
assert((file1.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
assert((file2.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
// check we can still read the files
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file1) => 0;
lfsr_file_close(&lfs, &file2) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file1, "jellyfish", LFS_O_RDONLY) => 0;
lfsr_file_open(&lfs, &file2, "octopus", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file1) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf1, SIZE) == 0);
lfsr_file_rewind(&lfs, &file2) => 0;
lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf2, SIZE) == 0);
}
lfsr_file_close(&lfs, &file1) => 0;
lfsr_file_close(&lfs, &file2) => 0;
lfsr_unmount(&lfs) => 0;
'''
[cases.test_traversal_compact_mtree]
defines.CKMETA = [false, true]
defines.CK = [false, true]
defines.LOOKAHEAD = [false, true]
defines.SIZE = 'FILE_BUFFER_SIZE/2'
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
defines.COMPACTSET = 'range(0x7)'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create three files
lfsr_file_t file1;
lfsr_file_open(&lfs, &file1, "cuttlefish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf1[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file1) => 0;
lfsr_file_t file2;
lfsr_file_open(&lfs, &file2, "jellyfish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf2[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file2) => 0;
lfsr_file_t file3;
lfsr_file_open(&lfs, &file3, "octopus",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf3[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf3[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file3, wbuf3, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file3) => 0;
// create enough files for mroot to split twice
lfs_size_t i = 0;
while (lfs.mtree.u.weight == 0x80000000) {
char name[256];
sprintf(name, "hydroid%03x", i);
lfsr_file_t file;
lfsr_file_open(&lfs, &file, name,
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
lfsr_file_close(&lfs, &file) => 0;
i += 1;
}
i = 0;
lfs_size_t orig = lfs.mtree.u.weight;
while (lfs.mtree.u.weight == orig) {
char name[256];
sprintf(name, "medusa%03x", i);
lfsr_file_t file;
lfsr_file_open(&lfs, &file, name,
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
lfsr_file_close(&lfs, &file) => 0;
i += 1;
}
// write to each file until mdir >gc_compact_thresh full
if (COMPACTSET & 0x1) {
// hack, don't use the internals like this
while ((file1.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
lfsr_file_rewind(&lfs, &file1) => 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file1) => 0;
}
}
if (COMPACTSET & 0x2) {
// hack, don't use the internals like this
while ((file2.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
lfsr_file_rewind(&lfs, &file2) => 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file2) => 0;
}
}
if (COMPACTSET & 0x4) {
// hack, don't use the internals like this
while ((file3.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH) {
lfsr_file_rewind(&lfs, &file3) => 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf3[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file3, wbuf3, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file3) => 0;
}
}
// try traversing and compacting
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t,
LFS_T_COMPACT
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
// traverse mroot
struct lfs_tinfo tinfo;
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
// traverse mtree
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_BTREE);
// traverse mdir
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
// traverse mdir
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
// traverse mdir
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
lfsr_traversal_close(&lfs, &t) => 0;
// mdirs should have been compacted
assert((file1.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
assert((file2.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
assert((file3.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
// check we can still read the files
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file1) => 0;
lfsr_file_close(&lfs, &file2) => 0;
lfsr_file_close(&lfs, &file3) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file1, "cuttlefish", LFS_O_RDONLY) => 0;
lfsr_file_open(&lfs, &file2, "jellyfish", LFS_O_RDONLY) => 0;
lfsr_file_open(&lfs, &file3, "octopus", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file1) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf1, SIZE) == 0);
lfsr_file_rewind(&lfs, &file2) => 0;
lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf2, SIZE) == 0);
lfsr_file_rewind(&lfs, &file3) => 0;
lfsr_file_read(&lfs, &file3, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf3, SIZE) == 0);
}
lfsr_file_close(&lfs, &file1) => 0;
lfsr_file_close(&lfs, &file2) => 0;
lfsr_file_close(&lfs, &file3) => 0;
lfsr_unmount(&lfs) => 0;
'''
[cases.test_traversal_compact_mtree_split]
defines.CKMETA = [false, true]
defines.CK = [false, true]
defines.LOOKAHEAD = [false, true]
defines.SIZE = 'FILE_BUFFER_SIZE/2'
# set compact thresh to minimum
defines.GC_COMPACT_THRESH = 'BLOCK_SIZE/2'
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
uint32_t prng = 42;
// create four files
lfsr_file_t file1;
lfsr_file_open(&lfs, &file1, "cuttlefish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf1[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf1[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file1, wbuf1, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file1) => 0;
lfsr_file_t file2;
lfsr_file_open(&lfs, &file2, "jellyfish",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf2[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf2[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file2, wbuf2, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file2) => 0;
lfsr_file_t file3;
lfsr_file_open(&lfs, &file3, "octopus",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf3[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf3[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file3, wbuf3, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file3) => 0;
lfsr_file_t file4;
lfsr_file_open(&lfs, &file4, "squid",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf4[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf4[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file4, wbuf4, SIZE) => SIZE;
lfsr_file_sync(&lfs, &file4) => 0;
// create enough files for mroot to split twice
lfs_size_t i = 0;
while (lfs.mtree.u.weight == 0x80000000) {
char name[256];
sprintf(name, "hydroid%03x", i);
lfsr_file_t file;
lfsr_file_open(&lfs, &file, name,
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
lfsr_file_close(&lfs, &file) => 0;
i += 1;
}
i = 0;
lfs_size_t orig = lfs.mtree.u.weight;
while (lfs.mtree.u.weight == orig) {
char name[256];
sprintf(name, "polyp%03x", i);
lfsr_file_t file;
lfsr_file_open(&lfs, &file, name,
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
lfsr_file_close(&lfs, &file) => 0;
i += 1;
}
// create enough files to both compact and split
i = 0;
while (true) {
// we need internals to check this
lfs_ssize_t estimate = lfsr_mdir_estimate__(&lfs,
&file2.o.mdir, -1, -1,
NULL);
assert(estimate >= 0);
if ((file2.o.mdir.rbyd.eoff & 0x7fffffff) > GC_COMPACT_THRESH
&& estimate > BLOCK_SIZE/2) {
break;
}
char name[256];
sprintf(name, "medusaaaaaaaaaaaaaaaaaaaaaaaaaaaa%03x", i);
lfsr_file_t file;
lfsr_file_open(&lfs, &file, name,
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
lfsr_file_close(&lfs, &file) => 0;
i += 1;
}
// try traversing and compacting
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t,
LFS_T_COMPACT
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
// traverse mroot
struct lfs_tinfo tinfo;
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
assert(tinfo.block == 0 || tinfo.block == 1);
// traverse mtree
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_BTREE);
// traverse mdir
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
// traverse mdir
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
// should have split, traverse mdir
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
// traverse mdir
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
assert(tinfo.btype == LFS_BTYPE_MDIR);
lfsr_traversal_close(&lfs, &t) => 0;
// mdirs should have been compacted
assert((file1.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
assert((file2.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
assert((file3.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
assert((file4.o.mdir.rbyd.eoff & 0x7fffffff) <= GC_COMPACT_THRESH);
// check we can still read the files
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_file_close(&lfs, &file1) => 0;
lfsr_file_close(&lfs, &file2) => 0;
lfsr_file_close(&lfs, &file3) => 0;
lfsr_file_close(&lfs, &file4) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file1, "cuttlefish", LFS_O_RDONLY) => 0;
lfsr_file_open(&lfs, &file2, "jellyfish", LFS_O_RDONLY) => 0;
lfsr_file_open(&lfs, &file3, "octopus", LFS_O_RDONLY) => 0;
lfsr_file_open(&lfs, &file4, "squid", LFS_O_RDONLY) => 0;
}
lfsr_file_rewind(&lfs, &file1) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file1, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf1, SIZE) == 0);
lfsr_file_rewind(&lfs, &file2) => 0;
lfsr_file_read(&lfs, &file2, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf2, SIZE) == 0);
lfsr_file_rewind(&lfs, &file3) => 0;
lfsr_file_read(&lfs, &file3, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf3, SIZE) == 0);
lfsr_file_rewind(&lfs, &file4) => 0;
lfsr_file_read(&lfs, &file4, rbuf, SIZE) => SIZE;
assert(memcmp(rbuf, wbuf4, SIZE) == 0);
}
lfsr_file_close(&lfs, &file1) => 0;
lfsr_file_close(&lfs, &file2) => 0;
lfsr_file_close(&lfs, &file3) => 0;
lfsr_file_close(&lfs, &file4) => 0;
lfsr_unmount(&lfs) => 0;
'''
# many/fuzz tests mixed with traversals
#
# these should hopefully test a bunch of messy traversal state
@@ -5051,6 +5788,7 @@ defines.EXCL = [false, true]
defines.CKMETA = [true]
defines.CK = [true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256]
code = '''
// test creating directories
@@ -5064,7 +5802,8 @@ code = '''
((EXCL) ? LFS_T_EXCL : 0)
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_T_COMPACT : 0)) => 0;
// make this many directories
for (lfs_size_t i = 0; i < N; i++) {
@@ -5157,6 +5896,7 @@ defines.EXCL = [false, true]
defines.CKMETA = [true]
defines.CK = [true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256]
defines.OPS = '2*N'
defines.SEED = 42
@@ -5177,7 +5917,8 @@ code = '''
((EXCL) ? LFS_T_EXCL : 0)
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_T_COMPACT : 0)) => 0;
uint32_t prng = SEED;
for (lfs_size_t i = 0; i < OPS; i++) {
@@ -5334,6 +6075,7 @@ defines.EXCL = [false, true]
defines.CKMETA = [true]
defines.CK = [true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
defines.N = [1, 2, 4, 8, 16, 32, 64]
defines.SIZE = [
'0',
@@ -5357,7 +6099,8 @@ code = '''
((EXCL) ? LFS_T_EXCL : 0)
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_T_COMPACT : 0)) => 0;
// create this many files
uint32_t prng = 42;
@@ -5434,6 +6177,7 @@ defines.EXCL = [false, true]
defines.CKMETA = [true]
defines.CK = [true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
defines.N = [1, 2, 4, 8, 16, 32, 64]
defines.OPS = '2*N'
defines.SIZE = [
@@ -5465,7 +6209,8 @@ code = '''
((EXCL) ? LFS_T_EXCL : 0)
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_T_COMPACT : 0)) => 0;
uint32_t prng = SEED;
for (lfs_size_t i = 0; i < OPS; i++) {
@@ -5673,6 +6418,7 @@ defines.EXCL = [false, true]
defines.CKMETA = [true]
defines.CK = [true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
defines.OPS = 20
defines.SIZE = [
'FILE_BUFFER_SIZE/2',
@@ -5736,7 +6482,8 @@ code = '''
((EXCL) ? LFS_T_EXCL : 0)
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_T_COMPACT : 0)) => 0;
for (lfs_size_t i = 0; i < OPS; i++) {
// choose a random location
@@ -5832,6 +6579,7 @@ defines.EXCL = [false, true]
defines.CKMETA = [true]
defines.CK = [true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
defines.N = [1, 2, 4, 8, 16, 32, 64]
defines.OPS = '2*N'
defines.SIZE = [
@@ -5873,7 +6621,8 @@ code = '''
((EXCL) ? LFS_T_EXCL : 0)
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_T_COMPACT : 0)) => 0;
uint32_t prng = SEED;
for (lfs_size_t i = 0; i < OPS; i++) {
@@ -6199,6 +6948,7 @@ defines.EXCL = [false, true]
defines.CKMETA = [true]
defines.CK = [true]
defines.LOOKAHEAD = [false, true]
defines.COMPACT = [false, true]
defines.N = [1, 2, 4, 8, 16, 32, 64]
defines.OPS = '2*N'
defines.SIZE = [
@@ -6241,7 +6991,8 @@ code = '''
((EXCL) ? LFS_T_EXCL : 0)
| ((CKMETA) ? LFS_T_CKMETA : 0)
| ((CK) ? LFS_T_CK : 0)
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)) => 0;
| ((LOOKAHEAD) ? LFS_T_LOOKAHEAD : 0)
| ((COMPACT) ? LFS_T_COMPACT : 0)) => 0;
uint32_t prng = SEED;
for (lfs_size_t i = 0; i < OPS; i++) {