Github/littlefs
1
Fork 0
Code Actions Activity

t: Implemented rudimentary lfsr_traversal_t and related functions

Browse Source 
This adds the lfsr_traversal_t object, which encapsulates a traversal
over all blocks in the filesystem.

This replaces the earlier lfs_fs_traverse function, but is sort of
"inside-out" in that instead of taking a callback, an lfsr_traversal_t
object can be read from to return lfs_tinfo structs that describe the
blocks in our system:

  lfsr_traversal_open(&lfs, &t) => 0;
  lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
  tinfo.btype => LFS_BTYPE_MDIR;
  tinfo.block => 0x0;
  lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
  tinfo.btype => LFS_BTYPE_MDIR;
  tinfo.block => 0x1;
  lfsr_traversal_read(&lfs, &t, &tinfo) => 0;
  tinfo.btype => LFS_BTYPE_DATA;
  tinfo.block => 0x42;
  lfsr_traversal_read(&lfs, &t, &tinfo) => LFS_ERR_NOENT;
  lfsr_traversal_close(&lfs, &t) => 0;

This is more flexible, allowing for aborted traversals, yielding,
rewinding, etc, but also more complicated to implement, since it
requires all traversal state to be stored explicitly.

Fortunately, since we needed to reimplement filesystem traversals
anyways, I was able to build this into the new system from the start
using a small state machine to drive the traversal internally. So all
that was really needed was a bit of window dressing, adding
LFS_TYPE_TRAVERSAL to track open traversals, logic to handle
invalidating traversals on file close, mutation, etc...

Which, uh, that last one is not implemented yet. Interactions with other
filesystem operations gets messy, so I figured I'd go ahead and commit
what is currently working.

Ugh, and tests. The biggest downside of adding lfsr_traversal_t is how
many more corner-cases it adds to the system...

lfsr_traversal_t is going to be a work-in-progress for a bit...

---

lfsr_traversal_t also adds a really interesting path towards more access
to advanced low-level operations, such as checking metadata/data
checksums, incrementally progressing the garbage collector, even
repairing bad metadata/data blocks eventually.

Currently implemented is LFS_T_CKMETADATA and LFS_T_CKDATA to check
metadata and data checksums respectively. This is the first feature that
actually allows you to validate data checksums.

Code changes so far:

           code          stack
  before: 33886           2560
  after:  34226 (+1.0%)   2560 (+0.0%)
This commit is contained in:
Christopher Haster
2024-06-14 01:53:31 -05:00
parent 74d382b48f
commit 670b9fbf99
6 changed files with 2526 additions and 580 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_alloc.toml
+67 -69
View File
@@ -108,7 +108,7 @@ code = '''
# clobber tests test that our traversal algorithm works
[cases.test_alloc_clobber_dirs]
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
defines.VALIDATE = [false, true]
defines.CKMETADATA = [false, true]
defines.REMOUNT = [false, true]
in = 'lfs.c'
code = '''
@@ -166,43 +166,43 @@ code = '''
uint8_t *seen = malloc((BLOCK_COUNT+7)/8);
memset(seen, 0, (BLOCK_COUNT+7)/8);
lfsr_traversal_t traversal = LFSR_TRAVERSAL(
(VALIDATE) ? LFSR_TRAVERSAL_VALIDATE : 0);
lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(
(CKMETADATA) ? LFS_T_CKMETADATA : 0);
for (lfs_block_t i = 0;; i++) {
// a bit hacky, but this catches infinite loops
assert(i < 2*BLOCK_COUNT);
lfsr_tinfo_t tinfo;
int err = lfsr_traversal_read(&lfs, &traversal, &tinfo);
lfsr_mtinfo_t mtinfo;
int err = lfsr_fs_traverse(&lfs, &mt, &mtinfo);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
if (tinfo.tag == LFSR_TAG_MDIR) {
if (mtinfo.tag == LFSR_TAG_MDIR) {
printf("traversal: 0x%x mdir 0x{%x,%x}\n",
tinfo.tag,
tinfo.u.mdir.rbyd.blocks[0],
tinfo.u.mdir.rbyd.blocks[1]);
mtinfo.tag,
mtinfo.u.mdir.rbyd.blocks[0],
mtinfo.u.mdir.rbyd.blocks[1]);
// keep track of seen blocks
seen[tinfo.u.mdir.rbyd.blocks[1] / 8]
|= 1 << (tinfo.u.mdir.rbyd.blocks[1] % 8);
seen[tinfo.u.mdir.rbyd.blocks[0] / 8]
|= 1 << (tinfo.u.mdir.rbyd.blocks[0] % 8);
seen[mtinfo.u.mdir.rbyd.blocks[1] / 8]
|= 1 << (mtinfo.u.mdir.rbyd.blocks[1] % 8);
seen[mtinfo.u.mdir.rbyd.blocks[0] / 8]
|= 1 << (mtinfo.u.mdir.rbyd.blocks[0] % 8);
} else if (tinfo.tag == LFSR_TAG_BRANCH) {
} else if (mtinfo.tag == LFSR_TAG_BRANCH) {
printf("traversal: 0x%x btree 0x%x.%x\n",
tinfo.tag,
tinfo.u.rbyd.blocks[0], tinfo.u.rbyd.trunk);
mtinfo.tag,
mtinfo.u.rbyd.blocks[0], mtinfo.u.rbyd.trunk);
// keep track of seen blocks
seen[tinfo.u.rbyd.blocks[0] / 8]
|= 1 << (tinfo.u.rbyd.blocks[0] % 8);
seen[mtinfo.u.rbyd.blocks[0] / 8]
|= 1 << (mtinfo.u.rbyd.blocks[0] % 8);
} else {
// this shouldn't happen
printf("traversal: 0x%x\n", tinfo.tag);
printf("traversal: 0x%x\n", mtinfo.tag);
assert(false);
}
}
@@ -271,7 +271,7 @@ defines.SIZE = [
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
defines.VALIDATE = [false, true]
defines.CKMETADATA = [false, true]
defines.REMOUNT = [false, true]
in = 'lfs.c'
if = '(SIZE*N)/BLOCK_SIZE <= 32'
@@ -334,53 +334,52 @@ code = '''
uint8_t *seen = malloc((BLOCK_COUNT+7)/8);
memset(seen, 0, (BLOCK_COUNT+7)/8);
lfsr_traversal_t traversal = LFSR_TRAVERSAL(
((VALIDATE) ? LFSR_TRAVERSAL_VALIDATE : 0)
| LFSR_TRAVERSAL_ALL);
lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(
((CKMETADATA) ? LFS_T_CKMETADATA : 0));
for (lfs_block_t i = 0;; i++) {
// a bit hacky, but this catches infinite loops
assert(i < 2*BLOCK_COUNT);
lfsr_tinfo_t tinfo;
int err = lfsr_traversal_read(&lfs, &traversal, &tinfo);
lfsr_mtinfo_t mtinfo;
int err = lfsr_fs_traverse(&lfs, &mt, &mtinfo);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
if (tinfo.tag == LFSR_TAG_MDIR) {
if (mtinfo.tag == LFSR_TAG_MDIR) {
printf("traversal: 0x%x mdir 0x{%x,%x}\n",
tinfo.tag,
tinfo.u.mdir.rbyd.blocks[0],
tinfo.u.mdir.rbyd.blocks[1]);
mtinfo.tag,
mtinfo.u.mdir.rbyd.blocks[0],
mtinfo.u.mdir.rbyd.blocks[1]);
// keep track of seen blocks
seen[tinfo.u.mdir.rbyd.blocks[1] / 8]
|= 1 << (tinfo.u.mdir.rbyd.blocks[1] % 8);
seen[tinfo.u.mdir.rbyd.blocks[0] / 8]
|= 1 << (tinfo.u.mdir.rbyd.blocks[0] % 8);
seen[mtinfo.u.mdir.rbyd.blocks[1] / 8]
|= 1 << (mtinfo.u.mdir.rbyd.blocks[1] % 8);
seen[mtinfo.u.mdir.rbyd.blocks[0] / 8]
|= 1 << (mtinfo.u.mdir.rbyd.blocks[0] % 8);
} else if (tinfo.tag == LFSR_TAG_BRANCH) {
} else if (mtinfo.tag == LFSR_TAG_BRANCH) {
printf("traversal: 0x%x btree 0x%x.%x\n",
tinfo.tag,
tinfo.u.rbyd.blocks[0], tinfo.u.rbyd.trunk);
mtinfo.tag,
mtinfo.u.rbyd.blocks[0], mtinfo.u.rbyd.trunk);
// keep track of seen blocks
seen[tinfo.u.rbyd.blocks[0] / 8]
|= 1 << (tinfo.u.rbyd.blocks[0] % 8);
seen[mtinfo.u.rbyd.blocks[0] / 8]
|= 1 << (mtinfo.u.rbyd.blocks[0] % 8);
} else if (tinfo.tag == LFSR_TAG_BLOCK) {
} else if (mtinfo.tag == LFSR_TAG_BLOCK) {
printf("traversal: 0x%x block 0x%x\n",
tinfo.tag,
tinfo.u.bptr.data.u.disk.block);
mtinfo.tag,
mtinfo.u.bptr.data.u.disk.block);
// keep track of seen blocks
seen[tinfo.u.bptr.data.u.disk.block / 8]
|= 1 << (tinfo.u.bptr.data.u.disk.block % 8);
seen[mtinfo.u.bptr.data.u.disk.block / 8]
|= 1 << (mtinfo.u.bptr.data.u.disk.block % 8);
} else {
// this shouldn't happen
printf("traversal: 0x%x\n", tinfo.tag);
printf("traversal: 0x%x\n", mtinfo.tag);
assert(false);
}
}
@@ -445,7 +444,7 @@ defines.SIZE = [
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
defines.VALIDATE = [false, true]
defines.CKMETADATA = [false, true]
in = 'lfs.c'
if = '(SIZE*N)/BLOCK_SIZE <= 32'
code = '''
@@ -489,53 +488,52 @@ code = '''
uint8_t *seen = malloc((BLOCK_COUNT+7)/8);
memset(seen, 0, (BLOCK_COUNT+7)/8);
lfsr_traversal_t traversal = LFSR_TRAVERSAL(
((VALIDATE) ? LFSR_TRAVERSAL_VALIDATE : 0)
| LFSR_TRAVERSAL_ALL);
lfsr_mtraversal_t mt = LFSR_MTRAVERSAL(
((CKMETADATA) ? LFS_T_CKMETADATA : 0));
for (lfs_block_t i = 0;; i++) {
// a bit hacky, but this catches infinite loops
assert(i < 2*BLOCK_COUNT);
lfsr_tinfo_t tinfo;
int err = lfsr_traversal_read(&lfs, &traversal, &tinfo);
lfsr_mtinfo_t mtinfo;
int err = lfsr_fs_traverse(&lfs, &mt, &mtinfo);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
if (tinfo.tag == LFSR_TAG_MDIR) {
if (mtinfo.tag == LFSR_TAG_MDIR) {
printf("traversal: 0x%x mdir 0x{%x,%x}\n",
tinfo.tag,
tinfo.u.mdir.rbyd.blocks[0],
tinfo.u.mdir.rbyd.blocks[1]);
mtinfo.tag,
mtinfo.u.mdir.rbyd.blocks[0],
mtinfo.u.mdir.rbyd.blocks[1]);
// keep track of seen blocks
seen[tinfo.u.mdir.rbyd.blocks[1] / 8]
|= 1 << (tinfo.u.mdir.rbyd.blocks[1] % 8);
seen[tinfo.u.mdir.rbyd.blocks[0] / 8]
|= 1 << (tinfo.u.mdir.rbyd.blocks[0] % 8);
seen[mtinfo.u.mdir.rbyd.blocks[1] / 8]
|= 1 << (mtinfo.u.mdir.rbyd.blocks[1] % 8);
seen[mtinfo.u.mdir.rbyd.blocks[0] / 8]
|= 1 << (mtinfo.u.mdir.rbyd.blocks[0] % 8);
} else if (tinfo.tag == LFSR_TAG_BRANCH) {
} else if (mtinfo.tag == LFSR_TAG_BRANCH) {
printf("traversal: 0x%x btree 0x%x.%x\n",
tinfo.tag,
tinfo.u.rbyd.blocks[0], tinfo.u.rbyd.trunk);
mtinfo.tag,
mtinfo.u.rbyd.blocks[0], mtinfo.u.rbyd.trunk);
// keep track of seen blocks
seen[tinfo.u.rbyd.blocks[0] / 8]
|= 1 << (tinfo.u.rbyd.blocks[0] % 8);
seen[mtinfo.u.rbyd.blocks[0] / 8]
|= 1 << (mtinfo.u.rbyd.blocks[0] % 8);
} else if (tinfo.tag == LFSR_TAG_BLOCK) {
} else if (mtinfo.tag == LFSR_TAG_BLOCK) {
printf("traversal: 0x%x block 0x%x\n",
tinfo.tag,
tinfo.u.bptr.data.u.disk.block);
mtinfo.tag,
mtinfo.u.bptr.data.u.disk.block);
// keep track of seen blocks
seen[tinfo.u.bptr.data.u.disk.block / 8]
|= 1 << (tinfo.u.bptr.data.u.disk.block % 8);
seen[mtinfo.u.bptr.data.u.disk.block / 8]
|= 1 << (mtinfo.u.bptr.data.u.disk.block % 8);
} else {
// this shouldn't happen
printf("traversal: 0x%x\n", tinfo.tag);
printf("traversal: 0x%x\n", mtinfo.tag);
assert(false);
}
}