Github/littlefs
1
Fork 0
Code Actions Activity

Added lfsr_file_ckmeta/ckdata

Browse Source 
These are basically the same as lfsr_fs_ckmeta/ckdata but limited to a
single file. They may be useful when you need to validate a file but
don't want to bother validating the entire filesystem:

  // Check a file for metadata errors
  int lfsr_file_ckmeta(lfs_t *lfs, lfsr_file_t *file);

  // Check a file for metadata + data errors
  int lfsr_file_ckdata(lfs_t *lfs, lfsr_file_t *file);

I've also added test_ck to test these and added some more
lfsr_fs_ckmeta/ckdata tests there. These currently just test simple
full-block clobbering, but we should eventually test more interesting
error patterns.

Unfortunately lfsr_file_ckmeta/ckdata can't reuse the internal
lfsr_mtree_traverse in quite the same way lfsr_fs_ckmeta/ckdata can, so
they're actually a bit more expensive. Though keep in mind with
link-time gc you won't pay the cost unless you call these functions:

           code          stack
  before: 36024           2696
  after:  36368 (+1.0%)   2664 (-1.2%)

Oh, and the multiple calls to lfsr_btree/bshrub_traverse apparently
uninlined it out of lfsr_mtree_traverse, saving the stack cost in the
stack hot-path... Yay?
This commit is contained in:
Christopher Haster
2024-07-26 14:10:22 -05:00
parent e812ac4a8c
commit e2c238c30d
5 changed files with 524 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+61
View File
@@ -11744,6 +11744,67 @@ failed:;
return err;
}
// file check functions
static int lfsr_file_ck(lfs_t *lfs, lfsr_file_t *file, uint32_t flags) {
// traverse the file's btree
lfsr_btraversal_t bt = LFSR_BTRAVERSAL();
while (true) {
lfsr_tag_t tag;
lfsr_bptr_t bptr;
int err = lfsr_bshrub_traverse(lfs,
&file->o.o.mdir, &file->o.bshrub,
&bt,
NULL, &tag, &bptr);
if (err) {
if (err == LFS_ERR_NOENT) {
break;
}
return err;
}
// validate btree nodes?
if ((lfsr_t_isckmeta(flags)
|| lfsr_t_isckdata(flags))
&& tag == LFSR_TAG_BRANCH) {
lfsr_rbyd_t *rbyd = (lfsr_rbyd_t*)bptr.data.u.buffer;
err = lfsr_rbyd_fetchck(lfs, rbyd,
rbyd->blocks[0], rbyd->trunk,
rbyd->cksum);
if (err) {
return err;
}
}
// validate data blocks?
if (lfsr_t_isckdata(flags)
&& tag == LFSR_TAG_BLOCK) {
err = lfsr_bptr_ck(lfs, &bptr);
if (err) {
return err;
}
}
}
return 0;
}
int lfsr_file_ckmeta(lfs_t *lfs, lfsr_file_t *file) {
LFS_ASSERT(lfsr_omdir_isopen(lfs, &file->o.o));
// can't read from writeonly files
LFS_ASSERT(!lfsr_o_iswronly(file->o.o.flags));
return lfsr_file_ck(lfs, file, LFS_T_CKMETA);
}
int lfsr_file_ckdata(lfs_t *lfs, lfsr_file_t *file) {
LFS_ASSERT(lfsr_omdir_isopen(lfs, &file->o.o));
// can't read from writeonly files
LFS_ASSERT(!lfsr_o_iswronly(file->o.o.flags));
return lfsr_file_ck(lfs, file, LFS_T_CKMETA | LFS_T_CKDATA);
}
lfs.h
+12
View File
@@ -1013,6 +1013,18 @@ int lfsr_file_rewind(lfs_t *lfs, lfsr_file_t *file);
//lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file);
lfs_soff_t lfsr_file_size(lfs_t *lfs, lfsr_file_t *file);
// Check a file for metadata errors
//
// Returns LFS_ERR_CORRUPT if a checksum mismatch is found, or a negative
// error code on failure.
int lfsr_file_ckmeta(lfs_t *lfs, lfsr_file_t *file);
// Check a file for metadata + data errors
//
// Returns LFS_ERR_CORRUPT if a checksum mismatch is found, or a negative
// error code on failure.
int lfsr_file_ckdata(lfs_t *lfs, lfsr_file_t *file);
/// Directory operations ///
tests/test_ck.toml
+447
View File
@@ -0,0 +1,447 @@
# Test checksum validation things
after = ['test_traversal', 'test_gc', 'test_mount']
# Test filesystem-level checksum things
# test we can detect at least fully clobbered blocks
[cases.test_ck_ckmeta_easy]
# METHOD=0 => lfsr_fs_ckmeta
# METHOD=1 => lfsr_fs_gc
# METHOD=2 => lfsr_traversal_read
# METHOD=3 => lfsr_mount
defines.METHOD = [0, 1, 2, 3]
defines.N = [1, 2, 4, 8, 16, 32, 64]
defines.SIZE = [
'0',
'FILE_BUFFER_SIZE/2',
'2*FILE_BUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
if = '(SIZE*N)/BLOCK_SIZE <= 32'
code = '''
lfs_block_t i = 0;
while (true) {
// a bit hacky, but this catches infinite loops
assert(i < 2*BLOCK_COUNT);
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0;
// create an interesting filesystem
uint32_t prng = 42;
for (lfs_size_t i = 0; i < N; i++) {
char name[256];
sprintf(name, "squid%03x", i);
uint8_t wbuf[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_t file;
lfsr_file_open(&lfs, &file, name,
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_close(&lfs, &file) => 0;
}
// traverse to find blocks
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t, 0) => 0;
lfs_block_t k = 0;
for (lfs_block_t j = 0;; j++) {
assert(j < 2*BLOCK_COUNT);
struct lfs_tinfo tinfo;
int err = lfsr_traversal_read(&lfs, &t, &tinfo);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
lfsr_traversal_close(&lfs, &t) => 0;
lfsr_unmount(&lfs) => 0;
goto done;
}
// this gets a bit tricky be cause we need to clobber both
// blocks in mdir pairs
if (tinfo.btype == LFS_BTYPE_MDIR
|| tinfo.btype == LFS_BTYPE_BTREE) {
if (k == i || k == i+1) {
// clobber this block
printf("clobbering 0x%x\n", tinfo.block);
uint8_t clobber_buf[BLOCK_SIZE];
memset(clobber_buf, 0xcc, BLOCK_SIZE);
CFG->erase(CFG, tinfo.block) => 0;
CFG->prog(CFG, tinfo.block, 0,
clobber_buf, BLOCK_SIZE) => 0;
if (tinfo.btype != LFS_BTYPE_MDIR || k == i+1) {
i += (tinfo.btype == LFS_BTYPE_MDIR) ? 2 : 1;
lfsr_traversal_close(&lfs, &t) => 0;
goto clobbered;
}
}
k += 1;
}
}
clobbered:;
// find clobbered blocks with lfsr_fs_ckmeta
if (METHOD == 0) {
lfsr_fs_ckmeta(&lfs) => LFS_ERR_CORRUPT;
// find clobbered blocks with lfsr_fs_gc
} else if (METHOD == 1) {
lfsr_fs_gc(&lfs, -1, LFS_GC_CKMETA) => LFS_ERR_CORRUPT;
// find clobbered blocks with lfsr_traversal_read
} else if (METHOD == 2) {
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t, LFS_T_CKMETA) => 0;
for (lfs_block_t i = 0;; i++) {
// a bit hacky, but this catches infinite loops
LFS_ASSERT(i < 2*BLOCK_COUNT);
struct lfs_tinfo tinfo;
int err = lfsr_traversal_read(&lfs, &t, &tinfo);
assert(!err || err == LFS_ERR_CORRUPT);
if (err == LFS_ERR_CORRUPT) {
break;
}
}
lfsr_traversal_close(&lfs, &t) => 0;
// find clobbered blocks with lfsr_mount
} else if (METHOD == 3) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs,
LFS_M_RDWR
| LFS_M_CKMETA,
CFG) => LFS_ERR_CORRUPT;
} else {
assert(false);
}
if (METHOD != 3) {
lfsr_unmount(&lfs) => 0;
}
}
done:;
'''
[cases.test_ck_ckdata_easy]
# METHOD=0 => lfsr_fs_ckdata
# METHOD=1 => lfsr_fs_gc
# METHOD=2 => lfsr_traversal_read
# METHOD=3 => lfsr_mount
defines.METHOD = [0, 1, 2, 3]
defines.N = [1, 2, 4, 8, 16, 32, 64]
defines.SIZE = [
'0',
'FILE_BUFFER_SIZE/2',
'2*FILE_BUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
if = '(SIZE*N)/BLOCK_SIZE <= 32'
code = '''
lfs_block_t i = 0;
while (true) {
// a bit hacky, but this catches infinite loops
assert(i < 2*BLOCK_COUNT);
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0;
// create an interesting filesystem
uint32_t prng = 42;
for (lfs_size_t i = 0; i < N; i++) {
char name[256];
sprintf(name, "squid%03x", i);
uint8_t wbuf[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_t file;
lfsr_file_open(&lfs, &file, name,
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_close(&lfs, &file) => 0;
}
// traverse to find blocks
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t, 0) => 0;
lfs_block_t k = 0;
for (lfs_block_t j = 0;; j++) {
assert(j < 2*BLOCK_COUNT);
struct lfs_tinfo tinfo;
int err = lfsr_traversal_read(&lfs, &t, &tinfo);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
lfsr_traversal_close(&lfs, &t) => 0;
lfsr_unmount(&lfs) => 0;
goto done;
}
// this gets a bit tricky be cause we need to clobber both
// blocks in mdir pairs
if (tinfo.btype == LFS_BTYPE_MDIR
|| tinfo.btype == LFS_BTYPE_BTREE
|| tinfo.btype == LFS_BTYPE_DATA) {
if (k == i || k == i+1) {
// clobber this block
printf("clobbering 0x%x\n", tinfo.block);
uint8_t clobber_buf[BLOCK_SIZE];
memset(clobber_buf, 0xcc, BLOCK_SIZE);
CFG->erase(CFG, tinfo.block) => 0;
CFG->prog(CFG, tinfo.block, 0,
clobber_buf, BLOCK_SIZE) => 0;
if (tinfo.btype != LFS_BTYPE_MDIR || k == i+1) {
i += (tinfo.btype == LFS_BTYPE_MDIR) ? 2 : 1;
lfsr_traversal_close(&lfs, &t) => 0;
goto clobbered;
}
}
k += 1;
}
}
clobbered:;
// find clobbered blocks with lfsr_fs_ckmeta
if (METHOD == 0) {
lfsr_fs_ckdata(&lfs) => LFS_ERR_CORRUPT;
// find clobbered blocks with lfsr_fs_gc
} else if (METHOD == 1) {
lfsr_fs_gc(&lfs, -1, LFS_GC_CKDATA) => LFS_ERR_CORRUPT;
// find clobbered blocks with lfsr_traversal_read
} else if (METHOD == 2) {
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t, LFS_T_CKDATA) => 0;
for (lfs_block_t i = 0;; i++) {
// a bit hacky, but this catches infinite loops
LFS_ASSERT(i < 2*BLOCK_COUNT);
struct lfs_tinfo tinfo;
int err = lfsr_traversal_read(&lfs, &t, &tinfo);
assert(!err || err == LFS_ERR_CORRUPT);
if (err == LFS_ERR_CORRUPT) {
break;
}
}
lfsr_traversal_close(&lfs, &t) => 0;
// find clobbered blocks with lfsr_mount
} else if (METHOD == 3) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs,
LFS_M_RDWR
| LFS_M_CKDATA,
CFG) => LFS_ERR_CORRUPT;
} else {
assert(false);
}
if (METHOD != 3) {
lfsr_unmount(&lfs) => 0;
}
}
done:;
'''
# Test file-level checksum things
# test we can detect at least fully clobbered blocks
[cases.test_ck_file_ckmeta_easy]
# METHOD=0 => lfsr_file_ckmeta
# METHOD=1 => lfsr_file_close+open+ckmeta
defines.METHOD = [0, 1]
defines.N = [1, 2, 4, 8, 16, 32, 64]
defines.SIZE = [
'0',
'FILE_BUFFER_SIZE/2',
'2*FILE_BUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
code = '''
for (lfs_block_t i = 0;; i++) {
// a bit hacky, but this catches infinite loops
assert(i < 2*BLOCK_COUNT);
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0;
// create an interesting file
uint32_t prng = 42;
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "octopus",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
// traverse to find blocks
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t, 0) => 0;
lfs_block_t k = 0;
for (lfs_block_t j = 0;; j++) {
assert(j < 2*BLOCK_COUNT);
struct lfs_tinfo tinfo;
int err = lfsr_traversal_read(&lfs, &t, &tinfo);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
lfsr_traversal_close(&lfs, &t) => 0;
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
goto done;
}
if (tinfo.btype == LFS_BTYPE_BTREE) {
if (k == i) {
// clobber this block
printf("clobbering 0x%x\n", tinfo.block);
uint8_t clobber_buf[BLOCK_SIZE];
memset(clobber_buf, 0xcc, BLOCK_SIZE);
CFG->erase(CFG, tinfo.block) => 0;
CFG->prog(CFG, tinfo.block, 0,
clobber_buf, BLOCK_SIZE) => 0;
lfsr_traversal_close(&lfs, &t) => 0;
goto clobbered;
}
k += 1;
}
}
clobbered:;
// find clobbered blocks with lfsr_file_ckmeta
if (METHOD == 0) {
lfsr_file_ckmeta(&lfs, &file) => LFS_ERR_CORRUPT;
// find clobbered blocks with lfsr_file_close+open+ckmeta
} else if (METHOD == 1) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_file_open(&lfs, &file, "octopus", LFS_O_RDONLY) => 0;
lfsr_file_ckmeta(&lfs, &file) => LFS_ERR_CORRUPT;
} else {
assert(false);
}
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
}
done:;
'''
# test we can detect at least fully clobbered blocks
[cases.test_ck_file_ckdata_easy]
# METHOD=0 => lfsr_file_ckdata
# METHOD=1 => lfsr_file_close+open+ckdata
defines.METHOD = [0, 1]
defines.N = [1, 2, 4, 8, 16, 32, 64]
defines.SIZE = [
'0',
'FILE_BUFFER_SIZE/2',
'2*FILE_BUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'8*BLOCK_SIZE',
]
code = '''
for (lfs_block_t i = 0;; i++) {
// a bit hacky, but this catches infinite loops
assert(i < 2*BLOCK_COUNT);
lfs_t lfs;
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0;
// create an interesting file
uint32_t prng = 42;
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "octopus",
LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf[SIZE];
for (lfs_size_t j = 0; j < SIZE; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
// traverse to find blocks
lfsr_traversal_t t;
lfsr_traversal_open(&lfs, &t, 0) => 0;
lfs_block_t k = 0;
for (lfs_block_t j = 0;; j++) {
assert(j < 2*BLOCK_COUNT);
struct lfs_tinfo tinfo;
int err = lfsr_traversal_read(&lfs, &t, &tinfo);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
lfsr_traversal_close(&lfs, &t) => 0;
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
goto done;
}
if (tinfo.btype == LFS_BTYPE_BTREE
|| tinfo.btype == LFS_BTYPE_DATA) {
if (k == i) {
// clobber this block
printf("clobbering 0x%x\n", tinfo.block);
uint8_t clobber_buf[BLOCK_SIZE];
memset(clobber_buf, 0xcc, BLOCK_SIZE);
CFG->erase(CFG, tinfo.block) => 0;
CFG->prog(CFG, tinfo.block, 0,
clobber_buf, BLOCK_SIZE) => 0;
lfsr_traversal_close(&lfs, &t) => 0;
goto clobbered;
}
k += 1;
}
}
clobbered:;
// find clobbered blocks with lfsr_file_ckmeta
if (METHOD == 0) {
lfsr_file_ckdata(&lfs, &file) => LFS_ERR_CORRUPT;
// find clobbered blocks with lfsr_file_close+open+ckmeta
} else if (METHOD == 1) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_file_open(&lfs, &file, "octopus", LFS_O_RDONLY) => 0;
lfsr_file_ckdata(&lfs, &file) => LFS_ERR_CORRUPT;
} else {
assert(false);
}
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
}
done:;
'''
tests/test_gc.toml
+2
View File
@@ -1222,6 +1222,8 @@ code = '''
# test we can detect at least fully clobbered blocks
#
# these are tested more thoroughly in test_ck
[cases.test_gc_ckmeta]
defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000]
defines.N = [1, 2, 4, 8, 16, 32, 64]
tests/test_mount.toml
+2
View File
@@ -237,6 +237,8 @@ code = '''
'''
# test we can detect at least fully clobbered blocks
#
# these are tested more thoroughly in test_ck
[cases.test_mount_t_ckmeta]
defines.N = [1, 2, 4, 8, 16, 32, 64]
defines.SIZE = [