From e2c238c30de50ccca17fec21bf6eae4009fd9130 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 26 Jul 2024 14:10:22 -0500 Subject: [PATCH] Added lfsr_file_ckmeta/ckdata These are basically the same as lfsr_fs_ckmeta/ckdata but limited to a single file. They may be useful when you need to validate a file but don't want to bother validating the entire filesystem: // Check a file for metadata errors int lfsr_file_ckmeta(lfs_t *lfs, lfsr_file_t *file); // Check a file for metadata + data errors int lfsr_file_ckdata(lfs_t *lfs, lfsr_file_t *file); I've also added test_ck to test these and added some more lfsr_fs_ckmeta/ckdata tests there. These currently just test simple full-block clobbering, but we should eventually test more interesting error patterns. Unfortunately lfsr_file_ckmeta/ckdata can't reuse the internal lfsr_mtree_traverse in quite the same way lfsr_fs_ckmeta/ckdata can, so they're actually a bit more expensive. Though keep in mind with link-time gc you won't pay the cost unless you call these functions: code stack before: 36024 2696 after: 36368 (+1.0%) 2664 (-1.2%) Oh, and the multiple calls to lfsr_btree/bshrub_traverse apparently uninlined it out of lfsr_mtree_traverse, saving the stack cost in the stack hot-path... Yay? --- lfs.c | 61 ++++++ lfs.h | 12 ++ tests/test_ck.toml | 447 ++++++++++++++++++++++++++++++++++++++++++ tests/test_gc.toml | 2 + tests/test_mount.toml | 2 + 5 files changed, 524 insertions(+) create mode 100644 tests/test_ck.toml diff --git a/lfs.c b/lfs.c index b7644460..731bb58e 100644 --- a/lfs.c +++ b/lfs.c @@ -11744,6 +11744,67 @@ failed:; return err; } +// file check functions +static int lfsr_file_ck(lfs_t *lfs, lfsr_file_t *file, uint32_t flags) { + // traverse the file's btree + lfsr_btraversal_t bt = LFSR_BTRAVERSAL(); + while (true) { + lfsr_tag_t tag; + lfsr_bptr_t bptr; + int err = lfsr_bshrub_traverse(lfs, + &file->o.o.mdir, &file->o.bshrub, + &bt, + NULL, &tag, &bptr); + if (err) { + if (err == LFS_ERR_NOENT) { + break; + } + return err; + } + + // validate btree nodes? + if ((lfsr_t_isckmeta(flags) + || lfsr_t_isckdata(flags)) + && tag == LFSR_TAG_BRANCH) { + lfsr_rbyd_t *rbyd = (lfsr_rbyd_t*)bptr.data.u.buffer; + err = lfsr_rbyd_fetchck(lfs, rbyd, + rbyd->blocks[0], rbyd->trunk, + rbyd->cksum); + if (err) { + return err; + } + } + + // validate data blocks? + if (lfsr_t_isckdata(flags) + && tag == LFSR_TAG_BLOCK) { + err = lfsr_bptr_ck(lfs, &bptr); + if (err) { + return err; + } + } + } + + return 0; +} + +int lfsr_file_ckmeta(lfs_t *lfs, lfsr_file_t *file) { + LFS_ASSERT(lfsr_omdir_isopen(lfs, &file->o.o)); + // can't read from writeonly files + LFS_ASSERT(!lfsr_o_iswronly(file->o.o.flags)); + + return lfsr_file_ck(lfs, file, LFS_T_CKMETA); +} + +int lfsr_file_ckdata(lfs_t *lfs, lfsr_file_t *file) { + LFS_ASSERT(lfsr_omdir_isopen(lfs, &file->o.o)); + // can't read from writeonly files + LFS_ASSERT(!lfsr_o_iswronly(file->o.o.flags)); + + return lfsr_file_ck(lfs, file, LFS_T_CKMETA | LFS_T_CKDATA); +} + + diff --git a/lfs.h b/lfs.h index a9e8e330..d3c44bc7 100644 --- a/lfs.h +++ b/lfs.h @@ -1013,6 +1013,18 @@ int lfsr_file_rewind(lfs_t *lfs, lfsr_file_t *file); //lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file); lfs_soff_t lfsr_file_size(lfs_t *lfs, lfsr_file_t *file); +// Check a file for metadata errors +// +// Returns LFS_ERR_CORRUPT if a checksum mismatch is found, or a negative +// error code on failure. +int lfsr_file_ckmeta(lfs_t *lfs, lfsr_file_t *file); + +// Check a file for metadata + data errors +// +// Returns LFS_ERR_CORRUPT if a checksum mismatch is found, or a negative +// error code on failure. +int lfsr_file_ckdata(lfs_t *lfs, lfsr_file_t *file); + /// Directory operations /// diff --git a/tests/test_ck.toml b/tests/test_ck.toml new file mode 100644 index 00000000..50b69737 --- /dev/null +++ b/tests/test_ck.toml @@ -0,0 +1,447 @@ +# Test checksum validation things +after = ['test_traversal', 'test_gc', 'test_mount'] + + +# Test filesystem-level checksum things + +# test we can detect at least fully clobbered blocks +[cases.test_ck_ckmeta_easy] +# METHOD=0 => lfsr_fs_ckmeta +# METHOD=1 => lfsr_fs_gc +# METHOD=2 => lfsr_traversal_read +# METHOD=3 => lfsr_mount +defines.METHOD = [0, 1, 2, 3] +defines.N = [1, 2, 4, 8, 16, 32, 64] +defines.SIZE = [ + '0', + 'FILE_BUFFER_SIZE/2', + '2*FILE_BUFFER_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '8*BLOCK_SIZE', +] +if = '(SIZE*N)/BLOCK_SIZE <= 32' +code = ''' + lfs_block_t i = 0; + while (true) { + // a bit hacky, but this catches infinite loops + assert(i < 2*BLOCK_COUNT); + + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; + + // create an interesting filesystem + uint32_t prng = 42; + for (lfs_size_t i = 0; i < N; i++) { + char name[256]; + sprintf(name, "squid%03x", i); + + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + lfsr_file_close(&lfs, &file) => 0; + } + + // traverse to find blocks + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, 0) => 0; + lfs_block_t k = 0; + for (lfs_block_t j = 0;; j++) { + assert(j < 2*BLOCK_COUNT); + + struct lfs_tinfo tinfo; + int err = lfsr_traversal_read(&lfs, &t, &tinfo); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + lfsr_traversal_close(&lfs, &t) => 0; + lfsr_unmount(&lfs) => 0; + goto done; + } + + // this gets a bit tricky be cause we need to clobber both + // blocks in mdir pairs + if (tinfo.btype == LFS_BTYPE_MDIR + || tinfo.btype == LFS_BTYPE_BTREE) { + if (k == i || k == i+1) { + // clobber this block + printf("clobbering 0x%x\n", tinfo.block); + uint8_t clobber_buf[BLOCK_SIZE]; + memset(clobber_buf, 0xcc, BLOCK_SIZE); + CFG->erase(CFG, tinfo.block) => 0; + CFG->prog(CFG, tinfo.block, 0, + clobber_buf, BLOCK_SIZE) => 0; + if (tinfo.btype != LFS_BTYPE_MDIR || k == i+1) { + i += (tinfo.btype == LFS_BTYPE_MDIR) ? 2 : 1; + lfsr_traversal_close(&lfs, &t) => 0; + goto clobbered; + } + } + k += 1; + } + } + + clobbered:; + // find clobbered blocks with lfsr_fs_ckmeta + if (METHOD == 0) { + lfsr_fs_ckmeta(&lfs) => LFS_ERR_CORRUPT; + + // find clobbered blocks with lfsr_fs_gc + } else if (METHOD == 1) { + lfsr_fs_gc(&lfs, -1, LFS_GC_CKMETA) => LFS_ERR_CORRUPT; + + // find clobbered blocks with lfsr_traversal_read + } else if (METHOD == 2) { + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, LFS_T_CKMETA) => 0; + for (lfs_block_t i = 0;; i++) { + // a bit hacky, but this catches infinite loops + LFS_ASSERT(i < 2*BLOCK_COUNT); + + struct lfs_tinfo tinfo; + int err = lfsr_traversal_read(&lfs, &t, &tinfo); + assert(!err || err == LFS_ERR_CORRUPT); + if (err == LFS_ERR_CORRUPT) { + break; + } + } + lfsr_traversal_close(&lfs, &t) => 0; + + // find clobbered blocks with lfsr_mount + } else if (METHOD == 3) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, + LFS_M_RDWR + | LFS_M_CKMETA, + CFG) => LFS_ERR_CORRUPT; + + } else { + assert(false); + } + + if (METHOD != 3) { + lfsr_unmount(&lfs) => 0; + } + } +done:; +''' + +[cases.test_ck_ckdata_easy] +# METHOD=0 => lfsr_fs_ckdata +# METHOD=1 => lfsr_fs_gc +# METHOD=2 => lfsr_traversal_read +# METHOD=3 => lfsr_mount +defines.METHOD = [0, 1, 2, 3] +defines.N = [1, 2, 4, 8, 16, 32, 64] +defines.SIZE = [ + '0', + 'FILE_BUFFER_SIZE/2', + '2*FILE_BUFFER_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '8*BLOCK_SIZE', +] +if = '(SIZE*N)/BLOCK_SIZE <= 32' +code = ''' + lfs_block_t i = 0; + while (true) { + // a bit hacky, but this catches infinite loops + assert(i < 2*BLOCK_COUNT); + + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; + + // create an interesting filesystem + uint32_t prng = 42; + for (lfs_size_t i = 0; i < N; i++) { + char name[256]; + sprintf(name, "squid%03x", i); + + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + lfsr_file_close(&lfs, &file) => 0; + } + + // traverse to find blocks + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, 0) => 0; + lfs_block_t k = 0; + for (lfs_block_t j = 0;; j++) { + assert(j < 2*BLOCK_COUNT); + + struct lfs_tinfo tinfo; + int err = lfsr_traversal_read(&lfs, &t, &tinfo); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + lfsr_traversal_close(&lfs, &t) => 0; + lfsr_unmount(&lfs) => 0; + goto done; + } + + // this gets a bit tricky be cause we need to clobber both + // blocks in mdir pairs + if (tinfo.btype == LFS_BTYPE_MDIR + || tinfo.btype == LFS_BTYPE_BTREE + || tinfo.btype == LFS_BTYPE_DATA) { + if (k == i || k == i+1) { + // clobber this block + printf("clobbering 0x%x\n", tinfo.block); + uint8_t clobber_buf[BLOCK_SIZE]; + memset(clobber_buf, 0xcc, BLOCK_SIZE); + CFG->erase(CFG, tinfo.block) => 0; + CFG->prog(CFG, tinfo.block, 0, + clobber_buf, BLOCK_SIZE) => 0; + if (tinfo.btype != LFS_BTYPE_MDIR || k == i+1) { + i += (tinfo.btype == LFS_BTYPE_MDIR) ? 2 : 1; + lfsr_traversal_close(&lfs, &t) => 0; + goto clobbered; + } + } + k += 1; + } + } + + clobbered:; + // find clobbered blocks with lfsr_fs_ckmeta + if (METHOD == 0) { + lfsr_fs_ckdata(&lfs) => LFS_ERR_CORRUPT; + + // find clobbered blocks with lfsr_fs_gc + } else if (METHOD == 1) { + lfsr_fs_gc(&lfs, -1, LFS_GC_CKDATA) => LFS_ERR_CORRUPT; + + // find clobbered blocks with lfsr_traversal_read + } else if (METHOD == 2) { + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, LFS_T_CKDATA) => 0; + for (lfs_block_t i = 0;; i++) { + // a bit hacky, but this catches infinite loops + LFS_ASSERT(i < 2*BLOCK_COUNT); + + struct lfs_tinfo tinfo; + int err = lfsr_traversal_read(&lfs, &t, &tinfo); + assert(!err || err == LFS_ERR_CORRUPT); + if (err == LFS_ERR_CORRUPT) { + break; + } + } + lfsr_traversal_close(&lfs, &t) => 0; + + // find clobbered blocks with lfsr_mount + } else if (METHOD == 3) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, + LFS_M_RDWR + | LFS_M_CKDATA, + CFG) => LFS_ERR_CORRUPT; + + } else { + assert(false); + } + + if (METHOD != 3) { + lfsr_unmount(&lfs) => 0; + } + } +done:; +''' + + +# Test file-level checksum things + +# test we can detect at least fully clobbered blocks +[cases.test_ck_file_ckmeta_easy] +# METHOD=0 => lfsr_file_ckmeta +# METHOD=1 => lfsr_file_close+open+ckmeta +defines.METHOD = [0, 1] +defines.N = [1, 2, 4, 8, 16, 32, 64] +defines.SIZE = [ + '0', + 'FILE_BUFFER_SIZE/2', + '2*FILE_BUFFER_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '8*BLOCK_SIZE', +] +code = ''' + for (lfs_block_t i = 0;; i++) { + // a bit hacky, but this catches infinite loops + assert(i < 2*BLOCK_COUNT); + + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; + + // create an interesting file + uint32_t prng = 42; + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "octopus", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + + // traverse to find blocks + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, 0) => 0; + lfs_block_t k = 0; + for (lfs_block_t j = 0;; j++) { + assert(j < 2*BLOCK_COUNT); + + struct lfs_tinfo tinfo; + int err = lfsr_traversal_read(&lfs, &t, &tinfo); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + lfsr_traversal_close(&lfs, &t) => 0; + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; + goto done; + } + + if (tinfo.btype == LFS_BTYPE_BTREE) { + if (k == i) { + // clobber this block + printf("clobbering 0x%x\n", tinfo.block); + uint8_t clobber_buf[BLOCK_SIZE]; + memset(clobber_buf, 0xcc, BLOCK_SIZE); + CFG->erase(CFG, tinfo.block) => 0; + CFG->prog(CFG, tinfo.block, 0, + clobber_buf, BLOCK_SIZE) => 0; + lfsr_traversal_close(&lfs, &t) => 0; + goto clobbered; + } + k += 1; + } + } + + clobbered:; + // find clobbered blocks with lfsr_file_ckmeta + if (METHOD == 0) { + lfsr_file_ckmeta(&lfs, &file) => LFS_ERR_CORRUPT; + + // find clobbered blocks with lfsr_file_close+open+ckmeta + } else if (METHOD == 1) { + lfsr_file_close(&lfs, &file) => 0; + lfsr_file_open(&lfs, &file, "octopus", LFS_O_RDONLY) => 0; + lfsr_file_ckmeta(&lfs, &file) => LFS_ERR_CORRUPT; + + } else { + assert(false); + } + + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; + } +done:; +''' + +# test we can detect at least fully clobbered blocks +[cases.test_ck_file_ckdata_easy] +# METHOD=0 => lfsr_file_ckdata +# METHOD=1 => lfsr_file_close+open+ckdata +defines.METHOD = [0, 1] +defines.N = [1, 2, 4, 8, 16, 32, 64] +defines.SIZE = [ + '0', + 'FILE_BUFFER_SIZE/2', + '2*FILE_BUFFER_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '8*BLOCK_SIZE', +] +code = ''' + for (lfs_block_t i = 0;; i++) { + // a bit hacky, but this catches infinite loops + assert(i < 2*BLOCK_COUNT); + + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0; + + // create an interesting file + uint32_t prng = 42; + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "octopus", + LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0; + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + + // traverse to find blocks + lfsr_traversal_t t; + lfsr_traversal_open(&lfs, &t, 0) => 0; + lfs_block_t k = 0; + for (lfs_block_t j = 0;; j++) { + assert(j < 2*BLOCK_COUNT); + + struct lfs_tinfo tinfo; + int err = lfsr_traversal_read(&lfs, &t, &tinfo); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + lfsr_traversal_close(&lfs, &t) => 0; + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; + goto done; + } + + if (tinfo.btype == LFS_BTYPE_BTREE + || tinfo.btype == LFS_BTYPE_DATA) { + if (k == i) { + // clobber this block + printf("clobbering 0x%x\n", tinfo.block); + uint8_t clobber_buf[BLOCK_SIZE]; + memset(clobber_buf, 0xcc, BLOCK_SIZE); + CFG->erase(CFG, tinfo.block) => 0; + CFG->prog(CFG, tinfo.block, 0, + clobber_buf, BLOCK_SIZE) => 0; + lfsr_traversal_close(&lfs, &t) => 0; + goto clobbered; + } + k += 1; + } + } + + clobbered:; + // find clobbered blocks with lfsr_file_ckmeta + if (METHOD == 0) { + lfsr_file_ckdata(&lfs, &file) => LFS_ERR_CORRUPT; + + // find clobbered blocks with lfsr_file_close+open+ckmeta + } else if (METHOD == 1) { + lfsr_file_close(&lfs, &file) => 0; + lfsr_file_open(&lfs, &file, "octopus", LFS_O_RDONLY) => 0; + lfsr_file_ckdata(&lfs, &file) => LFS_ERR_CORRUPT; + + } else { + assert(false); + } + + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; + } +done:; +''' diff --git a/tests/test_gc.toml b/tests/test_gc.toml index 391131c1..71beaa23 100644 --- a/tests/test_gc.toml +++ b/tests/test_gc.toml @@ -1222,6 +1222,8 @@ code = ''' # test we can detect at least fully clobbered blocks +# +# these are tested more thoroughly in test_ck [cases.test_gc_ckmeta] defines.GC_STEPS = [-1, 1, 2, 10, 100, 1000] defines.N = [1, 2, 4, 8, 16, 32, 64] diff --git a/tests/test_mount.toml b/tests/test_mount.toml index 48ea5e93..325681bf 100644 --- a/tests/test_mount.toml +++ b/tests/test_mount.toml @@ -237,6 +237,8 @@ code = ''' ''' # test we can detect at least fully clobbered blocks +# +# these are tested more thoroughly in test_ck [cases.test_mount_t_ckmeta] defines.N = [1, 2, 4, 8, 16, 32, 64] defines.SIZE = [