From 3b33c33339424e6624a61965d99e3bdbe5f8313f Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 27 May 2024 16:06:50 -0500 Subject: [PATCH] Added pseudo-stateless *_pl_fuzz tests These provide useful file powerloss testing that scales linearly as long as progress can be made. They can still struggle a bit, especially with relocations which often fail to make progress, but they are _much_ better than the O(n^2) simulation-based fuzz tests: - test_files_pl_fuzz - 258734 pls - test_relocations_pl_fuzz - 928638 pls Our current problem with simulation-based fuzz testing is that we lose the simulation on powerloss. We could brute force this, repeatedly rerunning the simulation until it succeeds, but this grows O(n^2) with our linear powerloss heuristic. To avoid this, test_*_pl_fuzz doesn't bother with a simulation, instead relying on internal asserts to catch bugs. This is less rigorous, but realistically probably going to catch any powerloss related issues. Some notes: - We need to store some state on disk. If we don't we will still end up with O(n^2) behavior because we simply don't know how many operations we've accomplished so far. - Since we rely on file operations to store our test state, this makes this approach incompatible with the dir tests, which assume file operations may not yet be implemented. We still use O(n^2) powerloss testing in test_dirs, just with a small number of directories. - It's tempting to try to store a full simulation on disk. But you would quickly run into atomicity issues with the simulation itself. Powerloss resilience is tricky! - We can at least store a checksum in the files (currently just mod 26) to check that the file itself was not corrupted. This doesn't protect against swapped data though. --- Also, a bit of a tangent, but I needed to add -Wno-format-overflow to the test flags to avoid an annoying invalid format-overlow warning: struct lfs_info info; char name[256]; if (strlen(info.name) < 100) { // can't overflow!? sprintf(name, "test/%s", info.name); // <-- } warning: '%s' directive writing up to 255 bytes into a region of size 251 [-Wformat-overflow=] This seems like a GCC bug, because as far as I can tell there is no way to signal or hint that the size is in bounds without just disabling the warning completely... --- Makefile | 2 + tests/test_files.toml | 214 +++++++++++++++++++++++++++++++++++ tests/test_relocations.toml | 216 +++++++++++++++++++++++++++++++++++- 3 files changed, 430 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e2848cb6..96d5ba7c 100644 --- a/Makefile +++ b/Makefile @@ -356,6 +356,7 @@ summary-diff sizes-diff: $(OBJ) $(CI) ## Build the test-runner .PHONY: test-runner build-test test-runner build-test: CFLAGS+=-Wno-unused-function +test-runner build-test: CFLAGS+=-Wno-format-overflow ifdef COVGEN test-runner build-test: CFLAGS+=--coverage endif @@ -408,6 +409,7 @@ testmarks-diff: $(TEST_CSV) ## Build the bench-runner .PHONY: bench-runner build-bench bench-runner build-bench: CFLAGS+=-Wno-unused-function +bench-runner build-bench: CFLAGS+=-Wno-format-overflow ifdef COVGEN bench-runner build-bench: CFLAGS+=--coverage endif diff --git a/tests/test_files.toml b/tests/test_files.toml index 0061dffa..b4f41553 100644 --- a/tests/test_files.toml +++ b/tests/test_files.toml @@ -2603,6 +2603,220 @@ code = ''' ''' +# fuzz test file ops under powerloss +# +# Under powerloss, we can't really keep track of a sim reliably/ +# efficiently, instead just do random operations, store a counter in a +# special file so we know how much progress has been made, and hope for +# the best. Most likely an internal assert will trigger if anything goes +# wrong. +# +[cases.test_files_pl_fuzz] +defines.N = [1, 2, 4, 8, 16, 32, 64] +# do more ops than files to encourage file rewrites +defines.OPS = '2*N' +defines.SIZE = [ + '0', + 'FBUFFER_SIZE/2', + '2*FBUFFER_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.REMOUNT = [false, true] +defines.SEED = 'range(20)' +if = '(SIZE*N)/BLOCK_SIZE <= 16' +reentrant = true +code = ''' + // format once per test + lfs_t lfs; + int err = lfsr_mount(&lfs, CFG); + if (err) { + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + } + + // keep some test state on disk to survive powerloss + typedef struct fuzz_state { + lfs_size_t i; + uint32_t prng; + } fuzz_state_t; + fuzz_state_t state = {.i = 0, .prng = SEED}; + + lfsr_file_t state_file; + lfsr_file_open(&lfs, &state_file, "state", LFS_O_RDWR | LFS_O_CREAT) => 0; + lfs_ssize_t d = lfsr_file_read(&lfs, &state_file, &state, sizeof(state)); + assert(d == 0 || d == sizeof(state)); + + // keep test files in a separate directory + err = lfsr_mkdir(&lfs, "test"); + assert(!err || err == LFS_ERR_EXIST); + + uint32_t prng = state.prng; + for (lfs_size_t i = state.i; i < OPS; i++) { + // choose which operation to do + uint8_t op = TEST_PRNG(&prng) % 3; + + // how many files do we have? + lfs_size_t count = 0; + lfsr_dir_t dir; + lfsr_dir_open(&lfs, &dir, "test") => 0; + struct lfs_info info; + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, ".") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "..") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + while (true) { + int err = lfsr_dir_read(&lfs, &dir, &info); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + break; + } + assert(strlen(info.name) == strlen("amethyst...")); + assert(memcmp(info.name, "amethyst", strlen("amethyst")) == 0); + assert(info.type == LFS_TYPE_REG); + assert(info.size == SIZE); + count++; + } + lfsr_dir_close(&lfs, &dir) => 0; + + // creating a new file? + if (op == 0 || count == 0) { + // choose a pseudo-random number + lfs_size_t x = TEST_PRNG(&prng) % N; + uint32_t wprng = TEST_PRNG(&prng); + + // create a file here + char name[256]; + sprintf(name, "test/amethyst%03x", x); + uint8_t wbuf[SIZE]; + uint8_t ck = 0; + for (lfs_size_t j = 0; j < SIZE-1; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&wprng) % 26); + ck = (ck + (wbuf[j] - 'a')) % 26; + } + // make the sum equal to 'a' mod 26 + if (SIZE > 0) { + wbuf[SIZE-1] = 'a' + ((26 - ck) % 26); + } + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + lfsr_file_close(&lfs, &file) => 0; + + // deleting a file? + } else if (op == 1) { + // choose a random file to delete + lfs_size_t j = TEST_PRNG(&prng) % count; + // find the file + lfsr_dir_open(&lfs, &dir, "test") => 0; + lfsr_dir_read(&lfs, &dir, &info) => 0; + lfsr_dir_read(&lfs, &dir, &info) => 0; + for (lfs_size_t k = 0; k <= j; k++) { + lfsr_dir_read(&lfs, &dir, &info) => 0; + } + lfsr_dir_close(&lfs, &dir) => 0; + + // delete this file + char name[256]; + assert(strlen(info.name) == strlen("amethyst...")); + sprintf(name, "test/%s", info.name); + lfsr_remove(&lfs, name) => 0; + + // renaming a file? + } else { + // choose a random file to rename, and a random number to + // rename to + lfs_size_t j = TEST_PRNG(&prng) % count; + lfs_size_t y = TEST_PRNG(&prng) % N; + // find the file + lfsr_dir_open(&lfs, &dir, "test") => 0; + lfsr_dir_read(&lfs, &dir, &info) => 0; + lfsr_dir_read(&lfs, &dir, &info) => 0; + for (lfs_size_t k = 0; k <= j; k++) { + lfsr_dir_read(&lfs, &dir, &info) => 0; + } + lfsr_dir_close(&lfs, &dir) => 0; + + // rename this file + char old_name[256]; + assert(strlen(info.name) == strlen("amethyst...")); + sprintf(old_name, "test/%s", info.name); + char new_name[256]; + sprintf(new_name, "test/amethyst%03x", y); + lfsr_rename(&lfs, old_name, new_name) => 0; + } + + // update our state file + state.i = i; + state.prng = prng; + lfsr_file_rewind(&lfs, &state_file) => 0; + lfsr_file_write(&lfs, &state_file, &state, sizeof(state)) + => sizeof(state); + lfsr_file_sync(&lfs, &state_file) => 0; + } + + // go ahead and close our state file in case we remount + lfsr_file_close(&lfs, &state_file) => 0; + + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + } + + // check that things look more-or-less ok + lfsr_dir_t dir; + lfsr_dir_open(&lfs, &dir, "test") => 0; + struct lfs_info info; + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, ".") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "..") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + while (true) { + int err = lfsr_dir_read(&lfs, &dir, &info); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + break; + } + assert(strlen(info.name) == strlen("amethyst...")); + assert(memcmp(info.name, "amethyst", strlen("amethyst")) == 0); + assert(info.type == LFS_TYPE_REG); + assert(info.size == SIZE); + + // at least try to read the files + char name[256]; + sprintf(name, "test/%s", info.name); + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, LFS_O_RDONLY) => 0; + + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + // sum should be equal to 'a' mod 26 + uint8_t ck = 0; + for (lfs_size_t j = 0; j < SIZE; j++) { + ck = (ck + (rbuf[j] - 'a')) % 26; + } + assert(ck == 0); + lfsr_file_close(&lfs, &file) => 0; + } + lfsr_dir_close(&lfs, &dir) => 0; + + lfsr_unmount(&lfs) => 0; +''' + + # TODO # [cases.test_files_multi_readers] # [cases.test_files_multi_readers_one_writer] diff --git a/tests/test_relocations.toml b/tests/test_relocations.toml index 4773c46e..83a797a8 100644 --- a/tests/test_relocations.toml +++ b/tests/test_relocations.toml @@ -11,8 +11,6 @@ after = [ # in test_mtree. This mostly just covers high-level operations with # relatively aggressive wear-leveling. -# TODO power-loss testing? - # dirs + relocations may create problems for gstate [cases.test_relocations_dir_fuzz] defines.BLOCK_RECYCLES = [-1, 5, 1, 0] @@ -1127,6 +1125,220 @@ code = ''' lfsr_unmount(&lfs) => 0; ''' + +# and don't forget potential powerloss problems +# +# Under powerloss, we can't really keep track of a sim reliably/ +# efficiently, instead just do random operations, store a counter in a +# special file so we know how much progress has been made, and hope for +# the best. Most likely an internal assert will trigger if anything goes +# wrong. +# +[cases.test_relocations_pl_fuzz] +defines.BLOCK_RECYCLES = [-1, 5, 1, 0] +defines.N = [1, 2, 4, 8, 16, 32, 64] +defines.OPS = 256 +defines.SIZE = [ + '0', + 'FBUFFER_SIZE/2', + '2*FBUFFER_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.REMOUNT = [false, true] +defines.SEED = 'range(10)' +if = '(SIZE*N)/BLOCK_SIZE <= 16' +reentrant = true +code = ''' + // format once per test + lfs_t lfs; + int err = lfsr_mount(&lfs, CFG); + if (err) { + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + } + + // keep some test state on disk to survive powerloss + typedef struct fuzz_state { + lfs_size_t i; + uint32_t prng; + } fuzz_state_t; + fuzz_state_t state = {.i = 0, .prng = SEED}; + + lfsr_file_t state_file; + lfsr_file_open(&lfs, &state_file, "state", LFS_O_RDWR | LFS_O_CREAT) => 0; + lfs_ssize_t d = lfsr_file_read(&lfs, &state_file, &state, sizeof(state)); + assert(d == 0 || d == sizeof(state)); + + // keep test files in a separate directory + err = lfsr_mkdir(&lfs, "test"); + assert(!err || err == LFS_ERR_EXIST); + + uint32_t prng = state.prng; + for (lfs_size_t i = state.i; i < OPS; i++) { + // choose which operation to do + uint8_t op = TEST_PRNG(&prng) % 3; + + // how many files do we have? + lfs_size_t count = 0; + lfsr_dir_t dir; + lfsr_dir_open(&lfs, &dir, "test") => 0; + struct lfs_info info; + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, ".") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "..") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + while (true) { + int err = lfsr_dir_read(&lfs, &dir, &info); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + break; + } + assert(strlen(info.name) == strlen("amethyst...")); + assert(memcmp(info.name, "amethyst", strlen("amethyst")) == 0); + assert(info.type == LFS_TYPE_REG); + assert(info.size == SIZE); + count++; + } + lfsr_dir_close(&lfs, &dir) => 0; + + // creating a new file? + if (op == 0 || count == 0) { + // choose a pseudo-random number + lfs_size_t x = TEST_PRNG(&prng) % N; + uint32_t wprng = TEST_PRNG(&prng); + + // create a file here + char name[256]; + sprintf(name, "test/amethyst%03x", x); + uint8_t wbuf[SIZE]; + uint8_t ck = 0; + for (lfs_size_t j = 0; j < SIZE-1; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&wprng) % 26); + ck = (ck + (wbuf[j] - 'a')) % 26; + } + // make the sum equal to 'a' mod 26 + if (SIZE > 0) { + wbuf[SIZE-1] = 'a' + ((26 - ck) % 26); + } + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE; + lfsr_file_close(&lfs, &file) => 0; + + // deleting a file? + } else if (op == 1) { + // choose a random file to delete + lfs_size_t j = TEST_PRNG(&prng) % count; + // find the file + lfsr_dir_open(&lfs, &dir, "test") => 0; + lfsr_dir_read(&lfs, &dir, &info) => 0; + lfsr_dir_read(&lfs, &dir, &info) => 0; + for (lfs_size_t k = 0; k <= j; k++) { + lfsr_dir_read(&lfs, &dir, &info) => 0; + } + lfsr_dir_close(&lfs, &dir) => 0; + + // delete this file + char name[256]; + assert(strlen(info.name) == strlen("amethyst...")); + sprintf(name, "test/%s", info.name); + lfsr_remove(&lfs, name) => 0; + + // renaming a file? + } else { + // choose a random file to rename, and a random number to + // rename to + lfs_size_t j = TEST_PRNG(&prng) % count; + lfs_size_t y = TEST_PRNG(&prng) % N; + // find the file + lfsr_dir_open(&lfs, &dir, "test") => 0; + lfsr_dir_read(&lfs, &dir, &info) => 0; + lfsr_dir_read(&lfs, &dir, &info) => 0; + for (lfs_size_t k = 0; k <= j; k++) { + lfsr_dir_read(&lfs, &dir, &info) => 0; + } + lfsr_dir_close(&lfs, &dir) => 0; + + // rename this file + char old_name[256]; + assert(strlen(info.name) == strlen("amethyst...")); + sprintf(old_name, "test/%s", info.name); + char new_name[256]; + sprintf(new_name, "test/amethyst%03x", y); + lfsr_rename(&lfs, old_name, new_name) => 0; + } + + // update our state file + state.i = i; + state.prng = prng; + lfsr_file_rewind(&lfs, &state_file) => 0; + lfsr_file_write(&lfs, &state_file, &state, sizeof(state)) + => sizeof(state); + lfsr_file_sync(&lfs, &state_file) => 0; + } + + // go ahead and close our state file in case we remount + lfsr_file_close(&lfs, &state_file) => 0; + + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + } + + // check that things look more-or-less ok + lfsr_dir_t dir; + lfsr_dir_open(&lfs, &dir, "test") => 0; + struct lfs_info info; + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, ".") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "..") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + while (true) { + int err = lfsr_dir_read(&lfs, &dir, &info); + assert(!err || err == LFS_ERR_NOENT); + if (err == LFS_ERR_NOENT) { + break; + } + assert(strlen(info.name) == strlen("amethyst...")); + assert(memcmp(info.name, "amethyst", strlen("amethyst")) == 0); + assert(info.type == LFS_TYPE_REG); + assert(info.size == SIZE); + + // at least try to read the files + char name[256]; + sprintf(name, "test/%s", info.name); + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, LFS_O_RDONLY) => 0; + + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + // sum should be equal to 'a' mod 26 + uint8_t ck = 0; + for (lfs_size_t j = 0; j < SIZE; j++) { + ck = (ck + (rbuf[j] - 'a')) % 26; + } + assert(ck == 0); + lfsr_file_close(&lfs, &file) => 0; + } + lfsr_dir_close(&lfs, &dir) => 0; + + lfsr_unmount(&lfs) => 0; +''' + # TODO, but this needs bad blocks ## test that wear-leveling converges to a more-or-less even distribution #[cases.test_relocations_distribution_dirs]