Github/littlefs
1
Fork 0
Code Actions Activity

Added pseudo-stateless *_pl_fuzz tests

Browse Source 
These provide useful file powerloss testing that scales linearly as long
as progress can be made. They can still struggle a bit, especially with
relocations which often fail to make progress, but they are _much_ better
than the O(n^2) simulation-based fuzz tests:

- test_files_pl_fuzz - 258734 pls
- test_relocations_pl_fuzz - 928638 pls

Our current problem with simulation-based fuzz testing is that we lose
the simulation on powerloss. We could brute force this, repeatedly
rerunning the simulation until it succeeds, but this grows O(n^2) with
our linear powerloss heuristic.

To avoid this, test_*_pl_fuzz doesn't bother with a simulation, instead
relying on internal asserts to catch bugs. This is less rigorous, but
realistically probably going to catch any powerloss related issues.

Some notes:

- We need to store some state on disk. If we don't we will still end up
  with O(n^2) behavior because we simply don't know how many operations
  we've accomplished so far.

- Since we rely on file operations to store our test state, this makes
  this approach incompatible with the dir tests, which assume file
  operations may not yet be implemented.

  We still use O(n^2) powerloss testing in test_dirs, just with a small
  number of directories.

- It's tempting to try to store a full simulation on disk. But you
  would quickly run into atomicity issues with the simulation itself.
  Powerloss resilience is tricky!

- We can at least store a checksum in the files (currently just mod 26)
  to check that the file itself was not corrupted. This doesn't protect
  against swapped data though.

---

Also, a bit of a tangent, but I needed to add -Wno-format-overflow to
the test flags to avoid an annoying invalid format-overlow warning:

  struct lfs_info info;
  char name[256];
  if (strlen(info.name) < 100) { // can't overflow!?
      sprintf(name, "test/%s", info.name); // <--
  }

  warning: '%s' directive writing up to 255 bytes into a region of size
  251 [-Wformat-overflow=]

This seems like a GCC bug, because as far as I can tell there is no way
to signal or hint that the size is in bounds without just disabling the
warning completely...
This commit is contained in:
Christopher Haster
2024-05-27 16:06:50 -05:00
parent 7e62ebe18a
commit 3b33c33339
3 changed files with 430 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+2
View File
@@ -356,6 +356,7 @@ summary-diff sizes-diff: $(OBJ) $(CI)
## Build the test-runner
.PHONY: test-runner build-test
test-runner build-test: CFLAGS+=-Wno-unused-function
test-runner build-test: CFLAGS+=-Wno-format-overflow
ifdef COVGEN
test-runner build-test: CFLAGS+=--coverage
endif
@@ -408,6 +409,7 @@ testmarks-diff: $(TEST_CSV)
## Build the bench-runner
.PHONY: bench-runner build-bench
bench-runner build-bench: CFLAGS+=-Wno-unused-function
bench-runner build-bench: CFLAGS+=-Wno-format-overflow
ifdef COVGEN
bench-runner build-bench: CFLAGS+=--coverage
endif
tests/test_files.toml
+214
View File
@@ -2603,6 +2603,220 @@ code = '''
'''
# fuzz test file ops under powerloss
#
# Under powerloss, we can't really keep track of a sim reliably/
# efficiently, instead just do random operations, store a counter in a
# special file so we know how much progress has been made, and hope for
# the best. Most likely an internal assert will trigger if anything goes
# wrong.
#
[cases.test_files_pl_fuzz]
defines.N = [1, 2, 4, 8, 16, 32, 64]
# do more ops than files to encourage file rewrites
defines.OPS = '2*N'
defines.SIZE = [
'0',
'FBUFFER_SIZE/2',
'2*FBUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'4*BLOCK_SIZE',
]
defines.REMOUNT = [false, true]
defines.SEED = 'range(20)'
if = '(SIZE*N)/BLOCK_SIZE <= 16'
reentrant = true
code = '''
// format once per test
lfs_t lfs;
int err = lfsr_mount(&lfs, CFG);
if (err) {
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
}
// keep some test state on disk to survive powerloss
typedef struct fuzz_state {
lfs_size_t i;
uint32_t prng;
} fuzz_state_t;
fuzz_state_t state = {.i = 0, .prng = SEED};
lfsr_file_t state_file;
lfsr_file_open(&lfs, &state_file, "state", LFS_O_RDWR | LFS_O_CREAT) => 0;
lfs_ssize_t d = lfsr_file_read(&lfs, &state_file, &state, sizeof(state));
assert(d == 0 || d == sizeof(state));
// keep test files in a separate directory
err = lfsr_mkdir(&lfs, "test");
assert(!err || err == LFS_ERR_EXIST);
uint32_t prng = state.prng;
for (lfs_size_t i = state.i; i < OPS; i++) {
// choose which operation to do
uint8_t op = TEST_PRNG(&prng) % 3;
// how many files do we have?
lfs_size_t count = 0;
lfsr_dir_t dir;
lfsr_dir_open(&lfs, &dir, "test") => 0;
struct lfs_info info;
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, ".") == 0);
assert(info.type == LFS_TYPE_DIR);
assert(info.size == 0);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "..") == 0);
assert(info.type == LFS_TYPE_DIR);
assert(info.size == 0);
while (true) {
int err = lfsr_dir_read(&lfs, &dir, &info);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
assert(strlen(info.name) == strlen("amethyst..."));
assert(memcmp(info.name, "amethyst", strlen("amethyst")) == 0);
assert(info.type == LFS_TYPE_REG);
assert(info.size == SIZE);
count++;
}
lfsr_dir_close(&lfs, &dir) => 0;
// creating a new file?
if (op == 0 || count == 0) {
// choose a pseudo-random number
lfs_size_t x = TEST_PRNG(&prng) % N;
uint32_t wprng = TEST_PRNG(&prng);
// create a file here
char name[256];
sprintf(name, "test/amethyst%03x", x);
uint8_t wbuf[SIZE];
uint8_t ck = 0;
for (lfs_size_t j = 0; j < SIZE-1; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&wprng) % 26);
ck = (ck + (wbuf[j] - 'a')) % 26;
}
// make the sum equal to 'a' mod 26
if (SIZE > 0) {
wbuf[SIZE-1] = 'a' + ((26 - ck) % 26);
}
lfsr_file_t file;
lfsr_file_open(&lfs, &file, name,
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0;
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_close(&lfs, &file) => 0;
// deleting a file?
} else if (op == 1) {
// choose a random file to delete
lfs_size_t j = TEST_PRNG(&prng) % count;
// find the file
lfsr_dir_open(&lfs, &dir, "test") => 0;
lfsr_dir_read(&lfs, &dir, &info) => 0;
lfsr_dir_read(&lfs, &dir, &info) => 0;
for (lfs_size_t k = 0; k <= j; k++) {
lfsr_dir_read(&lfs, &dir, &info) => 0;
}
lfsr_dir_close(&lfs, &dir) => 0;
// delete this file
char name[256];
assert(strlen(info.name) == strlen("amethyst..."));
sprintf(name, "test/%s", info.name);
lfsr_remove(&lfs, name) => 0;
// renaming a file?
} else {
// choose a random file to rename, and a random number to
// rename to
lfs_size_t j = TEST_PRNG(&prng) % count;
lfs_size_t y = TEST_PRNG(&prng) % N;
// find the file
lfsr_dir_open(&lfs, &dir, "test") => 0;
lfsr_dir_read(&lfs, &dir, &info) => 0;
lfsr_dir_read(&lfs, &dir, &info) => 0;
for (lfs_size_t k = 0; k <= j; k++) {
lfsr_dir_read(&lfs, &dir, &info) => 0;
}
lfsr_dir_close(&lfs, &dir) => 0;
// rename this file
char old_name[256];
assert(strlen(info.name) == strlen("amethyst..."));
sprintf(old_name, "test/%s", info.name);
char new_name[256];
sprintf(new_name, "test/amethyst%03x", y);
lfsr_rename(&lfs, old_name, new_name) => 0;
}
// update our state file
state.i = i;
state.prng = prng;
lfsr_file_rewind(&lfs, &state_file) => 0;
lfsr_file_write(&lfs, &state_file, &state, sizeof(state))
=> sizeof(state);
lfsr_file_sync(&lfs, &state_file) => 0;
}
// go ahead and close our state file in case we remount
lfsr_file_close(&lfs, &state_file) => 0;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
}
// check that things look more-or-less ok
lfsr_dir_t dir;
lfsr_dir_open(&lfs, &dir, "test") => 0;
struct lfs_info info;
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, ".") == 0);
assert(info.type == LFS_TYPE_DIR);
assert(info.size == 0);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "..") == 0);
assert(info.type == LFS_TYPE_DIR);
assert(info.size == 0);
while (true) {
int err = lfsr_dir_read(&lfs, &dir, &info);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
assert(strlen(info.name) == strlen("amethyst..."));
assert(memcmp(info.name, "amethyst", strlen("amethyst")) == 0);
assert(info.type == LFS_TYPE_REG);
assert(info.size == SIZE);
// at least try to read the files
char name[256];
sprintf(name, "test/%s", info.name);
lfsr_file_t file;
lfsr_file_open(&lfs, &file, name, LFS_O_RDONLY) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
// sum should be equal to 'a' mod 26
uint8_t ck = 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
ck = (ck + (rbuf[j] - 'a')) % 26;
}
assert(ck == 0);
lfsr_file_close(&lfs, &file) => 0;
}
lfsr_dir_close(&lfs, &dir) => 0;
lfsr_unmount(&lfs) => 0;
'''
# TODO
# [cases.test_files_multi_readers]
# [cases.test_files_multi_readers_one_writer]
tests/test_relocations.toml
+214 -2
View File
@@ -11,8 +11,6 @@ after = [
# in test_mtree. This mostly just covers high-level operations with
# relatively aggressive wear-leveling.
# TODO power-loss testing?
# dirs + relocations may create problems for gstate
[cases.test_relocations_dir_fuzz]
defines.BLOCK_RECYCLES = [-1, 5, 1, 0]
@@ -1127,6 +1125,220 @@ code = '''
lfsr_unmount(&lfs) => 0;
'''
# and don't forget potential powerloss problems
#
# Under powerloss, we can't really keep track of a sim reliably/
# efficiently, instead just do random operations, store a counter in a
# special file so we know how much progress has been made, and hope for
# the best. Most likely an internal assert will trigger if anything goes
# wrong.
#
[cases.test_relocations_pl_fuzz]
defines.BLOCK_RECYCLES = [-1, 5, 1, 0]
defines.N = [1, 2, 4, 8, 16, 32, 64]
defines.OPS = 256
defines.SIZE = [
'0',
'FBUFFER_SIZE/2',
'2*FBUFFER_SIZE',
'BLOCK_SIZE/2',
'BLOCK_SIZE',
'2*BLOCK_SIZE',
'4*BLOCK_SIZE',
]
defines.REMOUNT = [false, true]
defines.SEED = 'range(10)'
if = '(SIZE*N)/BLOCK_SIZE <= 16'
reentrant = true
code = '''
// format once per test
lfs_t lfs;
int err = lfsr_mount(&lfs, CFG);
if (err) {
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
}
// keep some test state on disk to survive powerloss
typedef struct fuzz_state {
lfs_size_t i;
uint32_t prng;
} fuzz_state_t;
fuzz_state_t state = {.i = 0, .prng = SEED};
lfsr_file_t state_file;
lfsr_file_open(&lfs, &state_file, "state", LFS_O_RDWR | LFS_O_CREAT) => 0;
lfs_ssize_t d = lfsr_file_read(&lfs, &state_file, &state, sizeof(state));
assert(d == 0 || d == sizeof(state));
// keep test files in a separate directory
err = lfsr_mkdir(&lfs, "test");
assert(!err || err == LFS_ERR_EXIST);
uint32_t prng = state.prng;
for (lfs_size_t i = state.i; i < OPS; i++) {
// choose which operation to do
uint8_t op = TEST_PRNG(&prng) % 3;
// how many files do we have?
lfs_size_t count = 0;
lfsr_dir_t dir;
lfsr_dir_open(&lfs, &dir, "test") => 0;
struct lfs_info info;
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, ".") == 0);
assert(info.type == LFS_TYPE_DIR);
assert(info.size == 0);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "..") == 0);
assert(info.type == LFS_TYPE_DIR);
assert(info.size == 0);
while (true) {
int err = lfsr_dir_read(&lfs, &dir, &info);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
assert(strlen(info.name) == strlen("amethyst..."));
assert(memcmp(info.name, "amethyst", strlen("amethyst")) == 0);
assert(info.type == LFS_TYPE_REG);
assert(info.size == SIZE);
count++;
}
lfsr_dir_close(&lfs, &dir) => 0;
// creating a new file?
if (op == 0 || count == 0) {
// choose a pseudo-random number
lfs_size_t x = TEST_PRNG(&prng) % N;
uint32_t wprng = TEST_PRNG(&prng);
// create a file here
char name[256];
sprintf(name, "test/amethyst%03x", x);
uint8_t wbuf[SIZE];
uint8_t ck = 0;
for (lfs_size_t j = 0; j < SIZE-1; j++) {
wbuf[j] = 'a' + (TEST_PRNG(&wprng) % 26);
ck = (ck + (wbuf[j] - 'a')) % 26;
}
// make the sum equal to 'a' mod 26
if (SIZE > 0) {
wbuf[SIZE-1] = 'a' + ((26 - ck) % 26);
}
lfsr_file_t file;
lfsr_file_open(&lfs, &file, name,
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0;
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_close(&lfs, &file) => 0;
// deleting a file?
} else if (op == 1) {
// choose a random file to delete
lfs_size_t j = TEST_PRNG(&prng) % count;
// find the file
lfsr_dir_open(&lfs, &dir, "test") => 0;
lfsr_dir_read(&lfs, &dir, &info) => 0;
lfsr_dir_read(&lfs, &dir, &info) => 0;
for (lfs_size_t k = 0; k <= j; k++) {
lfsr_dir_read(&lfs, &dir, &info) => 0;
}
lfsr_dir_close(&lfs, &dir) => 0;
// delete this file
char name[256];
assert(strlen(info.name) == strlen("amethyst..."));
sprintf(name, "test/%s", info.name);
lfsr_remove(&lfs, name) => 0;
// renaming a file?
} else {
// choose a random file to rename, and a random number to
// rename to
lfs_size_t j = TEST_PRNG(&prng) % count;
lfs_size_t y = TEST_PRNG(&prng) % N;
// find the file
lfsr_dir_open(&lfs, &dir, "test") => 0;
lfsr_dir_read(&lfs, &dir, &info) => 0;
lfsr_dir_read(&lfs, &dir, &info) => 0;
for (lfs_size_t k = 0; k <= j; k++) {
lfsr_dir_read(&lfs, &dir, &info) => 0;
}
lfsr_dir_close(&lfs, &dir) => 0;
// rename this file
char old_name[256];
assert(strlen(info.name) == strlen("amethyst..."));
sprintf(old_name, "test/%s", info.name);
char new_name[256];
sprintf(new_name, "test/amethyst%03x", y);
lfsr_rename(&lfs, old_name, new_name) => 0;
}
// update our state file
state.i = i;
state.prng = prng;
lfsr_file_rewind(&lfs, &state_file) => 0;
lfsr_file_write(&lfs, &state_file, &state, sizeof(state))
=> sizeof(state);
lfsr_file_sync(&lfs, &state_file) => 0;
}
// go ahead and close our state file in case we remount
lfsr_file_close(&lfs, &state_file) => 0;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
}
// check that things look more-or-less ok
lfsr_dir_t dir;
lfsr_dir_open(&lfs, &dir, "test") => 0;
struct lfs_info info;
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, ".") == 0);
assert(info.type == LFS_TYPE_DIR);
assert(info.size == 0);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "..") == 0);
assert(info.type == LFS_TYPE_DIR);
assert(info.size == 0);
while (true) {
int err = lfsr_dir_read(&lfs, &dir, &info);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
assert(strlen(info.name) == strlen("amethyst..."));
assert(memcmp(info.name, "amethyst", strlen("amethyst")) == 0);
assert(info.type == LFS_TYPE_REG);
assert(info.size == SIZE);
// at least try to read the files
char name[256];
sprintf(name, "test/%s", info.name);
lfsr_file_t file;
lfsr_file_open(&lfs, &file, name, LFS_O_RDONLY) => 0;
uint8_t rbuf[SIZE];
lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE;
// sum should be equal to 'a' mod 26
uint8_t ck = 0;
for (lfs_size_t j = 0; j < SIZE; j++) {
ck = (ck + (rbuf[j] - 'a')) % 26;
}
assert(ck == 0);
lfsr_file_close(&lfs, &file) => 0;
}
lfsr_dir_close(&lfs, &dir) => 0;
lfsr_unmount(&lfs) => 0;
'''
# TODO, but this needs bad blocks
## test that wear-leveling converges to a more-or-less even distribution
#[cases.test_relocations_distribution_dirs]