Github/littlefs
1
Fork 0
Code Actions Activity

Changed how fuzz tests are iterated to allow powerloss-fuzz testing

Browse Source 
Instead of iterating over a number of seeds in the test itself, the
seeds are now permuted as a part of normal test defines.

This lets each seed take advantage of other test features, mainly the
ability to test powerlosses heuristically.

This is probably how it should have been done in the first place, but
the permutation tests can't do this since the number of permutations
changes as the size of the test input changes. The test define system
can't handle that very well.

The tradeoffs here are:

- We can't do cross-fuzz checks, such as the balance checks in the rbyd
  tests, though those really should be moved to benchmarks anyways.

- The large number of cheap fuzz permutations skews the total
  permutation count, though I'm not sure this matters.

  before: 3083 permutations (-Gnor)
  after: 409893 permutations (-Gnor)
This commit is contained in:
Christopher Haster
2023-07-18 21:13:30 -05:00
parent c928ed131f
commit c5e84e874f
6 changed files with 2795 additions and 3124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bench.py
+7 -5
View File
@@ -758,20 +758,22 @@ def find_ids(runner, bench_ids=[], **args):
# find suite/case by id
bench_ids_ = []
for id in bench_ids:
# strip permutation
name, *_ = id.split(':', 1)
bench_ids__ = []
# resolve globs
if '*' in id:
if '*' in name:
bench_ids__.extend(suite
for suite in expected_suite_perms.keys()
if fnmatch.fnmatch(suite, id))
if fnmatch.fnmatch(suite, name))
bench_ids__.extend(case_
for case_ in expected_case_perms.keys()
if fnmatch.fnmatch(case_, id))
if fnmatch.fnmatch(case_, name))
# literal suite
elif id in expected_suite_perms:
elif name in expected_suite_perms:
bench_ids__.append(id)
# literal case
elif id in expected_case_perms:
elif name in expected_case_perms:
bench_ids__.append(id)
# no suite/case found? error
scripts/test.py
+7 -5
View File
@@ -767,20 +767,22 @@ def find_ids(runner, test_ids=[], **args):
# find suite/case by id
test_ids_ = []
for id in test_ids:
# strip permutation
name, *_ = id.split(':', 1)
test_ids__ = []
# resolve globs
if '*' in id:
if '*' in name:
test_ids__.extend(suite
for suite in expected_suite_perms.keys()
if fnmatch.fnmatch(suite, id))
if fnmatch.fnmatch(suite, name))
test_ids__.extend(case_
for case_ in expected_case_perms.keys()
if fnmatch.fnmatch(case_, id))
if fnmatch.fnmatch(case_, name))
# literal suite
elif id in expected_suite_perms:
elif name in expected_suite_perms:
test_ids__.append(id)
# literal case
elif id in expected_case_perms:
elif name in expected_case_perms:
test_ids__.append(id)
# no suite/case found? error
tests/t1_rbyd.toml
+410 -533
View File
File diff suppressed because it is too large Load Diff
tests/t2_btree.toml
+1827 -1991
View File
File diff suppressed because it is too large Load Diff
tests/t3_mtree.toml
+457 -501
View File
File diff suppressed because it is too large Load Diff
tests/t5_dirs.toml
+87 -89
View File
@@ -706,99 +706,97 @@ code = '''
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
defines.PARENT = [false, true]
defines.REMOUNT = [false, true]
defines.SAMPLES = 10
# -1 => all pseudo-random seeds
# n => reproduce a specific seed
defines.SEED = -1
defines.SEED = 'range(10)'
reentrant = true
code = '''
// iterate through severals seeds that we can reproduce easily
for (uint32_t seed = (SEED == -1 ? 1 : SEED);
(SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
seed++) {
printf("--- seed: %d ---\n", seed);
// reset lfs here each iteration
lfs_t lfs;
// format once per test
lfs_t lfs;
int err = lfsr_mount(&lfs, cfg);
if (err) {
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
if (PARENT) {
lfsr_mkdir(&lfs, "parent") => 0;
}
// set up a simulation to compare against
lfs_size_t *sim = malloc(N*sizeof(lfs_size_t));
lfs_size_t sim_size = 0;
uint32_t prng = seed;
for (lfs_size_t i = 0; i < N; i++) {
// choose a pseudo-random number, truncate to 4 decimals
lfs_size_t x = TEST_PRNG(&prng) % 1000;
// insert into our sim
for (lfs_size_t j = 0;; j++) {
if (j >= sim_size || sim[j] >= x) {
// already seen? skip
if (sim[j] == x) {
goto next;
}
// insert
memmove(&sim[j+1], &sim[j],
(sim_size-j)*sizeof(lfs_size_t));
sim_size += 1;
sim[j] = x;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
break;
}
}
// create a directory here
char name[256];
sprintf(name, "%s/dir%04d", (PARENT ? "parent" : ""), x);
lfsr_mkdir(&lfs, name) => 0;
next:;
}
// test that our directories match our simulation
for (lfs_size_t j = 0; j < sim_size; j++) {
char name[256];
sprintf(name, "%s/dir%04d", (PARENT ? "parent" : ""), sim[j]);
struct lfs_info info;
lfsr_stat(&lfs, name, &info) => 0;
char name2[256];
sprintf(name2, "dir%04d", sim[j]);
assert(strcmp(info.name, name2) == 0);
assert(info.type == LFS_TYPE_DIR);
}
lfsr_dir_t dir;
lfsr_dir_open(&lfs, &dir, (PARENT ? "parent" : "/")) => 0;
struct lfs_info info;
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, ".") == 0);
assert(info.type == LFS_TYPE_DIR);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "..") == 0);
assert(info.type == LFS_TYPE_DIR);
for (lfs_size_t j = 0; j < sim_size; j++) {
char name[256];
sprintf(name, "dir%04d", sim[j]);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, name) == 0);
assert(info.type == LFS_TYPE_DIR);
}
lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT;
// clean up sim/lfs
free(sim);
lfsr_unmount(&lfs) => 0;
}
if (PARENT) {
err = lfsr_mkdir(&lfs, "parent");
assert(!err || (TEST_PL && err == LFS_ERR_EXIST));
}
// set up a simulation to compare against
lfs_size_t *sim = malloc(N*sizeof(lfs_size_t));
lfs_size_t sim_size = 0;
uint32_t prng = SEED;
for (lfs_size_t i = 0; i < N; i++) {
// choose a pseudo-random number, truncate to 4 decimals
lfs_size_t x = TEST_PRNG(&prng) % 1000;
// insert into our sim
for (lfs_size_t j = 0;; j++) {
if (j >= sim_size || sim[j] >= x) {
// already seen? skip
if (j < sim_size && sim[j] == x) {
goto next;
}
// insert
memmove(&sim[j+1], &sim[j],
(sim_size-j)*sizeof(lfs_size_t));
sim_size += 1;
sim[j] = x;
break;
}
}
// create a directory here
char name[256];
sprintf(name, "%s/dir%04d", (PARENT ? "parent" : ""), x);
err = lfsr_mkdir(&lfs, name);
assert(!err || (TEST_PL && err == LFS_ERR_EXIST));
next:;
}
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
// test that our directories match our simulation
for (lfs_size_t j = 0; j < sim_size; j++) {
char name[256];
sprintf(name, "%s/dir%04d", (PARENT ? "parent" : ""), sim[j]);
struct lfs_info info;
lfsr_stat(&lfs, name, &info) => 0;
char name2[256];
sprintf(name2, "dir%04d", sim[j]);
assert(strcmp(info.name, name2) == 0);
assert(info.type == LFS_TYPE_DIR);
}
lfsr_dir_t dir;
lfsr_dir_open(&lfs, &dir, (PARENT ? "parent" : "/")) => 0;
struct lfs_info info;
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, ".") == 0);
assert(info.type == LFS_TYPE_DIR);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "..") == 0);
assert(info.type == LFS_TYPE_DIR);
for (lfs_size_t j = 0; j < sim_size; j++) {
char name[256];
sprintf(name, "dir%04d", sim[j]);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, name) == 0);
assert(info.type == LFS_TYPE_DIR);
}
lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT;
// clean up sim/lfs
free(sim);
lfsr_unmount(&lfs) => 0;
'''