Github/littlefs
1
Fork 0
Code Actions Activity

runners: Added both run+compile-time --no-internal/reentrant/fuzz flags

Browse Source 
--no-internal has already proven useful for skipping internal tests for
refactoring, so it makes sense to add --no-reentrant/fuzz flags as well.
--no-fuzz seems particularly useful for when you want to skip the less
targeted fuzz tests:

- with fuzz tests: 634616/634616 passed, in 1239.90s
- with --no-fuzz:    85434/85434 passed, in  423.41s

I also added runtime variants to test/bench_runner and test/bench.py.
These may be useful to skip tests without needing to recompile the
runner.

---

Also tweaked -s/--step to filter permutations in any --list-* flags, for
consistency.
This commit is contained in:
Christopher Haster
2026-02-05 01:14:17 -06:00
parent 9974656c5c
commit ebde2c7063
4 changed files with 127 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runners/bench_runner.c
+37 -10
View File
@@ -439,7 +439,9 @@ size_t bench_id_count = 1;
size_t bench_step_start = 0;
size_t bench_step_stop = -1;
size_t bench_step_step = 1;
size_t bench_step = 0; // incremented every permutation
bool bench_force = false;
bench_flags_t bench_mask = 0;
const char *bench_disk_path = NULL;
const char *bench_trace_path = NULL;
@@ -1234,8 +1236,23 @@ void perm_count(
struct perm_count_state *state = data;
(void)suite;
// masked? consider this lower-level than filtering
if (case_->flags & bench_mask) {
return;
}
// skip this step?
if (!(bench_step >= bench_step_start
&& bench_step < bench_step_stop
&& (bench_step-bench_step_start) % bench_step_step == 0)) {
bench_step += 1;
return;
}
bench_step += 1;
state->total += 1;
// filter? this includes ifdef (run=NULL) and if checks
if (!case_->run || !(bench_force || !case_->if_ || case_->if_())) {
return;
}
@@ -1807,15 +1824,18 @@ int bench_bd_sync(const struct lfs3_cfg *cfg) {
// global bench step count
size_t bench_step = 0;
// main permutation runner
void perm_run(
void *data,
const struct bench_suite *suite,
const struct bench_case *case_) {
(void)data;
// masked? consider this lower-level than filtering
if (case_->flags & bench_mask) {
return;
}
// skip this step?
if (!(bench_step >= bench_step_start
&& bench_step < bench_step_stop
@@ -1825,7 +1845,7 @@ void perm_run(
}
bench_step += 1;
// filter?
// filter? this includes ifdef (run=NULL) and if checks
if (!case_->run || !(bench_force || !case_->if_ || case_->if_())) {
printf("skipped ");
perm_printid(suite, case_);
@@ -1962,14 +1982,15 @@ enum opt_flags {
OPT_DEFINE_DEPTH = 6,
OPT_STEP = 's',
OPT_FORCE = 7,
OPT_NO_INTERNAL = 8,
OPT_DISK = 'd',
OPT_TRACE = 't',
OPT_TRACE_BACKTRACE = 8,
OPT_TRACE_PERIOD = 9,
OPT_TRACE_FREQ = 10,
OPT_READ_SLEEP = 11,
OPT_PROG_SLEEP = 12,
OPT_ERASE_SLEEP = 13,
OPT_TRACE_BACKTRACE = 9,
OPT_TRACE_PERIOD = 10,
OPT_TRACE_FREQ = 11,
OPT_READ_SLEEP = 12,
OPT_PROG_SLEEP = 13,
OPT_ERASE_SLEEP = 14,
};
const char *short_opts = "hYlLD:s:d:t:";
@@ -1990,6 +2011,7 @@ const struct option long_opts[] = {
{"define-depth", required_argument, NULL, OPT_DEFINE_DEPTH},
{"step", required_argument, NULL, OPT_STEP},
{"force", no_argument, NULL, OPT_FORCE},
{"no-internal", no_argument, NULL, OPT_NO_INTERNAL},
{"disk", required_argument, NULL, OPT_DISK},
{"trace", required_argument, NULL, OPT_TRACE},
{"trace-backtrace", no_argument, NULL, OPT_TRACE_BACKTRACE},
@@ -2015,6 +2037,7 @@ const char *const help_text[] = {
"How deep to evaluate recursive defines before erroring.",
"Comma-separated range of permutations to run.",
"Ignore bench filters.",
"Don't run internal benches.",
"Direct block device operations to this file.",
"Direct trace output to this file.",
"Include a backtrace with every trace statement.",
@@ -2341,6 +2364,10 @@ int main(int argc, char **argv) {
bench_force = true;
break;
case OPT_NO_INTERNAL:;
bench_mask |= BENCH_INTERNAL;
break;
case OPT_DISK:;
bench_disk_path = optarg;
break;
runners/test_runner.c
+52 -10
View File
@@ -450,7 +450,9 @@ size_t test_id_count = 1;
size_t test_step_start = 0;
size_t test_step_stop = -1;
size_t test_step_step = 1;
size_t test_step = 0; // incremented every permutation
bool test_force = false;
test_flags_t test_mask = 0;
const char *test_disk_path = NULL;
const char *test_trace_path = NULL;
@@ -465,7 +467,7 @@ test_ns_t test_read_sleep = 0.0;
test_ns_t test_prog_sleep = 0.0;
test_ns_t test_erase_sleep = 0.0;
volatile size_t TEST_PLS = 0;
volatile size_t TEST_PLS = 0; // incremented every powerloss
extern const test_powerloss_t *test_powerlosses;
extern size_t test_powerloss_count;
@@ -1041,10 +1043,25 @@ void perm_count(
struct perm_count_state *state = data;
(void)suite;
// masked? consider this lower-level than filtering
if (case_->flags & test_mask) {
return;
}
// skip this step?
if (!(test_step >= test_step_start
&& test_step < test_step_stop
&& (test_step-test_step_start) % test_step_step == 0)) {
test_step += 1;
return;
}
test_step += 1;
state->total += 1;
// set pls to 1 if running under powerloss so it useful for if predicates
TEST_PLS = (powerloss->run != run_powerloss_none);
// filter? this includes ifdef (run=NULL) and if checks
if (!case_->run || !(test_force || !case_->if_ || case_->if_())) {
return;
}
@@ -2259,9 +2276,8 @@ static void list_powerlosses(void) {
}
// global test step count
size_t test_step = 0;
// main permutation runner
void perm_run(
void *data,
const struct test_suite *suite,
@@ -2269,6 +2285,11 @@ void perm_run(
const test_powerloss_t *powerloss) {
(void)data;
// masked? consider this lower-level than filtering
if (case_->flags & test_mask) {
return;
}
// skip this step?
if (!(test_step >= test_step_start
&& test_step < test_step_stop
@@ -2280,7 +2301,7 @@ void perm_run(
// set pls to 1 if running under powerloss so it useful for if predicates
TEST_PLS = (powerloss->run != run_powerloss_none);
// filter?
// filter? this includes ifdef (run=NULL) and if checks
if (!case_->run || !(test_force || !case_->if_ || case_->if_())) {
printf("skipped ");
perm_printid(suite, case_, NULL, 0);
@@ -2340,14 +2361,17 @@ enum opt_flags {
OPT_POWERLOSS = 'P',
OPT_STEP = 's',
OPT_FORCE = 8,
OPT_NO_INTERNAL = 9,
OPT_NO_REENTRANT = 10,
OPT_NO_FUZZ = 11,
OPT_DISK = 'd',
OPT_TRACE = 't',
OPT_TRACE_BACKTRACE = 9,
OPT_TRACE_PERIOD = 10,
OPT_TRACE_FREQ = 11,
OPT_READ_SLEEP = 12,
OPT_PROG_SLEEP = 13,
OPT_ERASE_SLEEP = 14,
OPT_TRACE_BACKTRACE = 12,
OPT_TRACE_PERIOD = 13,
OPT_TRACE_FREQ = 14,
OPT_READ_SLEEP = 15,
OPT_PROG_SLEEP = 16,
OPT_ERASE_SLEEP = 17,
};
const char *short_opts = "hYlLD:P:s:d:t:";
@@ -2370,6 +2394,9 @@ const struct option long_opts[] = {
{"powerloss", required_argument, NULL, OPT_POWERLOSS},
{"step", required_argument, NULL, OPT_STEP},
{"force", no_argument, NULL, OPT_FORCE},
{"no-internal", no_argument, NULL, OPT_NO_INTERNAL},
{"no-reentrant", no_argument, NULL, OPT_NO_REENTRANT},
{"no-fuzz", no_argument, NULL, OPT_NO_FUZZ},
{"disk", required_argument, NULL, OPT_DISK},
{"trace", required_argument, NULL, OPT_TRACE},
{"trace-backtrace", no_argument, NULL, OPT_TRACE_BACKTRACE},
@@ -2397,6 +2424,9 @@ const char *const help_text[] = {
"Comma-separated list of powerloss scenarios to test.",
"Comma-separated range of permutations to run.",
"Ignore test filters.",
"Don't run internal tests.",
"Don't run reentrant tests.",
"Don't run fuzz tests.",
"Direct block device operations to this file.",
"Direct trace output to this file.",
"Include a backtrace with every trace statement.",
@@ -2894,6 +2924,18 @@ int main(int argc, char **argv) {
test_force = true;
break;
case OPT_NO_INTERNAL:;
test_mask |= TEST_INTERNAL;
break;
case OPT_NO_REENTRANT:;
test_mask |= TEST_REENTRANT;
break;
case OPT_NO_FUZZ:;
test_mask |= TEST_FUZZ;
break;
case OPT_DISK:;
test_disk_path = optarg;
break;