From ebde2c7063d15de2fdc0ccff0e20c7d30ac2ea5f Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 5 Feb 2026 01:14:17 -0600 Subject: [PATCH] runners: Added both run+compile-time --no-internal/reentrant/fuzz flags --no-internal has already proven useful for skipping internal tests for refactoring, so it makes sense to add --no-reentrant/fuzz flags as well. --no-fuzz seems particularly useful for when you want to skip the less targeted fuzz tests: - with fuzz tests: 634616/634616 passed, in 1239.90s - with --no-fuzz: 85434/85434 passed, in 423.41s I also added runtime variants to test/bench_runner and test/bench.py. These may be useful to skip tests without needing to recompile the runner. --- Also tweaked -s/--step to filter permutations in any --list-* flags, for consistency. --- runners/bench_runner.c | 47 +++++++++++++++++++++++++------- runners/test_runner.c | 62 +++++++++++++++++++++++++++++++++++------- scripts/bench.py | 10 +++++-- scripts/test.py | 30 ++++++++++++++++++++ 4 files changed, 127 insertions(+), 22 deletions(-) diff --git a/runners/bench_runner.c b/runners/bench_runner.c index ce145648..7ef31e50 100644 --- a/runners/bench_runner.c +++ b/runners/bench_runner.c @@ -439,7 +439,9 @@ size_t bench_id_count = 1; size_t bench_step_start = 0; size_t bench_step_stop = -1; size_t bench_step_step = 1; +size_t bench_step = 0; // incremented every permutation bool bench_force = false; +bench_flags_t bench_mask = 0; const char *bench_disk_path = NULL; const char *bench_trace_path = NULL; @@ -1234,8 +1236,23 @@ void perm_count( struct perm_count_state *state = data; (void)suite; + // masked? consider this lower-level than filtering + if (case_->flags & bench_mask) { + return; + } + + // skip this step? + if (!(bench_step >= bench_step_start + && bench_step < bench_step_stop + && (bench_step-bench_step_start) % bench_step_step == 0)) { + bench_step += 1; + return; + } + bench_step += 1; + state->total += 1; + // filter? this includes ifdef (run=NULL) and if checks if (!case_->run || !(bench_force || !case_->if_ || case_->if_())) { return; } @@ -1807,15 +1824,18 @@ int bench_bd_sync(const struct lfs3_cfg *cfg) { -// global bench step count -size_t bench_step = 0; - +// main permutation runner void perm_run( void *data, const struct bench_suite *suite, const struct bench_case *case_) { (void)data; + // masked? consider this lower-level than filtering + if (case_->flags & bench_mask) { + return; + } + // skip this step? if (!(bench_step >= bench_step_start && bench_step < bench_step_stop @@ -1825,7 +1845,7 @@ void perm_run( } bench_step += 1; - // filter? + // filter? this includes ifdef (run=NULL) and if checks if (!case_->run || !(bench_force || !case_->if_ || case_->if_())) { printf("skipped "); perm_printid(suite, case_); @@ -1962,14 +1982,15 @@ enum opt_flags { OPT_DEFINE_DEPTH = 6, OPT_STEP = 's', OPT_FORCE = 7, + OPT_NO_INTERNAL = 8, OPT_DISK = 'd', OPT_TRACE = 't', - OPT_TRACE_BACKTRACE = 8, - OPT_TRACE_PERIOD = 9, - OPT_TRACE_FREQ = 10, - OPT_READ_SLEEP = 11, - OPT_PROG_SLEEP = 12, - OPT_ERASE_SLEEP = 13, + OPT_TRACE_BACKTRACE = 9, + OPT_TRACE_PERIOD = 10, + OPT_TRACE_FREQ = 11, + OPT_READ_SLEEP = 12, + OPT_PROG_SLEEP = 13, + OPT_ERASE_SLEEP = 14, }; const char *short_opts = "hYlLD:s:d:t:"; @@ -1990,6 +2011,7 @@ const struct option long_opts[] = { {"define-depth", required_argument, NULL, OPT_DEFINE_DEPTH}, {"step", required_argument, NULL, OPT_STEP}, {"force", no_argument, NULL, OPT_FORCE}, + {"no-internal", no_argument, NULL, OPT_NO_INTERNAL}, {"disk", required_argument, NULL, OPT_DISK}, {"trace", required_argument, NULL, OPT_TRACE}, {"trace-backtrace", no_argument, NULL, OPT_TRACE_BACKTRACE}, @@ -2015,6 +2037,7 @@ const char *const help_text[] = { "How deep to evaluate recursive defines before erroring.", "Comma-separated range of permutations to run.", "Ignore bench filters.", + "Don't run internal benches.", "Direct block device operations to this file.", "Direct trace output to this file.", "Include a backtrace with every trace statement.", @@ -2341,6 +2364,10 @@ int main(int argc, char **argv) { bench_force = true; break; + case OPT_NO_INTERNAL:; + bench_mask |= BENCH_INTERNAL; + break; + case OPT_DISK:; bench_disk_path = optarg; break; diff --git a/runners/test_runner.c b/runners/test_runner.c index efe7a59b..922c58a3 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -450,7 +450,9 @@ size_t test_id_count = 1; size_t test_step_start = 0; size_t test_step_stop = -1; size_t test_step_step = 1; +size_t test_step = 0; // incremented every permutation bool test_force = false; +test_flags_t test_mask = 0; const char *test_disk_path = NULL; const char *test_trace_path = NULL; @@ -465,7 +467,7 @@ test_ns_t test_read_sleep = 0.0; test_ns_t test_prog_sleep = 0.0; test_ns_t test_erase_sleep = 0.0; -volatile size_t TEST_PLS = 0; +volatile size_t TEST_PLS = 0; // incremented every powerloss extern const test_powerloss_t *test_powerlosses; extern size_t test_powerloss_count; @@ -1041,10 +1043,25 @@ void perm_count( struct perm_count_state *state = data; (void)suite; + // masked? consider this lower-level than filtering + if (case_->flags & test_mask) { + return; + } + + // skip this step? + if (!(test_step >= test_step_start + && test_step < test_step_stop + && (test_step-test_step_start) % test_step_step == 0)) { + test_step += 1; + return; + } + test_step += 1; + state->total += 1; // set pls to 1 if running under powerloss so it useful for if predicates TEST_PLS = (powerloss->run != run_powerloss_none); + // filter? this includes ifdef (run=NULL) and if checks if (!case_->run || !(test_force || !case_->if_ || case_->if_())) { return; } @@ -2259,9 +2276,8 @@ static void list_powerlosses(void) { } -// global test step count -size_t test_step = 0; +// main permutation runner void perm_run( void *data, const struct test_suite *suite, @@ -2269,6 +2285,11 @@ void perm_run( const test_powerloss_t *powerloss) { (void)data; + // masked? consider this lower-level than filtering + if (case_->flags & test_mask) { + return; + } + // skip this step? if (!(test_step >= test_step_start && test_step < test_step_stop @@ -2280,7 +2301,7 @@ void perm_run( // set pls to 1 if running under powerloss so it useful for if predicates TEST_PLS = (powerloss->run != run_powerloss_none); - // filter? + // filter? this includes ifdef (run=NULL) and if checks if (!case_->run || !(test_force || !case_->if_ || case_->if_())) { printf("skipped "); perm_printid(suite, case_, NULL, 0); @@ -2340,14 +2361,17 @@ enum opt_flags { OPT_POWERLOSS = 'P', OPT_STEP = 's', OPT_FORCE = 8, + OPT_NO_INTERNAL = 9, + OPT_NO_REENTRANT = 10, + OPT_NO_FUZZ = 11, OPT_DISK = 'd', OPT_TRACE = 't', - OPT_TRACE_BACKTRACE = 9, - OPT_TRACE_PERIOD = 10, - OPT_TRACE_FREQ = 11, - OPT_READ_SLEEP = 12, - OPT_PROG_SLEEP = 13, - OPT_ERASE_SLEEP = 14, + OPT_TRACE_BACKTRACE = 12, + OPT_TRACE_PERIOD = 13, + OPT_TRACE_FREQ = 14, + OPT_READ_SLEEP = 15, + OPT_PROG_SLEEP = 16, + OPT_ERASE_SLEEP = 17, }; const char *short_opts = "hYlLD:P:s:d:t:"; @@ -2370,6 +2394,9 @@ const struct option long_opts[] = { {"powerloss", required_argument, NULL, OPT_POWERLOSS}, {"step", required_argument, NULL, OPT_STEP}, {"force", no_argument, NULL, OPT_FORCE}, + {"no-internal", no_argument, NULL, OPT_NO_INTERNAL}, + {"no-reentrant", no_argument, NULL, OPT_NO_REENTRANT}, + {"no-fuzz", no_argument, NULL, OPT_NO_FUZZ}, {"disk", required_argument, NULL, OPT_DISK}, {"trace", required_argument, NULL, OPT_TRACE}, {"trace-backtrace", no_argument, NULL, OPT_TRACE_BACKTRACE}, @@ -2397,6 +2424,9 @@ const char *const help_text[] = { "Comma-separated list of powerloss scenarios to test.", "Comma-separated range of permutations to run.", "Ignore test filters.", + "Don't run internal tests.", + "Don't run reentrant tests.", + "Don't run fuzz tests.", "Direct block device operations to this file.", "Direct trace output to this file.", "Include a backtrace with every trace statement.", @@ -2894,6 +2924,18 @@ int main(int argc, char **argv) { test_force = true; break; + case OPT_NO_INTERNAL:; + test_mask |= TEST_INTERNAL; + break; + + case OPT_NO_REENTRANT:; + test_mask |= TEST_REENTRANT; + break; + + case OPT_NO_FUZZ:; + test_mask |= TEST_FUZZ; + break; + case OPT_DISK:; test_disk_path = optarg; break; diff --git a/scripts/bench.py b/scripts/bench.py index 6db50db3..9461612d 100755 --- a/scripts/bench.py +++ b/scripts/bench.py @@ -301,7 +301,7 @@ class BenchSuite: **config_}, args) - # skipping internal tests? + # skipping internal benches? if args.get('no_internal') and case.internal: continue @@ -796,6 +796,8 @@ def find_runner(runner, id=None, main=True, **args): cmd.append('--define-depth=%s' % args['define_depth']) if args.get('force'): cmd.append('--force') + if args.get('no_internal'): + cmd.append('--no-internal') # only one thread should write to disk/trace, otherwise the output # ends up clobbered and useless @@ -1708,6 +1710,10 @@ if __name__ == "__main__": '--force', action='store_true', help="Ignore bench filters.") + bench_parser.add_argument( + '--no-internal', + action='store_true', + help="Don't run internal benches.") bench_parser.add_argument( '-d', '--disk', help="Direct block device operations to this file.") @@ -1869,7 +1875,7 @@ if __name__ == "__main__": comp_parser.add_argument( '--no-internal', action='store_true', - help="Don't build internal tests.") + help="Don't build internal benches.") # do the thing args = parser.parse_intermixed_args() diff --git a/scripts/test.py b/scripts/test.py index 1e729368..ce4e80b6 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -317,6 +317,10 @@ class TestSuite: # skipping internal tests? if args.get('no_internal') and case.internal: continue + if args.get('no_reentrant') and case.reentrant: + continue + if args.get('no_fuzz') and case.fuzz: + continue self.cases.append(case) @@ -819,6 +823,12 @@ def find_runner(runner, id=None, main=True, **args): cmd.append('-P%s' % args['powerloss']) if args.get('force'): cmd.append('--force') + if args.get('no_internal'): + cmd.append('--no-internal') + if args.get('no_reentrant'): + cmd.append('--no-reentrant') + if args.get('no_fuzz'): + cmd.append('--no-fuzz') # only one thread should write to disk/trace, otherwise the output # ends up clobbered and useless @@ -1727,6 +1737,18 @@ if __name__ == "__main__": '--force', action='store_true', help="Ignore test filters.") + test_parser.add_argument( + '--no-internal', + action='store_true', + help="Don't run internal tests.") + test_parser.add_argument( + '--no-reentrant', + action='store_true', + help="Don't run reentrant tests.") + test_parser.add_argument( + '--no-fuzz', + action='store_true', + help="Don't run fuzz tests.") test_parser.add_argument( '-d', '--disk', help="Direct block device operations to this file.") @@ -1901,6 +1923,14 @@ if __name__ == "__main__": '--no-internal', action='store_true', help="Don't build internal tests.") + comp_parser.add_argument( + '--no-reentrant', + action='store_true', + help="Don't build reentrant tests.") + comp_parser.add_argument( + '--no-fuzz', + action='store_true', + help="Don't build fuzz tests.") # do the thing args = parser.parse_intermixed_args()