runners: Added both run+compile-time --no-internal/reentrant/fuzz flags

--no-internal has already proven useful for skipping internal tests for
refactoring, so it makes sense to add --no-reentrant/fuzz flags as well.
--no-fuzz seems particularly useful for when you want to skip the less
targeted fuzz tests:

- with fuzz tests: 634616/634616 passed, in 1239.90s
- with --no-fuzz:    85434/85434 passed, in  423.41s

I also added runtime variants to test/bench_runner and test/bench.py.
These may be useful to skip tests without needing to recompile the
runner.

---

Also tweaked -s/--step to filter permutations in any --list-* flags, for
consistency.

runners/bench_runner.c

@@ -439,7 +439,9 @@ size_t bench_id_count = 1;
 size_t bench_step_start = 0;
 size_t bench_step_stop = -1;
 size_t bench_step_step = 1;
+size_t bench_step = 0; // incremented every permutation
 bool bench_force = false;
+bench_flags_t bench_mask = 0;
 
 const char *bench_disk_path = NULL;
 const char *bench_trace_path = NULL;
@@ -1234,8 +1236,23 @@ void perm_count(
     struct perm_count_state *state = data;
     (void)suite;
 
+    // masked? consider this lower-level than filtering
+    if (case_->flags & bench_mask) {
+        return;
+    }
+
+    // skip this step?
+    if (!(bench_step >= bench_step_start
+            && bench_step < bench_step_stop
+            && (bench_step-bench_step_start) % bench_step_step == 0)) {
+        bench_step += 1;
+        return;
+    }
+    bench_step += 1;
+
     state->total += 1;
 
+    // filter? this includes ifdef (run=NULL) and if checks
     if (!case_->run || !(bench_force || !case_->if_ || case_->if_())) {
         return;
     }
@@ -1807,15 +1824,18 @@ int bench_bd_sync(const struct lfs3_cfg *cfg) {
 
 
 
-// global bench step count
+// main permutation runner
-size_t bench_step = 0;
-
 void perm_run(
         void *data,
         const struct bench_suite *suite,
         const struct bench_case *case_) {
     (void)data;
 
+    // masked? consider this lower-level than filtering
+    if (case_->flags & bench_mask) {
+        return;
+    }
+
     // skip this step?
     if (!(bench_step >= bench_step_start
             && bench_step < bench_step_stop
@@ -1825,7 +1845,7 @@ void perm_run(
     }
     bench_step += 1;
 
-    // filter?
+    // filter? this includes ifdef (run=NULL) and if checks
     if (!case_->run || !(bench_force || !case_->if_ || case_->if_())) {
         printf("skipped ");
         perm_printid(suite, case_);
@@ -1962,14 +1982,15 @@ enum opt_flags {
     OPT_DEFINE_DEPTH             = 6,
     OPT_STEP                     = 's',
     OPT_FORCE                    = 7,
+    OPT_NO_INTERNAL              = 8,
     OPT_DISK                     = 'd',
     OPT_TRACE                    = 't',
-    OPT_TRACE_BACKTRACE          = 8,
+    OPT_TRACE_BACKTRACE          = 9,
-    OPT_TRACE_PERIOD             = 9,
+    OPT_TRACE_PERIOD             = 10,
-    OPT_TRACE_FREQ               = 10,
+    OPT_TRACE_FREQ               = 11,
-    OPT_READ_SLEEP               = 11,
+    OPT_READ_SLEEP               = 12,
-    OPT_PROG_SLEEP               = 12,
+    OPT_PROG_SLEEP               = 13,
-    OPT_ERASE_SLEEP              = 13,
+    OPT_ERASE_SLEEP              = 14,
 };
 
 const char *short_opts = "hYlLD:s:d:t:";
@@ -1990,6 +2011,7 @@ const struct option long_opts[] = {
     {"define-depth",     required_argument, NULL, OPT_DEFINE_DEPTH},
     {"step",             required_argument, NULL, OPT_STEP},
     {"force",            no_argument,       NULL, OPT_FORCE},
+    {"no-internal",      no_argument,       NULL, OPT_NO_INTERNAL},
     {"disk",             required_argument, NULL, OPT_DISK},
     {"trace",            required_argument, NULL, OPT_TRACE},
     {"trace-backtrace",  no_argument,       NULL, OPT_TRACE_BACKTRACE},
@@ -2015,6 +2037,7 @@ const char *const help_text[] = {
     "How deep to evaluate recursive defines before erroring.",
     "Comma-separated range of permutations to run.",
     "Ignore bench filters.",
+    "Don't run internal benches.",
     "Direct block device operations to this file.",
     "Direct trace output to this file.",
     "Include a backtrace with every trace statement.",
@@ -2341,6 +2364,10 @@ int main(int argc, char **argv) {
             bench_force = true;
             break;
 
+        case OPT_NO_INTERNAL:;
+            bench_mask |= BENCH_INTERNAL;
+            break;
+
         case OPT_DISK:;
             bench_disk_path = optarg;
             break;

runners/test_runner.c

@@ -450,7 +450,9 @@ size_t test_id_count = 1;
 size_t test_step_start = 0;
 size_t test_step_stop = -1;
 size_t test_step_step = 1;
+size_t test_step = 0; // incremented every permutation
 bool test_force = false;
+test_flags_t test_mask = 0;
 
 const char *test_disk_path = NULL;
 const char *test_trace_path = NULL;
@@ -465,7 +467,7 @@ test_ns_t test_read_sleep = 0.0;
 test_ns_t test_prog_sleep = 0.0;
 test_ns_t test_erase_sleep = 0.0;
 
-volatile size_t TEST_PLS = 0;
+volatile size_t TEST_PLS = 0; // incremented every powerloss
 
 extern const test_powerloss_t *test_powerlosses;
 extern size_t test_powerloss_count;
@@ -1041,10 +1043,25 @@ void perm_count(
     struct perm_count_state *state = data;
     (void)suite;
 
+    // masked? consider this lower-level than filtering
+    if (case_->flags & test_mask) {
+        return;
+    }
+
+    // skip this step?
+    if (!(test_step >= test_step_start
+            && test_step < test_step_stop
+            && (test_step-test_step_start) % test_step_step == 0)) {
+        test_step += 1;
+        return;
+    }
+    test_step += 1;
+
     state->total += 1;
 
     // set pls to 1 if running under powerloss so it useful for if predicates
     TEST_PLS = (powerloss->run != run_powerloss_none);
+    // filter? this includes ifdef (run=NULL) and if checks
     if (!case_->run || !(test_force || !case_->if_ || case_->if_())) {
         return;
     }
@@ -2259,9 +2276,8 @@ static void list_powerlosses(void) {
 }
 
 
-// global test step count
-size_t test_step = 0;
 
+// main permutation runner
 void perm_run(
         void *data,
         const struct test_suite *suite,
@@ -2269,6 +2285,11 @@ void perm_run(
         const test_powerloss_t *powerloss) {
     (void)data;
 
+    // masked? consider this lower-level than filtering
+    if (case_->flags & test_mask) {
+        return;
+    }
+
     // skip this step?
     if (!(test_step >= test_step_start
             && test_step < test_step_stop
@@ -2280,7 +2301,7 @@ void perm_run(
 
     // set pls to 1 if running under powerloss so it useful for if predicates
     TEST_PLS = (powerloss->run != run_powerloss_none);
-    // filter?
+    // filter? this includes ifdef (run=NULL) and if checks
     if (!case_->run || !(test_force || !case_->if_ || case_->if_())) {
         printf("skipped ");
         perm_printid(suite, case_, NULL, 0);
@@ -2340,14 +2361,17 @@ enum opt_flags {
     OPT_POWERLOSS                = 'P',
     OPT_STEP                     = 's',
     OPT_FORCE                    = 8,
+    OPT_NO_INTERNAL              = 9,
+    OPT_NO_REENTRANT             = 10,
+    OPT_NO_FUZZ                  = 11,
     OPT_DISK                     = 'd',
     OPT_TRACE                    = 't',
-    OPT_TRACE_BACKTRACE          = 9,
+    OPT_TRACE_BACKTRACE          = 12,
-    OPT_TRACE_PERIOD             = 10,
+    OPT_TRACE_PERIOD             = 13,
-    OPT_TRACE_FREQ               = 11,
+    OPT_TRACE_FREQ               = 14,
-    OPT_READ_SLEEP               = 12,
+    OPT_READ_SLEEP               = 15,
-    OPT_PROG_SLEEP               = 13,
+    OPT_PROG_SLEEP               = 16,
-    OPT_ERASE_SLEEP              = 14,
+    OPT_ERASE_SLEEP              = 17,
 };
 
 const char *short_opts = "hYlLD:P:s:d:t:";
@@ -2370,6 +2394,9 @@ const struct option long_opts[] = {
     {"powerloss",        required_argument, NULL, OPT_POWERLOSS},
     {"step",             required_argument, NULL, OPT_STEP},
     {"force",            no_argument,       NULL, OPT_FORCE},
+    {"no-internal",      no_argument,       NULL, OPT_NO_INTERNAL},
+    {"no-reentrant",     no_argument,       NULL, OPT_NO_REENTRANT},
+    {"no-fuzz",          no_argument,       NULL, OPT_NO_FUZZ},
     {"disk",             required_argument, NULL, OPT_DISK},
     {"trace",            required_argument, NULL, OPT_TRACE},
     {"trace-backtrace",  no_argument,       NULL, OPT_TRACE_BACKTRACE},
@@ -2397,6 +2424,9 @@ const char *const help_text[] = {
     "Comma-separated list of powerloss scenarios to test.",
     "Comma-separated range of permutations to run.",
     "Ignore test filters.",
+    "Don't run internal tests.",
+    "Don't run reentrant tests.",
+    "Don't run fuzz tests.",
     "Direct block device operations to this file.",
     "Direct trace output to this file.",
     "Include a backtrace with every trace statement.",
@@ -2894,6 +2924,18 @@ int main(int argc, char **argv) {
             test_force = true;
             break;
 
+        case OPT_NO_INTERNAL:;
+            test_mask |= TEST_INTERNAL;
+            break;
+
+        case OPT_NO_REENTRANT:;
+            test_mask |= TEST_REENTRANT;
+            break;
+
+        case OPT_NO_FUZZ:;
+            test_mask |= TEST_FUZZ;
+            break;
+
         case OPT_DISK:;
             test_disk_path = optarg;
             break;

scripts/bench.py

@@ -301,7 +301,7 @@ class BenchSuite:
                             **config_},
                         args)
 
-                # skipping internal tests?
+                # skipping internal benches?
                 if args.get('no_internal') and case.internal:
                     continue
 
@@ -796,6 +796,8 @@ def find_runner(runner, id=None, main=True, **args):
         cmd.append('--define-depth=%s' % args['define_depth'])
     if args.get('force'):
         cmd.append('--force')
+    if args.get('no_internal'):
+        cmd.append('--no-internal')
 
     # only one thread should write to disk/trace, otherwise the output
     # ends up clobbered and useless
@@ -1708,6 +1710,10 @@ if __name__ == "__main__":
             '--force',
             action='store_true',
             help="Ignore bench filters.")
+    bench_parser.add_argument(
+            '--no-internal',
+            action='store_true',
+            help="Don't run internal benches.")
     bench_parser.add_argument(
             '-d', '--disk',
             help="Direct block device operations to this file.")
@@ -1869,7 +1875,7 @@ if __name__ == "__main__":
     comp_parser.add_argument(
             '--no-internal',
             action='store_true',
-            help="Don't build internal tests.")
+            help="Don't build internal benches.")
 
     # do the thing
     args = parser.parse_intermixed_args()

scripts/test.py

@@ -317,6 +317,10 @@ class TestSuite:
                 # skipping internal tests?
                 if args.get('no_internal') and case.internal:
                     continue
+                if args.get('no_reentrant') and case.reentrant:
+                    continue
+                if args.get('no_fuzz') and case.fuzz:
+                    continue
 
                 self.cases.append(case)
 
@@ -819,6 +823,12 @@ def find_runner(runner, id=None, main=True, **args):
         cmd.append('-P%s' % args['powerloss'])
     if args.get('force'):
         cmd.append('--force')
+    if args.get('no_internal'):
+        cmd.append('--no-internal')
+    if args.get('no_reentrant'):
+        cmd.append('--no-reentrant')
+    if args.get('no_fuzz'):
+        cmd.append('--no-fuzz')
 
     # only one thread should write to disk/trace, otherwise the output
     # ends up clobbered and useless
@@ -1727,6 +1737,18 @@ if __name__ == "__main__":
             '--force',
             action='store_true',
             help="Ignore test filters.")
+    test_parser.add_argument(
+            '--no-internal',
+            action='store_true',
+            help="Don't run internal tests.")
+    test_parser.add_argument(
+            '--no-reentrant',
+            action='store_true',
+            help="Don't run reentrant tests.")
+    test_parser.add_argument(
+            '--no-fuzz',
+            action='store_true',
+            help="Don't run fuzz tests.")
     test_parser.add_argument(
             '-d', '--disk',
             help="Direct block device operations to this file.")
@@ -1901,6 +1923,14 @@ if __name__ == "__main__":
             '--no-internal',
             action='store_true',
             help="Don't build internal tests.")
+    comp_parser.add_argument(
+            '--no-reentrant',
+            action='store_true',
+            help="Don't build reentrant tests.")
+    comp_parser.add_argument(
+            '--no-fuzz',
+            action='store_true',
+            help="Don't build fuzz tests.")
 
     # do the thing
     args = parser.parse_intermixed_args()