Github/littlefs
1
Fork 0
Code Actions Activity

Added -a/--all to test.py/bench.py for bypass test/bench filters

Browse Source 
These really shouldn't be used all that often. Test filters are usually
used to protect against invalid test configurations, so if you bypass
test filters, expect things to fail!

But some filters just prevent test cases from taking too long. In these
cases being able to manually bypass the filter is useful for debugging/
benchmarking/etc...
This commit is contained in:
Christopher Haster
2024-05-28 16:45:26 -05:00
parent 37f738cc71
commit 31eebc1328
4 changed files with 36 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runners/bench_runner.c
+12 -4
View File
@@ -428,6 +428,7 @@ size_t bench_id_count = 1;
size_t bench_step_start = 0;
size_t bench_step_stop = -1;
size_t bench_step_step = 1;
bool bench_all = false;
const char *bench_disk_path = NULL;
const char *bench_trace_path = NULL;
@@ -848,7 +849,7 @@ void perm_count(
state->total += 1;
if (case_->if_ && !case_->if_()) {
if (!bench_all && case_->if_ && !case_->if_()) {
return;
}
@@ -1333,7 +1334,7 @@ void perm_run(
bench_step += 1;
// filter?
if (case_->if_ && !case_->if_()) {
if (!bench_all && case_->if_ && !case_->if_()) {
printf("skipped ");
perm_printid(suite, case_);
printf("\n");
@@ -1431,6 +1432,7 @@ enum opt_flags {
OPT_DEFINE = 'D',
OPT_DEFINE_DEPTH = 6,
OPT_STEP = 's',
OPT_ALL = 'a',
OPT_DISK = 'd',
OPT_TRACE = 't',
OPT_TRACE_BACKTRACE = 7,
@@ -1441,7 +1443,7 @@ enum opt_flags {
OPT_ERASE_SLEEP = 12,
};
const char *short_opts = "hYlLD:s:d:t:";
const char *short_opts = "hYlLD:s:ad:t:";
const struct option long_opts[] = {
{"help", no_argument, NULL, OPT_HELP},
@@ -1458,6 +1460,7 @@ const struct option long_opts[] = {
{"define", required_argument, NULL, OPT_DEFINE},
{"define-depth", required_argument, NULL, OPT_DEFINE_DEPTH},
{"step", required_argument, NULL, OPT_STEP},
{"all", no_argument, NULL, OPT_ALL},
{"disk", required_argument, NULL, OPT_DISK},
{"trace", required_argument, NULL, OPT_TRACE},
{"trace-backtrace", no_argument, NULL, OPT_TRACE_BACKTRACE},
@@ -1481,7 +1484,8 @@ const char *const help_text[] = {
"List implicit defines in this bench-runner.",
"Override a bench define.",
"How deep to evaluate recursive defines before erroring.",
"Comma-separated range of bench permutations to run (start,stop,step).",
"Comma-separated range of permutations to run.",
"Ignore bench filters.",
"Direct block device operations to this file.",
"Direct trace output to this file.",
"Include a backtrace with every trace statement.",
@@ -1804,6 +1808,10 @@ int main(int argc, char **argv) {
fprintf(stderr, "error: invalid step: %s\n", optarg);
exit(-1);
case OPT_ALL:;
bench_all = true;
break;
case OPT_DISK:;
bench_disk_path = optarg;
break;
runners/test_runner.c
+12 -4
View File
@@ -439,6 +439,7 @@ size_t test_id_count = 1;
size_t test_step_start = 0;
size_t test_step_stop = -1;
size_t test_step_step = 1;
bool test_all = false;
const char *test_disk_path = NULL;
const char *test_trace_path = NULL;
@@ -811,7 +812,7 @@ void perm_count(
// set pls to 1 if running under powerloss so it useful for if predicates
TEST_PLS = (powerloss->run != run_powerloss_none);
if (case_->if_ && !case_->if_()) {
if (!test_all && case_->if_ && !case_->if_()) {
return;
}
@@ -1811,7 +1812,7 @@ void perm_run(
// set pls to 1 if running under powerloss so it useful for if predicates
TEST_PLS = (powerloss->run != run_powerloss_none);
// filter?
if (case_->if_ && !case_->if_()) {
if (!test_all && case_->if_ && !case_->if_()) {
printf("skipped ");
perm_printid(suite, case_, NULL, 0);
printf("\n");
@@ -1869,6 +1870,7 @@ enum opt_flags {
OPT_DEFINE_DEPTH = 7,
OPT_POWERLOSS = 'P',
OPT_STEP = 's',
OPT_ALL = 'a',
OPT_DISK = 'd',
OPT_TRACE = 't',
OPT_TRACE_BACKTRACE = 8,
@@ -1879,7 +1881,7 @@ enum opt_flags {
OPT_ERASE_SLEEP = 13,
};
const char *short_opts = "hYlLD:P:s:d:t:";
const char *short_opts = "hYlLD:P:s:ad:t:";
const struct option long_opts[] = {
{"help", no_argument, NULL, OPT_HELP},
@@ -1898,6 +1900,7 @@ const struct option long_opts[] = {
{"define-depth", required_argument, NULL, OPT_DEFINE_DEPTH},
{"powerloss", required_argument, NULL, OPT_POWERLOSS},
{"step", required_argument, NULL, OPT_STEP},
{"all", no_argument, NULL, OPT_ALL},
{"disk", required_argument, NULL, OPT_DISK},
{"trace", required_argument, NULL, OPT_TRACE},
{"trace-backtrace", no_argument, NULL, OPT_TRACE_BACKTRACE},
@@ -1923,7 +1926,8 @@ const char *const help_text[] = {
"Override a test define.",
"How deep to evaluate recursive defines before erroring.",
"Comma-separated list of power-loss scenarios to test.",
"Comma-separated range of test permutations to run (start,stop,step).",
"Comma-separated range of permutations to run.",
"Ignore test filters.",
"Direct block device operations to this file.",
"Direct trace output to this file.",
"Include a backtrace with every trace statement.",
@@ -2411,6 +2415,10 @@ int main(int argc, char **argv) {
fprintf(stderr, "error: invalid step: %s\n", optarg);
exit(-1);
case OPT_ALL:;
test_all = true;
break;
case OPT_DISK:;
test_disk_path = optarg;
break;
scripts/bench.py
+6
View File
@@ -606,6 +606,8 @@ def find_runner(runner, id=None, **args):
# other context
if args.get('define_depth'):
cmd.append('--define-depth=%s' % args['define_depth'])
if args.get('all'):
cmd.append('-a')
if args.get('disk'):
cmd.append('-d%s' % args['disk'])
if args.get('trace'):
@@ -1481,6 +1483,10 @@ if __name__ == "__main__":
bench_parser.add_argument(
'--define-depth',
help="How deep to evaluate recursive defines before erroring.")
bench_parser.add_argument(
'-a', '--all',
action='store_true',
help="Ignore test filters.")
bench_parser.add_argument(
'-d', '--disk',
help="Direct block device operations to this file.")
scripts/test.py
+6
View File
@@ -623,6 +623,8 @@ def find_runner(runner, id=None, **args):
cmd.append('--define-depth=%s' % args['define_depth'])
if args.get('powerloss'):
cmd.append('-P%s' % args['powerloss'])
if args.get('all'):
cmd.append('-a')
if args.get('disk'):
cmd.append('-d%s' % args['disk'])
if args.get('trace'):
@@ -1510,6 +1512,10 @@ if __name__ == "__main__":
test_parser.add_argument(
'-P', '--powerloss',
help="Comma-separated list of power-loss scenarios to test.")
test_parser.add_argument(
'-a', '--all',
action='store_true',
help="Ignore test filters.")
test_parser.add_argument(
'-d', '--disk',
help="Direct block device operations to this file.")