Github/littlefs
1
Fork 0
Code Actions Activity

runners: Added -Q/--query-define and friends

Browse Source 
The fact that we don't include implicit defines in bench/test output
means we need to query the runner for these surprisingly often. So it'd
be nice to have an easier API than sedding the list output.

Some examples:

  $ ./scripts/test.py -QBLOCK_SIZE
  4096
  32768

  $ ./scripts/test.py --query-implicit-define=BLOCK_SIZE
  4096

  $ ./scripts/test.py --query-permutation-define=BLOCK_SIZE
  32768

  $ ./scripts/test.py -QBLOCK_SIZZLE
  (errors)

Unlike --list-*defines, --query-*defines:

- Separates by newline

- Errors if define is not found

Other than that, --query-*defines uses more-or-less the same code
internally.
This commit is contained in:
Christopher Haster
2026-02-12 18:16:47 -06:00
parent fb8d5a83aa
commit cb91b6b2a6
5 changed files with 465 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+3 -6
View File
@@ -684,20 +684,17 @@ bench-widths: $(BENCH_CSV)
float(bench_readed)/float(bench_reads), \
max(1, $$( \
./scripts/bench.py -R$(BENCH_RUNNER) $(BENCHFLAGS) \
--list-implicit-defines \
| sed -n 's/^READ_WIDTH=\(.*\)/\1/p')))))" \
--query-implicit-define=READ_WIDTH)))))" \
-fprogged="avg(saturate(ffrac( \
float(bench_progged)/float(bench_progs), \
max(1, $$( \
./scripts/bench.py -R$(BENCH_RUNNER) $(BENCHFLAGS) \
--list-implicit-defines \
| sed -n 's/^PROG_WIDTH=\(.*\)/\1/p')))))" \
--query-implicit-define=PROG_WIDTH)))))" \
-ferased="avg(saturate(ffrac( \
float(bench_erased)/float(bench_erases), \
max(1, $$( \
./scripts/bench.py -R$(BENCH_RUNNER) $(BENCHFLAGS) \
--list-implicit-defines \
| sed -n 's/^ERASE_WIDTH=\(.*\)/\1/p')))))" \
--query-implicit-define=ERASE_WIDTH)))))" \
$(SUMMARYFLAGS))
## Show heap/stack/disk usage
runners/bench_runner.c
+210 -26
View File
@@ -1913,6 +1913,14 @@ static void list_defines_add(
define_->value_capacity = 1;
}
static void list_defines_cleanup(
struct list_defines_defines *defines) {
for (size_t i = 0; i < defines->define_count; i++) {
free(defines->defines[i].values);
}
free(defines->defines);
}
void perm_list_defines(
void *data,
const struct bench_suite *suite,
@@ -1984,10 +1992,7 @@ static void list_defines(void) {
printf("\n");
}
for (size_t i = 0; i < defines.define_count; i++) {
free(defines.defines[i].values);
}
free(defines.defines);
list_defines_cleanup(&defines);
}
static void list_permutation_defines(void) {
@@ -2029,10 +2034,7 @@ static void list_permutation_defines(void) {
printf("\n");
}
for (size_t i = 0; i < defines.define_count; i++) {
free(defines.defines[i].values);
}
free(defines.defines);
list_defines_cleanup(&defines);
}
static void list_implicit_defines(void) {
@@ -2069,10 +2071,7 @@ static void list_implicit_defines(void) {
printf("\n");
}
for (size_t i = 0; i < defines.define_count; i++) {
free(defines.defines[i].values);
}
free(defines.defines);
list_defines_cleanup(&defines);
}
static void list_probes(void) {
@@ -2234,6 +2233,165 @@ static void list_case_probes(void) {
}
}
static const char *query_define_query = NULL;
void perm_query_define(
void *data,
const struct bench_suite *suite,
const struct bench_case *case_) {
struct list_defines_defines *defines = data;
(void)suite;
(void)case_;
// collect defines
for (size_t d = 0; d < bench_define_count; d++) {
if (bench_define_isdefined(bench_defines[d])
&& strcmp(bench_defines[d]->name, query_define_query) == 0) {
list_defines_add(defines, bench_defines[d]);
}
}
}
void perm_query_permutation_define(
void *data,
const struct bench_suite *suite,
const struct bench_case *case_) {
struct list_defines_defines *defines = data;
(void)suite;
(void)case_;
// collect permutation_defines
for (size_t d = 0; d < bench_define_count; d++) {
if (bench_define_ispermutation(bench_defines[d])
&& strcmp(bench_defines[d]->name, query_define_query) == 0) {
list_defines_add(defines, bench_defines[d]);
}
}
}
static void query_define(void) {
struct list_defines_defines defines = {NULL, 0, 0};
// add defines
for (size_t t = 0; t < bench_id_count; t++) {
for (size_t i = 0; i < bench_suite_count; i++) {
bench_define_suite(&bench_ids[t], bench_suites[i]);
for (size_t j = 0; j < bench_suites[i]->case_count; j++) {
// does neither suite nor case name match?
if (bench_ids[t].name && !(
strcmp(bench_ids[t].name,
bench_suites[i]->name) == 0
|| strcmp(bench_ids[t].name,
bench_suites[i]->cases[j].name) == 0)) {
continue;
}
case_forperm(
&bench_ids[t],
bench_suites[i],
&bench_suites[i]->cases[j],
perm_query_define,
&defines);
}
}
}
// none found?
if (defines.define_count == 0) {
exit(1);
}
// print what was found
assert(defines.define_count == 1);
for (size_t j = 0; j < defines.defines[0].value_count; j++) {
printf("%jd\n", defines.defines[0].values[j]);
}
list_defines_cleanup(&defines);
}
static void query_permutation_define(void) {
struct list_defines_defines defines = {NULL, 0, 0};
// add permutation defines
for (size_t t = 0; t < bench_id_count; t++) {
for (size_t i = 0; i < bench_suite_count; i++) {
bench_define_suite(&bench_ids[t], bench_suites[i]);
for (size_t j = 0; j < bench_suites[i]->case_count; j++) {
// does neither suite nor case name match?
if (bench_ids[t].name && !(
strcmp(bench_ids[t].name,
bench_suites[i]->name) == 0
|| strcmp(bench_ids[t].name,
bench_suites[i]->cases[j].name) == 0)) {
continue;
}
case_forperm(
&bench_ids[t],
bench_suites[i],
&bench_suites[i]->cases[j],
perm_query_permutation_define,
&defines);
}
}
}
// none found?
if (defines.define_count == 0) {
exit(1);
}
// print what was found
assert(defines.define_count == 1);
for (size_t j = 0; j < defines.defines[0].value_count; j++) {
printf("%jd\n", defines.defines[0].values[j]);
}
list_defines_cleanup(&defines);
}
static void query_implicit_define(void) {
struct list_defines_defines defines = {NULL, 0, 0};
// yes we do need to define a suite/case, these do a bit of bookeeping
// around mapping defines
bench_define_suite(NULL,
&(const struct bench_suite){0});
bench_define_case(NULL,
&(const struct bench_suite){0},
&(const struct bench_case){0},
0);
size_t permutations = bench_define_permutations();
for (size_t p = 0; p < permutations; p++) {
// define permutation permutation
bench_define_permutation(p);
// add implicit defines
for (size_t d = 0; d < bench_define_count; d++) {
if (strcmp(bench_defines[d]->name, query_define_query) == 0) {
list_defines_add(&defines, bench_defines[d]);
}
}
}
// none found?
if (defines.define_count == 0) {
exit(1);
}
// print what was found
assert(defines.define_count == 1);
for (size_t j = 0; j < defines.defines[0].value_count; j++) {
printf("%jd\n", defines.defines[0].values[j]);
}
list_defines_cleanup(&defines);
}
// bench bd wrappers for heap/stack tracking
@@ -2459,28 +2617,31 @@ enum opt_flags {
OPT_LIST_PROBES = 6,
OPT_LIST_SUITE_PROBES = 7,
OPT_LIST_CASE_PROBES = 8,
OPT_QUERY_DEFINE = 'Q',
OPT_QUERY_PERMUTATION_DEFINE = 9,
OPT_QUERY_IMPLICIT_DEFINE = 10,
OPT_DEFINE = 'D',
OPT_DEFINE_DEPTH = 9,
OPT_DEFINE_DEPTH = 11,
OPT_PROBE = 'S',
OPT_PROBE_STEP = 'x',
OPT_PROBE_RUNFREQ = 10,
OPT_PROBE_RUNFREQ = 12,
OPT_PROBE_SIMFREQ = 'X',
OPT_STEP = 11,
OPT_FORCE = 12,
OPT_NO_INTERNAL = 13,
OPT_NO_LITMUS = 14,
OPT_STEP = 13,
OPT_FORCE = 14,
OPT_NO_INTERNAL = 15,
OPT_NO_LITMUS = 16,
OPT_DISK = 'd',
OPT_TRACE = 't',
OPT_TRACE_BACKTRACE = 15,
OPT_TRACE_STEP = 16,
OPT_TRACE_RUNFREQ = 17,
OPT_TRACE_SIMFREQ = 18,
OPT_READ_SLEEP = 19,
OPT_PROG_SLEEP = 20,
OPT_ERASE_SLEEP = 21,
OPT_TRACE_BACKTRACE = 17,
OPT_TRACE_STEP = 18,
OPT_TRACE_RUNFREQ = 19,
OPT_TRACE_SIMFREQ = 20,
OPT_READ_SLEEP = 21,
OPT_PROG_SLEEP = 22,
OPT_ERASE_SLEEP = 23,
};
const char *short_opts = "hYlLD:S:x:X:d:t:";
const char *short_opts = "hYlLQ:D:S:x:X:d:t:";
const struct option long_opts[] = {
{"help", no_argument, NULL, OPT_HELP},
@@ -2498,6 +2659,11 @@ const struct option long_opts[] = {
{"list-suite-probes",
no_argument, NULL, OPT_LIST_SUITE_PROBES},
{"list-case-probes", no_argument, NULL, OPT_LIST_CASE_PROBES},
{"query-define", required_argument, NULL, OPT_QUERY_DEFINE},
{"query-permutation-define",
required_argument, NULL, OPT_QUERY_PERMUTATION_DEFINE},
{"query-implicit-define",
required_argument, NULL, OPT_QUERY_IMPLICIT_DEFINE},
{"define", required_argument, NULL, OPT_DEFINE},
{"define-depth", required_argument, NULL, OPT_DEFINE_DEPTH},
{"probe", required_argument, NULL, OPT_PROBE},
@@ -2533,6 +2699,9 @@ const char *const help_text[] = {
"List estimated probes.",
"List estimated probes for each bench suite.",
"List estimated probes for each bench case.",
"Query a bench define.",
"Query a permutation bench define.",
"Query an implicit bench define.",
"Override a bench define.",
"How deep to evaluate recursive defines before erroring.",
"Specify a probe to sample.",
@@ -2662,6 +2831,21 @@ int main(int argc, char **argv) {
op = list_case_probes;
break;
case OPT_QUERY_DEFINE:;
op = query_define;
query_define_query = optarg;
break;
case OPT_QUERY_PERMUTATION_DEFINE:;
op = query_permutation_define;
query_define_query = optarg;
break;
case OPT_QUERY_IMPLICIT_DEFINE:;
op = query_implicit_define;
query_define_query = optarg;
break;
// configuration
case OPT_DEFINE:;
// allocate space
runners/test_runner.c
+212 -24
View File
@@ -1185,6 +1185,14 @@ static void list_defines_add(
define_->value_capacity = 1;
}
static void list_defines_cleanup(
struct list_defines_defines *defines) {
for (size_t i = 0; i < defines->define_count; i++) {
free(defines->defines[i].values);
}
free(defines->defines);
}
void perm_list_defines(
void *data,
const struct test_suite *suite,
@@ -1260,10 +1268,7 @@ static void list_defines(void) {
printf("\n");
}
for (size_t i = 0; i < defines.define_count; i++) {
free(defines.defines[i].values);
}
free(defines.defines);
list_defines_cleanup(&defines);
}
static void list_permutation_defines(void) {
@@ -1305,10 +1310,7 @@ static void list_permutation_defines(void) {
printf("\n");
}
for (size_t i = 0; i < defines.define_count; i++) {
free(defines.defines[i].values);
}
free(defines.defines);
list_defines_cleanup(&defines);
}
static void list_implicit_defines(void) {
@@ -1345,10 +1347,170 @@ static void list_implicit_defines(void) {
printf("\n");
}
for (size_t i = 0; i < defines.define_count; i++) {
free(defines.defines[i].values);
list_defines_cleanup(&defines);
}
static const char *query_define_query = NULL;
void perm_query_define(
void *data,
const struct test_suite *suite,
const struct test_case *case_,
const test_powerloss_t *powerloss) {
struct list_defines_defines *defines = data;
(void)suite;
(void)case_;
(void)powerloss;
// collect defines
for (size_t d = 0; d < test_define_count; d++) {
if (test_define_isdefined(test_defines[d])
&& strcmp(test_defines[d]->name, query_define_query) == 0) {
list_defines_add(defines, test_defines[d]);
}
}
free(defines.defines);
}
void perm_query_permutation_define(
void *data,
const struct test_suite *suite,
const struct test_case *case_,
const test_powerloss_t *powerloss) {
struct list_defines_defines *defines = data;
(void)suite;
(void)case_;
(void)powerloss;
// collect permutation_defines
for (size_t d = 0; d < test_define_count; d++) {
if (test_define_ispermutation(test_defines[d])
&& strcmp(test_defines[d]->name, query_define_query) == 0) {
list_defines_add(defines, test_defines[d]);
}
}
}
static void query_define(void) {
struct list_defines_defines defines = {NULL, 0, 0};
// add defines
for (size_t t = 0; t < test_id_count; t++) {
for (size_t i = 0; i < test_suite_count; i++) {
test_define_suite(&test_ids[t], test_suites[i]);
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
// does neither suite nor case name match?
if (test_ids[t].name && !(
strcmp(test_ids[t].name,
test_suites[i]->name) == 0
|| strcmp(test_ids[t].name,
test_suites[i]->cases[j].name) == 0)) {
continue;
}
case_forperm(
&test_ids[t],
test_suites[i],
&test_suites[i]->cases[j],
perm_query_define,
&defines);
}
}
}
// none found?
if (defines.define_count == 0) {
exit(1);
}
// print what was found
assert(defines.define_count == 1);
for (size_t j = 0; j < defines.defines[0].value_count; j++) {
printf("%jd\n", defines.defines[0].values[j]);
}
list_defines_cleanup(&defines);
}
static void query_permutation_define(void) {
struct list_defines_defines defines = {NULL, 0, 0};
// add permutation defines
for (size_t t = 0; t < test_id_count; t++) {
for (size_t i = 0; i < test_suite_count; i++) {
test_define_suite(&test_ids[t], test_suites[i]);
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
// does neither suite nor case name match?
if (test_ids[t].name && !(
strcmp(test_ids[t].name,
test_suites[i]->name) == 0
|| strcmp(test_ids[t].name,
test_suites[i]->cases[j].name) == 0)) {
continue;
}
case_forperm(
&test_ids[t],
test_suites[i],
&test_suites[i]->cases[j],
perm_query_permutation_define,
&defines);
}
}
}
// none found?
if (defines.define_count == 0) {
exit(1);
}
// print what was found
assert(defines.define_count == 1);
for (size_t j = 0; j < defines.defines[0].value_count; j++) {
printf("%jd\n", defines.defines[0].values[j]);
}
list_defines_cleanup(&defines);
}
static void query_implicit_define(void) {
struct list_defines_defines defines = {NULL, 0, 0};
// yes we do need to define a suite/case, these do a bit of bookeeping
// around mapping defines
test_define_suite(NULL,
&(const struct test_suite){0});
test_define_case(NULL,
&(const struct test_suite){0},
&(const struct test_case){0},
0);
size_t permutations = test_define_permutations();
for (size_t p = 0; p < permutations; p++) {
// define permutation permutation
test_define_permutation(p);
// add implicit defines
for (size_t d = 0; d < test_define_count; d++) {
if (strcmp(test_defines[d]->name, query_define_query) == 0) {
list_defines_add(&defines, test_defines[d]);
}
}
}
// none found?
if (defines.define_count == 0) {
exit(1);
}
// print what was found
assert(defines.define_count == 1);
for (size_t j = 0; j < defines.defines[0].value_count; j++) {
printf("%jd\n", defines.defines[0].values[j]);
}
list_defines_cleanup(&defines);
}
@@ -2058,25 +2220,28 @@ enum opt_flags {
OPT_LIST_PERMUTATION_DEFINES = 4,
OPT_LIST_IMPLICIT_DEFINES = 5,
OPT_LIST_POWERLOSSES = 6,
OPT_QUERY_DEFINE = 'Q',
OPT_QUERY_PERMUTATION_DEFINE = 7,
OPT_QUERY_IMPLICIT_DEFINE = 8,
OPT_DEFINE = 'D',
OPT_DEFINE_DEPTH = 7,
OPT_DEFINE_DEPTH = 9,
OPT_POWERLOSS = 'P',
OPT_STEP = 8,
OPT_FORCE = 9,
OPT_NO_INTERNAL = 10,
OPT_NO_REENTRANT = 11,
OPT_NO_FUZZ = 12,
OPT_STEP = 10,
OPT_FORCE = 11,
OPT_NO_INTERNAL = 12,
OPT_NO_REENTRANT = 13,
OPT_NO_FUZZ = 14,
OPT_DISK = 'd',
OPT_TRACE = 't',
OPT_TRACE_BACKTRACE = 13,
OPT_TRACE_STEP = 14,
OPT_TRACE_RUNFREQ = 15,
OPT_READ_SLEEP = 16,
OPT_PROG_SLEEP = 17,
OPT_ERASE_SLEEP = 18,
OPT_TRACE_BACKTRACE = 15,
OPT_TRACE_STEP = 16,
OPT_TRACE_RUNFREQ = 17,
OPT_READ_SLEEP = 18,
OPT_PROG_SLEEP = 19,
OPT_ERASE_SLEEP = 20,
};
const char *short_opts = "hYlLD:P:d:t:";
const char *short_opts = "hYlLQ:D:P:d:t:";
const struct option long_opts[] = {
{"help", no_argument, NULL, OPT_HELP},
@@ -2091,6 +2256,11 @@ const struct option long_opts[] = {
{"list-implicit-defines",
no_argument, NULL, OPT_LIST_IMPLICIT_DEFINES},
{"list-powerlosses", no_argument, NULL, OPT_LIST_POWERLOSSES},
{"query-define", required_argument, NULL, OPT_QUERY_DEFINE},
{"query-permutation-define",
required_argument, NULL, OPT_QUERY_PERMUTATION_DEFINE},
{"query-implicit-define",
required_argument, NULL, OPT_QUERY_IMPLICIT_DEFINE},
{"define", required_argument, NULL, OPT_DEFINE},
{"define-depth", required_argument, NULL, OPT_DEFINE_DEPTH},
{"powerloss", required_argument, NULL, OPT_POWERLOSS},
@@ -2121,6 +2291,9 @@ const char *const help_text[] = {
"List explicit defines in this test-runner.",
"List implicit defines in this test-runner.",
"List the available powerloss scenarios.",
"Query a test define.",
"Query a permutation test define.",
"Query an implicit test define.",
"Override a test define.",
"How deep to evaluate recursive defines before erroring.",
"Specify a powerloss scenario to test.",
@@ -2239,6 +2412,21 @@ int main(int argc, char **argv) {
op = list_powerlosses;
break;
case OPT_QUERY_DEFINE:;
op = query_define;
query_define_query = optarg;
break;
case OPT_QUERY_PERMUTATION_DEFINE:;
op = query_permutation_define;
query_define_query = optarg;
break;
case OPT_QUERY_IMPLICIT_DEFINE:;
op = query_implicit_define;
query_define_query = optarg;
break;
// configuration
case OPT_DEFINE:;
// allocate space
scripts/bench.py
+20 -1
View File
@@ -1096,6 +1096,13 @@ def list_(runner, bench_ids=[], **args):
cmd.append('--list-suite-probes')
if args.get('list_case_probes'):
cmd.append('--list-case-probes')
if args.get('query_define'): cmd.append('-Q%s' % args['query_define'])
if args.get('query_permutation_define'):
cmd.append('--query-permutation-define=%s'
% args['query_permutation_define'])
if args.get('query_implicit_define'):
cmd.append('--query-implicit-define=%s'
% args['query_implicit_define'])
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
@@ -1689,7 +1696,10 @@ def main(**args):
or args.get('list_implicit_defines')
or args.get('list_probes')
or args.get('list_suite_probes')
or args.get('list_case_probes')):
or args.get('list_case_probes')
or args.get('query_define')
or args.get('query_permutation_define')
or args.get('query_implicit_define')):
return list_(**args)
else:
return run(**args)
@@ -1775,6 +1785,15 @@ if __name__ == "__main__":
'--list-case-probes',
action='store_true',
help="List estimated probes for each bench case.")
bench_parser.add_argument(
'-Q', '--query-define',
help="Query a bench define.")
bench_parser.add_argument(
'--query-permutation-define',
help="Query a permutation bench define.")
bench_parser.add_argument(
'--query-implicit-define',
help="Query an implicit bench define.")
bench_parser.add_argument(
'-D', '--define',
action='append',
scripts/test.py
+20 -1
View File
@@ -1084,6 +1084,13 @@ def list_(runner, test_ids=[], **args):
if args.get('list_implicit_defines'):
cmd.append('--list-implicit-defines')
if args.get('list_powerlosses'): cmd.append('--list-powerlosses')
if args.get('query_define'): cmd.append('-Q%s' % args['query_define'])
if args.get('query_permutation_define'):
cmd.append('--query-permutation-define=%s'
% args['query_permutation_define'])
if args.get('query_implicit_define'):
cmd.append('--query-implicit-define=%s'
% args['query_implicit_define'])
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
@@ -1654,7 +1661,10 @@ def main(**args):
or args.get('list_defines')
or args.get('list_permutation_defines')
or args.get('list_implicit_defines')
or args.get('list_powerlosses')):
or args.get('list_powerlosses')
or args.get('query_define')
or args.get('query_permutation_define')
or args.get('query_implicit_define')):
return list_(**args)
else:
return run(**args)
@@ -1732,6 +1742,15 @@ if __name__ == "__main__":
'--list-powerlosses',
action='store_true',
help="List the available powerloss scenarios.")
test_parser.add_argument(
'-Q', '--query-define',
help="Query a test define.")
test_parser.add_argument(
'--query-permutation-define',
help="Query a permutation test define.")
test_parser.add_argument(
'--query-implicit-define',
help="Query an implicit test define.")
test_parser.add_argument(
'-D', '--define',
action='append',