Github/littlefs
1
Fork 0
Code Actions Activity

runners: bench: Added best effort --list-probes, --list-case-probes, etc

Browse Source 
Adds a set of flags to query the bench_runner for available probes:

- --list-probes       - List estimated probes
- --list-suite-probes - List estimated probes for each bench suite
- --list-case-probes  - List estimated probes for each bench case

What's fun though, is we don't actually know the bench probes at compile
time, since the BENCH_* macros take a C string. But we're already
preprocessing bench_*.toml with Python, so guessing what probes are
available is easy with a bit of regex:

  BENCH_(?:STOP|F?RESULT)\( *"((?:\\.|[^"])*)"

This does make the --list*probes flags best effort, but I think unlikely
to break in practice.
This commit is contained in:
Christopher Haster
2026-02-08 05:16:23 -06:00
parent 95fddd3c18
commit 81d681cab2
3 changed files with 235 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runners/bench_runner.c
+194 -13
View File
@@ -1552,9 +1552,9 @@ static void list_suite_paths(void) {
|| strcmp(bench_ids[t].name,
bench_suites[i]->cases[j].name) == 0)) {
continue;
cases += 1;
}
cases += 1;
}
// no benches found?
@@ -1819,6 +1819,165 @@ static void list_implicit_defines(void) {
free(defines.defines);
}
static void list_probes(void) {
// find relevant probes
const char **probes = NULL;
size_t probe_count = 0;
size_t probe_capacity = 0;
for (size_t t = 0; t < bench_id_count; t++) {
for (size_t i = 0; i < bench_suite_count; i++) {
for (size_t j = 0; j < bench_suites[i]->case_count; j++) {
// does neither suite nor case name match?
if (bench_ids[t].name && !(
strcmp(bench_ids[t].name,
bench_suites[i]->name) == 0
|| strcmp(bench_ids[t].name,
bench_suites[i]->cases[j].name) == 0)) {
continue;
}
// add unseen probes
for (size_t p = 0;
p < bench_suites[i]->cases[j].probe_count;
p++) {
for (size_t q = 0; q < probe_count; q++) {
if (strcmp(probes[q],
bench_suites[i]->cases[j].probes[p]) == 0) {
goto next;
}
}
const char **probe = mappend(
(void**)&probes,
sizeof(const char*),
&probe_count,
&probe_capacity);
*probe = bench_suites[i]->cases[j].probes[p];
}
next:;
}
}
}
for (size_t p = 0; p < probe_count; p++) {
printf("%s\n", probes[p]);
}
free(probes);
}
static void list_suite_probes(void) {
// at least size so that names fit
unsigned name_width = 23;
for (size_t i = 0; i < bench_suite_count; i++) {
size_t len = strlen(bench_suites[i]->name);
if (len > name_width) {
name_width = len;
}
}
name_width = 4*((name_width+1+4-1)/4)-1;
printf("%-*s %s\n", name_width, "suite", "probes");
// find relevant probes
for (size_t t = 0; t < bench_id_count; t++) {
for (size_t i = 0; i < bench_suite_count; i++) {
const char **probes = NULL;
size_t probe_count = 0;
size_t probe_capacity = 0;
for (size_t j = 0; j < bench_suites[i]->case_count; j++) {
// does neither suite nor case name match?
if (bench_ids[t].name && !(
strcmp(bench_ids[t].name,
bench_suites[i]->name) == 0
|| strcmp(bench_ids[t].name,
bench_suites[i]->cases[j].name) == 0)) {
continue;
}
// add unseen probes
for (size_t p = 0;
p < bench_suites[i]->cases[j].probe_count;
p++) {
for (size_t q = 0; q < probe_count; q++) {
if (strcmp(probes[q],
bench_suites[i]->cases[j].probes[p]) == 0) {
goto next;
}
}
const char **probe = mappend(
(void**)&probes,
sizeof(const char*),
&probe_count,
&probe_capacity);
*probe = bench_suites[i]->cases[j].probes[p];
}
next:;
}
printf("%-*s ",
name_width,
bench_suites[i]->name);
for (size_t p = 0; p < probe_count; p++) {
printf("%s", probes[p]);
if (p != probe_count-1) {
printf(",");
}
}
printf("\n");
free(probes);
}
}
}
static void list_case_probes(void) {
// at least size so that names fit
unsigned name_width = 23;
for (size_t i = 0; i < bench_suite_count; i++) {
for (size_t j = 0; j < bench_suites[i]->case_count; j++) {
size_t len = strlen(bench_suites[i]->cases[j].name);
if (len > name_width) {
name_width = len;
}
}
}
name_width = 4*((name_width+1+4-1)/4)-1;
printf("%-*s %s\n", name_width, "case", "probes");
// find relevant probes
for (size_t t = 0; t < bench_id_count; t++) {
for (size_t i = 0; i < bench_suite_count; i++) {
for (size_t j = 0; j < bench_suites[i]->case_count; j++) {
// does neither suite nor case name match?
if (bench_ids[t].name && !(
strcmp(bench_ids[t].name,
bench_suites[i]->name) == 0
|| strcmp(bench_ids[t].name,
bench_suites[i]->cases[j].name) == 0)) {
continue;
}
printf("%-*s ",
name_width,
bench_suites[i]->cases[j].name);
for (size_t p = 0;
p < bench_suites[i]->cases[j].probe_count;
p++) {
printf("%s", bench_suites[i]->cases[j].probes[p]);
if (p != bench_suites[i]->cases[j].probe_count-1) {
printf(",");
}
}
printf("\n");
}
}
}
}
// bench bd wrappers for heap/stack tracking
@@ -2041,20 +2200,23 @@ enum opt_flags {
OPT_LIST_DEFINES = 3,
OPT_LIST_PERMUTATION_DEFINES = 4,
OPT_LIST_IMPLICIT_DEFINES = 5,
OPT_LIST_PROBES = 6,
OPT_LIST_SUITE_PROBES = 7,
OPT_LIST_CASE_PROBES = 8,
OPT_DEFINE = 'D',
OPT_DEFINE_DEPTH = 6,
OPT_STEP = 7,
OPT_FORCE = 8,
OPT_NO_INTERNAL = 9,
OPT_NO_LITMUS = 10,
OPT_DEFINE_DEPTH = 9,
OPT_STEP = 10,
OPT_FORCE = 11,
OPT_NO_INTERNAL = 12,
OPT_NO_LITMUS = 13,
OPT_DISK = 'd',
OPT_TRACE = 't',
OPT_TRACE_BACKTRACE = 11,
OPT_TRACE_STEP = 12,
OPT_TRACE_RUNFREQ = 13,
OPT_READ_SLEEP = 14,
OPT_PROG_SLEEP = 15,
OPT_ERASE_SLEEP = 16,
OPT_TRACE_BACKTRACE = 14,
OPT_TRACE_STEP = 15,
OPT_TRACE_RUNFREQ = 16,
OPT_READ_SLEEP = 17,
OPT_PROG_SLEEP = 18,
OPT_ERASE_SLEEP = 19,
};
const char *short_opts = "hYlLD:d:t:";
@@ -2071,6 +2233,10 @@ const struct option long_opts[] = {
no_argument, NULL, OPT_LIST_PERMUTATION_DEFINES},
{"list-implicit-defines",
no_argument, NULL, OPT_LIST_IMPLICIT_DEFINES},
{"list-probes", no_argument, NULL, OPT_LIST_PROBES},
{"list-suite-probes",
no_argument, NULL, OPT_LIST_SUITE_PROBES},
{"list-case-probes", no_argument, NULL, OPT_LIST_CASE_PROBES},
{"define", required_argument, NULL, OPT_DEFINE},
{"define-depth", required_argument, NULL, OPT_DEFINE_DEPTH},
{"step", required_argument, NULL, OPT_STEP},
@@ -2098,6 +2264,9 @@ const char *const help_text[] = {
"List all defines in this bench-runner.",
"List explicit defines in this bench-runner.",
"List implicit defines in this bench-runner.",
"List estimated probes.",
"List estimated probes for each bench suite.",
"List estimated probes for each bench case.",
"Override a bench define.",
"How deep to evaluate recursive defines before erroring.",
"Comma-separated range of permutations to run.",
@@ -2213,6 +2382,18 @@ int main(int argc, char **argv) {
op = list_implicit_defines;
break;
case OPT_LIST_PROBES:;
op = list_probes;
break;
case OPT_LIST_SUITE_PROBES:;
op = list_suite_probes;
break;
case OPT_LIST_CASE_PROBES:;
op = list_case_probes;
break;
// configuration
case OPT_DEFINE:;
// allocate space
runners/bench_runner.h
+3
View File
@@ -116,6 +116,9 @@ struct bench_case {
const bench_define_t *defines;
size_t permutations;
const char **probes;
size_t probe_count;
bool (*if_)(void);
void (*run)(const struct lfs3_cfg *cfg);
};
scripts/bench.py
+38 -1
View File
@@ -177,6 +177,12 @@ class BenchCase:
self.defines = defines__
self.permutations = permutations__
# we have the source code, so try to guess what probes are in
# use, note this is only informative
self.probes = list(co.OrderedDict.fromkeys(re.findall(
'BENCH_(?:STOP|F?RESULT)\( *"((?:\\.|[^"])*)"',
self.code)).keys())
for k in config.keys():
print('%swarning:%s in %s, found unused key %r' % (
'\x1b[1;33m' if args['color'] else '',
@@ -674,6 +680,16 @@ def compile(bench_paths, **args):
f.writeln(12*' '+'},')
f.writeln(12*' '+'.permutations = %d,' % (
len(case.permutations)))
# list case probes
if case.probes:
f.writeln(12*' '+'.probes'
' = (const char*[%d]){' % (
len(case.probes)))
for p in case.probes:
f.writeln(16*' '+'"%s",' % p)
f.writeln(12*' '+'},')
f.writeln(12*' '+'.probe_count = %d,' % (
len(case.probes)))
if suite.if_ or case.if_:
f.writeln(12*' '+'.if_ = __bench__%s__if,' % (
case.name))
@@ -1065,6 +1081,12 @@ def list_(runner, bench_ids=[], **args):
cmd.append('--list-permutation-defines')
if args.get('list_implicit_defines'):
cmd.append('--list-implicit-defines')
if args.get('list_probes'):
cmd.append('--list-probes')
if args.get('list_suite_probes'):
cmd.append('--list-suite-probes')
if args.get('list_case_probes'):
cmd.append('--list-case-probes')
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
@@ -1640,7 +1662,10 @@ def main(**args):
or args.get('list_case_paths')
or args.get('list_defines')
or args.get('list_permutation_defines')
or args.get('list_implicit_defines')):
or args.get('list_implicit_defines')
or args.get('list_probes')
or args.get('list_suite_probes')
or args.get('list_case_probes')):
return list_(**args)
else:
return run(**args)
@@ -1714,6 +1739,18 @@ if __name__ == "__main__":
'--list-implicit-defines',
action='store_true',
help="List implicit defines in this bench-runner.")
bench_parser.add_argument(
'--list-probes',
action='store_true',
help="List estimated probes.")
bench_parser.add_argument(
'--list-suite-probes',
action='store_true',
help="List estimated probes for each bench suite.")
bench_parser.add_argument(
'--list-case-probes',
action='store_true',
help="List estimated probes for each bench case.")
bench_parser.add_argument(
'-D', '--define',
action='append',