Github/littlefs
1
Fork 0
Code Actions Activity

runners: bench: Added litmus flag, default to disabled

Browse Source 
The litmus benches are really only intended for introspection/debugging/
cool plots/etc. They're interesting to poke around with and cover a wide
range of littlefs's data-structures, but are not very rigorous.

To make this more clear for new users, added a new litmus flag for
benches:

  litmus = true

This doesn't change anything about how the bench is run, but serves as a
marker to hint that the bench is intended for non-rigorous benchmarking.

---

In the makefile, litmus tests are disabled by default at runtime
(--no-litmus). This is to limit `make bench` to benches that are useful
for performance comparisons.

With --no-litmus at runtime, the litmus benches are at least compiled
into the bench_runner, which should hopefully encourage keeping them up
to date with code changes. Eventually we should also run them in CI, but
only to check for runtime errors.

Unlike our tests, we're not really worried about compile time at the
moment due to how few/small our benches are.
This commit is contained in:
Christopher Haster
2026-02-07 01:40:46 -06:00
parent c31be08708
commit 238c2babe4
9 changed files with 78 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runners/bench_runner.c
+27 -17
View File
@@ -1306,9 +1306,10 @@ static void summary(void) {
char perm_buf[64];
sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total);
char flag_buf[64];
sprintf(flag_buf, "%s%s",
(flags & BENCH_INTERNAL) ? "i" : "",
(!flags) ? "-" : "");
sprintf(flag_buf, "%s%s%s",
(flags & BENCH_INTERNAL) ? "i" : "",
(flags & BENCH_LITMUS) ? "l" : "",
(!flags) ? "-" : "");
printf("%-23s %7s %7zu %7zu %15s\n",
"TOTAL",
flag_buf,
@@ -1363,10 +1364,12 @@ static void list_suites(void) {
char perm_buf[64];
sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total);
bench_flags_t flags = bench_suites[i]->flags;
char flag_buf[64];
sprintf(flag_buf, "%s%s",
(bench_suites[i]->flags & BENCH_INTERNAL) ? "i" : "",
(!bench_suites[i]->flags) ? "-" : "");
sprintf(flag_buf, "%s%s%s",
(flags & BENCH_INTERNAL) ? "i" : "",
(flags & BENCH_LITMUS) ? "l" : "",
(!flags) ? "-" : "");
printf("%-*s %7s %7zu %15s\n",
name_width,
bench_suites[i]->name,
@@ -1415,12 +1418,12 @@ static void list_cases(void) {
char perm_buf[64];
sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total);
bench_flags_t flags = bench_suites[i]->cases[j].flags;
char flag_buf[64];
sprintf(flag_buf, "%s%s",
(bench_suites[i]->cases[j].flags & BENCH_INTERNAL)
? "i" : "",
(!bench_suites[i]->cases[j].flags)
? "-" : "");
sprintf(flag_buf, "%s%s%s",
(flags & BENCH_INTERNAL) ? "i" : "",
(flags & BENCH_LITMUS) ? "l" : "",
(!flags) ? "-" : "");
printf("%-*s %7s %15s\n",
name_width,
bench_suites[i]->cases[j].name,
@@ -1948,14 +1951,15 @@ enum opt_flags {
OPT_STEP = 's',
OPT_FORCE = 7,
OPT_NO_INTERNAL = 8,
OPT_NO_LITMUS = 9,
OPT_DISK = 'd',
OPT_TRACE = 't',
OPT_TRACE_BACKTRACE = 9,
OPT_TRACE_PERIOD = 10,
OPT_TRACE_FREQ = 11,
OPT_READ_SLEEP = 12,
OPT_PROG_SLEEP = 13,
OPT_ERASE_SLEEP = 14,
OPT_TRACE_BACKTRACE = 10,
OPT_TRACE_PERIOD = 11,
OPT_TRACE_FREQ = 12,
OPT_READ_SLEEP = 13,
OPT_PROG_SLEEP = 14,
OPT_ERASE_SLEEP = 15,
};
const char *short_opts = "hYlLD:s:d:t:";
@@ -1977,6 +1981,7 @@ const struct option long_opts[] = {
{"step", required_argument, NULL, OPT_STEP},
{"force", no_argument, NULL, OPT_FORCE},
{"no-internal", no_argument, NULL, OPT_NO_INTERNAL},
{"no-litmus", no_argument, NULL, OPT_NO_LITMUS},
{"disk", required_argument, NULL, OPT_DISK},
{"trace", required_argument, NULL, OPT_TRACE},
{"trace-backtrace", no_argument, NULL, OPT_TRACE_BACKTRACE},
@@ -2003,6 +2008,7 @@ const char *const help_text[] = {
"Comma-separated range of permutations to run.",
"Ignore bench filters.",
"Don't run internal benches.",
"Don't run litmus benches.",
"Direct block device operations to this file.",
"Direct trace output to this file.",
"Include a backtrace with every trace statement.",
@@ -2333,6 +2339,10 @@ int main(int argc, char **argv) {
bench_mask |= BENCH_INTERNAL;
break;
case OPT_NO_LITMUS:;
bench_mask |= BENCH_LITMUS;
break;
case OPT_DISK:;
bench_disk_path = optarg;
break;
runners/bench_runner.h
+1
View File
@@ -83,6 +83,7 @@ struct lfs3_cfg;
enum bench_flags {
BENCH_INTERNAL = 0x1,
BENCH_LITMUS = 0x2,
};
typedef uint8_t bench_flags_t;
runners/test_runner.c
+10 -12
View File
@@ -1206,12 +1206,13 @@ static void list_suites(void) {
char perm_buf[64];
sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total);
test_flags_t flags = test_suites[i]->flags;
char flag_buf[64];
sprintf(flag_buf, "%s%s%s%s",
(test_suites[i]->flags & TEST_INTERNAL) ? "i" : "",
(test_suites[i]->flags & TEST_REENTRANT) ? "r" : "",
(test_suites[i]->flags & TEST_FUZZ) ? "f" : "",
(!test_suites[i]->flags) ? "-" : "");
(flags & TEST_INTERNAL) ? "i" : "",
(flags & TEST_REENTRANT) ? "r" : "",
(flags & TEST_FUZZ) ? "f" : "",
(!flags) ? "-" : "");
printf("%-*s %7s %7zu %15s\n",
name_width,
test_suites[i]->name,
@@ -1260,16 +1261,13 @@ static void list_cases(void) {
char perm_buf[64];
sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total);
test_flags_t flags = test_suites[i]->cases[j].flags;
char flag_buf[64];
sprintf(flag_buf, "%s%s%s%s",
(test_suites[i]->cases[j].flags & TEST_INTERNAL)
? "i" : "",
(test_suites[i]->cases[j].flags & TEST_REENTRANT)
? "r" : "",
(test_suites[i]->cases[j].flags & TEST_FUZZ)
? "f" : "",
(!test_suites[i]->cases[j].flags)
? "-" : "");
(flags & TEST_INTERNAL) ? "i" : "",
(flags & TEST_REENTRANT) ? "r" : "",
(flags & TEST_FUZZ) ? "f" : "",
(!flags) ? "-" : "");
printf("%-*s %7s %15s\n",
name_width,
test_suites[i]->cases[j].name,