From 238c2babe4951e29e16a12297b469b659d18a502 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 7 Feb 2026 01:40:46 -0600 Subject: [PATCH] runners: bench: Added litmus flag, default to disabled The litmus benches are really only intended for introspection/debugging/ cool plots/etc. They're interesting to poke around with and cover a wide range of littlefs's data-structures, but are not very rigorous. To make this more clear for new users, added a new litmus flag for benches: litmus = true This doesn't change anything about how the bench is run, but serves as a marker to hint that the bench is intended for non-rigorous benchmarking. --- In the makefile, litmus tests are disabled by default at runtime (--no-litmus). This is to limit `make bench` to benches that are useful for performance comparisons. With --no-litmus at runtime, the litmus benches are at least compiled into the bench_runner, which should hopefully encourage keeping them up to date with code changes. Eventually we should also run them in CI, but only to check for runtime errors. Unlike our tests, we're not really worried about compile time at the moment due to how few/small our benches are. --- Makefile | 5 +++++ benches/bench_btree.toml | 4 ++++ benches/bench_dir.toml | 2 ++ benches/bench_file.toml | 2 ++ benches/bench_rbyd.toml | 4 ++++ runners/bench_runner.c | 44 ++++++++++++++++++++++++---------------- runners/bench_runner.h | 1 + runners/test_runner.c | 22 +++++++++----------- scripts/bench.py | 24 +++++++++++++++++++++- 9 files changed, 78 insertions(+), 30 deletions(-) diff --git a/Makefile b/Makefile index c8ae6ab1..4208d79b 100644 --- a/Makefile +++ b/Makefile @@ -201,6 +201,11 @@ TESTFLAGS += --perf-path="$(PERF)" BENCHFLAGS += --perf-path="$(PERF)" endif +# default to not running litmus benches +ifndef BENCH_ALL +BENCHFLAGS += --no-litmus +endif + # alternative bench geometries (defaults to NOR flash) ifdef BENCH_NOR BENCHFLAGS += -DDISK_GEOMETRY=0 diff --git a/benches/bench_btree.toml b/benches/bench_btree.toml index 3bde191f..92682bfb 100644 --- a/benches/bench_btree.toml +++ b/benches/bench_btree.toml @@ -23,6 +23,8 @@ defines.SEED = 42 # 0x2 => lookup # 0x8 => usage defines.MASK = 0xb +# not the most rigorous +litmus = true in = 'lfs3.c' code = ''' lfs3_t lfs3; @@ -124,6 +126,8 @@ defines.SEED = 42 # 0x4 => namelookup # 0x8 => usage defines.MASK = 0xf +# not the most rigorous +litmus = true in = 'lfs3.c' code = ''' lfs3_t lfs3; diff --git a/benches/bench_dir.toml b/benches/bench_dir.toml index b614f382..c1d6f2e7 100644 --- a/benches/bench_dir.toml +++ b/benches/bench_dir.toml @@ -22,6 +22,8 @@ defines.SEED = 42 # 0x4 => read # 0x8 => usage defines.MASK = 0xf +# not the most rigorous +litmus = true code = ''' lfs3_t lfs3; lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0; diff --git a/benches/bench_file.toml b/benches/bench_file.toml index 0a75719e..6a0bda08 100644 --- a/benches/bench_file.toml +++ b/benches/bench_file.toml @@ -24,6 +24,8 @@ defines.SEED = 42 # 0x2 => read # 0x4 => usage defines.MASK = 0x7 +# not the most rigorous +litmus = true code = ''' lfs3_t lfs3; lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0; diff --git a/benches/bench_rbyd.toml b/benches/bench_rbyd.toml index 1887cfae..b2d61878 100644 --- a/benches/bench_rbyd.toml +++ b/benches/bench_rbyd.toml @@ -27,6 +27,8 @@ defines.SEED = 42 # 0x08 => lookup # 0x10 => usage defines.MASK = 0x1f +# not the most rigorous +litmus = true in = 'lfs3.c' code = ''' lfs3_t lfs3; @@ -138,6 +140,8 @@ defines.SEED = 42 # 0x08 => lookup # 0x10 => usage defines.MASK = 0x1f +# not the most rigorous +litmus = true in = 'lfs3.c' code = ''' lfs3_t lfs3; diff --git a/runners/bench_runner.c b/runners/bench_runner.c index de91668e..92c97b4c 100644 --- a/runners/bench_runner.c +++ b/runners/bench_runner.c @@ -1306,9 +1306,10 @@ static void summary(void) { char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); char flag_buf[64]; - sprintf(flag_buf, "%s%s", - (flags & BENCH_INTERNAL) ? "i" : "", - (!flags) ? "-" : ""); + sprintf(flag_buf, "%s%s%s", + (flags & BENCH_INTERNAL) ? "i" : "", + (flags & BENCH_LITMUS) ? "l" : "", + (!flags) ? "-" : ""); printf("%-23s %7s %7zu %7zu %15s\n", "TOTAL", flag_buf, @@ -1363,10 +1364,12 @@ static void list_suites(void) { char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); + bench_flags_t flags = bench_suites[i]->flags; char flag_buf[64]; - sprintf(flag_buf, "%s%s", - (bench_suites[i]->flags & BENCH_INTERNAL) ? "i" : "", - (!bench_suites[i]->flags) ? "-" : ""); + sprintf(flag_buf, "%s%s%s", + (flags & BENCH_INTERNAL) ? "i" : "", + (flags & BENCH_LITMUS) ? "l" : "", + (!flags) ? "-" : ""); printf("%-*s %7s %7zu %15s\n", name_width, bench_suites[i]->name, @@ -1415,12 +1418,12 @@ static void list_cases(void) { char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); + bench_flags_t flags = bench_suites[i]->cases[j].flags; char flag_buf[64]; - sprintf(flag_buf, "%s%s", - (bench_suites[i]->cases[j].flags & BENCH_INTERNAL) - ? "i" : "", - (!bench_suites[i]->cases[j].flags) - ? "-" : ""); + sprintf(flag_buf, "%s%s%s", + (flags & BENCH_INTERNAL) ? "i" : "", + (flags & BENCH_LITMUS) ? "l" : "", + (!flags) ? "-" : ""); printf("%-*s %7s %15s\n", name_width, bench_suites[i]->cases[j].name, @@ -1948,14 +1951,15 @@ enum opt_flags { OPT_STEP = 's', OPT_FORCE = 7, OPT_NO_INTERNAL = 8, + OPT_NO_LITMUS = 9, OPT_DISK = 'd', OPT_TRACE = 't', - OPT_TRACE_BACKTRACE = 9, - OPT_TRACE_PERIOD = 10, - OPT_TRACE_FREQ = 11, - OPT_READ_SLEEP = 12, - OPT_PROG_SLEEP = 13, - OPT_ERASE_SLEEP = 14, + OPT_TRACE_BACKTRACE = 10, + OPT_TRACE_PERIOD = 11, + OPT_TRACE_FREQ = 12, + OPT_READ_SLEEP = 13, + OPT_PROG_SLEEP = 14, + OPT_ERASE_SLEEP = 15, }; const char *short_opts = "hYlLD:s:d:t:"; @@ -1977,6 +1981,7 @@ const struct option long_opts[] = { {"step", required_argument, NULL, OPT_STEP}, {"force", no_argument, NULL, OPT_FORCE}, {"no-internal", no_argument, NULL, OPT_NO_INTERNAL}, + {"no-litmus", no_argument, NULL, OPT_NO_LITMUS}, {"disk", required_argument, NULL, OPT_DISK}, {"trace", required_argument, NULL, OPT_TRACE}, {"trace-backtrace", no_argument, NULL, OPT_TRACE_BACKTRACE}, @@ -2003,6 +2008,7 @@ const char *const help_text[] = { "Comma-separated range of permutations to run.", "Ignore bench filters.", "Don't run internal benches.", + "Don't run litmus benches.", "Direct block device operations to this file.", "Direct trace output to this file.", "Include a backtrace with every trace statement.", @@ -2333,6 +2339,10 @@ int main(int argc, char **argv) { bench_mask |= BENCH_INTERNAL; break; + case OPT_NO_LITMUS:; + bench_mask |= BENCH_LITMUS; + break; + case OPT_DISK:; bench_disk_path = optarg; break; diff --git a/runners/bench_runner.h b/runners/bench_runner.h index 39be99ff..311c791d 100644 --- a/runners/bench_runner.h +++ b/runners/bench_runner.h @@ -83,6 +83,7 @@ struct lfs3_cfg; enum bench_flags { BENCH_INTERNAL = 0x1, + BENCH_LITMUS = 0x2, }; typedef uint8_t bench_flags_t; diff --git a/runners/test_runner.c b/runners/test_runner.c index 65400700..1d1968c1 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -1206,12 +1206,13 @@ static void list_suites(void) { char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); + test_flags_t flags = test_suites[i]->flags; char flag_buf[64]; sprintf(flag_buf, "%s%s%s%s", - (test_suites[i]->flags & TEST_INTERNAL) ? "i" : "", - (test_suites[i]->flags & TEST_REENTRANT) ? "r" : "", - (test_suites[i]->flags & TEST_FUZZ) ? "f" : "", - (!test_suites[i]->flags) ? "-" : ""); + (flags & TEST_INTERNAL) ? "i" : "", + (flags & TEST_REENTRANT) ? "r" : "", + (flags & TEST_FUZZ) ? "f" : "", + (!flags) ? "-" : ""); printf("%-*s %7s %7zu %15s\n", name_width, test_suites[i]->name, @@ -1260,16 +1261,13 @@ static void list_cases(void) { char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); + test_flags_t flags = test_suites[i]->cases[j].flags; char flag_buf[64]; sprintf(flag_buf, "%s%s%s%s", - (test_suites[i]->cases[j].flags & TEST_INTERNAL) - ? "i" : "", - (test_suites[i]->cases[j].flags & TEST_REENTRANT) - ? "r" : "", - (test_suites[i]->cases[j].flags & TEST_FUZZ) - ? "f" : "", - (!test_suites[i]->cases[j].flags) - ? "-" : ""); + (flags & TEST_INTERNAL) ? "i" : "", + (flags & TEST_REENTRANT) ? "r" : "", + (flags & TEST_FUZZ) ? "f" : "", + (!flags) ? "-" : ""); printf("%-*s %7s %15s\n", name_width, test_suites[i]->cases[j].name, diff --git a/scripts/bench.py b/scripts/bench.py index 9461612d..d1ce9424 100755 --- a/scripts/bench.py +++ b/scripts/bench.py @@ -106,6 +106,10 @@ class BenchCase: config.pop('suite_internal', None)) if self.internal is None: self.internal = False + self.litmus = config.pop('litmus', + config.pop('suite_litmus', None)) + if self.litmus is None: + self.litmus = False # in implies internal self.internal |= bool(self.in_) @@ -287,6 +291,7 @@ class BenchSuite: # a couple of these we just forward to all cases defines = config.pop('defines', None) internal = config.pop('internal', None) + litmus = config.pop('litmus', None) self.cases = [] for name, config_ in cases.items(): @@ -298,12 +303,15 @@ class BenchSuite: 'suite_defines': defines, 'suite_in': self.in_, 'suite_internal': internal, + 'suite_litmus': litmus, **config_}, args) # skipping internal benches? if args.get('no_internal') and case.internal: continue + if args.get('no_litmus') and case.litmus: + continue self.cases.append(case) @@ -316,6 +324,7 @@ class BenchSuite: # combine other per-case things self.internal = any(case.internal for case in self.cases) + self.litmus = any(case.litmus for case in self.cases) for k in config.keys(): print('%swarning:%s in %s, found unused key %r' % ( @@ -596,7 +605,8 @@ def compile(bench_paths, **args): f.writeln(4*' '+'.path = "%s",' % suite.path) f.writeln(4*' '+'.flags = %s,' % ( ' | '.join(filter(None, [ - 'BENCH_INTERNAL' if suite.internal else None])) + 'BENCH_INTERNAL' if suite.internal else None, + 'BENCH_LITMUS' if suite.litmus else None])) or 0)) for ifdef in suite.ifdef: f.writeln(4*' '+'#if (%s)' % re.sub( @@ -628,6 +638,8 @@ def compile(bench_paths, **args): f.writeln(12*' '+'.flags = %s,' % ( ' | '.join(filter(None, [ 'BENCH_INTERNAL' if case.internal + else None, + 'BENCH_LITMUS' if case.litmus else None])) or 0)) for ifdef in it.chain(suite.ifdef, case.ifdef): @@ -798,6 +810,8 @@ def find_runner(runner, id=None, main=True, **args): cmd.append('--force') if args.get('no_internal'): cmd.append('--no-internal') + if args.get('no_litmus'): + cmd.append('--no-litmus') # only one thread should write to disk/trace, otherwise the output # ends up clobbered and useless @@ -1714,6 +1728,10 @@ if __name__ == "__main__": '--no-internal', action='store_true', help="Don't run internal benches.") + bench_parser.add_argument( + '--no-litmus', + action='store_true', + help="Don't run litmus benches.") bench_parser.add_argument( '-d', '--disk', help="Direct block device operations to this file.") @@ -1876,6 +1894,10 @@ if __name__ == "__main__": '--no-internal', action='store_true', help="Don't build internal benches.") + comp_parser.add_argument( + '--no-litmus', + action='store_true', + help="Don't build litmus benches.") # do the thing args = parser.parse_intermixed_args()