diff --git a/Makefile b/Makefile index c8ae6ab1..4208d79b 100644 --- a/Makefile +++ b/Makefile @@ -201,6 +201,11 @@ TESTFLAGS += --perf-path="$(PERF)" BENCHFLAGS += --perf-path="$(PERF)" endif +# default to not running litmus benches +ifndef BENCH_ALL +BENCHFLAGS += --no-litmus +endif + # alternative bench geometries (defaults to NOR flash) ifdef BENCH_NOR BENCHFLAGS += -DDISK_GEOMETRY=0 diff --git a/benches/bench_btree.toml b/benches/bench_btree.toml index 3bde191f..92682bfb 100644 --- a/benches/bench_btree.toml +++ b/benches/bench_btree.toml @@ -23,6 +23,8 @@ defines.SEED = 42 # 0x2 => lookup # 0x8 => usage defines.MASK = 0xb +# not the most rigorous +litmus = true in = 'lfs3.c' code = ''' lfs3_t lfs3; @@ -124,6 +126,8 @@ defines.SEED = 42 # 0x4 => namelookup # 0x8 => usage defines.MASK = 0xf +# not the most rigorous +litmus = true in = 'lfs3.c' code = ''' lfs3_t lfs3; diff --git a/benches/bench_dir.toml b/benches/bench_dir.toml index b614f382..c1d6f2e7 100644 --- a/benches/bench_dir.toml +++ b/benches/bench_dir.toml @@ -22,6 +22,8 @@ defines.SEED = 42 # 0x4 => read # 0x8 => usage defines.MASK = 0xf +# not the most rigorous +litmus = true code = ''' lfs3_t lfs3; lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0; diff --git a/benches/bench_file.toml b/benches/bench_file.toml index 0a75719e..6a0bda08 100644 --- a/benches/bench_file.toml +++ b/benches/bench_file.toml @@ -24,6 +24,8 @@ defines.SEED = 42 # 0x2 => read # 0x4 => usage defines.MASK = 0x7 +# not the most rigorous +litmus = true code = ''' lfs3_t lfs3; lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0; diff --git a/benches/bench_rbyd.toml b/benches/bench_rbyd.toml index 1887cfae..b2d61878 100644 --- a/benches/bench_rbyd.toml +++ b/benches/bench_rbyd.toml @@ -27,6 +27,8 @@ defines.SEED = 42 # 0x08 => lookup # 0x10 => usage defines.MASK = 0x1f +# not the most rigorous +litmus = true in = 'lfs3.c' code = ''' lfs3_t lfs3; @@ -138,6 +140,8 @@ defines.SEED = 42 # 0x08 => lookup # 0x10 => usage defines.MASK = 0x1f +# not the most rigorous +litmus = true in = 'lfs3.c' code = ''' lfs3_t lfs3; diff --git a/runners/bench_runner.c b/runners/bench_runner.c index de91668e..92c97b4c 100644 --- a/runners/bench_runner.c +++ b/runners/bench_runner.c @@ -1306,9 +1306,10 @@ static void summary(void) { char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); char flag_buf[64]; - sprintf(flag_buf, "%s%s", - (flags & BENCH_INTERNAL) ? "i" : "", - (!flags) ? "-" : ""); + sprintf(flag_buf, "%s%s%s", + (flags & BENCH_INTERNAL) ? "i" : "", + (flags & BENCH_LITMUS) ? "l" : "", + (!flags) ? "-" : ""); printf("%-23s %7s %7zu %7zu %15s\n", "TOTAL", flag_buf, @@ -1363,10 +1364,12 @@ static void list_suites(void) { char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); + bench_flags_t flags = bench_suites[i]->flags; char flag_buf[64]; - sprintf(flag_buf, "%s%s", - (bench_suites[i]->flags & BENCH_INTERNAL) ? "i" : "", - (!bench_suites[i]->flags) ? "-" : ""); + sprintf(flag_buf, "%s%s%s", + (flags & BENCH_INTERNAL) ? "i" : "", + (flags & BENCH_LITMUS) ? "l" : "", + (!flags) ? "-" : ""); printf("%-*s %7s %7zu %15s\n", name_width, bench_suites[i]->name, @@ -1415,12 +1418,12 @@ static void list_cases(void) { char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); + bench_flags_t flags = bench_suites[i]->cases[j].flags; char flag_buf[64]; - sprintf(flag_buf, "%s%s", - (bench_suites[i]->cases[j].flags & BENCH_INTERNAL) - ? "i" : "", - (!bench_suites[i]->cases[j].flags) - ? "-" : ""); + sprintf(flag_buf, "%s%s%s", + (flags & BENCH_INTERNAL) ? "i" : "", + (flags & BENCH_LITMUS) ? "l" : "", + (!flags) ? "-" : ""); printf("%-*s %7s %15s\n", name_width, bench_suites[i]->cases[j].name, @@ -1948,14 +1951,15 @@ enum opt_flags { OPT_STEP = 's', OPT_FORCE = 7, OPT_NO_INTERNAL = 8, + OPT_NO_LITMUS = 9, OPT_DISK = 'd', OPT_TRACE = 't', - OPT_TRACE_BACKTRACE = 9, - OPT_TRACE_PERIOD = 10, - OPT_TRACE_FREQ = 11, - OPT_READ_SLEEP = 12, - OPT_PROG_SLEEP = 13, - OPT_ERASE_SLEEP = 14, + OPT_TRACE_BACKTRACE = 10, + OPT_TRACE_PERIOD = 11, + OPT_TRACE_FREQ = 12, + OPT_READ_SLEEP = 13, + OPT_PROG_SLEEP = 14, + OPT_ERASE_SLEEP = 15, }; const char *short_opts = "hYlLD:s:d:t:"; @@ -1977,6 +1981,7 @@ const struct option long_opts[] = { {"step", required_argument, NULL, OPT_STEP}, {"force", no_argument, NULL, OPT_FORCE}, {"no-internal", no_argument, NULL, OPT_NO_INTERNAL}, + {"no-litmus", no_argument, NULL, OPT_NO_LITMUS}, {"disk", required_argument, NULL, OPT_DISK}, {"trace", required_argument, NULL, OPT_TRACE}, {"trace-backtrace", no_argument, NULL, OPT_TRACE_BACKTRACE}, @@ -2003,6 +2008,7 @@ const char *const help_text[] = { "Comma-separated range of permutations to run.", "Ignore bench filters.", "Don't run internal benches.", + "Don't run litmus benches.", "Direct block device operations to this file.", "Direct trace output to this file.", "Include a backtrace with every trace statement.", @@ -2333,6 +2339,10 @@ int main(int argc, char **argv) { bench_mask |= BENCH_INTERNAL; break; + case OPT_NO_LITMUS:; + bench_mask |= BENCH_LITMUS; + break; + case OPT_DISK:; bench_disk_path = optarg; break; diff --git a/runners/bench_runner.h b/runners/bench_runner.h index 39be99ff..311c791d 100644 --- a/runners/bench_runner.h +++ b/runners/bench_runner.h @@ -83,6 +83,7 @@ struct lfs3_cfg; enum bench_flags { BENCH_INTERNAL = 0x1, + BENCH_LITMUS = 0x2, }; typedef uint8_t bench_flags_t; diff --git a/runners/test_runner.c b/runners/test_runner.c index 65400700..1d1968c1 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -1206,12 +1206,13 @@ static void list_suites(void) { char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); + test_flags_t flags = test_suites[i]->flags; char flag_buf[64]; sprintf(flag_buf, "%s%s%s%s", - (test_suites[i]->flags & TEST_INTERNAL) ? "i" : "", - (test_suites[i]->flags & TEST_REENTRANT) ? "r" : "", - (test_suites[i]->flags & TEST_FUZZ) ? "f" : "", - (!test_suites[i]->flags) ? "-" : ""); + (flags & TEST_INTERNAL) ? "i" : "", + (flags & TEST_REENTRANT) ? "r" : "", + (flags & TEST_FUZZ) ? "f" : "", + (!flags) ? "-" : ""); printf("%-*s %7s %7zu %15s\n", name_width, test_suites[i]->name, @@ -1260,16 +1261,13 @@ static void list_cases(void) { char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); + test_flags_t flags = test_suites[i]->cases[j].flags; char flag_buf[64]; sprintf(flag_buf, "%s%s%s%s", - (test_suites[i]->cases[j].flags & TEST_INTERNAL) - ? "i" : "", - (test_suites[i]->cases[j].flags & TEST_REENTRANT) - ? "r" : "", - (test_suites[i]->cases[j].flags & TEST_FUZZ) - ? "f" : "", - (!test_suites[i]->cases[j].flags) - ? "-" : ""); + (flags & TEST_INTERNAL) ? "i" : "", + (flags & TEST_REENTRANT) ? "r" : "", + (flags & TEST_FUZZ) ? "f" : "", + (!flags) ? "-" : ""); printf("%-*s %7s %15s\n", name_width, test_suites[i]->cases[j].name, diff --git a/scripts/bench.py b/scripts/bench.py index 9461612d..d1ce9424 100755 --- a/scripts/bench.py +++ b/scripts/bench.py @@ -106,6 +106,10 @@ class BenchCase: config.pop('suite_internal', None)) if self.internal is None: self.internal = False + self.litmus = config.pop('litmus', + config.pop('suite_litmus', None)) + if self.litmus is None: + self.litmus = False # in implies internal self.internal |= bool(self.in_) @@ -287,6 +291,7 @@ class BenchSuite: # a couple of these we just forward to all cases defines = config.pop('defines', None) internal = config.pop('internal', None) + litmus = config.pop('litmus', None) self.cases = [] for name, config_ in cases.items(): @@ -298,12 +303,15 @@ class BenchSuite: 'suite_defines': defines, 'suite_in': self.in_, 'suite_internal': internal, + 'suite_litmus': litmus, **config_}, args) # skipping internal benches? if args.get('no_internal') and case.internal: continue + if args.get('no_litmus') and case.litmus: + continue self.cases.append(case) @@ -316,6 +324,7 @@ class BenchSuite: # combine other per-case things self.internal = any(case.internal for case in self.cases) + self.litmus = any(case.litmus for case in self.cases) for k in config.keys(): print('%swarning:%s in %s, found unused key %r' % ( @@ -596,7 +605,8 @@ def compile(bench_paths, **args): f.writeln(4*' '+'.path = "%s",' % suite.path) f.writeln(4*' '+'.flags = %s,' % ( ' | '.join(filter(None, [ - 'BENCH_INTERNAL' if suite.internal else None])) + 'BENCH_INTERNAL' if suite.internal else None, + 'BENCH_LITMUS' if suite.litmus else None])) or 0)) for ifdef in suite.ifdef: f.writeln(4*' '+'#if (%s)' % re.sub( @@ -628,6 +638,8 @@ def compile(bench_paths, **args): f.writeln(12*' '+'.flags = %s,' % ( ' | '.join(filter(None, [ 'BENCH_INTERNAL' if case.internal + else None, + 'BENCH_LITMUS' if case.litmus else None])) or 0)) for ifdef in it.chain(suite.ifdef, case.ifdef): @@ -798,6 +810,8 @@ def find_runner(runner, id=None, main=True, **args): cmd.append('--force') if args.get('no_internal'): cmd.append('--no-internal') + if args.get('no_litmus'): + cmd.append('--no-litmus') # only one thread should write to disk/trace, otherwise the output # ends up clobbered and useless @@ -1714,6 +1728,10 @@ if __name__ == "__main__": '--no-internal', action='store_true', help="Don't run internal benches.") + bench_parser.add_argument( + '--no-litmus', + action='store_true', + help="Don't run litmus benches.") bench_parser.add_argument( '-d', '--disk', help="Direct block device operations to this file.") @@ -1876,6 +1894,10 @@ if __name__ == "__main__": '--no-internal', action='store_true', help="Don't build internal benches.") + comp_parser.add_argument( + '--no-litmus', + action='store_true', + help="Don't build litmus benches.") # do the thing args = parser.parse_intermixed_args()