From 56a990336b1d214ecc1ba51ec7e9f3e764714f42 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 16 Apr 2022 13:32:35 -0500 Subject: [PATCH 01/81] Created new test_runner.c and test_.py This is to try a different design for testing, the goals are to make the test infrastructure a bit simpler, with clear stages for building and running, and faster, by avoiding rebuilding lfs.c n-times. --- Makefile | 33 +++++- runners/test_runner.c | 178 ++++++++++++++++++++++++++++ runners/test_runner.h | 26 ++++ scripts/test_.py | 269 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 505 insertions(+), 1 deletion(-) create mode 100644 runners/test_runner.c create mode 100644 runners/test_runner.h create mode 100755 scripts/test_.py diff --git a/Makefile b/Makefile index 7cc59f8a..9d3b1444 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ override BUILDDIR := $(BUILDDIR)/ $(if $(findstring n,$(MAKEFLAGS)),, $(shell mkdir -p \ $(BUILDDIR) \ $(BUILDDIR)bd \ + $(BUILDDIR)runners \ $(BUILDDIR)tests)) endif @@ -25,12 +26,19 @@ NM ?= nm OBJDUMP ?= objdump LCOV ?= lcov -SRC ?= $(wildcard *.c) +SRC ?= $(filter-out $(wildcard *.*.c),$(wildcard *.c)) OBJ := $(SRC:%.c=$(BUILDDIR)%.o) DEP := $(SRC:%.c=$(BUILDDIR)%.d) ASM := $(SRC:%.c=$(BUILDDIR)%.s) CGI := $(SRC:%.c=$(BUILDDIR)%.ci) +TESTS ?= $(wildcard tests_/*.toml) +TEST_TSRC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) \ + $(SRC:%.c=$(BUILDDIR)%.t.c) \ + $(BUILDDIR)runners/test_runner.t.c +TEST_TASRC := $(TEST_TSRC:%.t.c=%.t.a.c) +TEST_TAOBJ := $(TEST_TASRC:%.t.a.c=%.t.a.o) + ifdef DEBUG override CFLAGS += -O0 else @@ -103,6 +111,9 @@ test: test%: tests/test$$(firstword $$(subst \#, ,%)).toml ./scripts/test.py $@ $(TESTFLAGS) +.PHONY: test_ +test_: $(BUILDDIR)runners/test_runner + .PHONY: code code: $(OBJ) ./scripts/code.py $^ -S $(CODEFLAGS) @@ -131,6 +142,7 @@ summary: $(BUILDDIR)lfs.csv # rules -include $(DEP) .SUFFIXES: +.SECONDARY: $(BUILDDIR)lfs: $(OBJ) $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ @@ -147,6 +159,9 @@ $(BUILDDIR)lfs.csv: $(OBJ) $(CGI) ./scripts/coverage.py $(BUILDDIR)tests/*.toml.info \ -q -m $@ $(COVERAGEFLAGS) -o $@) +$(BUILDDIR)runners/test_runner: $(TEST_TAOBJ) + $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ + $(BUILDDIR)%.o: %.c $(CC) -c -MMD $(CFLAGS) $< -o $@ @@ -160,14 +175,30 @@ $(BUILDDIR)%.s: %.c $(BUILDDIR)%.ci: %.c | $(BUILDDIR)%.o $(CC) -c -MMD -fcallgraph-info=su $(CFLAGS) $< -o $| +$(BUILDDIR)%.a.c: %.c + ./scripts/explode_asserts.py $< -o $@ + +$(BUILDDIR)%.a.c: $(BUILDDIR)%.c + ./scripts/explode_asserts.py $< -o $@ + +$(BUILDDIR)%.t.c: %.toml + ./scripts/test_.py -c $< -o $@ + +$(BUILDDIR)%.t.c: %.c $(TESTS) + ./scripts/test_.py -c $(TESTS) -s $< -o $@ + # clean everything .PHONY: clean clean: rm -f $(BUILDDIR)lfs rm -f $(BUILDDIR)lfs.a rm -f $(BUILDDIR)lfs.csv + rm -f $(BUILDDIR)runners/test_runner rm -f $(OBJ) rm -f $(CGI) rm -f $(DEP) rm -f $(ASM) rm -f $(BUILDDIR)tests/*.toml.* + rm -f $(TEST_TSRC) + rm -f $(TEST_TASRC) + rm -f $(TEST_TAOBJ) diff --git a/runners/test_runner.c b/runners/test_runner.c new file mode 100644 index 00000000..758e1fe3 --- /dev/null +++ b/runners/test_runner.c @@ -0,0 +1,178 @@ + +#include "runners/test_runner.h" +#include + + +// disk geometries +struct test_geometry { + const char *name; + lfs_size_t read_size; + lfs_size_t prog_size; + lfs_size_t erase_size; + lfs_size_t erase_count; +}; + +const struct test_geometry test_geometries[] = { + // Made up geometries that works well for testing + {"small", 16, 16, 512, (1024*1024)/512}, + {"medium", 16, 16, 4096, (1024*1024)/4096}, + {"big", 16, 16, 32*1024, (1024*1024)/(32*1024)}, + // EEPROM/NVRAM + {"eeprom", 1, 1, 512, (1024*1024)/512}, + // SD/eMMC + {"emmc", 512, 512, 512, (1024*1024)/512}, + // NOR flash + {"nor", 1, 1, 4096, (1024*1024)/4096}, + // NAND flash + {"nand", 4096, 4096, 32*1024, (1024*1024)/(32*1024)}, +}; +const size_t test_geometry_count = ( + sizeof(test_geometries) / sizeof(test_geometries[0])); + + +// option handling +enum opt_flags { + OPT_HELP = 'h', + OPT_LIST = 'l', + OPT_LIST_PATHS = 1, + OPT_LIST_DEFINES = 2, + OPT_LIST_GEOMETRIES = 3, +}; + +const struct option long_opts[] = { + {"help", no_argument, NULL, OPT_HELP}, + {"list", no_argument, NULL, OPT_LIST}, + {"list-paths", no_argument, NULL, OPT_LIST_PATHS}, + {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, + {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, + {NULL, 0, NULL, 0}, +}; + +const char *const help_text[] = { + "Show this help message.", + "List test cases.", + "List the path for each test case.", + "List the defines for each test permutation.", + "List the disk geometries used for testing.", +}; + +int main(int argc, char **argv) { + bool list = false; + bool list_paths = false; + bool list_defines = false; + bool list_geometries = false; + + // parse options + while (true) { + int index = 0; + int c = getopt_long(argc, argv, "hl", long_opts, &index); + switch (c) { + // generate help message + case OPT_HELP: { + printf("usage: %s [options] [test_case]\n", argv[0]); + printf("\n"); + + printf("options:\n"); + size_t i = 0; + while (long_opts[i].name) { + size_t indent; + if (long_opts[i].val >= '0' && long_opts[i].val < 'z') { + printf(" -%c, --%-16s", + long_opts[i].val, + long_opts[i].name); + indent = 8+strlen(long_opts[i].name); + } else { + printf(" --%-20s", long_opts[i].name); + indent = 4+strlen(long_opts[i].name); + } + + // a quick, hacky, byte-level method for text wrapping + size_t len = strlen(help_text[i]); + size_t j = 0; + if (indent < 24) { + printf("%.80s\n", &help_text[i][j]); + j += 80; + } + + while (j < len) { + printf("%24s%.80s\n", "", &help_text[i][j]); + j += 80; + } + + i += 1; + } + + printf("\n"); + exit(0); + } + // list flags + case OPT_LIST: + list = true; + break; + case OPT_LIST_PATHS: + list_paths = true; + break; + case OPT_LIST_DEFINES: + list_defines = true; + break; + case OPT_LIST_GEOMETRIES: + list_geometries = true; + break; + // done parsing + case -1: + goto getopt_done; + // unknown arg, getopt prints a message for us + default: + exit(-1); + } + } +getopt_done: + + // what do we need to do? + if (list) { + printf("%-36s %-12s %-12s %7s %7s\n", + "id", "suite", "case", "type", "perms"); + for (size_t i = 0; i < test_suite_count; i++) { + for (size_t j = 0; j < test_suites[i]->case_count; j++) { + printf("%-36s %-12s %-12s %7s %7d\n", + test_suites[i]->cases[j]->id, + test_suites[i]->name, + test_suites[i]->cases[j]->name, + "n", // TODO + test_suites[i]->cases[j]->permutations); + } + } + + } else if (list_paths) { + printf("%-36s %-36s\n", "id", "path"); + for (size_t i = 0; i < test_suite_count; i++) { + for (size_t j = 0; j < test_suites[i]->case_count; j++) { + printf("%-36s %-36s\n", + test_suites[i]->cases[j]->id, + test_suites[i]->cases[j]->path); + } + } + } else if (list_defines) { + // TODO + } else if (list_geometries) { + printf("%-12s %7s %7s %7s %7s %7s\n", + "name", "read", "prog", "erase", "count", "size"); + for (size_t i = 0; i < test_geometry_count; i++) { + printf("%-12s %7d %7d %7d %7d %7d\n", + test_geometries[i].name, + test_geometries[i].read_size, + test_geometries[i].prog_size, + test_geometries[i].erase_size, + test_geometries[i].erase_count, + test_geometries[i].erase_size + * test_geometries[i].erase_count); + } + } else { + printf("remaining: "); + for (int i = optind; i < argc; i++) { + printf("%s ", argv[i]); + } + printf("\n"); + } +} + diff --git a/runners/test_runner.h b/runners/test_runner.h new file mode 100644 index 00000000..c830e36b --- /dev/null +++ b/runners/test_runner.h @@ -0,0 +1,26 @@ +#ifndef TEST_RUNNER_H +#define TEST_RUNNER_H + +#include "lfs.h" + + +struct test_case { + const char *id; + const char *name; + const char *path; + uint32_t permutations; + void (*run)(struct lfs_config *cfg, uint32_t perm); +}; + +struct test_suite { + const char *id; + const char *name; + const char *path; + const struct test_case *const *cases; + size_t case_count; +}; + +extern const struct test_suite *test_suites[]; +extern const size_t test_suite_count; + +#endif diff --git a/scripts/test_.py b/scripts/test_.py new file mode 100755 index 00000000..c411d0ca --- /dev/null +++ b/scripts/test_.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +# +# Script to compile and runs tests. +# + +import glob +import itertools as it +import os +import re +import shutil +import toml + +TEST_PATHS = ['tests_'] + +SUITE_PROLOGUE = """ +//////// AUTOGENERATED //////// +#include "runners/test_runner.h" +#include +""" +# TODO handle indention implicity? +# TODO change cfg to be not by value? maybe not? +CASE_PROLOGUE = """ + lfs_t lfs; + struct lfs_config cfg = *cfg_; +""" +CASE_EPILOGUE = """ +""" + + +# TODO +# def testpath(path): +# def testcase(path): +# def testperm(path): + +def testsuite(path): + name = os.path.basename(path) + if name.endswith('.toml'): + name = name[:-len('.toml')] + return name + +# TODO move this out in other files +def openio(path, mode='r'): + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) + +class TestCase: + # create a TestCase object from a config + def __init__(self, config, args={}): + self.name = config.pop('name') + self.path = config.pop('path') + self.suite = config.pop('suite') + self.lineno = config.pop('lineno', None) + self.code = config.pop('code') + self.code_lineno = config.pop('code_lineno', None) + + self.permutations = 1 + + for k in config.keys(): + print('warning: in %s, found unused key %r' % (self.id(), k), + file=sys.stderr) + + def id(self): + return '%s#%s' % (self.suite, self.name) + + +class TestSuite: + # create a TestSuite object from a toml file + def __init__(self, path, args={}): + self.name = testsuite(path) + self.path = path + + # load toml file and parse test cases + with open(self.path) as f: + # load tests + config = toml.load(f) + + # find line numbers + f.seek(0) + case_linenos = [] + code_linenos = [] + for i, line in enumerate(f): + match = re.match( + '(?P\[\s*cases\s*\.\s*(?P\w+)\s*\])' + + '|(?Pcode\s*=\s*(?:\'\'\'|"""))', + line) + if match and match.group('case'): + case_linenos.append((i+1, match.group('name'))) + elif match and match.group('code'): + code_linenos.append(i+2) + + # sort in case toml parsing did not retain order + case_linenos.sort() + + cases = config.pop('cases', []) + for (lineno, name), (nlineno, _) in it.zip_longest( + case_linenos, case_linenos[1:], + fillvalue=(float('inf'), None)): + code_lineno = min( + (l for l in code_linenos if l >= lineno and l < nlineno), + default=None) + cases[name]['lineno'] = lineno + cases[name]['code_lineno'] = code_lineno + + self.code = config.pop('code', None) + self.code_lineno = min( + (l for l in code_linenos + if not case_linenos or l < case_linenos[0][0]), + default=None) + + self.cases = [] + for name, case in sorted(cases.items(), + key=lambda c: c[1].get('lineno')): + self.cases.append(TestCase(config={ + 'name': name, + 'path': path + (':%d' % case['lineno'] + if 'lineno' in case else ''), + 'suite': self.name, + **case})) + + for k in config.keys(): + print('warning: in %s, found unused key %r' % (self.id(), k), + file=sys.stderr) + + def id(self): + return self.name + + + +def compile(**args): + # find .toml files + paths = [] + for path in args['test_paths']: + if os.path.isdir(path): + path = path + '/*.toml' + + for path in glob.glob(path): + paths.append(path) + + if not paths: + print('no test suites found in %r?' % args['test_paths']) + sys.exit(-1) + + if not args.get('source'): + if len(paths) > 1: + print('more than one test suite for compilation? (%r)' + % args['test_paths']) + sys.exit(-1) + + # write out a test suite + suite = TestSuite(paths[0]) + if 'output' in args: + with openio(args['output'], 'w') as f: + f.write(SUITE_PROLOGUE) + f.write('\n') + if suite.code is not None: + if suite.code_lineno is not None: + f.write('#line %d "%s"\n' + % (suite.code_lineno, suite.path)) + f.write(suite.code) + f.write('\n') + + # create test functions and case structs + for case in suite.cases: + f.write('void __test__%s__%s(' + '__attribute__((unused)) struct lfs_config *cfg_, ' + '__attribute__((unused)) uint32_t perm) {\n' + % (suite.name, case.name)) + f.write(CASE_PROLOGUE) + f.write('\n') + f.write(4*' '+'// test case %s\n' % case.id()) + if case.code_lineno is not None: + f.write(4*' '+'#line %d "%s"\n' + % (case.code_lineno, suite.path)) + f.write(case.code) + f.write('\n') + f.write(CASE_EPILOGUE) + f.write('}\n') + f.write('\n') + + f.write('const struct test_case __test__%s__%s__case = {\n' + % (suite.name, case.name)) + f.write(4*' '+'.id = "%s",\n' % case.id()) + f.write(4*' '+'.name = "%s",\n' % case.name) + f.write(4*' '+'.path = "%s",\n' % case.path) + f.write(4*' '+'.permutations = %d,\n' % case.permutations) + f.write(4*' '+'.run = __test__%s__%s,\n' + % (suite.name, case.name)) + f.write('};\n') + f.write('\n') + + # create suite struct + f.write('const struct test_suite __test__%s__suite = {\n' + % (suite.name)) + f.write(4*' '+'.id = "%s",\n' % suite.id()) + f.write(4*' '+'.name = "%s",\n' % suite.name) + f.write(4*' '+'.path = "%s",\n' % suite.path) + f.write(4*' '+'.cases = (const struct test_case *const []){\n') + for case in suite.cases: + f.write(8*' '+'&__test__%s__%s__case,\n' + % (suite.name, case.name)) + f.write(4*' '+'},\n') + f.write(4*' '+'.case_count = %d,\n' % len(suite.cases)) + f.write('};\n') + f.write('\n') + + else: + # load all suites + suites = [TestSuite(path) for path in paths] + suites.sort(key=lambda s: s.name) + + # write out a test source + if 'output' in args: + with openio(args['output'], 'w') as f: + f.write(SUITE_PROLOGUE) + f.write('\n') + f.write('#line 1 "%s"\n' % args['source']) + with open(args['source']) as sf: + shutil.copyfileobj(sf, f) + + # add suite info to test_runner.c + if args['source'] == 'runners/test_runner.c': + f.write('\n') + for suite in suites: + f.write('extern const struct test_suite ' + '__test__%s__suite;\n' % suite.name) + f.write('const struct test_suite *test_suites[] = {\n') + for suite in suites: + f.write(4*' '+'&__test__%s__suite,\n' % suite.name) + f.write('};\n') + f.write('const size_t test_suite_count = %d;\n' + % len(suites)) + +def run(**args): + pass + +def main(**args): + if args.get('compile'): + compile(**args) + else: + run(**args) + +if __name__ == "__main__": + import argparse + import sys + parser = argparse.ArgumentParser( + description="Build and run tests.") + # TODO document test case/perm specifier + parser.add_argument('test_paths', nargs='*', default=TEST_PATHS, + help="Description of test(s) to run. May be a directory, a path, or \ + test identifier. Defaults to all tests in %r." % TEST_PATHS) + # test flags + test_parser = parser.add_argument_group('test options') + # compilation flags + comp_parser = parser.add_argument_group('compilation options') + comp_parser.add_argument('-c', '--compile', action='store_true', + help="Compile a test suite or source file.") + comp_parser.add_argument('-s', '--source', + help="Source file to compile, possibly injecting internal tests.") + comp_parser.add_argument('-o', '--output', + help="Output file.") + # TODO apply this to other scripts? + sys.exit(main(**{k: v + for k, v in vars(parser.parse_args()).items() + if v is not None})) From d683f1c76c04db3541e4a6a86344ff4059de154e Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 17 Apr 2022 21:41:38 -0500 Subject: [PATCH 02/81] Reintroduced test-defines into the new test_runner This moves defines entirely into the runtime of the test_runner, simplifying thing and reducing the amount of generated code that needs to be build, at the cost of limiting test-defines to uintmax_t types. This is implemented using a set of index-based scopes (created by test.py) that allow different layers to override defines from other layers, accessible through the global `test_define` function. layers: 1. command-line overrides 2. per-case defines 3. per-geometry defines --- .gitignore | 2 + Makefile | 10 +- runners/test_runner.c | 456 ++++++++++++++++++++++++++++++++++-------- runners/test_runner.h | 37 +++- scripts/test_.py | 136 +++++++++++-- 5 files changed, 542 insertions(+), 99 deletions(-) diff --git a/.gitignore b/.gitignore index 3f7b860e..6bee32df 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ *.a *.ci *.csv +*.t.c +*.a.c # Testing things blocks/ diff --git a/Makefile b/Makefile index 9d3b1444..943735bc 100644 --- a/Makefile +++ b/Makefile @@ -33,11 +33,13 @@ ASM := $(SRC:%.c=$(BUILDDIR)%.s) CGI := $(SRC:%.c=$(BUILDDIR)%.ci) TESTS ?= $(wildcard tests_/*.toml) -TEST_TSRC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) \ - $(SRC:%.c=$(BUILDDIR)%.t.c) \ - $(BUILDDIR)runners/test_runner.t.c +TEST_SRC ?= $(SRC) \ + $(filter-out $(wildcard bd/*.*.c),$(wildcard bd/*.c)) \ + runners/test_runner.c +TEST_TSRC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) $(TEST_SRC:%.c=$(BUILDDIR)%.t.c) TEST_TASRC := $(TEST_TSRC:%.t.c=%.t.a.c) TEST_TAOBJ := $(TEST_TASRC:%.t.a.c=%.t.a.o) +TEST_TADEP := $(TEST_TASRC:%.t.a.c=%.t.a.d) ifdef DEBUG override CFLAGS += -O0 @@ -141,6 +143,7 @@ summary: $(BUILDDIR)lfs.csv # rules -include $(DEP) +-include $(TEST_TADEP) .SUFFIXES: .SECONDARY: @@ -202,3 +205,4 @@ clean: rm -f $(TEST_TSRC) rm -f $(TEST_TASRC) rm -f $(TEST_TAOBJ) + rm -f $(TEST_TADEP) diff --git a/runners/test_runner.c b/runners/test_runner.c index 758e1fe3..7ab433f7 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -1,71 +1,363 @@ #include "runners/test_runner.h" -#include +#include "bd/lfs_testbd.h" +#include +#include // disk geometries struct test_geometry { const char *name; - lfs_size_t read_size; - lfs_size_t prog_size; - lfs_size_t erase_size; - lfs_size_t erase_count; + const uintmax_t *defines; }; +// Note this includes the default configuration for test pre-defines +#define TEST_GEOMETRY(name, read, prog, erase, count) \ + {name, (const uintmax_t[]){ \ + /* READ_SIZE */ read, \ + /* PROG_SIZE */ prog, \ + /* BLOCK_SIZE */ erase, \ + /* BLOCK_COUNT */ count, \ + /* BLOCK_CYCLES */ -1, \ + /* CACHE_SIZE */ (64 % (prog) == 0) ? 64 : (prog), \ + /* LOOKAHEAD_SIZE */ 16, \ + /* ERASE_VALUE */ 0xff, \ + /* ERASE_CYCLES */ 0, \ + /* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \ + }} + const struct test_geometry test_geometries[] = { - // Made up geometries that works well for testing - {"small", 16, 16, 512, (1024*1024)/512}, - {"medium", 16, 16, 4096, (1024*1024)/4096}, - {"big", 16, 16, 32*1024, (1024*1024)/(32*1024)}, + // Made up geometry that works well for testing + TEST_GEOMETRY("small", 16, 16, 512, (1024*1024)/512), + TEST_GEOMETRY("medium", 16, 16, 4096, (1024*1024)/4096), + TEST_GEOMETRY("big", 16, 16, 32*1024, (1024*1024)/(32*1024)), // EEPROM/NVRAM - {"eeprom", 1, 1, 512, (1024*1024)/512}, + TEST_GEOMETRY("eeprom", 1, 1, 512, (1024*1024)/512), // SD/eMMC - {"emmc", 512, 512, 512, (1024*1024)/512}, - // NOR flash - {"nor", 1, 1, 4096, (1024*1024)/4096}, - // NAND flash - {"nand", 4096, 4096, 32*1024, (1024*1024)/(32*1024)}, + TEST_GEOMETRY("emmc", 512, 512, 512, (1024*1024)/512), + // NOR flash + TEST_GEOMETRY("nor", 1, 1, 4096, (1024*1024)/4096), + // NAND flash + TEST_GEOMETRY("nand", 4096, 4096, 32*1024, (1024*1024)/(32*1024)), }; + const size_t test_geometry_count = ( sizeof(test_geometries) / sizeof(test_geometries[0])); +// test define lookup and management +const uintmax_t *test_defines[3] = {NULL}; +const bool *test_define_masks[2] = {NULL}; + +uintmax_t test_define(size_t define) { + if (test_define_masks[0] && test_define_masks[0][define]) { + return test_defines[0][define]; + } else if (test_define_masks[1] && test_define_masks[1][define]) { + return test_defines[1][define]; + } else { + return test_defines[2][define]; + } +} + +void test_define_geometry(const struct test_geometry *geometry) { + if (geometry) { + test_defines[2] = geometry->defines; + } else { + test_defines[2] = NULL; + } +} + +void test_define_case(const struct test_case *case_, size_t perm) { + if (case_ && case_->defines) { + test_defines[1] = case_->defines[perm]; + test_define_masks[1] = case_->define_mask; + } else { + test_defines[1] = NULL; + test_define_masks[1] = NULL; + } +} + +struct override { + const char *name; + uintmax_t override; +}; + +void test_define_overrides( + const struct test_suite *suite, + const struct override *overrides, + size_t override_count) { + if (overrides && override_count > 0) { + uintmax_t *defines = malloc(suite->define_count * sizeof(uintmax_t)); + memset(defines, 0, suite->define_count * sizeof(uintmax_t)); + bool *define_mask = malloc(suite->define_count * sizeof(bool)); + memset(define_mask, 0, suite->define_count * sizeof(bool)); + + // lookup each override in the suite defines, they may have a + // different index in each suite + for (size_t i = 0; i < override_count; i++) { + ssize_t index = -1; + for (size_t j = 0; j < suite->define_count; j++) { + if (strcmp(overrides[i].name, suite->define_names[j]) == 0) { + index = j; + break; + } + } + + if (index >= 0) { + defines[index] = overrides[i].override; + define_mask[index] = true; + } + } + + test_defines[0] = defines; + test_define_masks[0] = define_mask; + } else { + free((uintmax_t *)test_defines[0]); + test_defines[0] = NULL; + free((bool *)test_define_masks[0]); + test_define_masks[0] = NULL; + } +} + + +// operations we can do +void summary( + struct override *overrides, + size_t override_count) { + (void)overrides; + (void)override_count; + printf("%-36s %7s %7s %7s %7s\n", + "", "geoms", "suites", "cases", "perms"); + size_t cases = 0; + size_t perms = 0; + for (size_t i = 0; i < test_suite_count; i++) { + cases += test_suites[i]->case_count; + + for (size_t j = 0; j < test_suites[i]->case_count; j++) { + perms += test_suites[i]->cases[j]->permutations; + } + } + + printf("%-36s %7zu %7zu %7zu %7zu\n", + "TOTAL", + test_geometry_count, + test_suite_count, + cases, + test_geometry_count*perms); +} + +void list_suites( + struct override *overrides, + size_t override_count) { + (void)overrides; + (void)override_count; + printf("%-36s %-12s %7s %7s %7s\n", + "id", "suite", "types", "cases", "perms"); + for (size_t i = 0; i < test_suite_count; i++) { + size_t perms = 0; + for (size_t j = 0; j < test_suites[i]->case_count; j++) { + perms += test_suites[i]->cases[j]->permutations; + } + + printf("%-36s %-12s %7s %7zu %7zu\n", + test_suites[i]->id, + test_suites[i]->name, + "n", // TODO + test_suites[i]->case_count, + test_geometry_count*perms); + } +} + +void list_cases( + struct override *overrides, + size_t override_count) { + (void)overrides; + (void)override_count; + printf("%-36s %-12s %-12s %7s %7s\n", + "id", "suite", "case", "types", "perms"); + for (size_t i = 0; i < test_suite_count; i++) { + for (size_t j = 0; j < test_suites[i]->case_count; j++) { + printf("%-36s %-12s %-12s %7s %7zu\n", + test_suites[i]->cases[j]->id, + test_suites[i]->name, + test_suites[i]->cases[j]->name, + "n", // TODO + test_geometry_count + * test_suites[i]->cases[j]->permutations); + } + } +} + +void list_paths( + struct override *overrides, + size_t override_count) { + (void)overrides; + (void)override_count; + printf("%-36s %-36s\n", "id", "path"); + for (size_t i = 0; i < test_suite_count; i++) { + for (size_t j = 0; j < test_suites[i]->case_count; j++) { + printf("%-36s %-36s\n", + test_suites[i]->cases[j]->id, + test_suites[i]->cases[j]->path); + } + } +} + +void list_defines( + struct override *overrides, + size_t override_count) { + (void)overrides; + (void)override_count; + // TODO +} + +void list_geometries( + struct override *overrides, + size_t override_count) { + (void)overrides; + (void)override_count; + printf("%-36s %7s %7s %7s %7s %7s\n", + "name", "read", "prog", "erase", "count", "size"); + for (size_t i = 0; i < test_geometry_count; i++) { + test_define_geometry(&test_geometries[i]); + + printf("%-36s %7ju %7ju %7ju %7ju %7ju\n", + test_geometries[i].name, + READ_SIZE, + PROG_SIZE, + BLOCK_SIZE, + BLOCK_COUNT, + BLOCK_SIZE*BLOCK_COUNT); + } +} + +void run( + struct override *overrides, + size_t override_count) { + for (size_t i = 0; i < test_suite_count; i++) { + test_define_overrides(test_suites[i], overrides, override_count); + + for (size_t j = 0; j < test_suites[i]->case_count; j++) { + for (size_t perm = 0; + perm < test_geometry_count + * test_suites[i]->cases[j]->permutations; + perm++) { + size_t case_perm = perm / test_geometry_count; + size_t geom_perm = perm % test_geometry_count; + + // setup defines + test_define_geometry(&test_geometries[geom_perm]); + test_define_case(test_suites[i]->cases[j], case_perm); + + // create block device and configuration + lfs_testbd_t bd; + + struct lfs_config cfg = { + .context = &bd, + .read = lfs_testbd_read, + .prog = lfs_testbd_prog, + .erase = lfs_testbd_erase, + .sync = lfs_testbd_sync, + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .block_size = BLOCK_SIZE, + .block_count = BLOCK_COUNT, + .block_cycles = BLOCK_CYCLES, + .cache_size = CACHE_SIZE, + .lookahead_size = LOOKAHEAD_SIZE, + }; + + struct lfs_testbd_config bdcfg = { + .erase_value = ERASE_VALUE, + .erase_cycles = ERASE_CYCLES, + .badblock_behavior = BADBLOCK_BEHAVIOR, + .power_cycles = 0, + }; + + lfs_testbd_createcfg(&cfg, NULL, &bdcfg) => 0; + + // filter? + if (test_suites[i]->cases[j]->filter) { + bool filter = test_suites[i]->cases[j]->filter( + &cfg, case_perm); + if (!filter) { + printf("skipped %s#%zu\n", + test_suites[i]->cases[j]->id, + perm); + continue; + } + } + + // run the test + printf("running %s#%zu\n", test_suites[i]->cases[j]->id, perm); + + test_suites[i]->cases[j]->run(&cfg, case_perm); + + printf("finished %s#%zu\n", test_suites[i]->cases[j]->id, perm); + + // cleanup + lfs_testbd_destroy(&cfg) => 0; + + test_define_geometry(NULL); + test_define_case(NULL, 0); + } + } + + test_define_overrides(NULL, NULL, 0); + } +} + + + + // option handling enum opt_flags { OPT_HELP = 'h', - OPT_LIST = 'l', - OPT_LIST_PATHS = 1, - OPT_LIST_DEFINES = 2, - OPT_LIST_GEOMETRIES = 3, + OPT_SUMMARY = 'Y', + OPT_LIST_SUITES = 1, + OPT_LIST_CASES = 'l', + OPT_LIST_PATHS = 2, + OPT_LIST_DEFINES = 3, + OPT_LIST_GEOMETRIES = 4, + OPT_DEFINE = 'D', }; +const char *short_opts = "hYlD:"; + const struct option long_opts[] = { - {"help", no_argument, NULL, OPT_HELP}, - {"list", no_argument, NULL, OPT_LIST}, - {"list-paths", no_argument, NULL, OPT_LIST_PATHS}, - {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, - {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, + {"help", no_argument, NULL, OPT_HELP}, + {"summary", no_argument, NULL, OPT_SUMMARY}, + {"list-suites", no_argument, NULL, OPT_LIST_SUITES}, + {"list-cases", no_argument, NULL, OPT_LIST_CASES}, + {"list-paths", no_argument, NULL, OPT_LIST_PATHS}, + {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, + {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, + {"define", required_argument, NULL, OPT_DEFINE}, {NULL, 0, NULL, 0}, }; const char *const help_text[] = { "Show this help message.", + "Show quick summary.", + "List test suites.", "List test cases.", "List the path for each test case.", "List the defines for each test permutation.", "List the disk geometries used for testing.", + "Override a test define.", }; int main(int argc, char **argv) { - bool list = false; - bool list_paths = false; - bool list_defines = false; - bool list_geometries = false; + void (*op)( + struct override *overrides, + size_t override_count) = run; + struct override *overrides = NULL; + size_t override_count = 0; + size_t override_cap = 0; // parse options while (true) { - int index = 0; - int c = getopt_long(argc, argv, "hl", long_opts, &index); + int c = getopt_long(argc, argv, short_opts, long_opts, NULL); switch (c) { // generate help message case OPT_HELP: { @@ -105,19 +397,56 @@ int main(int argc, char **argv) { printf("\n"); exit(0); } - // list flags - case OPT_LIST: - list = true; + // summary/list flags + case OPT_SUMMARY: + op = summary; + break; + case OPT_LIST_SUITES: + op = list_suites; + break; + case OPT_LIST_CASES: + op = list_cases; break; case OPT_LIST_PATHS: - list_paths = true; + op = list_paths; break; case OPT_LIST_DEFINES: - list_defines = true; + op = list_defines; break; case OPT_LIST_GEOMETRIES: - list_geometries = true; + op = list_geometries; break; + // configuration + case OPT_DEFINE: { + // realloc if necessary + override_count += 1; + if (override_count > override_cap) { + override_cap = (2*override_cap > 4) ? 2*override_cap : 4; + overrides = realloc(overrides, override_cap + * sizeof(struct override)); + } + + // parse into string key/uintmax_t value, cannibalizing the + // arg in the process + char *sep = strchr(optarg, '='); + char *parsed = NULL; + if (!sep) { + goto invalid_define; + } + overrides[override_count-1].override + = strtoumax(sep+1, &parsed, 0); + if (parsed == sep+1) { + goto invalid_define; + } + + overrides[override_count-1].name = optarg; + *sep = '\0'; + break; + +invalid_define: + fprintf(stderr, "error: invalid define: %s\n", optarg); + exit(-1); + } // done parsing case -1: goto getopt_done; @@ -128,51 +457,16 @@ int main(int argc, char **argv) { } getopt_done: - // what do we need to do? - if (list) { - printf("%-36s %-12s %-12s %7s %7s\n", - "id", "suite", "case", "type", "perms"); - for (size_t i = 0; i < test_suite_count; i++) { - for (size_t j = 0; j < test_suites[i]->case_count; j++) { - printf("%-36s %-12s %-12s %7s %7d\n", - test_suites[i]->cases[j]->id, - test_suites[i]->name, - test_suites[i]->cases[j]->name, - "n", // TODO - test_suites[i]->cases[j]->permutations); - } - } - - } else if (list_paths) { - printf("%-36s %-36s\n", "id", "path"); - for (size_t i = 0; i < test_suite_count; i++) { - for (size_t j = 0; j < test_suites[i]->case_count; j++) { - printf("%-36s %-36s\n", - test_suites[i]->cases[j]->id, - test_suites[i]->cases[j]->path); - } - } - } else if (list_defines) { - // TODO - } else if (list_geometries) { - printf("%-12s %7s %7s %7s %7s %7s\n", - "name", "read", "prog", "erase", "count", "size"); - for (size_t i = 0; i < test_geometry_count; i++) { - printf("%-12s %7d %7d %7d %7d %7d\n", - test_geometries[i].name, - test_geometries[i].read_size, - test_geometries[i].prog_size, - test_geometries[i].erase_size, - test_geometries[i].erase_count, - test_geometries[i].erase_size - * test_geometries[i].erase_count); - } - } else { - printf("remaining: "); - for (int i = optind; i < argc; i++) { - printf("%s ", argv[i]); - } - printf("\n"); + for (size_t i = 0; i < override_count; i++) { + printf("define: %s %ju\n", overrides[i].name, overrides[i].override); } + + // do the thing + op( + overrides, + override_count); + + // cleanup (need to be done for valgrind testing) + free(overrides); } diff --git a/runners/test_runner.h b/runners/test_runner.h index c830e36b..d394dbc9 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -4,11 +4,24 @@ #include "lfs.h" +// generated test configurations +enum test_type { + TEST_NORMAL = 0x1, + TEST_REENTRANT = 0x2, + TEST_VALGRIND = 0x4, +}; + struct test_case { const char *id; const char *name; const char *path; - uint32_t permutations; + uint8_t types; + size_t permutations; + + const uintmax_t *const *defines; + const bool *define_mask; + + bool (*filter)(struct lfs_config *cfg, uint32_t perm); void (*run)(struct lfs_config *cfg, uint32_t perm); }; @@ -16,11 +29,33 @@ struct test_suite { const char *id; const char *name; const char *path; + + const char *const *define_names; + size_t define_count; + const struct test_case *const *cases; size_t case_count; }; +// TODO remove this indirection extern const struct test_suite *test_suites[]; extern const size_t test_suite_count; + +// access generated test defines +uintmax_t test_define(size_t define); + +// a few preconfigured defines that control how tests run +#define READ_SIZE test_define(0) +#define PROG_SIZE test_define(1) +#define BLOCK_SIZE test_define(2) +#define BLOCK_COUNT test_define(3) +#define BLOCK_CYCLES test_define(4) +#define CACHE_SIZE test_define(5) +#define LOOKAHEAD_SIZE test_define(6) +#define ERASE_VALUE test_define(7) +#define ERASE_CYCLES test_define(8) +#define BADBLOCK_BEHAVIOR test_define(9) + + #endif diff --git a/scripts/test_.py b/scripts/test_.py index c411d0ca..3ab9b201 100755 --- a/scripts/test_.py +++ b/scripts/test_.py @@ -5,6 +5,7 @@ import glob import itertools as it +import math as m import os import re import shutil @@ -17,15 +18,25 @@ SUITE_PROLOGUE = """ #include "runners/test_runner.h" #include """ -# TODO handle indention implicity? -# TODO change cfg to be not by value? maybe not? CASE_PROLOGUE = """ - lfs_t lfs; - struct lfs_config cfg = *cfg_; +lfs_t lfs; """ CASE_EPILOGUE = """ """ +PRE_DEFINES = [ + 'READ_SIZE', + 'PROG_SIZE', + 'BLOCK_SIZE', + 'BLOCK_COUNT', + 'BLOCK_CYCLES', + 'CACHE_SIZE', + 'LOOKAHEAD_SIZE', + 'ERASE_VALUE', + 'ERASE_CYCLES', + 'BADBLOCK_BEHAVIOR', +] + # TODO # def testpath(path): @@ -58,7 +69,25 @@ class TestCase: self.code = config.pop('code') self.code_lineno = config.pop('code_lineno', None) - self.permutations = 1 + # figure out defines and the number of resulting permutations + self.defines = {} + for k, v in config.pop('defines', {}).items(): + try: + v = eval(v) + except: + v = v + + if not isinstance(v, str): + try: + v = list(v) + except: + v = [v] + else: + v = [v] + + self.defines[k] = v + + self.permutations = m.prod(len(v) for v in self.defines.values()) for k in config.keys(): print('warning: in %s, found unused key %r' % (self.id(), k), @@ -122,6 +151,10 @@ class TestSuite: 'suite': self.name, **case})) + # combine pre-defines and per-case defines + self.defines = PRE_DEFINES + sorted( + set.union(*(set(case.defines) for case in self.cases))) + for k in config.keys(): print('warning: in %s, found unused key %r' % (self.id(), k), file=sys.stderr) @@ -129,6 +162,8 @@ class TestSuite: def id(self): return self.name + + def compile(**args): @@ -164,13 +199,62 @@ def compile(**args): f.write(suite.code) f.write('\n') - # create test functions and case structs + for i, define in it.islice( + enumerate(suite.defines), + len(PRE_DEFINES), None): + f.write('#define %-24s test_define(%d)\n' % (define, i)) + f.write('\n') + for case in suite.cases: - f.write('void __test__%s__%s(' - '__attribute__((unused)) struct lfs_config *cfg_, ' + # create case defines + if case.defines: + for perm, defines in enumerate( + it.product(*( + [(k, v) for v in vs] + for k, vs in case.defines.items()))): + f.write('const uintmax_t ' + '__test__%s__%s__%d__defines[] = {\n' + % (suite.name, case.name, perm)) + for k, v in sorted(defines): + f.write(4*' '+'[%d] = %s,\n' + % (suite.defines.index(k), v)) + f.write('};\n') + f.write('\n') + + f.write('const uintmax_t *const ' + '__test__%s__%s__defines[] = {\n' + % (suite.name, case.name)) + for perm in range(case.permutations): + f.write(4*' '+'__test__%s__%s__%d__defines,\n' + % (suite.name, case.name, perm)) + f.write('};\n') + f.write('\n') + + f.write('const bool ' + '__test__%s__%s__define_mask[] = {\n' + % (suite.name, case.name)) + for i, k in enumerate(suite.defines): + f.write(4*' '+'%s,\n' + % ('true' if k in case.defines else 'false')) + f.write('};\n') + f.write('\n') + + # create case filter function + f.write('bool __test__%s__%s__filter(' + '__attribute__((unused)) struct lfs_config *cfg, ' '__attribute__((unused)) uint32_t perm) {\n' % (suite.name, case.name)) - f.write(CASE_PROLOGUE) + f.write(4*' '+'return true;\n') + f.write('}\n') + f.write('\n') + + # create case run function + f.write('void __test__%s__%s__run(' + '__attribute__((unused)) struct lfs_config *cfg, ' + '__attribute__((unused)) uint32_t perm) {\n' + % (suite.name, case.name)) + f.write(4*' '+'%s\n' + % CASE_PROLOGUE.strip().replace('\n', '\n'+4*' ')) f.write('\n') f.write(4*' '+'// test case %s\n' % case.id()) if case.code_lineno is not None: @@ -178,27 +262,49 @@ def compile(**args): % (case.code_lineno, suite.path)) f.write(case.code) f.write('\n') - f.write(CASE_EPILOGUE) + f.write(4*' '+'%s\n' + % CASE_EPILOGUE.strip().replace('\n', '\n'+4*' ')) f.write('}\n') f.write('\n') + # create case struct f.write('const struct test_case __test__%s__%s__case = {\n' % (suite.name, case.name)) f.write(4*' '+'.id = "%s",\n' % case.id()) f.write(4*' '+'.name = "%s",\n' % case.name) f.write(4*' '+'.path = "%s",\n' % case.path) + f.write(4*' '+'.types = TEST_NORMAL,\n') f.write(4*' '+'.permutations = %d,\n' % case.permutations) - f.write(4*' '+'.run = __test__%s__%s,\n' + if case.defines: + f.write(4*' '+'.defines = __test__%s__%s__defines,\n' + % (suite.name, case.name)) + f.write(4*' '+'.define_mask = ' + '__test__%s__%s__define_mask,\n' + % (suite.name, case.name)) + f.write(4*' '+'.filter = __test__%s__%s__filter,\n' + % (suite.name, case.name)) + f.write(4*' '+'.run = __test__%s__%s__run,\n' % (suite.name, case.name)) f.write('};\n') f.write('\n') + # create suite define names + f.write('const char *const __test__%s__define_names[] = {\n' + % suite.name) + for k in suite.defines: + f.write(4*' '+'"%s",\n' % k) + f.write('};\n') + f.write('\n') + # create suite struct f.write('const struct test_suite __test__%s__suite = {\n' - % (suite.name)) + % suite.name) f.write(4*' '+'.id = "%s",\n' % suite.id()) f.write(4*' '+'.name = "%s",\n' % suite.name) f.write(4*' '+'.path = "%s",\n' % suite.path) + f.write(4*' '+'.define_names = __test__%s__define_names,\n' + % suite.name) + f.write(4*' '+'.define_count = %d,\n' % len(suite.defines)) f.write(4*' '+'.cases = (const struct test_case *const []){\n') for case in suite.cases: f.write(8*' '+'&__test__%s__%s__case,\n' @@ -216,11 +322,13 @@ def compile(**args): # write out a test source if 'output' in args: with openio(args['output'], 'w') as f: - f.write(SUITE_PROLOGUE) - f.write('\n') f.write('#line 1 "%s"\n' % args['source']) with open(args['source']) as sf: shutil.copyfileobj(sf, f) + f.write('\n') + + f.write(SUITE_PROLOGUE) + f.write('\n') # add suite info to test_runner.c if args['source'] == 'runners/test_runner.c': From 4b0aa6272e985fa8bf592701ddd0332ab9b9d043 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 18 Apr 2022 00:09:01 -0500 Subject: [PATCH 03/81] Some more minor improvements to the test_runner - Indirect index map instead of bitmap+sparse array - test_define_t and test_type_t - Added back conditional filtering - Added suite-level defines and filtering --- runners/test_runner.c | 131 +++++++++++++++++++++++------------------- runners/test_runner.h | 13 +++-- scripts/test_.py | 104 +++++++++++++++++++++------------ 3 files changed, 147 insertions(+), 101 deletions(-) diff --git a/runners/test_runner.c b/runners/test_runner.c index 7ab433f7..dcc550b3 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -8,12 +8,12 @@ // disk geometries struct test_geometry { const char *name; - const uintmax_t *defines; + const test_define_t *defines; }; // Note this includes the default configuration for test pre-defines #define TEST_GEOMETRY(name, read, prog, erase, count) \ - {name, (const uintmax_t[]){ \ + {name, (const test_define_t[]){ \ /* READ_SIZE */ read, \ /* PROG_SIZE */ prog, \ /* BLOCK_SIZE */ erase, \ @@ -46,14 +46,14 @@ const size_t test_geometry_count = ( // test define lookup and management -const uintmax_t *test_defines[3] = {NULL}; -const bool *test_define_masks[2] = {NULL}; +const test_define_t *test_defines[3] = {NULL}; +const uint8_t *test_define_maps[2] = {NULL}; -uintmax_t test_define(size_t define) { - if (test_define_masks[0] && test_define_masks[0][define]) { - return test_defines[0][define]; - } else if (test_define_masks[1] && test_define_masks[1][define]) { - return test_defines[1][define]; +test_define_t test_define(size_t define) { + if (test_define_maps[0] && test_define_maps[0][define] != 0xff) { + return test_defines[0][test_define_maps[0][define]]; + } else if (test_define_maps[1] && test_define_maps[1][define] != 0xff) { + return test_defines[1][test_define_maps[1][define]]; } else { return test_defines[2][define]; } @@ -70,61 +70,54 @@ void test_define_geometry(const struct test_geometry *geometry) { void test_define_case(const struct test_case *case_, size_t perm) { if (case_ && case_->defines) { test_defines[1] = case_->defines[perm]; - test_define_masks[1] = case_->define_mask; + test_define_maps[1] = case_->define_map; } else { test_defines[1] = NULL; - test_define_masks[1] = NULL; + test_define_maps[1] = NULL; } } -struct override { - const char *name; - uintmax_t override; -}; - void test_define_overrides( const struct test_suite *suite, - const struct override *overrides, + const char *const *override_names, + const test_define_t *override_defines, size_t override_count) { - if (overrides && override_count > 0) { - uintmax_t *defines = malloc(suite->define_count * sizeof(uintmax_t)); - memset(defines, 0, suite->define_count * sizeof(uintmax_t)); - bool *define_mask = malloc(suite->define_count * sizeof(bool)); - memset(define_mask, 0, suite->define_count * sizeof(bool)); + if (override_names && override_defines && override_count > 0) { + uint8_t *define_map = malloc(suite->define_count * sizeof(uint8_t)); + memset(define_map, 0xff, suite->define_count * sizeof(bool)); // lookup each override in the suite defines, they may have a // different index in each suite for (size_t i = 0; i < override_count; i++) { - ssize_t index = -1; - for (size_t j = 0; j < suite->define_count; j++) { - if (strcmp(overrides[i].name, suite->define_names[j]) == 0) { - index = j; + size_t j = 0; + for (; j < suite->define_count; j++) { + if (strcmp(override_names[i], suite->define_names[j]) == 0) { break; } } - if (index >= 0) { - defines[index] = overrides[i].override; - define_mask[index] = true; + if (j < suite->define_count) { + define_map[j] = i; } } - test_defines[0] = defines; - test_define_masks[0] = define_mask; + test_defines[0] = override_defines; + test_define_maps[0] = define_map; } else { - free((uintmax_t *)test_defines[0]); test_defines[0] = NULL; - free((bool *)test_define_masks[0]); - test_define_masks[0] = NULL; + free((uint8_t *)test_define_maps[0]); + test_define_maps[0] = NULL; } } // operations we can do void summary( - struct override *overrides, + const char *const *override_names, + const test_define_t *override_defines, size_t override_count) { - (void)overrides; + (void)override_names; + (void)override_defines; (void)override_count; printf("%-36s %7s %7s %7s %7s\n", "", "geoms", "suites", "cases", "perms"); @@ -147,9 +140,11 @@ void summary( } void list_suites( - struct override *overrides, + const char *const *override_names, + const test_define_t *override_defines, size_t override_count) { - (void)overrides; + (void)override_names; + (void)override_defines; (void)override_count; printf("%-36s %-12s %7s %7s %7s\n", "id", "suite", "types", "cases", "perms"); @@ -169,9 +164,11 @@ void list_suites( } void list_cases( - struct override *overrides, + const char *const *override_names, + const test_define_t *override_defines, size_t override_count) { - (void)overrides; + (void)override_names; + (void)override_defines; (void)override_count; printf("%-36s %-12s %-12s %7s %7s\n", "id", "suite", "case", "types", "perms"); @@ -189,9 +186,11 @@ void list_cases( } void list_paths( - struct override *overrides, + const char *const *override_names, + const test_define_t *override_defines, size_t override_count) { - (void)overrides; + (void)override_names; + (void)override_defines; (void)override_count; printf("%-36s %-36s\n", "id", "path"); for (size_t i = 0; i < test_suite_count; i++) { @@ -204,17 +203,21 @@ void list_paths( } void list_defines( - struct override *overrides, + const char *const *override_names, + const test_define_t *override_defines, size_t override_count) { - (void)overrides; + (void)override_names; + (void)override_defines; (void)override_count; // TODO } void list_geometries( - struct override *overrides, + const char *const *override_names, + const test_define_t *override_defines, size_t override_count) { - (void)overrides; + (void)override_names; + (void)override_defines; (void)override_count; printf("%-36s %7s %7s %7s %7s %7s\n", "name", "read", "prog", "erase", "count", "size"); @@ -232,10 +235,13 @@ void list_geometries( } void run( - struct override *overrides, + const char *const *override_names, + const test_define_t *override_defines, size_t override_count) { for (size_t i = 0; i < test_suite_count; i++) { - test_define_overrides(test_suites[i], overrides, override_count); + test_define_overrides( + test_suites[i], + override_names, override_defines, override_count); for (size_t j = 0; j < test_suites[i]->case_count; j++) { for (size_t perm = 0; @@ -303,7 +309,7 @@ void run( } } - test_define_overrides(NULL, NULL, 0); + test_define_overrides(NULL, NULL, NULL, 0); } } @@ -349,9 +355,11 @@ const char *const help_text[] = { int main(int argc, char **argv) { void (*op)( - struct override *overrides, + const char *const *override_names, + const test_define_t *override_defines, size_t override_count) = run; - struct override *overrides = NULL; + const char **override_names = NULL; + test_define_t *override_defines = NULL; size_t override_count = 0; size_t override_cap = 0; @@ -422,24 +430,26 @@ int main(int argc, char **argv) { override_count += 1; if (override_count > override_cap) { override_cap = (2*override_cap > 4) ? 2*override_cap : 4; - overrides = realloc(overrides, override_cap - * sizeof(struct override)); + override_names = realloc(override_names, override_cap + * sizeof(const char *)); + override_defines = realloc(override_defines, override_cap + * sizeof(test_define_t)); } - // parse into string key/uintmax_t value, cannibalizing the + // parse into string key/test_define_t value, cannibalizing the // arg in the process char *sep = strchr(optarg, '='); char *parsed = NULL; if (!sep) { goto invalid_define; } - overrides[override_count-1].override + override_defines[override_count-1] = strtoumax(sep+1, &parsed, 0); if (parsed == sep+1) { goto invalid_define; } - overrides[override_count-1].name = optarg; + override_names[override_count-1] = optarg; *sep = '\0'; break; @@ -458,15 +468,16 @@ invalid_define: getopt_done: for (size_t i = 0; i < override_count; i++) { - printf("define: %s %ju\n", overrides[i].name, overrides[i].override); + printf("define: %s %ju\n", override_names[i], override_defines[i]); } // do the thing op( - overrides, + override_names, + override_defines, override_count); // cleanup (need to be done for valgrind testing) - free(overrides); -} + free(override_names); + free(override_defines);} diff --git a/runners/test_runner.h b/runners/test_runner.h index d394dbc9..fa5e2128 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -5,21 +5,24 @@ // generated test configurations -enum test_type { +enum test_types { TEST_NORMAL = 0x1, TEST_REENTRANT = 0x2, TEST_VALGRIND = 0x4, }; +typedef uint8_t test_types_t; +typedef uintmax_t test_define_t; + struct test_case { const char *id; const char *name; const char *path; - uint8_t types; + test_types_t types; size_t permutations; - const uintmax_t *const *defines; - const bool *define_mask; + const test_define_t *const *defines; + const uint8_t *define_map; bool (*filter)(struct lfs_config *cfg, uint32_t perm); void (*run)(struct lfs_config *cfg, uint32_t perm); @@ -43,7 +46,7 @@ extern const size_t test_suite_count; // access generated test defines -uintmax_t test_define(size_t define); +test_define_t test_define(size_t define); // a few preconfigured defines that control how tests run #define READ_SIZE test_define(0) diff --git a/scripts/test_.py b/scripts/test_.py index 3ab9b201..2b3b4377 100755 --- a/scripts/test_.py +++ b/scripts/test_.py @@ -66,23 +66,19 @@ class TestCase: self.path = config.pop('path') self.suite = config.pop('suite') self.lineno = config.pop('lineno', None) + self.if_ = config.pop('if', None) + if isinstance(self.if_, bool): + self.if_ = 'true' if self.if_ else 'false' + self.if_lineno = config.pop('if_lineno', None) self.code = config.pop('code') self.code_lineno = config.pop('code_lineno', None) # figure out defines and the number of resulting permutations self.defines = {} - for k, v in config.pop('defines', {}).items(): - try: - v = eval(v) - except: - v = v - - if not isinstance(v, str): - try: - v = list(v) - except: - v = [v] - else: + for k, v in ( + config.pop('suite_defines', {}) + | config.pop('defines', {})).items(): + if not isinstance(v, list): v = [v] self.defines[k] = v @@ -111,14 +107,18 @@ class TestSuite: # find line numbers f.seek(0) case_linenos = [] + if_linenos = [] code_linenos = [] for i, line in enumerate(f): match = re.match( '(?P\[\s*cases\s*\.\s*(?P\w+)\s*\])' + - '|(?Pcode\s*=\s*(?:\'\'\'|"""))', + '|(?Pif\s*=)' + '|(?Pcode\s*=)', line) if match and match.group('case'): case_linenos.append((i+1, match.group('name'))) + elif match and match.group('if'): + if_linenos.append(i+1) elif match and match.group('code'): code_linenos.append(i+2) @@ -129,18 +129,33 @@ class TestSuite: for (lineno, name), (nlineno, _) in it.zip_longest( case_linenos, case_linenos[1:], fillvalue=(float('inf'), None)): + if_lineno = min( + (l for l in if_linenos if l >= lineno and l < nlineno), + default=None) code_lineno = min( (l for l in code_linenos if l >= lineno and l < nlineno), default=None) cases[name]['lineno'] = lineno + cases[name]['if_lineno'] = if_lineno cases[name]['code_lineno'] = code_lineno + self.if_ = config.pop('if', None) + if isinstance(self.if_, bool): + self.if_ = 'true' if self.if_ else 'false' + self.if_lineno = min( + (l for l in if_linenos + if not case_linenos or l < case_linenos[0][0]), + default=None) + self.code = config.pop('code', None) self.code_lineno = min( (l for l in code_linenos if not case_linenos or l < case_linenos[0][0]), default=None) + # a couple of these we just forward to all cases + defines = config.pop('defines', {}) + self.cases = [] for name, case in sorted(cases.items(), key=lambda c: c[1].get('lineno')): @@ -149,6 +164,7 @@ class TestSuite: 'path': path + (':%d' % case['lineno'] if 'lineno' in case else ''), 'suite': self.name, + 'suite_defines': defines, **case})) # combine pre-defines and per-case defines @@ -162,8 +178,6 @@ class TestSuite: def id(self): return self.name - - def compile(**args): @@ -208,20 +222,21 @@ def compile(**args): for case in suite.cases: # create case defines if case.defines: + sorted_defines = sorted(case.defines.items()) + for perm, defines in enumerate( it.product(*( [(k, v) for v in vs] - for k, vs in case.defines.items()))): - f.write('const uintmax_t ' + for k, vs in sorted_defines))): + f.write('const test_define_t ' '__test__%s__%s__%d__defines[] = {\n' % (suite.name, case.name, perm)) - for k, v in sorted(defines): - f.write(4*' '+'[%d] = %s,\n' - % (suite.defines.index(k), v)) + for k, v in defines: + f.write(4*' '+'%s,\n' % v) f.write('};\n') f.write('\n') - f.write('const uintmax_t *const ' + f.write('const test_define_t *const ' '__test__%s__%s__defines[] = {\n' % (suite.name, case.name)) for perm in range(case.permutations): @@ -230,23 +245,39 @@ def compile(**args): f.write('};\n') f.write('\n') - f.write('const bool ' - '__test__%s__%s__define_mask[] = {\n' + f.write('const uint8_t ' + '__test__%s__%s__define_map[] = {\n' % (suite.name, case.name)) - for i, k in enumerate(suite.defines): + for k in suite.defines: f.write(4*' '+'%s,\n' - % ('true' if k in case.defines else 'false')) + % ([k for k, _ in sorted_defines].index(k) + if k in case.defines else '0xff')) f.write('};\n') f.write('\n') # create case filter function - f.write('bool __test__%s__%s__filter(' - '__attribute__((unused)) struct lfs_config *cfg, ' - '__attribute__((unused)) uint32_t perm) {\n' - % (suite.name, case.name)) - f.write(4*' '+'return true;\n') - f.write('}\n') - f.write('\n') + if suite.if_ is not None or case.if_ is not None: + f.write('bool __test__%s__%s__filter(' + '__attribute__((unused)) struct lfs_config *cfg, ' + '__attribute__((unused)) uint32_t perm) {\n' + % (suite.name, case.name)) + if suite.if_ is not None: + f.write(4*' '+'#line %d "%s"\n' + % (suite.if_lineno, suite.path)) + f.write(4*' '+'if (!(%s)) {\n' % suite.if_) + f.write(8*' '+'return false;\n') + f.write(4*' '+'}\n') + f.write('\n') + if case.if_ is not None: + f.write(4*' '+'#line %d "%s"\n' + % (case.if_lineno, suite.path)) + f.write(4*' '+'if (!(%s)) {\n' % case.if_) + f.write(8*' '+'return false;\n') + f.write(4*' '+'}\n') + f.write('\n') + f.write(4*' '+'return true;\n') + f.write('}\n') + f.write('\n') # create case run function f.write('void __test__%s__%s__run(' @@ -278,11 +309,12 @@ def compile(**args): if case.defines: f.write(4*' '+'.defines = __test__%s__%s__defines,\n' % (suite.name, case.name)) - f.write(4*' '+'.define_mask = ' - '__test__%s__%s__define_mask,\n' + f.write(4*' '+'.define_map = ' + '__test__%s__%s__define_map,\n' + % (suite.name, case.name)) + if suite.if_ is not None or case.if_ is not None: + f.write(4*' '+'.filter = __test__%s__%s__filter,\n' % (suite.name, case.name)) - f.write(4*' '+'.filter = __test__%s__%s__filter,\n' - % (suite.name, case.name)) f.write(4*' '+'.run = __test__%s__%s__run,\n' % (suite.name, case.name)) f.write('};\n') From 9281ce26a79de46e650488660e2fb37656624545 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 18 Apr 2022 14:45:41 -0500 Subject: [PATCH 04/81] More test_runner progress - Added filtering based on suite, case, perm, type, geometry - Added --skip, --count, and --every (will be used for parallelism) - Implemented --list-defines - Better helptext for flags with arguments - Other minor tweaks --- runners/test_runner.c | 549 +++++++++++++++++++++++++++++++++--------- runners/test_runner.h | 5 +- scripts/test_.py | 36 ++- 3 files changed, 464 insertions(+), 126 deletions(-) diff --git a/runners/test_runner.c b/runners/test_runner.c index dcc550b3..38c6896d 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -59,7 +59,7 @@ test_define_t test_define(size_t define) { } } -void test_define_geometry(const struct test_geometry *geometry) { +static void test_define_geometry(const struct test_geometry *geometry) { if (geometry) { test_defines[2] = geometry->defines; } else { @@ -67,7 +67,7 @@ void test_define_geometry(const struct test_geometry *geometry) { } } -void test_define_case(const struct test_case *case_, size_t perm) { +static void test_define_case(const struct test_case *case_, size_t perm) { if (case_ && case_->defines) { test_defines[1] = case_->defines[perm]; test_define_maps[1] = case_->define_map; @@ -77,7 +77,7 @@ void test_define_case(const struct test_case *case_, size_t perm) { } } -void test_define_overrides( +static void test_define_overrides( const struct test_suite *suite, const char *const *override_names, const test_define_t *override_defines, @@ -111,90 +111,230 @@ void test_define_overrides( } -// operations we can do -void summary( - const char *const *override_names, - const test_define_t *override_defines, - size_t override_count) { - (void)override_names; - (void)override_defines; - (void)override_count; - printf("%-36s %7s %7s %7s %7s\n", - "", "geoms", "suites", "cases", "perms"); - size_t cases = 0; - size_t perms = 0; - for (size_t i = 0; i < test_suite_count; i++) { - cases += test_suites[i]->case_count; +// other miscellany +static const char *test_suite = NULL; +static const char *test_case = NULL; +static size_t test_perm = -1; +static const char *test_geometry = NULL; +static test_types_t test_types = 0; +static size_t test_skip = 0; +static size_t test_count = -1; +static size_t test_every = 1; - for (size_t j = 0; j < test_suites[i]->case_count; j++) { - perms += test_suites[i]->cases[j]->permutations; +static const char **override_names = NULL; +static test_define_t *override_defines = NULL; +static size_t override_count = 0; +static size_t override_cap = 0; + +// note, these skips are different than filtered tests +static bool test_suite_skip(const struct test_suite *suite) { + return (test_suite && strcmp(suite->name, test_suite) != 0) + || (test_types && (suite->types & test_types) == 0); +} + +static bool test_case_skip(const struct test_case *case_) { + return (test_case && strcmp(case_->name, test_case) != 0) + || (test_types && (case_->types & test_types) == 0); +} + +static bool test_perm_skip(size_t perm) { + size_t geom_perm = perm % test_geometry_count; + return (test_perm != (size_t)-1 && perm != test_perm) + || (test_geometry && (strcmp( + test_geometries[geom_perm].name, + test_geometry) != 0)); +} + +static bool test_step_skip(size_t step) { + return !(step >= test_skip + && (step-test_skip) < test_count + && (step-test_skip) % test_every == 0); +} + +static void test_case_sumpermutations( + const struct test_case *case_, + size_t *perms, + size_t *filtered) { + size_t perms_ = 0; + size_t filtered_ = 0; + + for (size_t perm = 0; + perm < test_geometry_count + * case_->permutations; + perm++) { + if (test_perm_skip(perm)) { + continue; } + + perms_ += 1; + + // setup defines + size_t case_perm = perm / test_geometry_count; + size_t geom_perm = perm % test_geometry_count; + test_define_geometry(&test_geometries[geom_perm]); + test_define_case(case_, case_perm); + + if (case_->filter) { + if (!case_->filter(case_perm)) { + test_define_geometry(NULL); + test_define_case(NULL, 0); + continue; + } + } + + filtered_ += 1; + + test_define_geometry(NULL); + test_define_case(NULL, 0); } - printf("%-36s %7zu %7zu %7zu %7zu\n", + *perms += perms_; + *filtered += filtered_; +} + + +// operations we can do +static void summary(void) { + printf("%-36s %7s %7s %7s %11s\n", + "", "types", "suites", "cases", "perms"); + size_t cases = 0; + test_types_t types = 0; + size_t perms = 0; + size_t filtered = 0; + for (size_t i = 0; i < test_suite_count; i++) { + if (test_suite_skip(test_suites[i])) { + continue; + } + + test_define_overrides( + test_suites[i], + override_names, override_defines, override_count); + + for (size_t j = 0; j < test_suites[i]->case_count; j++) { + if (test_case_skip(test_suites[i]->cases[j])) { + continue; + } + + test_case_sumpermutations(test_suites[i]->cases[j], + &perms, &filtered); + } + + test_define_overrides(NULL, NULL, NULL, 0); + + cases += test_suites[i]->case_count; + types |= test_suites[i]->types; + } + + char perm_buf[64]; + sprintf(perm_buf, "%zu/%zu", filtered, perms); + char type_buf[64]; + sprintf(type_buf, "%s%s%s", + (types & TEST_NORMAL) ? "n" : "", + (types & TEST_REENTRANT) ? "r" : "", + (types & TEST_VALGRIND) ? "V" : ""); + printf("%-36s %7s %7zu %7zu %11s\n", "TOTAL", - test_geometry_count, + type_buf, test_suite_count, cases, - test_geometry_count*perms); + perm_buf); } -void list_suites( - const char *const *override_names, - const test_define_t *override_defines, - size_t override_count) { - (void)override_names; - (void)override_defines; - (void)override_count; - printf("%-36s %-12s %7s %7s %7s\n", +static void list_suites(void) { + printf("%-36s %-12s %7s %7s %11s\n", "id", "suite", "types", "cases", "perms"); for (size_t i = 0; i < test_suite_count; i++) { - size_t perms = 0; - for (size_t j = 0; j < test_suites[i]->case_count; j++) { - perms += test_suites[i]->cases[j]->permutations; + if (test_suite_skip(test_suites[i])) { + continue; } - printf("%-36s %-12s %7s %7zu %7zu\n", + test_define_overrides( + test_suites[i], + override_names, override_defines, override_count); + + size_t perms = 0; + size_t filtered = 0; + for (size_t j = 0; j < test_suites[i]->case_count; j++) { + if (test_case_skip(test_suites[i]->cases[j])) { + continue; + } + + test_case_sumpermutations(test_suites[i]->cases[j], + &perms, &filtered); + } + + test_define_overrides(NULL, NULL, NULL, 0); + + char perm_buf[64]; + sprintf(perm_buf, "%zu/%zu", filtered, perms); + char type_buf[64]; + sprintf(type_buf, "%s%s%s", + (test_suites[i]->types & TEST_NORMAL) ? "n" : "", + (test_suites[i]->types & TEST_REENTRANT) ? "r" : "", + (test_suites[i]->types & TEST_VALGRIND) ? "V" : ""); + printf("%-36s %-12s %7s %7zu %11s\n", test_suites[i]->id, test_suites[i]->name, - "n", // TODO + type_buf, test_suites[i]->case_count, - test_geometry_count*perms); + perm_buf); } } -void list_cases( - const char *const *override_names, - const test_define_t *override_defines, - size_t override_count) { - (void)override_names; - (void)override_defines; - (void)override_count; - printf("%-36s %-12s %-12s %7s %7s\n", +static void list_cases(void) { + printf("%-36s %-12s %-12s %7s %11s\n", "id", "suite", "case", "types", "perms"); for (size_t i = 0; i < test_suite_count; i++) { + if (test_suite_skip(test_suites[i])) { + continue; + } + + test_define_overrides( + test_suites[i], + override_names, override_defines, override_count); + for (size_t j = 0; j < test_suites[i]->case_count; j++) { - printf("%-36s %-12s %-12s %7s %7zu\n", + if (test_case_skip(test_suites[i]->cases[j])) { + continue; + } + + size_t perms = 0; + size_t filtered = 0; + test_case_sumpermutations(test_suites[i]->cases[j], + &perms, &filtered); + test_types_t types = test_suites[i]->cases[j]->types; + + char perm_buf[64]; + sprintf(perm_buf, "%zu/%zu", filtered, perms); + char type_buf[64]; + sprintf(type_buf, "%s%s%s", + (types & TEST_NORMAL) ? "n" : "", + (types & TEST_REENTRANT) ? "r" : "", + (types & TEST_VALGRIND) ? "V" : ""); + printf("%-36s %-12s %-12s %7s %11s\n", test_suites[i]->cases[j]->id, test_suites[i]->name, test_suites[i]->cases[j]->name, - "n", // TODO - test_geometry_count - * test_suites[i]->cases[j]->permutations); + type_buf, + perm_buf); } + + test_define_overrides(NULL, NULL, NULL, 0); } } -void list_paths( - const char *const *override_names, - const test_define_t *override_defines, - size_t override_count) { - (void)override_names; - (void)override_defines; - (void)override_count; +static void list_paths(void) { printf("%-36s %-36s\n", "id", "path"); for (size_t i = 0; i < test_suite_count; i++) { + if (test_suite_skip(test_suites[i])) { + continue; + } + for (size_t j = 0; j < test_suites[i]->case_count; j++) { + if (test_case_skip(test_suites[i]->cases[j])) { + continue; + } + printf("%-36s %-36s\n", test_suites[i]->cases[j]->id, test_suites[i]->cases[j]->path); @@ -202,26 +342,70 @@ void list_paths( } } -void list_defines( - const char *const *override_names, - const test_define_t *override_defines, - size_t override_count) { - (void)override_names; - (void)override_defines; - (void)override_count; - // TODO +static void list_defines(void) { + printf("%-36s %s\n", "id", "defines"); + for (size_t i = 0; i < test_suite_count; i++) { + if (test_suite_skip(test_suites[i])) { + continue; + } + + test_define_overrides( + test_suites[i], + override_names, override_defines, override_count); + + for (size_t j = 0; j < test_suites[i]->case_count; j++) { + if (test_case_skip(test_suites[i]->cases[j])) { + continue; + } + + for (size_t perm = 0; + perm < test_geometry_count + * test_suites[i]->cases[j]->permutations; + perm++) { + if (test_perm_skip(perm)) { + continue; + } + + // setup defines + size_t case_perm = perm / test_geometry_count; + size_t geom_perm = perm % test_geometry_count; + test_define_geometry(&test_geometries[geom_perm]); + test_define_case(test_suites[i]->cases[j], case_perm); + + // print each define + char id_buf[256]; + sprintf(id_buf, "%s#%zu", test_suites[i]->cases[j]->id, perm); + printf("%-36s ", id_buf); + for (size_t k = 0; k < test_suites[i]->define_count; k++) { + if (k >= TEST_PREDEFINE_COUNT && ( + !test_suites[i]->cases[j]->define_map + || test_suites[i]->cases[j]->define_map[k] + == 0xff)) { + continue; + } + + printf("%s=%jd ", + test_suites[i]->define_names[k], + test_define(k)); + } + printf("\n"); + } + } + + test_define_overrides(NULL, NULL, NULL, 0); + } } -void list_geometries( - const char *const *override_names, - const test_define_t *override_defines, - size_t override_count) { - (void)override_names; - (void)override_defines; - (void)override_count; +static void list_geometries(void) { printf("%-36s %7s %7s %7s %7s %7s\n", "name", "read", "prog", "erase", "count", "size"); for (size_t i = 0; i < test_geometry_count; i++) { + if (test_geometry && strcmp( + test_geometries[i].name, + test_geometry) != 0) { + continue; + } + test_define_geometry(&test_geometries[i]); printf("%-36s %7ju %7ju %7ju %7ju %7ju\n", @@ -234,27 +418,54 @@ void list_geometries( } } -void run( - const char *const *override_names, - const test_define_t *override_defines, - size_t override_count) { +static void run(void) { + size_t step = 0; for (size_t i = 0; i < test_suite_count; i++) { + if (test_suite_skip(test_suites[i])) { + continue; + } + test_define_overrides( test_suites[i], override_names, override_defines, override_count); for (size_t j = 0; j < test_suites[i]->case_count; j++) { + if (test_case_skip(test_suites[i]->cases[j])) { + continue; + } + for (size_t perm = 0; perm < test_geometry_count * test_suites[i]->cases[j]->permutations; perm++) { - size_t case_perm = perm / test_geometry_count; - size_t geom_perm = perm % test_geometry_count; + if (test_perm_skip(perm)) { + continue; + } + + if (test_step_skip(step)) { + step += 1; + continue; + } + step += 1; // setup defines + size_t case_perm = perm / test_geometry_count; + size_t geom_perm = perm % test_geometry_count; test_define_geometry(&test_geometries[geom_perm]); test_define_case(test_suites[i]->cases[j], case_perm); + // filter? + if (test_suites[i]->cases[j]->filter) { + if (!test_suites[i]->cases[j]->filter(case_perm)) { + printf("skipped %s#%zu\n", + test_suites[i]->cases[j]->id, + perm); + test_define_geometry(NULL); + test_define_case(NULL, 0); + continue; + } + } + // create block device and configuration lfs_testbd_t bd; @@ -282,18 +493,6 @@ void run( lfs_testbd_createcfg(&cfg, NULL, &bdcfg) => 0; - // filter? - if (test_suites[i]->cases[j]->filter) { - bool filter = test_suites[i]->cases[j]->filter( - &cfg, case_perm); - if (!filter) { - printf("skipped %s#%zu\n", - test_suites[i]->cases[j]->id, - perm); - continue; - } - } - // run the test printf("running %s#%zu\n", test_suites[i]->cases[j]->id, perm); @@ -320,15 +519,22 @@ void run( enum opt_flags { OPT_HELP = 'h', OPT_SUMMARY = 'Y', - OPT_LIST_SUITES = 1, - OPT_LIST_CASES = 'l', - OPT_LIST_PATHS = 2, - OPT_LIST_DEFINES = 3, - OPT_LIST_GEOMETRIES = 4, + OPT_LIST_SUITES = 'l', + OPT_LIST_CASES = 'L', + OPT_LIST_PATHS = 1, + OPT_LIST_DEFINES = 2, + OPT_LIST_GEOMETRIES = 3, OPT_DEFINE = 'D', + OPT_GEOMETRY = 'G', + OPT_NORMAL = 'n', + OPT_REENTRANT = 'r', + OPT_VALGRIND = 'V', + OPT_SKIP = 4, + OPT_COUNT = 5, + OPT_EVERY = 6, }; -const char *short_opts = "hYlD:"; +const char *short_opts = "hYlLD:G:nrV"; const struct option long_opts[] = { {"help", no_argument, NULL, OPT_HELP}, @@ -339,6 +545,13 @@ const struct option long_opts[] = { {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, {"define", required_argument, NULL, OPT_DEFINE}, + {"geometry", required_argument, NULL, OPT_GEOMETRY}, + {"normal", no_argument, NULL, OPT_NORMAL}, + {"reentrant", no_argument, NULL, OPT_REENTRANT}, + {"valgrind", no_argument, NULL, OPT_VALGRIND}, + {"skip", required_argument, NULL, OPT_SKIP}, + {"count", required_argument, NULL, OPT_COUNT}, + {"every", required_argument, NULL, OPT_EVERY}, {NULL, 0, NULL, 0}, }; @@ -351,17 +564,17 @@ const char *const help_text[] = { "List the defines for each test permutation.", "List the disk geometries used for testing.", "Override a test define.", + "Filter by geometry.", + "Filter for normal tests. Can be combined.", + "Filter for reentrant tests. Can be combined.", + "Filter for valgrind tests. Can be combined.", + "Skip the first n tests.", + "Stop after n tests.", + "Only run every n tests, calculated after --skip and --stop.", }; int main(int argc, char **argv) { - void (*op)( - const char *const *override_names, - const test_define_t *override_defines, - size_t override_count) = run; - const char **override_names = NULL; - test_define_t *override_defines = NULL; - size_t override_count = 0; - size_t override_cap = 0; + void (*op)(void) = run; // parse options while (true) { @@ -369,29 +582,47 @@ int main(int argc, char **argv) { switch (c) { // generate help message case OPT_HELP: { - printf("usage: %s [options] [test_case]\n", argv[0]); + printf("usage: %s [options] [test_id]\n", argv[0]); printf("\n"); printf("options:\n"); size_t i = 0; while (long_opts[i].name) { size_t indent; - if (long_opts[i].val >= '0' && long_opts[i].val < 'z') { - printf(" -%c, --%-16s", - long_opts[i].val, - long_opts[i].name); - indent = 8+strlen(long_opts[i].name); + if (long_opts[i].has_arg == no_argument) { + if (long_opts[i].val >= '0' && long_opts[i].val < 'z') { + indent = printf(" -%c, --%s ", + long_opts[i].val, + long_opts[i].name); + } else { + indent = printf(" --%s ", + long_opts[i].name); + } } else { - printf(" --%-20s", long_opts[i].name); - indent = 4+strlen(long_opts[i].name); + if (long_opts[i].val >= '0' && long_opts[i].val < 'z') { + indent = printf(" -%c %s, --%s %s ", + long_opts[i].val, + long_opts[i].name, + long_opts[i].name, + long_opts[i].name); + } else { + indent = printf(" --%s %s ", + long_opts[i].name, + long_opts[i].name); + } } // a quick, hacky, byte-level method for text wrapping size_t len = strlen(help_text[i]); size_t j = 0; if (indent < 24) { - printf("%.80s\n", &help_text[i][j]); + printf("%*s %.80s\n", + (int)(24-1-indent), + "", + &help_text[i][j]); j += 80; + } else { + printf("\n"); } while (j < len) { @@ -457,6 +688,45 @@ invalid_define: fprintf(stderr, "error: invalid define: %s\n", optarg); exit(-1); } + case OPT_GEOMETRY: + test_geometry = optarg; + break; + case OPT_NORMAL: + test_types |= TEST_NORMAL; + break; + case OPT_REENTRANT: + test_types |= TEST_REENTRANT; + break; + case OPT_VALGRIND: + test_types |= TEST_VALGRIND; + break; + case OPT_SKIP: { + char *parsed = NULL; + test_skip = strtoumax(optarg, &parsed, 0); + if (parsed == optarg) { + fprintf(stderr, "error: invalid skip: %s\n", optarg); + exit(-1); + } + break; + } + case OPT_COUNT: { + char *parsed = NULL; + test_count = strtoumax(optarg, &parsed, 0); + if (parsed == optarg) { + fprintf(stderr, "error: invalid count: %s\n", optarg); + exit(-1); + } + break; + } + case OPT_EVERY: { + char *parsed = NULL; + test_every = strtoumax(optarg, &parsed, 0); + if (parsed == optarg) { + fprintf(stderr, "error: invalid every: %s\n", optarg); + exit(-1); + } + break; + } // done parsing case -1: goto getopt_done; @@ -465,19 +735,58 @@ invalid_define: exit(-1); } } -getopt_done: +getopt_done: ; - for (size_t i = 0; i < override_count; i++) { - printf("define: %s %ju\n", override_names[i], override_defines[i]); + // parse test identifier, if any, cannibalizing the arg in the process + if (argc > optind) { + if (argc - optind > 1) { + fprintf(stderr, "error: more than one test identifier\n"); + exit(-1); + } + + // parse suite + char *suite = argv[optind]; + char *case_ = strchr(suite, '#'); + + if (case_) { + *case_ = '\0'; + case_ += 1; + + // parse case + char *perm = strchr(case_, '#'); + if (perm) { + *perm = '\0'; + perm += 1; + + char *parsed = NULL; + test_perm = strtoumax(perm, &parsed, 10); + if (parsed == perm) { + fprintf(stderr, "error: could not parse test identifier\n"); + exit(-1); + } + } + + test_case = case_; + } + + // remove optional path and .toml suffix + char *slash = strrchr(suite, '/'); + if (slash) { + suite = slash+1; + } + + size_t suite_len = strlen(suite); + if (suite_len > 5 && strcmp(&suite[suite_len-5], ".toml") == 0) { + suite[suite_len-5] = '\0'; + } + + test_suite = suite; } // do the thing - op( - override_names, - override_defines, - override_count); + op(); // cleanup (need to be done for valgrind testing) free(override_names); - free(override_defines);} - + free(override_defines); +} diff --git a/runners/test_runner.h b/runners/test_runner.h index fa5e2128..92d8d312 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -24,7 +24,7 @@ struct test_case { const test_define_t *const *defines; const uint8_t *define_map; - bool (*filter)(struct lfs_config *cfg, uint32_t perm); + bool (*filter)(uint32_t perm); void (*run)(struct lfs_config *cfg, uint32_t perm); }; @@ -32,6 +32,7 @@ struct test_suite { const char *id; const char *name; const char *path; + test_types_t types; const char *const *define_names; size_t define_count; @@ -60,5 +61,7 @@ test_define_t test_define(size_t define); #define ERASE_CYCLES test_define(8) #define BADBLOCK_BEHAVIOR test_define(9) +#define TEST_PREDEFINE_COUNT 10 + #endif diff --git a/scripts/test_.py b/scripts/test_.py index 2b3b4377..9007f1f7 100755 --- a/scripts/test_.py +++ b/scripts/test_.py @@ -24,7 +24,7 @@ lfs_t lfs; CASE_EPILOGUE = """ """ -PRE_DEFINES = [ +TEST_PREDEFINES = [ 'READ_SIZE', 'PROG_SIZE', 'BLOCK_SIZE', @@ -73,6 +73,13 @@ class TestCase: self.code = config.pop('code') self.code_lineno = config.pop('code_lineno', None) + self.normal = config.pop('normal', + config.pop('suite_normal', True)) + self.reentrant = config.pop('reentrant', + config.pop('suite_reentrant', False)) + self.valgrind = config.pop('valgrind', + config.pop('suite_valgrind', True)) + # figure out defines and the number of resulting permutations self.defines = {} for k, v in ( @@ -155,6 +162,9 @@ class TestSuite: # a couple of these we just forward to all cases defines = config.pop('defines', {}) + normal = config.pop('normal', True) + reentrant = config.pop('reentrant', False) + valgrind = config.pop('valgrind', True) self.cases = [] for name, case in sorted(cases.items(), @@ -165,12 +175,20 @@ class TestSuite: if 'lineno' in case else ''), 'suite': self.name, 'suite_defines': defines, + 'suite_normal': normal, + 'suite_reentrant': reentrant, + 'suite_valgrind': valgrind, **case})) # combine pre-defines and per-case defines - self.defines = PRE_DEFINES + sorted( + self.defines = TEST_PREDEFINES + sorted( set.union(*(set(case.defines) for case in self.cases))) + # combine other per-case things + self.normal = any(case.normal for case in self.cases) + self.reentrant = any(case.reentrant for case in self.cases) + self.valgrind = any(case.valgrind for case in self.cases) + for k in config.keys(): print('warning: in %s, found unused key %r' % (self.id(), k), file=sys.stderr) @@ -215,7 +233,7 @@ def compile(**args): for i, define in it.islice( enumerate(suite.defines), - len(PRE_DEFINES), None): + len(TEST_PREDEFINES), None): f.write('#define %-24s test_define(%d)\n' % (define, i)) f.write('\n') @@ -258,7 +276,6 @@ def compile(**args): # create case filter function if suite.if_ is not None or case.if_ is not None: f.write('bool __test__%s__%s__filter(' - '__attribute__((unused)) struct lfs_config *cfg, ' '__attribute__((unused)) uint32_t perm) {\n' % (suite.name, case.name)) if suite.if_ is not None: @@ -304,7 +321,11 @@ def compile(**args): f.write(4*' '+'.id = "%s",\n' % case.id()) f.write(4*' '+'.name = "%s",\n' % case.name) f.write(4*' '+'.path = "%s",\n' % case.path) - f.write(4*' '+'.types = TEST_NORMAL,\n') + f.write(4*' '+'.types = %s,\n' + % ' | '.join(filter(None, [ + 'TEST_NORMAL' if case.normal else None, + 'TEST_REENTRANT' if case.reentrant else None, + 'TEST_VALGRIND' if case.valgrind else None]))) f.write(4*' '+'.permutations = %d,\n' % case.permutations) if case.defines: f.write(4*' '+'.defines = __test__%s__%s__defines,\n' @@ -334,6 +355,11 @@ def compile(**args): f.write(4*' '+'.id = "%s",\n' % suite.id()) f.write(4*' '+'.name = "%s",\n' % suite.name) f.write(4*' '+'.path = "%s",\n' % suite.path) + f.write(4*' '+'.types = %s,\n' + % ' | '.join(filter(None, [ + 'TEST_NORMAL' if suite.normal else None, + 'TEST_REENTRANT' if suite.reentrant else None, + 'TEST_VALGRIND' if suite.valgrind else None]))) f.write(4*' '+'.define_names = __test__%s__define_names,\n' % suite.name) f.write(4*' '+'.define_count = %d,\n' % len(suite.defines)) From 92a600a980438c4506ff46c95588c5af347ac879 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 19 Apr 2022 02:05:08 -0500 Subject: [PATCH 05/81] Added trace and persist flags to test_runner --- bd/lfs_filebd.h | 2 ++ bd/lfs_rambd.h | 2 ++ bd/lfs_testbd.h | 2 ++ runners/test_runner.c | 41 ++++++++++++++++++++++++++++++++++++++--- scripts/test_.py | 32 ++++++++++++++++++++++++++++++-- 5 files changed, 74 insertions(+), 5 deletions(-) diff --git a/bd/lfs_filebd.h b/bd/lfs_filebd.h index 1a9456c5..0ed1909a 100644 --- a/bd/lfs_filebd.h +++ b/bd/lfs_filebd.h @@ -18,11 +18,13 @@ extern "C" // Block device specific tracing +#ifndef LFS_FILEBD_TRACE #ifdef LFS_FILEBD_YES_TRACE #define LFS_FILEBD_TRACE(...) LFS_TRACE(__VA_ARGS__) #else #define LFS_FILEBD_TRACE(...) #endif +#endif // filebd config (optional) struct lfs_filebd_config { diff --git a/bd/lfs_rambd.h b/bd/lfs_rambd.h index 3a70bc6e..b7629a9b 100644 --- a/bd/lfs_rambd.h +++ b/bd/lfs_rambd.h @@ -18,11 +18,13 @@ extern "C" // Block device specific tracing +#ifndef LFS_RAMBD_TRACE #ifdef LFS_RAMBD_YES_TRACE #define LFS_RAMBD_TRACE(...) LFS_TRACE(__VA_ARGS__) #else #define LFS_RAMBD_TRACE(...) #endif +#endif // rambd config (optional) struct lfs_rambd_config { diff --git a/bd/lfs_testbd.h b/bd/lfs_testbd.h index 61679e5e..06794e72 100644 --- a/bd/lfs_testbd.h +++ b/bd/lfs_testbd.h @@ -21,11 +21,13 @@ extern "C" // Block device specific tracing +#ifndef LFS_TESTBD_TRACE #ifdef LFS_TESTBD_YES_TRACE #define LFS_TESTBD_TRACE(...) LFS_TRACE(__VA_ARGS__) #else #define LFS_TESTBD_TRACE(...) #endif +#endif // Mode determining how "bad blocks" behave during testing. This simulates // some real-world circumstances such as progs not sticking (prog-noop), diff --git a/runners/test_runner.c b/runners/test_runner.c index 38c6896d..185fa8aa 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -4,6 +4,7 @@ #include #include +#include // disk geometries struct test_geometry { @@ -126,6 +127,9 @@ static test_define_t *override_defines = NULL; static size_t override_count = 0; static size_t override_cap = 0; +static const char *test_persist = NULL; +FILE *test_trace = NULL; + // note, these skips are different than filtered tests static bool test_suite_skip(const struct test_suite *suite) { return (test_suite && strcmp(suite->name, test_suite) != 0) @@ -491,7 +495,12 @@ static void run(void) { .power_cycles = 0, }; - lfs_testbd_createcfg(&cfg, NULL, &bdcfg) => 0; + int err = lfs_testbd_createcfg(&cfg, test_persist, &bdcfg); + if (err) { + fprintf(stderr, "error: " + "could not create block device: %d\n", err); + exit(-1); + } // run the test printf("running %s#%zu\n", test_suites[i]->cases[j]->id, perm); @@ -501,7 +510,12 @@ static void run(void) { printf("finished %s#%zu\n", test_suites[i]->cases[j]->id, perm); // cleanup - lfs_testbd_destroy(&cfg) => 0; + err = lfs_testbd_destroy(&cfg); + if (err) { + fprintf(stderr, "error: " + "could not destroy block device: %d\n", err); + exit(-1); + } test_define_geometry(NULL); test_define_case(NULL, 0); @@ -532,9 +546,11 @@ enum opt_flags { OPT_SKIP = 4, OPT_COUNT = 5, OPT_EVERY = 6, + OPT_PERSIST = 'p', + OPT_TRACE = 't', }; -const char *short_opts = "hYlLD:G:nrV"; +const char *short_opts = "hYlLD:G:nrVp:t:"; const struct option long_opts[] = { {"help", no_argument, NULL, OPT_HELP}, @@ -552,6 +568,8 @@ const struct option long_opts[] = { {"skip", required_argument, NULL, OPT_SKIP}, {"count", required_argument, NULL, OPT_COUNT}, {"every", required_argument, NULL, OPT_EVERY}, + {"persist", required_argument, NULL, OPT_PERSIST}, + {"trace", required_argument, NULL, OPT_TRACE}, {NULL, 0, NULL, 0}, }; @@ -571,6 +589,8 @@ const char *const help_text[] = { "Skip the first n tests.", "Stop after n tests.", "Only run every n tests, calculated after --skip and --stop.", + "Persist the disk to this file.", + "Redirect trace output to this file.", }; int main(int argc, char **argv) { @@ -727,6 +747,21 @@ invalid_define: } break; } + case OPT_PERSIST: + test_persist = optarg; + break; + case OPT_TRACE: + if (strcmp(optarg, "-") == 0) { + test_trace = stdout; + } else { + test_trace = fopen(optarg, "w"); + if (!test_trace) { + fprintf(stderr, "error: could not open for trace: %d\n", + -errno); + exit(-1); + } + } + break; // done parsing case -1: goto getopt_done; diff --git a/scripts/test_.py b/scripts/test_.py index 9007f1f7..e6458760 100755 --- a/scripts/test_.py +++ b/scripts/test_.py @@ -14,7 +14,6 @@ import toml TEST_PATHS = ['tests_'] SUITE_PROLOGUE = """ -//////// AUTOGENERATED //////// #include "runners/test_runner.h" #include """ @@ -222,7 +221,21 @@ def compile(**args): suite = TestSuite(paths[0]) if 'output' in args: with openio(args['output'], 'w') as f: - f.write(SUITE_PROLOGUE) + # redirect littlefs tracing + f.write('#define LFS_TRACE_(fmt, ...) do { \\\n') + f.write(8*' '+'extern FILE *test_trace; \\\n') + f.write(8*' '+'if (test_trace) { \\\n') + f.write(12*' '+'fprintf(test_trace, ' + '"%s:%d:trace: " fmt "%s\\n", \\\n') + f.write(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\\n') + f.write(8*' '+'} \\\n') + f.write(4*' '+'} while (0)\n') + f.write('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")\n') + f.write('#define LFS_TESTBD_TRACE(...) ' + 'LFS_TRACE_(__VA_ARGS__, "")\n') + f.write('\n') + + f.write('%s\n' % SUITE_PROLOGUE.strip()) f.write('\n') if suite.code is not None: if suite.code_lineno is not None: @@ -380,6 +393,21 @@ def compile(**args): # write out a test source if 'output' in args: with openio(args['output'], 'w') as f: + # redirect littlefs tracing + f.write('#define LFS_TRACE_(fmt, ...) do { \\\n') + f.write(8*' '+'extern FILE *test_trace; \\\n') + f.write(8*' '+'if (test_trace) { \\\n') + f.write(12*' '+'fprintf(test_trace, ' + '"%s:%d:trace: " fmt "%s\\n", \\\n') + f.write(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\\n') + f.write(8*' '+'} \\\n') + f.write(4*' '+'} while (0)\n') + f.write('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")\n') + f.write('#define LFS_TESTBD_TRACE(...) ' + 'LFS_TRACE_(__VA_ARGS__, "")\n') + f.write('\n') + + # copy source f.write('#line 1 "%s"\n' % args['source']) with open(args['source']) as sf: shutil.copyfileobj(sf, f) From 64436933e2845da3d691eaebb558fc9939da6af3 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 24 Apr 2022 01:45:22 -0500 Subject: [PATCH 06/81] Putting together rewritten test.py script --- Makefile | 12 +- runners/test_runner.c | 6 +- scripts/test_.py | 407 +++++++++++++++++++++++++++++++++++++++++- tests/test_files.toml | 2 +- 4 files changed, 412 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 943735bc..04fe8bdc 100644 --- a/Makefile +++ b/Makefile @@ -62,12 +62,15 @@ override DATAFLAGS += -v override STACKFLAGS += -v override STRUCTSFLAGS += -v override COVERAGEFLAGS += -v +override TESTFLAGS_ += -v +override TESTCFLAGS_ += -v endif ifdef EXEC -override TESTFLAGS += --exec="$(EXEC)" +override TESTFLAGS_ += --exec="$(EXEC)" endif ifdef COVERAGE override TESTFLAGS += --coverage +override TESTFLAGS_ += --coverage endif ifdef BUILDDIR override TESTFLAGS += --build-dir="$(BUILDDIR:/=)" @@ -85,6 +88,8 @@ endif ifneq ($(OBJDUMP),objdump) override STRUCTSFLAGS += --objdump-tool="$(OBJDUMP)" endif +# forward -j flag +override TESTFLAGS_ += $(filter -j%,$(MAKEFLAGS)) # commands @@ -115,6 +120,7 @@ test%: tests/test$$(firstword $$(subst \#, ,%)).toml .PHONY: test_ test_: $(BUILDDIR)runners/test_runner + ./scripts/test_.py --runner=$< $(TESTFLAGS_) .PHONY: code code: $(OBJ) @@ -185,10 +191,10 @@ $(BUILDDIR)%.a.c: $(BUILDDIR)%.c ./scripts/explode_asserts.py $< -o $@ $(BUILDDIR)%.t.c: %.toml - ./scripts/test_.py -c $< -o $@ + ./scripts/test_.py -c $< $(TESTCFLAGS_) -o $@ $(BUILDDIR)%.t.c: %.c $(TESTS) - ./scripts/test_.py -c $(TESTS) -s $< -o $@ + ./scripts/test_.py -c $(TESTS) -s $< $(TESTCFLAGS_) -o $@ # clean everything .PHONY: clean diff --git a/runners/test_runner.c b/runners/test_runner.c index 185fa8aa..1406574e 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -29,9 +29,7 @@ struct test_geometry { const struct test_geometry test_geometries[] = { // Made up geometry that works well for testing - TEST_GEOMETRY("small", 16, 16, 512, (1024*1024)/512), - TEST_GEOMETRY("medium", 16, 16, 4096, (1024*1024)/4096), - TEST_GEOMETRY("big", 16, 16, 32*1024, (1024*1024)/(32*1024)), + TEST_GEOMETRY("test", 16, 16, 512, (1024*1024)/512), // EEPROM/NVRAM TEST_GEOMETRY("eeprom", 1, 1, 512, (1024*1024)/512), // SD/eMMC @@ -585,7 +583,7 @@ const char *const help_text[] = { "Filter by geometry.", "Filter for normal tests. Can be combined.", "Filter for reentrant tests. Can be combined.", - "Filter for valgrind tests. Can be combined.", + "Filter for Valgrind tests. Can be combined.", "Skip the first n tests.", "Stop after n tests.", "Only run every n tests, calculated after --skip and --stop.", diff --git a/scripts/test_.py b/scripts/test_.py index e6458760..8d6e34ec 100755 --- a/scripts/test_.py +++ b/scripts/test_.py @@ -3,22 +3,30 @@ # Script to compile and runs tests. # +import collections as co +import errno import glob import itertools as it import math as m import os +import pty import re +import shlex import shutil +import subprocess as sp +import threading as th +import time import toml + TEST_PATHS = ['tests_'] +RUNNER_PATH = './runners/test_runner' SUITE_PROLOGUE = """ #include "runners/test_runner.h" #include """ CASE_PROLOGUE = """ -lfs_t lfs; """ CASE_EPILOGUE = """ """ @@ -194,13 +202,13 @@ class TestSuite: def id(self): return self.name - + def compile(**args): # find .toml files paths = [] - for path in args['test_paths']: + for path in args.get('test_paths', TEST_PATHS): if os.path.isdir(path): path = path + '/*.toml' @@ -429,26 +437,411 @@ def compile(**args): f.write('const size_t test_suite_count = %d;\n' % len(suites)) +def runner(**args): + cmd = args['runner'].copy() + # TODO multiple paths? + if 'test_paths' in args: + cmd.extend(args.get('test_paths')) + + if args.get('normal'): cmd.append('-n') + if args.get('reentrant'): cmd.append('-r') + if args.get('valgrind'): cmd.append('-V') + if args.get('geometry'): + cmd.append('-G%s' % args.get('geometry')) + if args.get('define'): + for define in args.get('define'): + cmd.append('-D%s' % define) + + return cmd + +def list_(**args): + cmd = runner(**args) + if args.get('summary'): cmd.append('--summary') + if args.get('list_suites'): cmd.append('--list-suites') + if args.get('list_cases'): cmd.append('--list-cases') + if args.get('list_paths'): cmd.append('--list-paths') + if args.get('list_defines'): cmd.append('--list-defines') + if args.get('list_geometries'): cmd.append('--list-geometries') + + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + sys.exit(sp.call(cmd)) + + +def find_cases(runner_, **args): + # first get suite/case/perm counts + cmd = runner_ + ['--list-cases'] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace') + expected_suite_perms = co.defaultdict(lambda: 0) + expected_case_perms = co.defaultdict(lambda: 0) + expected_perms = 0 + total_perms = 0 + pattern = re.compile( + '^(?P(?P[^#]+)#[^ #]+) +' + '[^ ]+ +[^ ]+ +[^ ]+ +' + '(?P[0-9]+)/(?P[0-9]+)$') + # skip the first line + next(proc.stdout) + for line in proc.stdout: + m = pattern.match(line) + if m: + filtered = int(m.group('filtered')) + expected_suite_perms[m.group('suite')] += filtered + expected_case_perms[m.group('id')] += filtered + expected_perms += filtered + total_perms += int(m.group('perms')) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return ( + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) + +class TestFailure(Exception): + def __init__(self, id, returncode, stdout, assert_=None): + self.id = id + self.returncode = returncode + self.stdout = stdout + self.assert_ = assert_ + +def run_step(name, runner_, **args): + # get expected suite/case/perm counts + expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( + find_cases(runner_, **args)) + + # TODO persist/trace + # TODO valgrind/gdb/exec + passed_suite_perms = co.defaultdict(lambda: 0) + passed_case_perms = co.defaultdict(lambda: 0) + passed_perms = 0 + failures = [] + + pattern = re.compile('^(?:' + '(?Prunning|finished|skipped) ' + '(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)' + '|' '(?P[^:]+):(?P[0-9]+):(?Passert):' + ' *(?P.*)' ')$') + locals = th.local() + # TODO use process group instead of this set? + children = set() + + def run_runner(runner_): + nonlocal passed_suite_perms + nonlocal passed_case_perms + nonlocal passed_perms + nonlocal locals + + # run the tests! + cmd = runner_ + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + mpty, spty = pty.openpty() + proc = sp.Popen(cmd, stdout=spty, stderr=spty) + os.close(spty) + mpty = os.fdopen(mpty, 'r', 1) + children.add(proc) + + last_id = None + last_stdout = [] + last_assert = None + try: + while True: + # parse a line for state changes + try: + line = mpty.readline() + except OSError as e: + if e.errno == errno.EIO: + break + raise + if not line: + break + last_stdout.append(line) + if args.get('verbose'): + sys.stdout.write(line) + + m = pattern.match(line) + if m: + op = m.group('op') or m.group('op_') + if op == 'running': + locals.seen_perms += 1 + last_id = m.group('id') + last_stdout = [] + last_assert = None + elif op == 'finished': + passed_suite_perms[m.group('suite')] += 1 + passed_case_perms[m.group('case')] += 1 + passed_perms += 1 + elif op == 'skipped': + locals.seen_perms += 1 + elif op == 'assert': + last_assert = ( + m.group('path'), + int(m.group('lineno')), + m.group('message')) + # TODO why is kill _so_ much faster than terminate? + proc.kill() + except KeyboardInterrupt: + raise TestFailure(last_id, 1, last_stdout) + finally: + children.remove(proc) + mpty.close() + + proc.wait() + if proc.returncode != 0: + raise TestFailure( + last_id, + proc.returncode, + last_stdout, + last_assert) + + def run_job(runner, skip=None, every=None): + nonlocal failures + nonlocal locals + + while (skip or 0) < total_perms: + runner_ = runner.copy() + if skip is not None: + runner_.append('--skip=%d' % skip) + if every is not None: + runner_.append('--every=%d' % every) + + try: + # run the tests + locals.seen_perms = 0 + run_runner(runner_) + + except TestFailure as failure: + # race condition for multiple failures? + if failures and not args.get('keep_going'): + break + + failures.append(failure) + + if args.get('keep_going'): + # resume after failed test + skip = (skip or 0) + locals.seen_perms*(every or 1) + continue + else: + # stop other tests + for child in children: + # TODO why is kill _so_ much faster than terminate? + child.kill() + + break + + + # parallel jobs? + runners = [] + if 'jobs' in args: + for job in range(args['jobs']): + runners.append(th.Thread( + target=run_job, args=(runner_, job, args['jobs']))) + else: + runners.append(th.Thread( + target=run_job, args=(runner_, None, None))) + + for r in runners: + r.start() + + needs_newline = False + try: + while any(r.is_alive() for r in runners): + time.sleep(0.01) + + if not args.get('verbose'): + sys.stdout.write('\r\x1b[K' + 'running \x1b[%dm%s\x1b[m: ' + '%d/%d suites, %d/%d cases, %d/%d perms%s ' + % (32 if not failures else 31, + name, + sum(passed_suite_perms[k] == v + for k, v in expected_suite_perms.items()), + len(expected_suite_perms), + sum(passed_case_perms[k] == v + for k, v in expected_case_perms.items()), + len(expected_case_perms), + passed_perms, + expected_perms, + ', \x1b[31m%d/%d failures\x1b[m' + % (len(failures), expected_perms) + if failures else '')) + sys.stdout.flush() + needs_newline = True + finally: + if needs_newline: + print() + + for r in runners: + r.join() + + return ( + expected_perms, + passed_perms, + failures) + + def run(**args): - pass + start = time.time() + + runner_ = runner(**args) + print('using runner `%s`' + % ' '.join(shlex.quote(c) for c in runner_)) + expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( + find_cases(runner_, **args)) + print('found %d suites, %d cases, %d/%d permutations' + % (len(expected_suite_perms), + len(expected_case_perms), + expected_perms, + total_perms)) + print() + + expected = 0 + passed = 0 + failures = [] + if args.get('by_suites'): + for type in ['normal', 'reentrant', 'valgrind']: + for suite in expected_suite_perms.keys(): + expected_, passed_, failures_ = run_step( + '%s %s' % (type, suite), + runner_ + ['--%s' % type, suite], + **args) + expected += expected_ + passed += passed_ + failures.extend(failures_) + if failures and not args.get('keep_going'): + break + if failures and not args.get('keep_going'): + break + elif args.get('by_cases'): + for type in ['normal', 'reentrant', 'valgrind']: + for case in expected_case_perms.keys(): + expected_, passed_, failures_ = run_step( + '%s %s' % (type, case), + runner_ + ['--%s' % type, case], + **args) + expected += expected_ + passed += passed_ + failures.extend(failures_) + if failures and not args.get('keep_going'): + break + if failures and not args.get('keep_going'): + break + else: + for type in ['normal', 'reentrant', 'valgrind']: + expected_, passed_, failures_ = run_step( + '%s tests' % type, + runner_ + ['--%s' % type], + **args) + expected += expected_ + passed += passed_ + failures.extend(failures_) + if failures and not args.get('keep_going'): + break + + # show summary + print() + print('\x1b[%dmdone\x1b[m: %d/%d passed, %d/%d failed, in %.2fs' + % (32 if not failures else 31, + passed, expected, len(failures), expected, + time.time()-start)) + print() + + # print each failure + # TODO get line, defines, path + for failure in failures: +# print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s failed' +# # TODO this should be the suite path and lineno +# % (failure.assert + + if failure.assert_ is not None: + path, lineno, message = failure.assert_ + print('\x1b[01m%s:%d:\x1b[01;31massert:\x1b[m %s' + % (path, lineno, message)) + with open(path) as f: + line = next(it.islice(f, lineno-1, None)).strip('\n') + print(line) + print() + + return 1 if failures else 0 + def main(**args): if args.get('compile'): compile(**args) + elif (args.get('summary') + or args.get('list_suites') + or args.get('list_cases') + or args.get('list_paths') + or args.get('list_defines') + or args.get('list_geometries')): + list_(**args) else: run(**args) + if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( description="Build and run tests.") # TODO document test case/perm specifier - parser.add_argument('test_paths', nargs='*', default=TEST_PATHS, - help="Description of test(s) to run. May be a directory, a path, or \ - test identifier. Defaults to all tests in %r." % TEST_PATHS) + parser.add_argument('test_paths', nargs='*', + help="Description of testis to run. May be a directory, path, or \ + test identifier. Defaults to %r." % TEST_PATHS) + parser.add_argument('-v', '--verbose', action='store_true', + help="Output commands that run behind the scenes.") # test flags test_parser = parser.add_argument_group('test options') + test_parser.add_argument('-Y', '--summary', action='store_true', + help="Show quick summary.") + test_parser.add_argument('-l', '--list-suites', action='store_true', + help="List test suites.") + test_parser.add_argument('-L', '--list-cases', action='store_true', + help="List test cases.") + test_parser.add_argument('--list-paths', action='store_true', + help="List the path for each test case.") + test_parser.add_argument('--list-defines', action='store_true', + help="List the defines for each test permutation.") + test_parser.add_argument('--list-geometries', action='store_true', + help="List the disk geometries used for testing.") + test_parser.add_argument('-D', '--define', action='append', + help="Override a test define.") + test_parser.add_argument('-G', '--geometry', + help="Filter by geometry.") + test_parser.add_argument('-n', '--normal', action='store_true', + help="Filter for normal tests. Can be combined.") + test_parser.add_argument('-r', '--reentrant', action='store_true', + help="Filter for reentrant tests. Can be combined.") + test_parser.add_argument('-V', '--valgrind', action='store_true', + help="Filter for Valgrind tests. Can be combined.") + test_parser.add_argument('-p', '--persist', + help="Persist the disk to this file.") + test_parser.add_argument('-t', '--trace', + help="Redirect trace output to this file.") + test_parser.add_argument('--runner', default=[RUNNER_PATH], + type=lambda x: x.split(), + help="Path to runner, defaults to %r" % RUNNER_PATH) + test_parser.add_argument('-j', '--jobs', nargs='?', type=int, + const=len(os.sched_getaffinity(0)), + help="Number of parallel runners to run.") + test_parser.add_argument('-k', '--keep-going', action='store_true', + help="Don't stop on first error.") + test_parser.add_argument('-b', '--by-suites', action='store_true', + help="Step through tests by suite.") + test_parser.add_argument('-B', '--by-cases', action='store_true', + help="Step through tests by case.") # compilation flags comp_parser = parser.add_argument_group('compilation options') comp_parser.add_argument('-c', '--compile', action='store_true', diff --git a/tests/test_files.toml b/tests/test_files.toml index 565e665b..54630546 100644 --- a/tests/test_files.toml +++ b/tests/test_files.toml @@ -65,7 +65,7 @@ code = ''' lfs_format(&lfs, &cfg) => 0; // write - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, &cfg) => 1; lfs_file_open(&lfs, &file, "avacado", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; srand(1); From 5812d2b5cfdbab12491c6eb00340a877fc9fa8de Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 24 Apr 2022 23:34:28 -0500 Subject: [PATCH 07/81] Reworked how multi-layered defines work in the test-runner In the test-runner, defines are parameterized constants (limited to integers) that are generated from the test suite tomls resulting in many permutations of each test. In order to make this efficient, these defines are implemented as multi-layered lookup tables, using per-layer/per-scope indirect mappings. This lets the test-runner and test suites define their own defines with compile-time indexes independently. It also makes building of the lookup tables very efficient, since they can be incrementally populated as we expand the test permutations. The four current define layers and when we need to build them: layer defines predefine_map define_map user-provided overrides per-run per-run per-suite per-permutation defines per-perm per-case per-perm per-geometry defines per-perm compile-time - default defines compile-time compile-time - --- .gitignore | 1 + Makefile | 5 +- runners/test_runner.c | 391 ++++++++++++++++++++---------------- runners/test_runner.h | 63 +++++- scripts/test_.py | 454 ++++++++++++++++++++++++------------------ 5 files changed, 541 insertions(+), 373 deletions(-) diff --git a/.gitignore b/.gitignore index 6bee32df..abd7eb8a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ test.c tests/*.toml.* scripts/__pycache__ .gdb_history +runners/test_runner diff --git a/Makefile b/Makefile index 04fe8bdc..513187dc 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,9 @@ override CFLAGS += -I. override CFLAGS += -std=c99 -Wall -pedantic override CFLAGS += -Wextra -Wshadow -Wjump-misses-init -Wundef +override TESTFLAGS_ += -b +# forward -j flag +override TESTFLAGS_ += $(filter -j%,$(MAKEFLAGS)) ifdef VERBOSE override TESTFLAGS += -v override CALLSFLAGS += -v @@ -88,8 +91,6 @@ endif ifneq ($(OBJDUMP),objdump) override STRUCTSFLAGS += --objdump-tool="$(OBJDUMP)" endif -# forward -j flag -override TESTFLAGS_ += $(filter -j%,$(MAKEFLAGS)) # commands diff --git a/runners/test_runner.c b/runners/test_runner.c index 1406574e..1c9b2550 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -6,106 +6,151 @@ #include #include -// disk geometries + +// test geometries struct test_geometry { const char *name; - const test_define_t *defines; + test_define_t defines[TEST_GEOMETRY_DEFINE_COUNT]; }; -// Note this includes the default configuration for test pre-defines -#define TEST_GEOMETRY(name, read, prog, erase, count) \ - {name, (const test_define_t[]){ \ - /* READ_SIZE */ read, \ - /* PROG_SIZE */ prog, \ - /* BLOCK_SIZE */ erase, \ - /* BLOCK_COUNT */ count, \ - /* BLOCK_CYCLES */ -1, \ - /* CACHE_SIZE */ (64 % (prog) == 0) ? 64 : (prog), \ - /* LOOKAHEAD_SIZE */ 16, \ - /* ERASE_VALUE */ 0xff, \ - /* ERASE_CYCLES */ 0, \ - /* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \ - }} - -const struct test_geometry test_geometries[] = { - // Made up geometry that works well for testing - TEST_GEOMETRY("test", 16, 16, 512, (1024*1024)/512), - // EEPROM/NVRAM - TEST_GEOMETRY("eeprom", 1, 1, 512, (1024*1024)/512), - // SD/eMMC - TEST_GEOMETRY("emmc", 512, 512, 512, (1024*1024)/512), - // NOR flash - TEST_GEOMETRY("nor", 1, 1, 4096, (1024*1024)/4096), - // NAND flash - TEST_GEOMETRY("nand", 4096, 4096, 32*1024, (1024*1024)/(32*1024)), -}; - -const size_t test_geometry_count = ( - sizeof(test_geometries) / sizeof(test_geometries[0])); - +const struct test_geometry test_geometries[TEST_GEOMETRY_COUNT] + = TEST_GEOMETRIES; // test define lookup and management -const test_define_t *test_defines[3] = {NULL}; -const uint8_t *test_define_maps[2] = {NULL}; +#define TEST_DEFINE_LAYERS 4 +const test_define_t *test_defines[TEST_DEFINE_LAYERS] = { + NULL, + NULL, + NULL, + (const test_define_t[TEST_DEFAULT_COUNT])TEST_DEFAULTS, +}; + +const uint8_t *test_predefine_maps[TEST_DEFINE_LAYERS] = { + NULL, + NULL, + (const uint8_t[TEST_PREDEFINE_COUNT])TEST_GEOMETRY_DEFINE_MAP, + (const uint8_t[TEST_PREDEFINE_COUNT])TEST_DEFAULT_MAP, +}; + +const uint8_t *test_define_maps[TEST_DEFINE_LAYERS] = { + NULL, + NULL, + NULL, + NULL, +}; + +uint8_t test_override_predefine_map[TEST_PREDEFINE_COUNT]; +uint8_t test_override_define_map[256]; +uint8_t test_case_predefine_map[TEST_PREDEFINE_COUNT]; + +const char *const *test_override_names; +size_t test_override_count; + +const char *const test_predefine_names[TEST_PREDEFINE_COUNT] + = TEST_PREDEFINE_NAMES; + +const char *const *test_define_names; +size_t test_define_count; + + +test_define_t test_predefine(size_t define) { + for (int i = 0; i < TEST_DEFINE_LAYERS; i++) { + if (test_defines[i] + && test_predefine_maps[i] + && test_predefine_maps[i][define] != 0xff) { + return test_defines[i][test_predefine_maps[i][define]]; + } + } + + fprintf(stderr, "error: undefined predefine %s\n", + test_predefine_names[define]); + assert(false); + exit(-1); +} test_define_t test_define(size_t define) { - if (test_define_maps[0] && test_define_maps[0][define] != 0xff) { - return test_defines[0][test_define_maps[0][define]]; - } else if (test_define_maps[1] && test_define_maps[1][define] != 0xff) { - return test_defines[1][test_define_maps[1][define]]; - } else { - return test_defines[2][define]; + for (int i = 0; i < TEST_DEFINE_LAYERS; i++) { + if (test_defines[i] + && test_define_maps[i] + && test_define_maps[i][define] != 0xff) { + return test_defines[i][test_define_maps[i][define]]; + } } + + fprintf(stderr, "error: undefined define %s\n", + test_define_names[define]); + assert(false); + exit(-1); } static void test_define_geometry(const struct test_geometry *geometry) { - if (geometry) { - test_defines[2] = geometry->defines; - } else { - test_defines[2] = NULL; - } -} - -static void test_define_case(const struct test_case *case_, size_t perm) { - if (case_ && case_->defines) { - test_defines[1] = case_->defines[perm]; - test_define_maps[1] = case_->define_map; - } else { - test_defines[1] = NULL; - test_define_maps[1] = NULL; - } + test_defines[2] = geometry->defines; } static void test_define_overrides( - const struct test_suite *suite, const char *const *override_names, const test_define_t *override_defines, size_t override_count) { - if (override_names && override_defines && override_count > 0) { - uint8_t *define_map = malloc(suite->define_count * sizeof(uint8_t)); - memset(define_map, 0xff, suite->define_count * sizeof(bool)); + test_defines[0] = override_defines; + test_override_names = override_names; + test_override_count = override_count; - // lookup each override in the suite defines, they may have a - // different index in each suite - for (size_t i = 0; i < override_count; i++) { - size_t j = 0; - for (; j < suite->define_count; j++) { - if (strcmp(override_names[i], suite->define_names[j]) == 0) { - break; - } - } - - if (j < suite->define_count) { - define_map[j] = i; + // map any predefines + memset(test_override_predefine_map, 0xff, TEST_PREDEFINE_COUNT); + for (size_t i = 0; i < override_count; i++) { + for (size_t j = 0; j < TEST_PREDEFINE_COUNT; j++) { + if (strcmp(override_names[i], test_predefine_names[j]) == 0) { + test_override_predefine_map[j] = i; } } + } + test_predefine_maps[0] = test_override_predefine_map; +} - test_defines[0] = override_defines; - test_define_maps[0] = define_map; +static void test_define_suite(const struct test_suite *suite) { + test_define_names = suite->define_names; + test_define_count = suite->define_count; + + // map any defines + memset(test_override_define_map, 0xff, suite->define_count); + for (size_t i = 0; i < test_override_count; i++) { + for (size_t j = 0; j < suite->define_count; j++) { + if (strcmp(test_override_names[i], suite->define_names[j]) == 0) { + test_override_define_map[j] = i; + } + } + } + test_define_maps[0] = test_override_define_map; +} + +static void test_define_case( + const struct test_suite *suite, + const struct test_case *case_) { + (void)suite; + // case_->define_map is already correct, but we need to do + // some fixup for the predefine map + test_define_maps[1] = case_->define_map; + + memset(test_case_predefine_map, 0xff, TEST_PREDEFINE_COUNT); + for (size_t i = 0; i < test_define_count; i++) { + for (size_t j = 0; j < TEST_PREDEFINE_COUNT; j++) { + if (strcmp(test_define_names[i], test_predefine_names[j]) == 0) { + test_case_predefine_map[j] = case_->define_map[i]; + } + } + } + test_predefine_maps[1] = test_case_predefine_map; +} + +static void test_define_perm( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm) { + (void)suite; + if (case_->defines) { + test_defines[1] = case_->defines[perm]; } else { - test_defines[0] = NULL; - free((uint8_t *)test_define_maps[0]); - test_define_maps[0] = NULL; + test_defines[1] = NULL; } } @@ -120,11 +165,6 @@ static size_t test_skip = 0; static size_t test_count = -1; static size_t test_every = 1; -static const char **override_names = NULL; -static test_define_t *override_defines = NULL; -static size_t override_count = 0; -static size_t override_cap = 0; - static const char *test_persist = NULL; FILE *test_trace = NULL; @@ -140,7 +180,7 @@ static bool test_case_skip(const struct test_case *case_) { } static bool test_perm_skip(size_t perm) { - size_t geom_perm = perm % test_geometry_count; + size_t geom_perm = perm % TEST_GEOMETRY_COUNT; return (test_perm != (size_t)-1 && perm != test_perm) || (test_geometry && (strcmp( test_geometries[geom_perm].name, @@ -153,7 +193,8 @@ static bool test_step_skip(size_t step) { && (step-test_skip) % test_every == 0); } -static void test_case_sumpermutations( +static void test_case_permcount( + const struct test_suite *suite, const struct test_case *case_, size_t *perms, size_t *filtered) { @@ -161,7 +202,7 @@ static void test_case_sumpermutations( size_t filtered_ = 0; for (size_t perm = 0; - perm < test_geometry_count + perm < TEST_GEOMETRY_COUNT * case_->permutations; perm++) { if (test_perm_skip(perm)) { @@ -171,23 +212,18 @@ static void test_case_sumpermutations( perms_ += 1; // setup defines - size_t case_perm = perm / test_geometry_count; - size_t geom_perm = perm % test_geometry_count; + size_t case_perm = perm / TEST_GEOMETRY_COUNT; + size_t geom_perm = perm % TEST_GEOMETRY_COUNT; + test_define_perm(suite, case_, case_perm); test_define_geometry(&test_geometries[geom_perm]); - test_define_case(case_, case_perm); if (case_->filter) { if (!case_->filter(case_perm)) { - test_define_geometry(NULL); - test_define_case(NULL, 0); continue; } } filtered_ += 1; - - test_define_geometry(NULL); - test_define_case(NULL, 0); } *perms += perms_; @@ -208,21 +244,18 @@ static void summary(void) { continue; } - test_define_overrides( - test_suites[i], - override_names, override_defines, override_count); + test_define_suite(test_suites[i]); for (size_t j = 0; j < test_suites[i]->case_count; j++) { if (test_case_skip(test_suites[i]->cases[j])) { continue; } - test_case_sumpermutations(test_suites[i]->cases[j], + test_define_case(test_suites[i], test_suites[i]->cases[j]); + test_case_permcount(test_suites[i], test_suites[i]->cases[j], &perms, &filtered); } - test_define_overrides(NULL, NULL, NULL, 0); - cases += test_suites[i]->case_count; types |= test_suites[i]->types; } @@ -243,16 +276,13 @@ static void summary(void) { } static void list_suites(void) { - printf("%-36s %-12s %7s %7s %11s\n", - "id", "suite", "types", "cases", "perms"); + printf("%-36s %7s %7s %11s\n", "suite", "types", "cases", "perms"); for (size_t i = 0; i < test_suite_count; i++) { if (test_suite_skip(test_suites[i])) { continue; } - test_define_overrides( - test_suites[i], - override_names, override_defines, override_count); + test_define_suite(test_suites[i]); size_t perms = 0; size_t filtered = 0; @@ -261,12 +291,11 @@ static void list_suites(void) { continue; } - test_case_sumpermutations(test_suites[i]->cases[j], + test_define_case(test_suites[i], test_suites[i]->cases[j]); + test_case_permcount(test_suites[i], test_suites[i]->cases[j], &perms, &filtered); } - test_define_overrides(NULL, NULL, NULL, 0); - char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", filtered, perms); char type_buf[64]; @@ -274,9 +303,8 @@ static void list_suites(void) { (test_suites[i]->types & TEST_NORMAL) ? "n" : "", (test_suites[i]->types & TEST_REENTRANT) ? "r" : "", (test_suites[i]->types & TEST_VALGRIND) ? "V" : ""); - printf("%-36s %-12s %7s %7zu %11s\n", + printf("%-36s %7s %7zu %11s\n", test_suites[i]->id, - test_suites[i]->name, type_buf, test_suites[i]->case_count, perm_buf); @@ -284,25 +312,24 @@ static void list_suites(void) { } static void list_cases(void) { - printf("%-36s %-12s %-12s %7s %11s\n", - "id", "suite", "case", "types", "perms"); + printf("%-36s %7s %11s\n", "case", "types", "perms"); for (size_t i = 0; i < test_suite_count; i++) { if (test_suite_skip(test_suites[i])) { continue; } - test_define_overrides( - test_suites[i], - override_names, override_defines, override_count); + test_define_suite(test_suites[i]); for (size_t j = 0; j < test_suites[i]->case_count; j++) { if (test_case_skip(test_suites[i]->cases[j])) { continue; } + test_define_case(test_suites[i], test_suites[i]->cases[j]); + size_t perms = 0; size_t filtered = 0; - test_case_sumpermutations(test_suites[i]->cases[j], + test_case_permcount(test_suites[i], test_suites[i]->cases[j], &perms, &filtered); test_types_t types = test_suites[i]->cases[j]->types; @@ -313,20 +340,15 @@ static void list_cases(void) { (types & TEST_NORMAL) ? "n" : "", (types & TEST_REENTRANT) ? "r" : "", (types & TEST_VALGRIND) ? "V" : ""); - printf("%-36s %-12s %-12s %7s %11s\n", + printf("%-36s %7s %11s\n", test_suites[i]->cases[j]->id, - test_suites[i]->name, - test_suites[i]->cases[j]->name, type_buf, perm_buf); } - - test_define_overrides(NULL, NULL, NULL, 0); } } static void list_paths(void) { - printf("%-36s %-36s\n", "id", "path"); for (size_t i = 0; i < test_suite_count; i++) { if (test_suite_skip(test_suites[i])) { continue; @@ -345,23 +367,22 @@ static void list_paths(void) { } static void list_defines(void) { - printf("%-36s %s\n", "id", "defines"); for (size_t i = 0; i < test_suite_count; i++) { if (test_suite_skip(test_suites[i])) { continue; } - test_define_overrides( - test_suites[i], - override_names, override_defines, override_count); + test_define_suite(test_suites[i]); for (size_t j = 0; j < test_suites[i]->case_count; j++) { if (test_case_skip(test_suites[i]->cases[j])) { continue; } + test_define_case(test_suites[i], test_suites[i]->cases[j]); + for (size_t perm = 0; - perm < test_geometry_count + perm < TEST_GEOMETRY_COUNT * test_suites[i]->cases[j]->permutations; perm++) { if (test_perm_skip(perm)) { @@ -369,39 +390,38 @@ static void list_defines(void) { } // setup defines - size_t case_perm = perm / test_geometry_count; - size_t geom_perm = perm % test_geometry_count; + size_t case_perm = perm / TEST_GEOMETRY_COUNT; + size_t geom_perm = perm % TEST_GEOMETRY_COUNT; + test_define_perm(test_suites[i], + test_suites[i]->cases[j], case_perm); test_define_geometry(&test_geometries[geom_perm]); - test_define_case(test_suites[i]->cases[j], case_perm); - // print each define + // print the case char id_buf[256]; sprintf(id_buf, "%s#%zu", test_suites[i]->cases[j]->id, perm); printf("%-36s ", id_buf); - for (size_t k = 0; k < test_suites[i]->define_count; k++) { - if (k >= TEST_PREDEFINE_COUNT && ( - !test_suites[i]->cases[j]->define_map - || test_suites[i]->cases[j]->define_map[k] - == 0xff)) { - continue; - } - printf("%s=%jd ", - test_suites[i]->define_names[k], - test_define(k)); + // special case for the current geometry + printf("GEOMETRY=%s ", test_geometries[geom_perm].name); + + // print each define + for (size_t k = 0; k < test_suites[i]->define_count; k++) { + if (test_suites[i]->cases[j]->define_map + && test_suites[i]->cases[j]->define_map[k] + != 0xff) { + printf("%s=%jd ", + test_suites[i]->define_names[k], + test_define(k)); + } } printf("\n"); } } - - test_define_overrides(NULL, NULL, NULL, 0); } } static void list_geometries(void) { - printf("%-36s %7s %7s %7s %7s %7s\n", - "name", "read", "prog", "erase", "count", "size"); - for (size_t i = 0; i < test_geometry_count; i++) { + for (size_t i = 0; i < TEST_GEOMETRY_COUNT; i++) { if (test_geometry && strcmp( test_geometries[i].name, test_geometry) != 0) { @@ -410,16 +430,33 @@ static void list_geometries(void) { test_define_geometry(&test_geometries[i]); - printf("%-36s %7ju %7ju %7ju %7ju %7ju\n", - test_geometries[i].name, - READ_SIZE, - PROG_SIZE, - BLOCK_SIZE, - BLOCK_COUNT, - BLOCK_SIZE*BLOCK_COUNT); + printf("%-36s ", test_geometries[i].name); + // print each define + for (size_t k = 0; k < TEST_PREDEFINE_COUNT; k++) { + if (test_predefine_maps[2][k] != 0xff) { + printf("%s=%jd ", + test_predefine_names[k], + test_predefine(k)); + } + } + printf("\n"); + } } +static void list_defaults(void) { + printf("%-36s ", "defaults"); + // print each define + for (size_t k = 0; k < TEST_PREDEFINE_COUNT; k++) { + if (test_predefine_maps[3][k] != 0xff) { + printf("%s=%jd ", + test_predefine_names[k], + test_predefine(k)); + } + } + printf("\n"); +} + static void run(void) { size_t step = 0; for (size_t i = 0; i < test_suite_count; i++) { @@ -427,23 +464,22 @@ static void run(void) { continue; } - test_define_overrides( - test_suites[i], - override_names, override_defines, override_count); + test_define_suite(test_suites[i]); for (size_t j = 0; j < test_suites[i]->case_count; j++) { if (test_case_skip(test_suites[i]->cases[j])) { continue; } + test_define_case(test_suites[i], test_suites[i]->cases[j]); + for (size_t perm = 0; - perm < test_geometry_count + perm < TEST_GEOMETRY_COUNT * test_suites[i]->cases[j]->permutations; perm++) { if (test_perm_skip(perm)) { continue; } - if (test_step_skip(step)) { step += 1; continue; @@ -451,10 +487,11 @@ static void run(void) { step += 1; // setup defines - size_t case_perm = perm / test_geometry_count; - size_t geom_perm = perm % test_geometry_count; + size_t case_perm = perm / TEST_GEOMETRY_COUNT; + size_t geom_perm = perm % TEST_GEOMETRY_COUNT; + test_define_perm(test_suites[i], + test_suites[i]->cases[j], case_perm); test_define_geometry(&test_geometries[geom_perm]); - test_define_case(test_suites[i]->cases[j], case_perm); // filter? if (test_suites[i]->cases[j]->filter) { @@ -462,8 +499,6 @@ static void run(void) { printf("skipped %s#%zu\n", test_suites[i]->cases[j]->id, perm); - test_define_geometry(NULL); - test_define_case(NULL, 0); continue; } } @@ -514,13 +549,8 @@ static void run(void) { "could not destroy block device: %d\n", err); exit(-1); } - - test_define_geometry(NULL); - test_define_case(NULL, 0); } } - - test_define_overrides(NULL, NULL, NULL, 0); } } @@ -536,14 +566,15 @@ enum opt_flags { OPT_LIST_PATHS = 1, OPT_LIST_DEFINES = 2, OPT_LIST_GEOMETRIES = 3, + OPT_LIST_DEFAULTS = 4, OPT_DEFINE = 'D', OPT_GEOMETRY = 'G', OPT_NORMAL = 'n', OPT_REENTRANT = 'r', OPT_VALGRIND = 'V', - OPT_SKIP = 4, - OPT_COUNT = 5, - OPT_EVERY = 6, + OPT_SKIP = 5, + OPT_COUNT = 6, + OPT_EVERY = 7, OPT_PERSIST = 'p', OPT_TRACE = 't', }; @@ -558,6 +589,7 @@ const struct option long_opts[] = { {"list-paths", no_argument, NULL, OPT_LIST_PATHS}, {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, + {"list-defaults", no_argument, NULL, OPT_LIST_DEFAULTS}, {"define", required_argument, NULL, OPT_DEFINE}, {"geometry", required_argument, NULL, OPT_GEOMETRY}, {"normal", no_argument, NULL, OPT_NORMAL}, @@ -579,6 +611,7 @@ const char *const help_text[] = { "List the path for each test case.", "List the defines for each test permutation.", "List the disk geometries used for testing.", + "List the default defines in this test-runner.", "Override a test define.", "Filter by geometry.", "Filter for normal tests. Can be combined.", @@ -594,6 +627,11 @@ const char *const help_text[] = { int main(int argc, char **argv) { void (*op)(void) = run; + static const char **override_names = NULL; + static test_define_t *override_defines = NULL; + static size_t override_count = 0; + static size_t override_cap = 0; + // parse options while (true) { int c = getopt_long(argc, argv, short_opts, long_opts, NULL); @@ -673,8 +711,18 @@ int main(int argc, char **argv) { case OPT_LIST_GEOMETRIES: op = list_geometries; break; + case OPT_LIST_DEFAULTS: + op = list_defaults; + break; // configuration case OPT_DEFINE: { + // special case for -DGEOMETRY=, we treat this the same + // as --geometry= + if (strncmp(optarg, "GEOMETRY=", strlen("GEOMETRY=")) == 0) { + test_geometry = &optarg[strlen("GEOMETRY=")]; + break; + } + // realloc if necessary override_count += 1; if (override_count > override_cap) { @@ -816,6 +864,9 @@ getopt_done: ; test_suite = suite; } + // register overrides + test_define_overrides(override_names, override_defines, override_count); + // do the thing op(); diff --git a/runners/test_runner.h b/runners/test_runner.h index 92d8d312..c413a85a 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -41,27 +41,68 @@ struct test_suite { size_t case_count; }; -// TODO remove this indirection extern const struct test_suite *test_suites[]; extern const size_t test_suite_count; // access generated test defines +test_define_t test_predefine(size_t define); test_define_t test_define(size_t define); // a few preconfigured defines that control how tests run -#define READ_SIZE test_define(0) -#define PROG_SIZE test_define(1) -#define BLOCK_SIZE test_define(2) -#define BLOCK_COUNT test_define(3) -#define BLOCK_CYCLES test_define(4) -#define CACHE_SIZE test_define(5) -#define LOOKAHEAD_SIZE test_define(6) -#define ERASE_VALUE test_define(7) -#define ERASE_CYCLES test_define(8) -#define BADBLOCK_BEHAVIOR test_define(9) +#define READ_SIZE test_predefine(0) +#define PROG_SIZE test_predefine(1) +#define BLOCK_SIZE test_predefine(2) +#define BLOCK_COUNT test_predefine(3) +#define CACHE_SIZE test_predefine(4) +#define LOOKAHEAD_SIZE test_predefine(5) +#define BLOCK_CYCLES test_predefine(6) +#define ERASE_VALUE test_predefine(7) +#define ERASE_CYCLES test_predefine(8) +#define BADBLOCK_BEHAVIOR test_predefine(9) +#define TEST_PREDEFINE_NAMES { \ + "READ_SIZE", \ + "PROG_SIZE", \ + "BLOCK_SIZE", \ + "BLOCK_COUNT", \ + "CACHE_SIZE", \ + "LOOKAHEAD_SIZE", \ + "BLOCK_CYCLES", \ + "ERASE_VALUE", \ + "ERASE_CYCLES", \ + "BADBLOCK_BEHAVIOR", \ +} #define TEST_PREDEFINE_COUNT 10 +// default predefines +#define TEST_DEFAULTS { \ + /* LOOKAHEAD_SIZE */ 16, \ + /* BLOCK_CYCLES */ -1, \ + /* ERASE_VALUE */ 0xff, \ + /* ERASE_CYCLES */ 0, \ + /* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \ +} +#define TEST_DEFAULT_COUNT 5 +#define TEST_DEFAULT_MAP { \ + 0xff, 0xff, 0xff, 0xff, 0xff, 0, 1, 2, 3, 4 \ +} + +// test geometries +#define TEST_GEOMETRIES { \ + /*geometry, read, write, erase, count, cache */ \ + {"test", { 16, 16, 512, (1024*1024)/512, 64}}, \ + {"eeprom", { 1, 1, 512, (1024*1024)/512, 64}}, \ + {"emmc", { 512, 512, 512, (1024*1024)/512, 512}}, \ + {"nor", { 1, 1, 4096, (1024*1024)/4096, 64}}, \ + {"nand", {4096, 4096, 32*1024, (1024*1024)/(32*1024), 4096}}, \ +} +#define TEST_GEOMETRY_COUNT 5 +#define TEST_GEOMETRY_DEFINE_COUNT 5 +#define TEST_GEOMETRY_DEFINE_MAP { \ + 0, 1, 2, 3, 4, 0xff, 0xff, 0xff, 0xff, 0xff \ +} + + #endif diff --git a/scripts/test_.py b/scripts/test_.py index 8d6e34ec..2e5b9841 100755 --- a/scripts/test_.py +++ b/scripts/test_.py @@ -31,30 +31,21 @@ CASE_PROLOGUE = """ CASE_EPILOGUE = """ """ -TEST_PREDEFINES = [ - 'READ_SIZE', - 'PROG_SIZE', - 'BLOCK_SIZE', - 'BLOCK_COUNT', - 'BLOCK_CYCLES', - 'CACHE_SIZE', - 'LOOKAHEAD_SIZE', - 'ERASE_VALUE', - 'ERASE_CYCLES', - 'BADBLOCK_BEHAVIOR', -] - -# TODO -# def testpath(path): -# def testcase(path): -# def testperm(path): +def testpath(path): + path, *_ = path.split('#', 1) + return path def testsuite(path): - name = os.path.basename(path) - if name.endswith('.toml'): - name = name[:-len('.toml')] - return name + suite = testpath(path) + suite = os.path.basename(suite) + if suite.endswith('.toml'): + suite = suite[:-len('.toml')] + return suite + +def testcase(path): + _, case, *_ = path.split('#', 2) + return '%s#%s' % (testsuite(path), case) # TODO move this out in other files def openio(path, mode='r'): @@ -111,7 +102,7 @@ class TestSuite: # create a TestSuite object from a toml file def __init__(self, path, args={}): self.name = testsuite(path) - self.path = path + self.path = testpath(path) # load toml file and parse test cases with open(self.path) as f: @@ -125,9 +116,9 @@ class TestSuite: code_linenos = [] for i, line in enumerate(f): match = re.match( - '(?P\[\s*cases\s*\.\s*(?P\w+)\s*\])' + - '|(?Pif\s*=)' - '|(?Pcode\s*=)', + '(?P\[\s*cases\s*\.\s*(?P\w+)\s*\])' + '|' '(?Pif\s*=)' + '|' '(?Pcode\s*=)', line) if match and match.group('case'): case_linenos.append((i+1, match.group('name'))) @@ -187,8 +178,8 @@ class TestSuite: 'suite_valgrind': valgrind, **case})) - # combine pre-defines and per-case defines - self.defines = TEST_PREDEFINES + sorted( + # combine per-case defines + self.defines = sorted( set.union(*(set(case.defines) for case in self.cases))) # combine other per-case things @@ -225,216 +216,222 @@ def compile(**args): % args['test_paths']) sys.exit(-1) - # write out a test suite + # load our suite suite = TestSuite(paths[0]) - if 'output' in args: - with openio(args['output'], 'w') as f: - # redirect littlefs tracing - f.write('#define LFS_TRACE_(fmt, ...) do { \\\n') - f.write(8*' '+'extern FILE *test_trace; \\\n') - f.write(8*' '+'if (test_trace) { \\\n') - f.write(12*' '+'fprintf(test_trace, ' - '"%s:%d:trace: " fmt "%s\\n", \\\n') - f.write(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\\n') - f.write(8*' '+'} \\\n') - f.write(4*' '+'} while (0)\n') - f.write('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")\n') - f.write('#define LFS_TESTBD_TRACE(...) ' - 'LFS_TRACE_(__VA_ARGS__, "")\n') - f.write('\n') + else: + # load all suites + suites = [TestSuite(path) for path in paths] + suites.sort(key=lambda s: s.name) - f.write('%s\n' % SUITE_PROLOGUE.strip()) - f.write('\n') + # write generated test source + if 'output' in args: + with openio(args['output'], 'w') as f: + _write = f.write + def write(s): + f.lineno += s.count('\n') + _write(s) + def writeln(s=''): + f.lineno += s.count('\n') + 1 + _write(s) + _write('\n') + f.lineno = 1 + f.write = write + f.writeln = writeln + + # redirect littlefs tracing + f.writeln('#define LFS_TRACE_(fmt, ...) do { \\') + f.writeln(8*' '+'extern FILE *test_trace; \\') + f.writeln(8*' '+'if (test_trace) { \\') + f.writeln(12*' '+'fprintf(test_trace, ' + '"%s:%d:trace: " fmt "%s\\n", \\') + f.writeln(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\') + f.writeln(8*' '+'} \\') + f.writeln(4*' '+'} while (0)') + f.writeln('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")') + f.writeln('#define LFS_TESTBD_TRACE(...) ' + 'LFS_TRACE_(__VA_ARGS__, "")') + f.writeln() + + if not args.get('source'): + # write test suite prologue + f.writeln('%s' % SUITE_PROLOGUE.strip()) + f.writeln() if suite.code is not None: if suite.code_lineno is not None: - f.write('#line %d "%s"\n' + f.writeln('#line %d "%s"' % (suite.code_lineno, suite.path)) f.write(suite.code) - f.write('\n') + if suite.code_lineno is not None: + f.writeln('#line %d "%s"' + % (f.lineno+1, args['output'])) + f.writeln() - for i, define in it.islice( - enumerate(suite.defines), - len(TEST_PREDEFINES), None): - f.write('#define %-24s test_define(%d)\n' % (define, i)) - f.write('\n') + for i, define in enumerate(suite.defines): + f.writeln('#ifndef %s' % define) + f.writeln('#define %-24s test_define(%d)' % (define, i)) + f.writeln('#endif') + f.writeln() for case in suite.cases: # create case defines if case.defines: sorted_defines = sorted(case.defines.items()) - for perm, defines in enumerate( - it.product(*( - [(k, v) for v in vs] - for k, vs in sorted_defines))): - f.write('const test_define_t ' - '__test__%s__%s__%d__defines[] = {\n' - % (suite.name, case.name, perm)) - for k, v in defines: - f.write(4*' '+'%s,\n' % v) - f.write('};\n') - f.write('\n') - - f.write('const test_define_t *const ' - '__test__%s__%s__defines[] = {\n' + f.writeln('const test_define_t *const ' + '__test__%s__%s__defines[] = {' % (suite.name, case.name)) - for perm in range(case.permutations): - f.write(4*' '+'__test__%s__%s__%d__defines,\n' - % (suite.name, case.name, perm)) - f.write('};\n') - f.write('\n') + for defines in it.product(*( + [(k, v) for v in vs] + for k, vs in sorted_defines)): + f.writeln(4*' '+'(const test_define_t[]){%s},' + % ', '.join('%s' % v for _, v in defines)) + f.writeln('};') + f.writeln() - f.write('const uint8_t ' - '__test__%s__%s__define_map[] = {\n' + f.writeln('const uint8_t ' + '__test__%s__%s__define_map[] = {' % (suite.name, case.name)) - for k in suite.defines: - f.write(4*' '+'%s,\n' - % ([k for k, _ in sorted_defines].index(k) - if k in case.defines else '0xff')) - f.write('};\n') - f.write('\n') + f.writeln(4*' '+'%s,' + % ', '.join( + '%s' % [k for k, _ in sorted_defines].index(k) + if k in case.defines else '0xff' + for k in suite.defines)) + f.writeln('};') + f.writeln() # create case filter function if suite.if_ is not None or case.if_ is not None: - f.write('bool __test__%s__%s__filter(' - '__attribute__((unused)) uint32_t perm) {\n' + f.writeln('bool __test__%s__%s__filter(' + '__attribute__((unused)) uint32_t perm) {' % (suite.name, case.name)) if suite.if_ is not None: - f.write(4*' '+'#line %d "%s"\n' - % (suite.if_lineno, suite.path)) - f.write(4*' '+'if (!(%s)) {\n' % suite.if_) - f.write(8*' '+'return false;\n') - f.write(4*' '+'}\n') - f.write('\n') + if suite.if_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (suite.if_lineno, suite.path)) + f.writeln(4*' '+'if (!(%s)) {' % suite.if_) + if suite.if_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (f.lineno+1, args['output'])) + f.writeln(8*' '+'return false;') + f.writeln(4*' '+'}') + f.writeln() if case.if_ is not None: - f.write(4*' '+'#line %d "%s"\n' - % (case.if_lineno, suite.path)) - f.write(4*' '+'if (!(%s)) {\n' % case.if_) - f.write(8*' '+'return false;\n') - f.write(4*' '+'}\n') - f.write('\n') - f.write(4*' '+'return true;\n') - f.write('}\n') - f.write('\n') + if case.if_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (case.if_lineno, suite.path)) + f.writeln(4*' '+'if (!(%s)) {' % case.if_) + if case.if_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (f.lineno+1, args['output'])) + f.writeln(8*' '+'return false;') + f.writeln(4*' '+'}') + f.writeln() + f.writeln(4*' '+'return true;') + f.writeln('}') + f.writeln() # create case run function - f.write('void __test__%s__%s__run(' + f.writeln('void __test__%s__%s__run(' '__attribute__((unused)) struct lfs_config *cfg, ' - '__attribute__((unused)) uint32_t perm) {\n' + '__attribute__((unused)) uint32_t perm) {' % (suite.name, case.name)) - f.write(4*' '+'%s\n' - % CASE_PROLOGUE.strip().replace('\n', '\n'+4*' ')) - f.write('\n') - f.write(4*' '+'// test case %s\n' % case.id()) + if CASE_PROLOGUE.strip(): + f.writeln(4*' '+'%s' + % CASE_PROLOGUE.strip().replace('\n', '\n'+4*' ')) + f.writeln() + f.writeln(4*' '+'// test case %s' % case.id()) if case.code_lineno is not None: - f.write(4*' '+'#line %d "%s"\n' + f.writeln(4*' '+'#line %d "%s"' % (case.code_lineno, suite.path)) f.write(case.code) - f.write('\n') - f.write(4*' '+'%s\n' - % CASE_EPILOGUE.strip().replace('\n', '\n'+4*' ')) - f.write('}\n') - f.write('\n') + if case.code_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (f.lineno+1, args['output'])) + if CASE_EPILOGUE.strip(): + f.writeln() + f.writeln(4*' '+'%s' + % CASE_EPILOGUE.strip().replace('\n', '\n'+4*' ')) + f.writeln('}') + f.writeln() # create case struct - f.write('const struct test_case __test__%s__%s__case = {\n' + f.writeln('const struct test_case __test__%s__%s__case = {' % (suite.name, case.name)) - f.write(4*' '+'.id = "%s",\n' % case.id()) - f.write(4*' '+'.name = "%s",\n' % case.name) - f.write(4*' '+'.path = "%s",\n' % case.path) - f.write(4*' '+'.types = %s,\n' + f.writeln(4*' '+'.id = "%s",' % case.id()) + f.writeln(4*' '+'.name = "%s",' % case.name) + f.writeln(4*' '+'.path = "%s",' % case.path) + f.writeln(4*' '+'.types = %s,' % ' | '.join(filter(None, [ 'TEST_NORMAL' if case.normal else None, 'TEST_REENTRANT' if case.reentrant else None, 'TEST_VALGRIND' if case.valgrind else None]))) - f.write(4*' '+'.permutations = %d,\n' % case.permutations) + f.writeln(4*' '+'.permutations = %d,' % case.permutations) if case.defines: - f.write(4*' '+'.defines = __test__%s__%s__defines,\n' + f.writeln(4*' '+'.defines = __test__%s__%s__defines,' % (suite.name, case.name)) - f.write(4*' '+'.define_map = ' - '__test__%s__%s__define_map,\n' + f.writeln(4*' '+'.define_map = ' + '__test__%s__%s__define_map,' % (suite.name, case.name)) if suite.if_ is not None or case.if_ is not None: - f.write(4*' '+'.filter = __test__%s__%s__filter,\n' + f.writeln(4*' '+'.filter = __test__%s__%s__filter,' % (suite.name, case.name)) - f.write(4*' '+'.run = __test__%s__%s__run,\n' + f.writeln(4*' '+'.run = __test__%s__%s__run,' % (suite.name, case.name)) - f.write('};\n') - f.write('\n') + f.writeln('};') + f.writeln() # create suite define names - f.write('const char *const __test__%s__define_names[] = {\n' + f.writeln('const char *const __test__%s__define_names[] = {' % suite.name) for k in suite.defines: - f.write(4*' '+'"%s",\n' % k) - f.write('};\n') - f.write('\n') + f.writeln(4*' '+'"%s",' % k) + f.writeln('};') + f.writeln() # create suite struct - f.write('const struct test_suite __test__%s__suite = {\n' + f.writeln('const struct test_suite __test__%s__suite = {' % suite.name) - f.write(4*' '+'.id = "%s",\n' % suite.id()) - f.write(4*' '+'.name = "%s",\n' % suite.name) - f.write(4*' '+'.path = "%s",\n' % suite.path) - f.write(4*' '+'.types = %s,\n' + f.writeln(4*' '+'.id = "%s",' % suite.id()) + f.writeln(4*' '+'.name = "%s",' % suite.name) + f.writeln(4*' '+'.path = "%s",' % suite.path) + f.writeln(4*' '+'.types = %s,' % ' | '.join(filter(None, [ 'TEST_NORMAL' if suite.normal else None, 'TEST_REENTRANT' if suite.reentrant else None, 'TEST_VALGRIND' if suite.valgrind else None]))) - f.write(4*' '+'.define_names = __test__%s__define_names,\n' + f.writeln(4*' '+'.define_names = __test__%s__define_names,' % suite.name) - f.write(4*' '+'.define_count = %d,\n' % len(suite.defines)) - f.write(4*' '+'.cases = (const struct test_case *const []){\n') + f.writeln(4*' '+'.define_count = %d,' % len(suite.defines)) + f.writeln(4*' '+'.cases = (const struct test_case *const []){') for case in suite.cases: - f.write(8*' '+'&__test__%s__%s__case,\n' + f.writeln(8*' '+'&__test__%s__%s__case,' % (suite.name, case.name)) - f.write(4*' '+'},\n') - f.write(4*' '+'.case_count = %d,\n' % len(suite.cases)) - f.write('};\n') - f.write('\n') - - else: - # load all suites - suites = [TestSuite(path) for path in paths] - suites.sort(key=lambda s: s.name) - - # write out a test source - if 'output' in args: - with openio(args['output'], 'w') as f: - # redirect littlefs tracing - f.write('#define LFS_TRACE_(fmt, ...) do { \\\n') - f.write(8*' '+'extern FILE *test_trace; \\\n') - f.write(8*' '+'if (test_trace) { \\\n') - f.write(12*' '+'fprintf(test_trace, ' - '"%s:%d:trace: " fmt "%s\\n", \\\n') - f.write(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\\n') - f.write(8*' '+'} \\\n') - f.write(4*' '+'} while (0)\n') - f.write('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")\n') - f.write('#define LFS_TESTBD_TRACE(...) ' - 'LFS_TRACE_(__VA_ARGS__, "")\n') - f.write('\n') + f.writeln(4*' '+'},') + f.writeln(4*' '+'.case_count = %d,' % len(suite.cases)) + f.writeln('};') + f.writeln() + else: # copy source - f.write('#line 1 "%s"\n' % args['source']) + f.writeln('#line 1 "%s"' % args['source']) with open(args['source']) as sf: shutil.copyfileobj(sf, f) - f.write('\n') + f.writeln() f.write(SUITE_PROLOGUE) - f.write('\n') + f.writeln() # add suite info to test_runner.c if args['source'] == 'runners/test_runner.c': - f.write('\n') + f.writeln() for suite in suites: - f.write('extern const struct test_suite ' - '__test__%s__suite;\n' % suite.name) - f.write('const struct test_suite *test_suites[] = {\n') + f.writeln('extern const struct test_suite ' + '__test__%s__suite;' % suite.name) + f.writeln('const struct test_suite *test_suites[] = {') for suite in suites: - f.write(4*' '+'&__test__%s__suite,\n' % suite.name) - f.write('};\n') - f.write('const size_t test_suite_count = %d;\n' + f.writeln(4*' '+'&__test__%s__suite,' % suite.name) + f.writeln('};') + f.writeln('const size_t test_suite_count = %d;' % len(suites)) def runner(**args): @@ -469,7 +466,7 @@ def list_(**args): def find_cases(runner_, **args): - # first get suite/case/perm counts + # query from runner cmd = runner_ + ['--list-cases'] if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) @@ -483,9 +480,8 @@ def find_cases(runner_, **args): expected_perms = 0 total_perms = 0 pattern = re.compile( - '^(?P(?P[^#]+)#[^ #]+) +' - '[^ ]+ +[^ ]+ +[^ ]+ +' - '(?P[0-9]+)/(?P[0-9]+)$') + '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' + '[^\s]+\s+(?P\d+)/(?P\d+)') # skip the first line next(proc.stdout) for line in proc.stdout: @@ -509,11 +505,69 @@ def find_cases(runner_, **args): expected_perms, total_perms) +def find_paths(runner_, **args): + # query from runner + cmd = runner_ + ['--list-paths'] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace') + paths = co.OrderedDict() + pattern = re.compile( + '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' + '(?P[^:]+):(?P\d+)') + # skip the first line + for line in proc.stdout: + m = pattern.match(line) + if m: + paths[m.group('id')] = (m.group('path'), int(m.group('lineno'))) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return paths + +def find_defines(runner_, **args): + # query from runner + cmd = runner_ + ['--list-defines'] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace') + defines = co.OrderedDict() + pattern = re.compile( + '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' + '(?P(?:\w+=\w+\s*)+)') + # skip the first line + for line in proc.stdout: + m = pattern.match(line) + if m: + defines[m.group('id')] = {k: v + for k, v in re.findall('(\w+)=(\w+)', m.group('defines'))} + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return defines + + class TestFailure(Exception): - def __init__(self, id, returncode, stdout, assert_=None): + def __init__(self, id, returncode, output, assert_=None): self.id = id self.returncode = returncode - self.stdout = stdout + self.output = output self.assert_ = assert_ def run_step(name, runner_, **args): @@ -531,7 +585,7 @@ def run_step(name, runner_, **args): pattern = re.compile('^(?:' '(?Prunning|finished|skipped) ' '(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)' - '|' '(?P[^:]+):(?P[0-9]+):(?Passert):' + '|' '(?P[^:]+):(?P\d+):(?Passert):' ' *(?P.*)' ')$') locals = th.local() # TODO use process group instead of this set? @@ -554,7 +608,7 @@ def run_step(name, runner_, **args): children.add(proc) last_id = None - last_stdout = [] + last_output = [] last_assert = None try: while True: @@ -567,7 +621,7 @@ def run_step(name, runner_, **args): raise if not line: break - last_stdout.append(line) + last_output.append(line) if args.get('verbose'): sys.stdout.write(line) @@ -577,7 +631,7 @@ def run_step(name, runner_, **args): if op == 'running': locals.seen_perms += 1 last_id = m.group('id') - last_stdout = [] + last_output = [] last_assert = None elif op == 'finished': passed_suite_perms[m.group('suite')] += 1 @@ -590,10 +644,11 @@ def run_step(name, runner_, **args): m.group('path'), int(m.group('lineno')), m.group('message')) - # TODO why is kill _so_ much faster than terminate? - proc.kill() + # go ahead and kill the process, aborting takes a while + if args.get('keep_going'): + proc.kill() except KeyboardInterrupt: - raise TestFailure(last_id, 1, last_stdout) + raise TestFailure(last_id, 1, last_output) finally: children.remove(proc) mpty.close() @@ -603,7 +658,7 @@ def run_step(name, runner_, **args): raise TestFailure( last_id, proc.returncode, - last_stdout, + last_output, last_assert) def run_job(runner, skip=None, every=None): @@ -636,7 +691,6 @@ def run_step(name, runner_, **args): else: # stop other tests for child in children: - # TODO why is kill _so_ much faster than terminate? child.kill() break @@ -759,11 +813,28 @@ def run(**args): print() # print each failure - # TODO get line, defines, path + if failures: + # get some extra info from runner + runner_paths = find_paths(runner_, **args) + runner_defines = find_defines(runner_, **args) + for failure in failures: -# print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s failed' -# # TODO this should be the suite path and lineno -# % (failure.assert + # show summary of failure + path, lineno = runner_paths[testcase(failure.id)] + defines = runner_defines[failure.id] + + print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s%s failed' + % (path, lineno, failure.id, + ' (%s)' % ', '.join( + '%s=%s' % (k, v) for k, v in defines.items()) + if defines else '')) + + if failure.output: + output = failure.output + if failure.assert_ is not None: + output = output[:-1] + for line in output[-5:]: + sys.stdout.write(line) if failure.assert_ is not None: path, lineno, message = failure.assert_ @@ -785,7 +856,8 @@ def main(**args): or args.get('list_cases') or args.get('list_paths') or args.get('list_defines') - or args.get('list_geometries')): + or args.get('list_geometries') + or args.get('list_defaults')): list_(**args) else: run(**args) @@ -816,6 +888,8 @@ if __name__ == "__main__": help="List the defines for each test permutation.") test_parser.add_argument('--list-geometries', action='store_true', help="List the disk geometries used for testing.") + test_parser.add_argument('--list-defaults', action='store_true', + help="List the default defines in this test-runner.") test_parser.add_argument('-D', '--define', action='append', help="Override a test define.") test_parser.add_argument('-G', '--geometry', From 5ee4b052ae16464ca5714d19e1d96a7c2ac167b6 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 25 Apr 2022 01:08:44 -0500 Subject: [PATCH 08/81] Misc test-runner improvements - Added --disk/--trace/--output options for information-heavy debugging - Renamed --skip/--count/--every to --start/--stop/--step. This matches common terms for ranges, and frees --skip for being used to skip test cases in the future. - Better handling of SIGTERM, now all tests are killed, reported as failures, and testing is halted irregardless of -k. This is a compromise, you throw away the rest of the tests, which is normally what -k is for, but prevents annoying-to-terminate processes when debugging, which is a very interactive process. --- runners/test_runner.c | 56 +++++++++++++++++----------------- scripts/test_.py | 70 +++++++++++++++++++++++++++---------------- 2 files changed, 72 insertions(+), 54 deletions(-) diff --git a/runners/test_runner.c b/runners/test_runner.c index 1c9b2550..4d65f633 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -161,11 +161,11 @@ static const char *test_case = NULL; static size_t test_perm = -1; static const char *test_geometry = NULL; static test_types_t test_types = 0; -static size_t test_skip = 0; -static size_t test_count = -1; -static size_t test_every = 1; +static size_t test_start = 0; +static size_t test_stop = -1; +static size_t test_step = 1; -static const char *test_persist = NULL; +static const char *test_disk = NULL; FILE *test_trace = NULL; // note, these skips are different than filtered tests @@ -188,9 +188,9 @@ static bool test_perm_skip(size_t perm) { } static bool test_step_skip(size_t step) { - return !(step >= test_skip - && (step-test_skip) < test_count - && (step-test_skip) % test_every == 0); + return !(step >= test_start + && step < test_stop + && (step-test_start) % test_step == 0); } static void test_case_permcount( @@ -528,7 +528,7 @@ static void run(void) { .power_cycles = 0, }; - int err = lfs_testbd_createcfg(&cfg, test_persist, &bdcfg); + int err = lfs_testbd_createcfg(&cfg, test_disk, &bdcfg); if (err) { fprintf(stderr, "error: " "could not create block device: %d\n", err); @@ -572,10 +572,10 @@ enum opt_flags { OPT_NORMAL = 'n', OPT_REENTRANT = 'r', OPT_VALGRIND = 'V', - OPT_SKIP = 5, - OPT_COUNT = 6, - OPT_EVERY = 7, - OPT_PERSIST = 'p', + OPT_START = 5, + OPT_STEP = 6, + OPT_STOP = 7, + OPT_DISK = 'd', OPT_TRACE = 't', }; @@ -595,10 +595,10 @@ const struct option long_opts[] = { {"normal", no_argument, NULL, OPT_NORMAL}, {"reentrant", no_argument, NULL, OPT_REENTRANT}, {"valgrind", no_argument, NULL, OPT_VALGRIND}, - {"skip", required_argument, NULL, OPT_SKIP}, - {"count", required_argument, NULL, OPT_COUNT}, - {"every", required_argument, NULL, OPT_EVERY}, - {"persist", required_argument, NULL, OPT_PERSIST}, + {"start", required_argument, NULL, OPT_START}, + {"stop", required_argument, NULL, OPT_STOP}, + {"step", required_argument, NULL, OPT_STEP}, + {"disk", required_argument, NULL, OPT_DISK}, {"trace", required_argument, NULL, OPT_TRACE}, {NULL, 0, NULL, 0}, }; @@ -617,10 +617,10 @@ const char *const help_text[] = { "Filter for normal tests. Can be combined.", "Filter for reentrant tests. Can be combined.", "Filter for Valgrind tests. Can be combined.", - "Skip the first n tests.", - "Stop after n tests.", - "Only run every n tests, calculated after --skip and --stop.", - "Persist the disk to this file.", + "Start at the nth test.", + "Stop before the nth test.", + "Only run every n tests, calculated after --start and --stop.", + "Use this file as the disk.", "Redirect trace output to this file.", }; @@ -766,35 +766,35 @@ invalid_define: case OPT_VALGRIND: test_types |= TEST_VALGRIND; break; - case OPT_SKIP: { + case OPT_START: { char *parsed = NULL; - test_skip = strtoumax(optarg, &parsed, 0); + test_start = strtoumax(optarg, &parsed, 0); if (parsed == optarg) { fprintf(stderr, "error: invalid skip: %s\n", optarg); exit(-1); } break; } - case OPT_COUNT: { + case OPT_STOP: { char *parsed = NULL; - test_count = strtoumax(optarg, &parsed, 0); + test_stop = strtoumax(optarg, &parsed, 0); if (parsed == optarg) { fprintf(stderr, "error: invalid count: %s\n", optarg); exit(-1); } break; } - case OPT_EVERY: { + case OPT_STEP: { char *parsed = NULL; - test_every = strtoumax(optarg, &parsed, 0); + test_step = strtoumax(optarg, &parsed, 0); if (parsed == optarg) { fprintf(stderr, "error: invalid every: %s\n", optarg); exit(-1); } break; } - case OPT_PERSIST: - test_persist = optarg; + case OPT_DISK: + test_disk = optarg; break; case OPT_TRACE: if (strcmp(optarg, "-") == 0) { diff --git a/scripts/test_.py b/scripts/test_.py index 2e5b9841..eb560010 100755 --- a/scripts/test_.py +++ b/scripts/test_.py @@ -570,17 +570,17 @@ class TestFailure(Exception): self.output = output self.assert_ = assert_ -def run_step(name, runner_, **args): +def run_stage(name, runner_, **args): # get expected suite/case/perm counts expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( find_cases(runner_, **args)) - # TODO persist/trace # TODO valgrind/gdb/exec passed_suite_perms = co.defaultdict(lambda: 0) passed_case_perms = co.defaultdict(lambda: 0) passed_perms = 0 failures = [] + killed = False pattern = re.compile('^(?:' '(?Prunning|finished|skipped) ' @@ -598,14 +598,20 @@ def run_step(name, runner_, **args): nonlocal locals # run the tests! - cmd = runner_ + cmd = runner_.copy() + if args.get('disk'): + cmd.append('--disk=%s' % args['disk']) + if args.get('trace'): + cmd.append('--trace=%s' % args['trace']) if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) mpty, spty = pty.openpty() proc = sp.Popen(cmd, stdout=spty, stderr=spty) os.close(spty) - mpty = os.fdopen(mpty, 'r', 1) children.add(proc) + mpty = os.fdopen(mpty, 'r', 1) + if args.get('output'): + output = openio(args['output'], 'w') last_id = None last_output = [] @@ -622,7 +628,9 @@ def run_step(name, runner_, **args): if not line: break last_output.append(line) - if args.get('verbose'): + if args.get('output'): + output.write(line) + elif args.get('verbose'): sys.stdout.write(line) m = pattern.match(line) @@ -652,6 +660,8 @@ def run_step(name, runner_, **args): finally: children.remove(proc) mpty.close() + if args.get('output'): + output.close() proc.wait() if proc.returncode != 0: @@ -661,16 +671,16 @@ def run_step(name, runner_, **args): last_output, last_assert) - def run_job(runner, skip=None, every=None): + def run_job(runner, start=None, step=None): nonlocal failures nonlocal locals - while (skip or 0) < total_perms: + while (start or 0) < total_perms: runner_ = runner.copy() - if skip is not None: - runner_.append('--skip=%d' % skip) - if every is not None: - runner_.append('--every=%d' % every) + if start is not None: + runner_.append('--start=%d' % start) + if step is not None: + runner_.append('--step=%d' % step) try: # run the tests @@ -684,13 +694,13 @@ def run_step(name, runner_, **args): failures.append(failure) - if args.get('keep_going'): + if args.get('keep_going') and not killed: # resume after failed test - skip = (skip or 0) + locals.seen_perms*(every or 1) + start = (start or 0) + locals.seen_perms*(step or 1) continue else: # stop other tests - for child in children: + for child in children.copy(): child.kill() break @@ -733,6 +743,10 @@ def run_step(name, runner_, **args): if failures else '')) sys.stdout.flush() needs_newline = True + except KeyboardInterrupt: + # this is handled by the runner threads, we just + # need to not abort here + killed = True finally: if needs_newline: print() @@ -743,7 +757,8 @@ def run_step(name, runner_, **args): return ( expected_perms, passed_perms, - failures) + failures, + killed) def run(**args): @@ -767,41 +782,41 @@ def run(**args): if args.get('by_suites'): for type in ['normal', 'reentrant', 'valgrind']: for suite in expected_suite_perms.keys(): - expected_, passed_, failures_ = run_step( + expected_, passed_, failures_, killed = run_stage( '%s %s' % (type, suite), runner_ + ['--%s' % type, suite], **args) expected += expected_ passed += passed_ failures.extend(failures_) - if failures and not args.get('keep_going'): + if (failures and not args.get('keep_going')) or killed: break - if failures and not args.get('keep_going'): + if (failures and not args.get('keep_going')) or killed: break elif args.get('by_cases'): for type in ['normal', 'reentrant', 'valgrind']: for case in expected_case_perms.keys(): - expected_, passed_, failures_ = run_step( + expected_, passed_, failures_, killed = run_stage( '%s %s' % (type, case), runner_ + ['--%s' % type, case], **args) expected += expected_ passed += passed_ failures.extend(failures_) - if failures and not args.get('keep_going'): + if (failures and not args.get('keep_going')) or killed: break - if failures and not args.get('keep_going'): + if (failures and not args.get('keep_going')) or killed: break else: for type in ['normal', 'reentrant', 'valgrind']: - expected_, passed_, failures_ = run_step( + expected_, passed_, failures_, killed = run_stage( '%s tests' % type, runner_ + ['--%s' % type], **args) expected += expected_ passed += passed_ failures.extend(failures_) - if failures and not args.get('keep_going'): + if (failures and not args.get('keep_going')) or killed: break # show summary @@ -867,7 +882,8 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Build and run tests.") + description="Build and run tests.", + conflict_handler='resolve') # TODO document test case/perm specifier parser.add_argument('test_paths', nargs='*', help="Description of testis to run. May be a directory, path, or \ @@ -900,10 +916,12 @@ if __name__ == "__main__": help="Filter for reentrant tests. Can be combined.") test_parser.add_argument('-V', '--valgrind', action='store_true', help="Filter for Valgrind tests. Can be combined.") - test_parser.add_argument('-p', '--persist', - help="Persist the disk to this file.") + test_parser.add_argument('-d', '--disk', + help="Use this file as the disk.") test_parser.add_argument('-t', '--trace', help="Redirect trace output to this file.") + test_parser.add_argument('-o', '--output', + help="Redirect stdout and stderr to this file.") test_parser.add_argument('--runner', default=[RUNNER_PATH], type=lambda x: x.split(), help="Path to runner, defaults to %r" % RUNNER_PATH) From 496282901719762726ea5331f357c37f749a33d0 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 1 May 2022 11:06:34 -0500 Subject: [PATCH 09/81] Continued progress toward feature parity with new test-runner - Expanded test defines to allow for lists of configurations These are useful for changing multi-dimensional test configurations without leading to extremely large and less useful configuration combinations. - Made warnings more visible durring test parsing - Add lfs_testbd.h to implicit test includes - Fixed issue with not closing files in ./scripts/explode_asserts.py - Add `make test_runner` and `make test_list` build rules for convenience --- Makefile | 11 ++++- scripts/explode_asserts.py | 54 ++++++++++++--------- scripts/test_.py | 96 +++++++++++++++++++++----------------- 3 files changed, 94 insertions(+), 67 deletions(-) diff --git a/Makefile b/Makefile index 513187dc..fc164516 100644 --- a/Makefile +++ b/Makefile @@ -119,9 +119,16 @@ test: test%: tests/test$$(firstword $$(subst \#, ,%)).toml ./scripts/test.py $@ $(TESTFLAGS) +.PHONY: test_runner +test_runner: $(BUILDDIR)runners/test_runner + .PHONY: test_ -test_: $(BUILDDIR)runners/test_runner - ./scripts/test_.py --runner=$< $(TESTFLAGS_) +test_: test_runner + ./scripts/test_.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS_) + +.PHONY: test_list +test_list: test_runner + ./scripts/test_.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS_) -l .PHONY: code code: $(OBJ) diff --git a/scripts/explode_asserts.py b/scripts/explode_asserts.py index 8a8e5b1c..e49dbc0c 100755 --- a/scripts/explode_asserts.py +++ b/scripts/explode_asserts.py @@ -134,6 +134,15 @@ TYPE = { } } +def openio(path, mode='r'): + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) + def mkdecls(outf, maxwidth=16): outf.write("#include \n") outf.write("#include \n") @@ -341,32 +350,31 @@ def pstmt(p): def main(args): - inf = open(args.input, 'r') if args.input else sys.stdin - outf = open(args.output, 'w') if args.output else sys.stdout + with openio(args.input or '-', 'r') as inf: + with openio(args.output or '-', 'w') as outf: + lexemes = LEX.copy() + if args.pattern: + lexemes['assert'] = args.pattern + p = Parse(inf, lexemes) - lexemes = LEX.copy() - if args.pattern: - lexemes['assert'] = args.pattern - p = Parse(inf, lexemes) + # write extra verbose asserts + mkdecls(outf, maxwidth=args.maxwidth) + if args.input: + outf.write("#line %d \"%s\"\n" % (1, args.input)) - # write extra verbose asserts - mkdecls(outf, maxwidth=args.maxwidth) - if args.input: - outf.write("#line %d \"%s\"\n" % (1, args.input)) + # parse and write out stmt at a time + try: + while True: + outf.write(pstmt(p)) + if p.accept('sep'): + outf.write(p.m) + else: + break + except ParseFailure as f: + pass - # parse and write out stmt at a time - try: - while True: - outf.write(pstmt(p)) - if p.accept('sep'): - outf.write(p.m) - else: - break - except ParseFailure as f: - pass - - for i in range(p.off, len(p.tokens)): - outf.write(p.tokens[i][1]) + for i in range(p.off, len(p.tokens)): + outf.write(p.tokens[i][1]) if __name__ == "__main__": import argparse diff --git a/scripts/test_.py b/scripts/test_.py index eb560010..55e12722 100755 --- a/scripts/test_.py +++ b/scripts/test_.py @@ -24,6 +24,7 @@ RUNNER_PATH = './runners/test_runner' SUITE_PROLOGUE = """ #include "runners/test_runner.h" +#include "bd/lfs_testbd.h" #include """ CASE_PROLOGUE = """ @@ -78,20 +79,30 @@ class TestCase: self.valgrind = config.pop('valgrind', config.pop('suite_valgrind', True)) - # figure out defines and the number of resulting permutations - self.defines = {} - for k, v in ( - config.pop('suite_defines', {}) - | config.pop('defines', {})).items(): - if not isinstance(v, list): - v = [v] + # figure out defines and build possible permutations + self.defines = set() + self.permutations = [] - self.defines[k] = v + suite_defines = config.pop('suite_defines', {}) + if not isinstance(suite_defines, list): + suite_defines = [suite_defines] + defines = config.pop('defines', {}) + if not isinstance(defines, list): + defines = [defines] - self.permutations = m.prod(len(v) for v in self.defines.values()) + # build possible permutations + for suite_defines_ in suite_defines: + self.defines |= suite_defines_.keys() + for defines_ in defines: + self.defines |= defines_.keys() + self.permutations.extend(map(dict, it.product(*( + [(k, v) for v in (vs if isinstance(vs, list) else [vs])] + for k, vs in sorted( + (suite_defines_ | defines_).items()))))) for k in config.keys(): - print('warning: in %s, found unused key %r' % (self.id(), k), + print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r' + % (self.id(), k), file=sys.stderr) def id(self): @@ -130,7 +141,7 @@ class TestSuite: # sort in case toml parsing did not retain order case_linenos.sort() - cases = config.pop('cases', []) + cases = config.pop('cases') for (lineno, name), (nlineno, _) in it.zip_longest( case_linenos, case_linenos[1:], fillvalue=(float('inf'), None)): @@ -179,8 +190,8 @@ class TestSuite: **case})) # combine per-case defines - self.defines = sorted( - set.union(*(set(case.defines) for case in self.cases))) + self.defines = set.union(*( + set(case.defines) for case in self.cases)) # combine other per-case things self.normal = any(case.normal for case in self.cases) @@ -188,7 +199,8 @@ class TestSuite: self.valgrind = any(case.valgrind for case in self.cases) for k in config.keys(): - print('warning: in %s, found unused key %r' % (self.id(), k), + print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r' + % (self.id(), k), file=sys.stderr) def id(self): @@ -266,7 +278,7 @@ def compile(**args): % (f.lineno+1, args['output'])) f.writeln() - for i, define in enumerate(suite.defines): + for i, define in enumerate(sorted(suite.defines)): f.writeln('#ifndef %s' % define) f.writeln('#define %-24s test_define(%d)' % (define, i)) f.writeln('#endif') @@ -275,16 +287,13 @@ def compile(**args): for case in suite.cases: # create case defines if case.defines: - sorted_defines = sorted(case.defines.items()) - f.writeln('const test_define_t *const ' '__test__%s__%s__defines[] = {' % (suite.name, case.name)) - for defines in it.product(*( - [(k, v) for v in vs] - for k, vs in sorted_defines)): + for permutation in case.permutations: f.writeln(4*' '+'(const test_define_t[]){%s},' - % ', '.join('%s' % v for _, v in defines)) + % ', '.join(str(v) for _, v in sorted( + permutation.items()))) f.writeln('};') f.writeln() @@ -293,9 +302,9 @@ def compile(**args): % (suite.name, case.name)) f.writeln(4*' '+'%s,' % ', '.join( - '%s' % [k for k, _ in sorted_defines].index(k) + str(sorted(case.defines).index(k)) if k in case.defines else '0xff' - for k in suite.defines)) + for k in sorted(suite.defines))) f.writeln('};') f.writeln() @@ -365,7 +374,8 @@ def compile(**args): 'TEST_NORMAL' if case.normal else None, 'TEST_REENTRANT' if case.reentrant else None, 'TEST_VALGRIND' if case.valgrind else None]))) - f.writeln(4*' '+'.permutations = %d,' % case.permutations) + f.writeln(4*' '+'.permutations = %d,' + % len(case.permutations)) if case.defines: f.writeln(4*' '+'.defines = __test__%s__%s__defines,' % (suite.name, case.name)) @@ -381,12 +391,13 @@ def compile(**args): f.writeln() # create suite define names - f.writeln('const char *const __test__%s__define_names[] = {' - % suite.name) - for k in suite.defines: - f.writeln(4*' '+'"%s",' % k) - f.writeln('};') - f.writeln() + if suite.defines: + f.writeln('const char *const __test__%s__define_names[] = {' + % suite.name) + for k in sorted(suite.defines): + f.writeln(4*' '+'"%s",' % k) + f.writeln('};') + f.writeln() # create suite struct f.writeln('const struct test_suite __test__%s__suite = {' @@ -399,8 +410,9 @@ def compile(**args): 'TEST_NORMAL' if suite.normal else None, 'TEST_REENTRANT' if suite.reentrant else None, 'TEST_VALGRIND' if suite.valgrind else None]))) - f.writeln(4*' '+'.define_names = __test__%s__define_names,' - % suite.name) + if suite.defines: + f.writeln(4*' '+'.define_names = __test__%s__define_names,' + % suite.name) f.writeln(4*' '+'.define_count = %d,' % len(suite.defines)) f.writeln(4*' '+'.cases = (const struct test_case *const []){') for case in suite.cases: @@ -726,7 +738,7 @@ def run_stage(name, runner_, **args): if not args.get('verbose'): sys.stdout.write('\r\x1b[K' - 'running \x1b[%dm%s\x1b[m: ' + 'running \x1b[%dm%s:\x1b[m ' '%d/%d suites, %d/%d cases, %d/%d perms%s ' % (32 if not failures else 31, name, @@ -779,12 +791,12 @@ def run(**args): expected = 0 passed = 0 failures = [] - if args.get('by_suites'): + if args.get('by_cases'): for type in ['normal', 'reentrant', 'valgrind']: - for suite in expected_suite_perms.keys(): + for case in expected_case_perms.keys(): expected_, passed_, failures_, killed = run_stage( - '%s %s' % (type, suite), - runner_ + ['--%s' % type, suite], + '%s %s' % (type, case), + runner_ + ['--%s' % type, case], **args) expected += expected_ passed += passed_ @@ -793,12 +805,12 @@ def run(**args): break if (failures and not args.get('keep_going')) or killed: break - elif args.get('by_cases'): + elif args.get('by_suites'): for type in ['normal', 'reentrant', 'valgrind']: - for case in expected_case_perms.keys(): + for suite in expected_suite_perms.keys(): expected_, passed_, failures_, killed = run_stage( - '%s %s' % (type, case), - runner_ + ['--%s' % type, case], + '%s %s' % (type, suite), + runner_ + ['--%s' % type, suite], **args) expected += expected_ passed += passed_ @@ -821,7 +833,7 @@ def run(**args): # show summary print() - print('\x1b[%dmdone\x1b[m: %d/%d passed, %d/%d failed, in %.2fs' + print('\x1b[%dmdone:\x1b[m %d/%d passed, %d/%d failed, in %.2fs' % (32 if not failures else 31, passed, expected, len(failures), expected, time.time()-start)) From be0e6ad5ebfeb340f553c5656092230304237d36 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 1 May 2022 21:13:13 -0500 Subject: [PATCH 10/81] More progress toward test-runner feature parity - Added internal tests, which can run tests inside other source files, allowing access to "private" functions and data Note this required a special bit of handling our defining and later undefining test configurations to not polute the namespace of the source file, since it can end up with test cases from different suites/configuration namespaces. - Removed unnecessary/unused permutation argument to generated test functions. - Some cleanup to progress output of test.py. --- runners/test_runner.c | 6 +- runners/test_runner.h | 4 +- scripts/test_.py | 238 ++++++++++++++++++++++++------------------ 3 files changed, 140 insertions(+), 108 deletions(-) diff --git a/runners/test_runner.c b/runners/test_runner.c index 4d65f633..7fbf6ec6 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -218,7 +218,7 @@ static void test_case_permcount( test_define_geometry(&test_geometries[geom_perm]); if (case_->filter) { - if (!case_->filter(case_perm)) { + if (!case_->filter()) { continue; } } @@ -495,7 +495,7 @@ static void run(void) { // filter? if (test_suites[i]->cases[j]->filter) { - if (!test_suites[i]->cases[j]->filter(case_perm)) { + if (!test_suites[i]->cases[j]->filter()) { printf("skipped %s#%zu\n", test_suites[i]->cases[j]->id, perm); @@ -538,7 +538,7 @@ static void run(void) { // run the test printf("running %s#%zu\n", test_suites[i]->cases[j]->id, perm); - test_suites[i]->cases[j]->run(&cfg, case_perm); + test_suites[i]->cases[j]->run(&cfg); printf("finished %s#%zu\n", test_suites[i]->cases[j]->id, perm); diff --git a/runners/test_runner.h b/runners/test_runner.h index c413a85a..addb898e 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -24,8 +24,8 @@ struct test_case { const test_define_t *const *defines; const uint8_t *define_map; - bool (*filter)(uint32_t perm); - void (*run)(struct lfs_config *cfg, uint32_t perm); + bool (*filter)(void); + void (*run)(struct lfs_config *cfg); }; struct test_suite { diff --git a/scripts/test_.py b/scripts/test_.py index 55e12722..6e94f901 100755 --- a/scripts/test_.py +++ b/scripts/test_.py @@ -71,13 +71,15 @@ class TestCase: self.if_lineno = config.pop('if_lineno', None) self.code = config.pop('code') self.code_lineno = config.pop('code_lineno', None) + self.in_ = config.pop('in', + config.pop('suite_in', None)) self.normal = config.pop('normal', - config.pop('suite_normal', True)) + config.pop('suite_normal', True)) self.reentrant = config.pop('reentrant', - config.pop('suite_reentrant', False)) + config.pop('suite_reentrant', False)) self.valgrind = config.pop('valgrind', - config.pop('suite_valgrind', True)) + config.pop('suite_valgrind', True)) # figure out defines and build possible permutations self.defines = set() @@ -171,6 +173,7 @@ class TestSuite: # a couple of these we just forward to all cases defines = config.pop('defines', {}) + in_ = config.pop('in', None) normal = config.pop('normal', True) reentrant = config.pop('reentrant', False) valgrind = config.pop('valgrind', True) @@ -184,6 +187,7 @@ class TestSuite: if 'lineno' in case else ''), 'suite': self.name, 'suite_defines': defines, + 'suite_in': in_, 'suite_normal': normal, 'suite_reentrant': reentrant, 'suite_valgrind': valgrind, @@ -264,54 +268,15 @@ def compile(**args): 'LFS_TRACE_(__VA_ARGS__, "")') f.writeln() - if not args.get('source'): - # write test suite prologue - f.writeln('%s' % SUITE_PROLOGUE.strip()) - f.writeln() - if suite.code is not None: - if suite.code_lineno is not None: - f.writeln('#line %d "%s"' - % (suite.code_lineno, suite.path)) - f.write(suite.code) - if suite.code_lineno is not None: - f.writeln('#line %d "%s"' - % (f.lineno+1, args['output'])) - f.writeln() - - for i, define in enumerate(sorted(suite.defines)): - f.writeln('#ifndef %s' % define) - f.writeln('#define %-24s test_define(%d)' % (define, i)) - f.writeln('#endif') - f.writeln() - - for case in suite.cases: - # create case defines - if case.defines: - f.writeln('const test_define_t *const ' - '__test__%s__%s__defines[] = {' - % (suite.name, case.name)) - for permutation in case.permutations: - f.writeln(4*' '+'(const test_define_t[]){%s},' - % ', '.join(str(v) for _, v in sorted( - permutation.items()))) - f.writeln('};') - f.writeln() - - f.writeln('const uint8_t ' - '__test__%s__%s__define_map[] = {' - % (suite.name, case.name)) - f.writeln(4*' '+'%s,' - % ', '.join( - str(sorted(case.defines).index(k)) - if k in case.defines else '0xff' - for k in sorted(suite.defines))) - f.writeln('};') - f.writeln() - + # write out generated functions, this can end up in different + # files depending on the "in" attribute + # + # note it's up to the specific generated file to declare + # the test defines + def write_case_functions(f, suite, case): # create case filter function if suite.if_ is not None or case.if_ is not None: - f.writeln('bool __test__%s__%s__filter(' - '__attribute__((unused)) uint32_t perm) {' + f.writeln('bool __test__%s__%s__filter(void) {' % (suite.name, case.name)) if suite.if_ is not None: if suite.if_lineno is not None: @@ -341,8 +306,7 @@ def compile(**args): # create case run function f.writeln('void __test__%s__%s__run(' - '__attribute__((unused)) struct lfs_config *cfg, ' - '__attribute__((unused)) uint32_t perm) {' + '__attribute__((unused)) struct lfs_config *cfg) {' % (suite.name, case.name)) if CASE_PROLOGUE.strip(): f.writeln(4*' '+'%s' @@ -363,6 +327,65 @@ def compile(**args): f.writeln('}') f.writeln() + if not args.get('source'): + # write test suite prologue + f.writeln('%s' % SUITE_PROLOGUE.strip()) + f.writeln() + if suite.code is not None: + if suite.code_lineno is not None: + f.writeln('#line %d "%s"' + % (suite.code_lineno, suite.path)) + f.write(suite.code) + if suite.code_lineno is not None: + f.writeln('#line %d "%s"' + % (f.lineno+1, args['output'])) + f.writeln() + + if suite.defines: + for i, define in enumerate(sorted(suite.defines)): + f.writeln('#ifndef %s' % define) + f.writeln('#define %-24s test_define(%d)' + % (define, i)) + f.writeln('#endif') + f.writeln() + + for case in suite.cases: + # create case defines + if case.defines: + f.writeln('const test_define_t *const ' + '__test__%s__%s__defines[] = {' + % (suite.name, case.name)) + for permutation in case.permutations: + f.writeln(4*' '+'(const test_define_t[]){%s},' + % ', '.join(str(v) for _, v in sorted( + permutation.items()))) + f.writeln('};') + f.writeln() + + f.writeln('const uint8_t ' + '__test__%s__%s__define_map[] = {' + % (suite.name, case.name)) + f.writeln(4*' '+'%s,' + % ', '.join( + str(sorted(case.defines).index(k)) + if k in case.defines else '0xff' + for k in sorted(suite.defines))) + f.writeln('};') + f.writeln() + + # create case functions + if case.in_ is None: + write_case_functions(f, suite, case) + else: + if suite.if_ is not None or case.if_ is not None: + f.writeln('extern bool __test__%s__%s__filter(' + 'void);' + % (suite.name, case.name)) + f.writeln('extern void __test__%s__%s__run(' + 'struct lfs_config *cfg);' + % (suite.name, case.name)) + f.writeln() + # create case struct f.writeln('const struct test_case __test__%s__%s__case = {' % (suite.name, case.name)) @@ -433,6 +456,35 @@ def compile(**args): f.write(SUITE_PROLOGUE) f.writeln() + # write any internal tests + for suite in suites: + for case in suite.cases: + if case.in_ == args.get('source'): + # write defines, but note we need to undef any + # new defines since we're in someone else's file + if suite.defines: + for i, define in enumerate( + sorted(suite.defines)): + f.writeln('#ifndef %s' % define) + f.writeln('#define %-24s test_define(%d)' + % (define, i)) + f.writeln('#define __TEST__%s__NEEDS_UNDEF' + % define) + f.writeln('#endif') + f.writeln() + + write_case_functions(f, suite, case) + + if suite.defines: + for define in sorted(suite.defines): + f.writeln('#ifdef __TEST__%s__NEEDS_UNDEF' + % define) + f.writeln('#undef __TEST__%s__NEEDS_UNDEF' + % define) + f.writeln('#undef %s' % define) + f.writeln('#endif') + f.writeln() + # add suite info to test_runner.c if args['source'] == 'runners/test_runner.c': f.writeln() @@ -738,21 +790,25 @@ def run_stage(name, runner_, **args): if not args.get('verbose'): sys.stdout.write('\r\x1b[K' - 'running \x1b[%dm%s:\x1b[m ' - '%d/%d suites, %d/%d cases, %d/%d perms%s ' + 'running \x1b[%dm%s:\x1b[m %s ' % (32 if not failures else 31, name, - sum(passed_suite_perms[k] == v - for k, v in expected_suite_perms.items()), - len(expected_suite_perms), - sum(passed_case_perms[k] == v - for k, v in expected_case_perms.items()), - len(expected_case_perms), - passed_perms, - expected_perms, - ', \x1b[31m%d/%d failures\x1b[m' - % (len(failures), expected_perms) - if failures else '')) + ', '.join(filter(None, [ + '%d/%d suites' % ( + sum(passed_suite_perms[k] == v + for k, v in expected_suite_perms.items()), + len(expected_suite_perms)) + if (not args.get('by_suites') + and not args.get('by_cases')) else None, + '%d/%d cases' % ( + sum(passed_case_perms[k] == v + for k, v in expected_case_perms.items()), + len(expected_case_perms)) + if not args.get('by_cases') else None, + '%d/%d perms' % (passed_perms, expected_perms), + '\x1b[31m%d/%d failures\x1b[m' + % (len(failures), expected_perms) + if failures else None])))) sys.stdout.flush() needs_newline = True except KeyboardInterrupt: @@ -791,45 +847,21 @@ def run(**args): expected = 0 passed = 0 failures = [] - if args.get('by_cases'): - for type in ['normal', 'reentrant', 'valgrind']: - for case in expected_case_perms.keys(): - expected_, passed_, failures_, killed = run_stage( - '%s %s' % (type, case), - runner_ + ['--%s' % type, case], - **args) - expected += expected_ - passed += passed_ - failures.extend(failures_) - if (failures and not args.get('keep_going')) or killed: - break - if (failures and not args.get('keep_going')) or killed: - break - elif args.get('by_suites'): - for type in ['normal', 'reentrant', 'valgrind']: - for suite in expected_suite_perms.keys(): - expected_, passed_, failures_, killed = run_stage( - '%s %s' % (type, suite), - runner_ + ['--%s' % type, suite], - **args) - expected += expected_ - passed += passed_ - failures.extend(failures_) - if (failures and not args.get('keep_going')) or killed: - break - if (failures and not args.get('keep_going')) or killed: - break - else: - for type in ['normal', 'reentrant', 'valgrind']: - expected_, passed_, failures_, killed = run_stage( - '%s tests' % type, - runner_ + ['--%s' % type], - **args) - expected += expected_ - passed += passed_ - failures.extend(failures_) - if (failures and not args.get('keep_going')) or killed: - break + for type, by in it.product( + ['normal', 'reentrant', 'valgrind'], + expected_case_perms.keys() if args.get('by_cases') + else expected_suite_perms.keys() if args.get('by_suites') + else [None]): + + expected_, passed_, failures_, killed = run_stage( + '%s %s' % (type, by or 'tests'), + runner_ + ['--%s' % type] + ([by] if by is not None else []), + **args) + expected += expected_ + passed += passed_ + failures.extend(failures_) + if (failures and not args.get('keep_going')) or killed: + break # show summary print() From 5a572ced3c3a531e97dda9c1d7591d09f6627a44 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 7 May 2022 19:13:52 -0500 Subject: [PATCH 11/81] Reworked how test defines are implemented to support recursion Previously test defines were implemented using layers of index-mapped uintmax_t arrays. This worked well for lookup, but limited defines to constants computed at compile-time. Since test defines themselves are actually calculated at _run-time_ (yeah, they have deviated quite a bit from the original, compile-time evaluated defines, which makes the name make less sense), this means defines can't depend on other defines. Which was limiting since a lot of test defines relied on defines generated from the geometry being tested. This new implementation uses callbacks for the per-case defines. This means they can easily contain full C statements, which can depend on other test defines. This does means you can create infinitely-recursive defines, but the test-runner will just break at run-time so don't do that. One concern is that there might be a performance hit for evaluating all defines through callbacks, but if there is it is well below the noise floor: - constants: 43.55s - callbacks: 42.05s --- runners/test_runner.c | 143 +++++++++++++++--------------------------- runners/test_runner.h | 16 ++--- scripts/test_.py | 99 +++++++++++------------------ 3 files changed, 93 insertions(+), 165 deletions(-) diff --git a/runners/test_runner.c b/runners/test_runner.c index 7fbf6ec6..c6f35a33 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -10,34 +10,18 @@ // test geometries struct test_geometry { const char *name; - test_define_t defines[TEST_GEOMETRY_DEFINE_COUNT]; + uintmax_t defines[TEST_GEOMETRY_DEFINE_COUNT]; }; const struct test_geometry test_geometries[TEST_GEOMETRY_COUNT] = TEST_GEOMETRIES; // test define lookup and management -#define TEST_DEFINE_LAYERS 4 -const test_define_t *test_defines[TEST_DEFINE_LAYERS] = { - NULL, - NULL, - NULL, - (const test_define_t[TEST_DEFAULT_COUNT])TEST_DEFAULTS, -}; - -const uint8_t *test_predefine_maps[TEST_DEFINE_LAYERS] = { - NULL, - NULL, - (const uint8_t[TEST_PREDEFINE_COUNT])TEST_GEOMETRY_DEFINE_MAP, - (const uint8_t[TEST_PREDEFINE_COUNT])TEST_DEFAULT_MAP, -}; - -const uint8_t *test_define_maps[TEST_DEFINE_LAYERS] = { - NULL, - NULL, - NULL, - NULL, -}; +const uintmax_t *test_override_defines; +uintmax_t (*const *test_case_defines)(void); +const uintmax_t *test_geometry_defines; +const uintmax_t test_default_defines[TEST_PREDEFINE_COUNT] + = TEST_DEFAULTS; uint8_t test_override_predefine_map[TEST_PREDEFINE_COUNT]; uint8_t test_override_define_map[256]; @@ -53,28 +37,28 @@ const char *const *test_define_names; size_t test_define_count; -test_define_t test_predefine(size_t define) { - for (int i = 0; i < TEST_DEFINE_LAYERS; i++) { - if (test_defines[i] - && test_predefine_maps[i] - && test_predefine_maps[i][define] != 0xff) { - return test_defines[i][test_predefine_maps[i][define]]; - } +uintmax_t test_predefine(size_t define) { + if (test_override_defines + && test_override_predefine_map[define] != 0xff) { + return test_override_defines[test_override_predefine_map[define]]; + } else if (test_case_defines + && test_case_predefine_map[define] != 0xff + && test_case_defines[test_case_predefine_map[define]]) { + return test_case_defines[test_case_predefine_map[define]](); + } else if (define < TEST_GEOMETRY_DEFINE_COUNT) { + return test_geometry_defines[define]; + } else { + return test_default_defines[define-TEST_GEOMETRY_DEFINE_COUNT]; } - - fprintf(stderr, "error: undefined predefine %s\n", - test_predefine_names[define]); - assert(false); - exit(-1); } -test_define_t test_define(size_t define) { - for (int i = 0; i < TEST_DEFINE_LAYERS; i++) { - if (test_defines[i] - && test_define_maps[i] - && test_define_maps[i][define] != 0xff) { - return test_defines[i][test_define_maps[i][define]]; - } +uintmax_t test_define(size_t define) { + if (test_override_defines + && test_override_define_map[define] != 0xff) { + return test_override_defines[test_override_define_map[define]]; + } else if (test_case_defines + && test_case_defines[define]) { + return test_case_defines[define](); } fprintf(stderr, "error: undefined define %s\n", @@ -84,34 +68,33 @@ test_define_t test_define(size_t define) { } static void test_define_geometry(const struct test_geometry *geometry) { - test_defines[2] = geometry->defines; + test_geometry_defines = geometry->defines; } static void test_define_overrides( const char *const *override_names, - const test_define_t *override_defines, + const uintmax_t *override_defines, size_t override_count) { - test_defines[0] = override_defines; + test_override_defines = override_defines; test_override_names = override_names; test_override_count = override_count; - // map any predefines + // map any override predefines memset(test_override_predefine_map, 0xff, TEST_PREDEFINE_COUNT); - for (size_t i = 0; i < override_count; i++) { + for (size_t i = 0; i < test_override_count; i++) { for (size_t j = 0; j < TEST_PREDEFINE_COUNT; j++) { - if (strcmp(override_names[i], test_predefine_names[j]) == 0) { + if (strcmp(test_override_names[i], test_predefine_names[j]) == 0) { test_override_predefine_map[j] = i; } } } - test_predefine_maps[0] = test_override_predefine_map; } static void test_define_suite(const struct test_suite *suite) { test_define_names = suite->define_names; test_define_count = suite->define_count; - // map any defines + // map any override defines memset(test_override_define_map, 0xff, suite->define_count); for (size_t i = 0; i < test_override_count; i++) { for (size_t j = 0; j < suite->define_count; j++) { @@ -120,26 +103,16 @@ static void test_define_suite(const struct test_suite *suite) { } } } - test_define_maps[0] = test_override_define_map; -} - -static void test_define_case( - const struct test_suite *suite, - const struct test_case *case_) { - (void)suite; - // case_->define_map is already correct, but we need to do - // some fixup for the predefine map - test_define_maps[1] = case_->define_map; + // map any suite/case predefines memset(test_case_predefine_map, 0xff, TEST_PREDEFINE_COUNT); - for (size_t i = 0; i < test_define_count; i++) { + for (size_t i = 0; i < suite->define_count; i++) { for (size_t j = 0; j < TEST_PREDEFINE_COUNT; j++) { - if (strcmp(test_define_names[i], test_predefine_names[j]) == 0) { - test_case_predefine_map[j] = case_->define_map[i]; + if (strcmp(suite->define_names[i], test_predefine_names[j]) == 0) { + test_case_predefine_map[j] = i; } } } - test_predefine_maps[1] = test_case_predefine_map; } static void test_define_perm( @@ -148,9 +121,9 @@ static void test_define_perm( size_t perm) { (void)suite; if (case_->defines) { - test_defines[1] = case_->defines[perm]; + test_case_defines = case_->defines[perm]; } else { - test_defines[1] = NULL; + test_case_defines = NULL; } } @@ -251,7 +224,6 @@ static void summary(void) { continue; } - test_define_case(test_suites[i], test_suites[i]->cases[j]); test_case_permcount(test_suites[i], test_suites[i]->cases[j], &perms, &filtered); } @@ -291,7 +263,6 @@ static void list_suites(void) { continue; } - test_define_case(test_suites[i], test_suites[i]->cases[j]); test_case_permcount(test_suites[i], test_suites[i]->cases[j], &perms, &filtered); } @@ -325,8 +296,6 @@ static void list_cases(void) { continue; } - test_define_case(test_suites[i], test_suites[i]->cases[j]); - size_t perms = 0; size_t filtered = 0; test_case_permcount(test_suites[i], test_suites[i]->cases[j], @@ -379,8 +348,6 @@ static void list_defines(void) { continue; } - test_define_case(test_suites[i], test_suites[i]->cases[j]); - for (size_t perm = 0; perm < TEST_GEOMETRY_COUNT * test_suites[i]->cases[j]->permutations; @@ -406,9 +373,9 @@ static void list_defines(void) { // print each define for (size_t k = 0; k < test_suites[i]->define_count; k++) { - if (test_suites[i]->cases[j]->define_map - && test_suites[i]->cases[j]->define_map[k] - != 0xff) { + if (test_suites[i]->cases[j]->defines + && test_suites[i]->cases[j] + ->defines[case_perm][k]) { printf("%s=%jd ", test_suites[i]->define_names[k], test_define(k)); @@ -432,12 +399,10 @@ static void list_geometries(void) { printf("%-36s ", test_geometries[i].name); // print each define - for (size_t k = 0; k < TEST_PREDEFINE_COUNT; k++) { - if (test_predefine_maps[2][k] != 0xff) { - printf("%s=%jd ", - test_predefine_names[k], - test_predefine(k)); - } + for (size_t k = 0; k < TEST_GEOMETRY_DEFINE_COUNT; k++) { + printf("%s=%jd ", + test_predefine_names[k], + test_predefine(k)); } printf("\n"); @@ -447,12 +412,10 @@ static void list_geometries(void) { static void list_defaults(void) { printf("%-36s ", "defaults"); // print each define - for (size_t k = 0; k < TEST_PREDEFINE_COUNT; k++) { - if (test_predefine_maps[3][k] != 0xff) { - printf("%s=%jd ", - test_predefine_names[k], - test_predefine(k)); - } + for (size_t k = 0; k < TEST_DEFAULT_DEFINE_COUNT; k++) { + printf("%s=%jd ", + test_predefine_names[k+TEST_GEOMETRY_DEFINE_COUNT], + test_predefine(k+TEST_GEOMETRY_DEFINE_COUNT)); } printf("\n"); } @@ -471,8 +434,6 @@ static void run(void) { continue; } - test_define_case(test_suites[i], test_suites[i]->cases[j]); - for (size_t perm = 0; perm < TEST_GEOMETRY_COUNT * test_suites[i]->cases[j]->permutations; @@ -628,7 +589,7 @@ int main(int argc, char **argv) { void (*op)(void) = run; static const char **override_names = NULL; - static test_define_t *override_defines = NULL; + static uintmax_t *override_defines = NULL; static size_t override_count = 0; static size_t override_cap = 0; @@ -730,10 +691,10 @@ int main(int argc, char **argv) { override_names = realloc(override_names, override_cap * sizeof(const char *)); override_defines = realloc(override_defines, override_cap - * sizeof(test_define_t)); + * sizeof(uintmax_t)); } - // parse into string key/test_define_t value, cannibalizing the + // parse into string key/uintmax_t value, cannibalizing the // arg in the process char *sep = strchr(optarg, '='); char *parsed = NULL; diff --git a/runners/test_runner.h b/runners/test_runner.h index addb898e..24ed3887 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -12,7 +12,6 @@ enum test_types { }; typedef uint8_t test_types_t; -typedef uintmax_t test_define_t; struct test_case { const char *id; @@ -21,8 +20,7 @@ struct test_case { test_types_t types; size_t permutations; - const test_define_t *const *defines; - const uint8_t *define_map; + uintmax_t (*const *const *defines)(void); bool (*filter)(void); void (*run)(struct lfs_config *cfg); @@ -46,8 +44,8 @@ extern const size_t test_suite_count; // access generated test defines -test_define_t test_predefine(size_t define); -test_define_t test_define(size_t define); +uintmax_t test_predefine(size_t define); +uintmax_t test_define(size_t define); // a few preconfigured defines that control how tests run #define READ_SIZE test_predefine(0) @@ -84,10 +82,7 @@ test_define_t test_define(size_t define); /* ERASE_CYCLES */ 0, \ /* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \ } -#define TEST_DEFAULT_COUNT 5 -#define TEST_DEFAULT_MAP { \ - 0xff, 0xff, 0xff, 0xff, 0xff, 0, 1, 2, 3, 4 \ -} +#define TEST_DEFAULT_DEFINE_COUNT 5 // test geometries #define TEST_GEOMETRIES { \ @@ -100,9 +95,6 @@ test_define_t test_define(size_t define); } #define TEST_GEOMETRY_COUNT 5 #define TEST_GEOMETRY_DEFINE_COUNT 5 -#define TEST_GEOMETRY_DEFINE_MAP { \ - 0, 1, 2, 3, 4, 0xff, 0xff, 0xff, 0xff, 0xff \ -} #endif diff --git a/scripts/test_.py b/scripts/test_.py index 6e94f901..e6e75d5c 100755 --- a/scripts/test_.py +++ b/scripts/test_.py @@ -68,7 +68,6 @@ class TestCase: self.if_ = config.pop('if', None) if isinstance(self.if_, bool): self.if_ = 'true' if self.if_ else 'false' - self.if_lineno = config.pop('if_lineno', None) self.code = config.pop('code') self.code_lineno = config.pop('code_lineno', None) self.in_ = config.pop('in', @@ -125,18 +124,14 @@ class TestSuite: # find line numbers f.seek(0) case_linenos = [] - if_linenos = [] code_linenos = [] for i, line in enumerate(f): match = re.match( '(?P\[\s*cases\s*\.\s*(?P\w+)\s*\])' - '|' '(?Pif\s*=)' '|' '(?Pcode\s*=)', line) if match and match.group('case'): case_linenos.append((i+1, match.group('name'))) - elif match and match.group('if'): - if_linenos.append(i+1) elif match and match.group('code'): code_linenos.append(i+2) @@ -147,23 +142,15 @@ class TestSuite: for (lineno, name), (nlineno, _) in it.zip_longest( case_linenos, case_linenos[1:], fillvalue=(float('inf'), None)): - if_lineno = min( - (l for l in if_linenos if l >= lineno and l < nlineno), - default=None) code_lineno = min( (l for l in code_linenos if l >= lineno and l < nlineno), default=None) cases[name]['lineno'] = lineno - cases[name]['if_lineno'] = if_lineno cases[name]['code_lineno'] = code_lineno self.if_ = config.pop('if', None) if isinstance(self.if_, bool): self.if_ = 'true' if self.if_ else 'false' - self.if_lineno = min( - (l for l in if_linenos - if not case_linenos or l < case_linenos[0][0]), - default=None) self.code = config.pop('code', None) self.code_lineno = min( @@ -274,33 +261,43 @@ def compile(**args): # note it's up to the specific generated file to declare # the test defines def write_case_functions(f, suite, case): + # create case define functions + if case.defines: + # deduplicate defines by value to try to reduce the + # number of functions we generate + define_cbs = {} + for i, defines in enumerate(case.permutations): + for k, v in sorted(defines.items()): + if v not in define_cbs: + name = ('__test__%s__%s__%s__%d' + % (suite.name, case.name, k, i)) + define_cbs[v] = name + f.writeln('uintmax_t %s(void) {' % name) + f.writeln(4*' '+'return %s;' % v) + f.writeln('}') + f.writeln() + f.writeln('uintmax_t (*const *const ' + '__test__%s__%s__defines[])(void) = {' + % (suite.name, case.name)) + for defines in case.permutations: + f.writeln(4*' '+'(uintmax_t (*const[])(void)){') + for define in sorted(suite.defines): + f.writeln(8*' '+'%s,' % ( + define_cbs[defines[define]] + if define in defines + else 'NULL')) + f.writeln(4*' '+'},') + f.writeln('};') + f.writeln() + # create case filter function if suite.if_ is not None or case.if_ is not None: f.writeln('bool __test__%s__%s__filter(void) {' % (suite.name, case.name)) - if suite.if_ is not None: - if suite.if_lineno is not None: - f.writeln(4*' '+'#line %d "%s"' - % (suite.if_lineno, suite.path)) - f.writeln(4*' '+'if (!(%s)) {' % suite.if_) - if suite.if_lineno is not None: - f.writeln(4*' '+'#line %d "%s"' - % (f.lineno+1, args['output'])) - f.writeln(8*' '+'return false;') - f.writeln(4*' '+'}') - f.writeln() - if case.if_ is not None: - if case.if_lineno is not None: - f.writeln(4*' '+'#line %d "%s"' - % (case.if_lineno, suite.path)) - f.writeln(4*' '+'if (!(%s)) {' % case.if_) - if case.if_lineno is not None: - f.writeln(4*' '+'#line %d "%s"' - % (f.lineno+1, args['output'])) - f.writeln(8*' '+'return false;') - f.writeln(4*' '+'}') - f.writeln() - f.writeln(4*' '+'return true;') + f.writeln(4*' '+'return %s;' + % ' && '.join('(%s)' % if_ + for if_ in [suite.if_, case.if_] + if if_ is not None)) f.writeln('}') f.writeln() @@ -350,33 +347,14 @@ def compile(**args): f.writeln() for case in suite.cases: - # create case defines - if case.defines: - f.writeln('const test_define_t *const ' - '__test__%s__%s__defines[] = {' - % (suite.name, case.name)) - for permutation in case.permutations: - f.writeln(4*' '+'(const test_define_t[]){%s},' - % ', '.join(str(v) for _, v in sorted( - permutation.items()))) - f.writeln('};') - f.writeln() - - f.writeln('const uint8_t ' - '__test__%s__%s__define_map[] = {' - % (suite.name, case.name)) - f.writeln(4*' '+'%s,' - % ', '.join( - str(sorted(case.defines).index(k)) - if k in case.defines else '0xff' - for k in sorted(suite.defines))) - f.writeln('};') - f.writeln() - # create case functions if case.in_ is None: write_case_functions(f, suite, case) else: + if case.defines: + f.writeln('extern uintmax_t (*const *const ' + '__test__%s__%s__defines[])(void);' + % (suite.name, case.name)) if suite.if_ is not None or case.if_ is not None: f.writeln('extern bool __test__%s__%s__filter(' 'void);' @@ -402,9 +380,6 @@ def compile(**args): if case.defines: f.writeln(4*' '+'.defines = __test__%s__%s__defines,' % (suite.name, case.name)) - f.writeln(4*' '+'.define_map = ' - '__test__%s__%s__define_map,' - % (suite.name, case.name)) if suite.if_ is not None or case.if_ is not None: f.writeln(4*' '+'.filter = __test__%s__%s__filter,' % (suite.name, case.name)) From d679fbb389935cf1bc83fb31b31c4de20fc9ade4 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 8 May 2022 09:51:14 -0500 Subject: [PATCH 12/81] In ./scripts/test.py, readded external commands, tweaked subprocesses - Added --exec for wrapping the test-runner with external commands, such as Qemu or Valgrind. - Added --valgrind, which just aliases --exec=valgrind with a few extra flags useful during testing. - Dropped the "valgrind" type for tests. These aren't separate tests that run in the test-runner, and I don't see a need for disabling Valgrind for any tests. This can be added back later if needed. - Readded support for dropping directly into gdb after a test failure, either at the assert failure, entry point of test case, or entry point of the test runner with --gdb, --gdb-case, or --gdb-main. - Added --isolate for running each test permutation in its own process, this is required for associating Valgrind errors with the right test case. - Fixed an issue where explicit test identifier conflicted with per-stage test identifiers generated as a part of --by-suite and --by-case. --- runners/test_runner.c | 21 ++----- runners/test_runner.h | 1 - scripts/test_.py | 134 ++++++++++++++++++++++++++++++------------ 3 files changed, 103 insertions(+), 53 deletions(-) diff --git a/runners/test_runner.c b/runners/test_runner.c index c6f35a33..1b3ea5c1 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -235,10 +235,9 @@ static void summary(void) { char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", filtered, perms); char type_buf[64]; - sprintf(type_buf, "%s%s%s", + sprintf(type_buf, "%s%s", (types & TEST_NORMAL) ? "n" : "", - (types & TEST_REENTRANT) ? "r" : "", - (types & TEST_VALGRIND) ? "V" : ""); + (types & TEST_REENTRANT) ? "r" : ""); printf("%-36s %7s %7zu %7zu %11s\n", "TOTAL", type_buf, @@ -270,10 +269,9 @@ static void list_suites(void) { char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", filtered, perms); char type_buf[64]; - sprintf(type_buf, "%s%s%s", + sprintf(type_buf, "%s%s", (test_suites[i]->types & TEST_NORMAL) ? "n" : "", - (test_suites[i]->types & TEST_REENTRANT) ? "r" : "", - (test_suites[i]->types & TEST_VALGRIND) ? "V" : ""); + (test_suites[i]->types & TEST_REENTRANT) ? "r" : ""); printf("%-36s %7s %7zu %11s\n", test_suites[i]->id, type_buf, @@ -305,10 +303,9 @@ static void list_cases(void) { char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", filtered, perms); char type_buf[64]; - sprintf(type_buf, "%s%s%s", + sprintf(type_buf, "%s%s", (types & TEST_NORMAL) ? "n" : "", - (types & TEST_REENTRANT) ? "r" : "", - (types & TEST_VALGRIND) ? "V" : ""); + (types & TEST_REENTRANT) ? "r" : ""); printf("%-36s %7s %11s\n", test_suites[i]->cases[j]->id, type_buf, @@ -532,7 +529,6 @@ enum opt_flags { OPT_GEOMETRY = 'G', OPT_NORMAL = 'n', OPT_REENTRANT = 'r', - OPT_VALGRIND = 'V', OPT_START = 5, OPT_STEP = 6, OPT_STOP = 7, @@ -555,7 +551,6 @@ const struct option long_opts[] = { {"geometry", required_argument, NULL, OPT_GEOMETRY}, {"normal", no_argument, NULL, OPT_NORMAL}, {"reentrant", no_argument, NULL, OPT_REENTRANT}, - {"valgrind", no_argument, NULL, OPT_VALGRIND}, {"start", required_argument, NULL, OPT_START}, {"stop", required_argument, NULL, OPT_STOP}, {"step", required_argument, NULL, OPT_STEP}, @@ -577,7 +572,6 @@ const char *const help_text[] = { "Filter by geometry.", "Filter for normal tests. Can be combined.", "Filter for reentrant tests. Can be combined.", - "Filter for Valgrind tests. Can be combined.", "Start at the nth test.", "Stop before the nth test.", "Only run every n tests, calculated after --start and --stop.", @@ -724,9 +718,6 @@ invalid_define: case OPT_REENTRANT: test_types |= TEST_REENTRANT; break; - case OPT_VALGRIND: - test_types |= TEST_VALGRIND; - break; case OPT_START: { char *parsed = NULL; test_start = strtoumax(optarg, &parsed, 0); diff --git a/runners/test_runner.h b/runners/test_runner.h index 24ed3887..22cbcaa4 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -8,7 +8,6 @@ enum test_types { TEST_NORMAL = 0x1, TEST_REENTRANT = 0x2, - TEST_VALGRIND = 0x4, }; typedef uint8_t test_types_t; diff --git a/scripts/test_.py b/scripts/test_.py index e6e75d5c..02147ef2 100755 --- a/scripts/test_.py +++ b/scripts/test_.py @@ -13,6 +13,7 @@ import pty import re import shlex import shutil +import signal import subprocess as sp import threading as th import time @@ -77,8 +78,6 @@ class TestCase: config.pop('suite_normal', True)) self.reentrant = config.pop('reentrant', config.pop('suite_reentrant', False)) - self.valgrind = config.pop('valgrind', - config.pop('suite_valgrind', True)) # figure out defines and build possible permutations self.defines = set() @@ -163,7 +162,6 @@ class TestSuite: in_ = config.pop('in', None) normal = config.pop('normal', True) reentrant = config.pop('reentrant', False) - valgrind = config.pop('valgrind', True) self.cases = [] for name, case in sorted(cases.items(), @@ -177,7 +175,6 @@ class TestSuite: 'suite_in': in_, 'suite_normal': normal, 'suite_reentrant': reentrant, - 'suite_valgrind': valgrind, **case})) # combine per-case defines @@ -187,7 +184,6 @@ class TestSuite: # combine other per-case things self.normal = any(case.normal for case in self.cases) self.reentrant = any(case.reentrant for case in self.cases) - self.valgrind = any(case.valgrind for case in self.cases) for k in config.keys(): print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r' @@ -202,7 +198,7 @@ class TestSuite: def compile(**args): # find .toml files paths = [] - for path in args.get('test_paths', TEST_PATHS): + for path in args.get('test_ids', TEST_PATHS): if os.path.isdir(path): path = path + '/*.toml' @@ -210,13 +206,13 @@ def compile(**args): paths.append(path) if not paths: - print('no test suites found in %r?' % args['test_paths']) + print('no test suites found in %r?' % args['test_ids']) sys.exit(-1) if not args.get('source'): if len(paths) > 1: print('more than one test suite for compilation? (%r)' - % args['test_paths']) + % args['test_ids']) sys.exit(-1) # load our suite @@ -373,8 +369,7 @@ def compile(**args): f.writeln(4*' '+'.types = %s,' % ' | '.join(filter(None, [ 'TEST_NORMAL' if case.normal else None, - 'TEST_REENTRANT' if case.reentrant else None, - 'TEST_VALGRIND' if case.valgrind else None]))) + 'TEST_REENTRANT' if case.reentrant else None]))) f.writeln(4*' '+'.permutations = %d,' % len(case.permutations)) if case.defines: @@ -406,8 +401,7 @@ def compile(**args): f.writeln(4*' '+'.types = %s,' % ' | '.join(filter(None, [ 'TEST_NORMAL' if suite.normal else None, - 'TEST_REENTRANT' if suite.reentrant else None, - 'TEST_VALGRIND' if suite.valgrind else None]))) + 'TEST_REENTRANT' if suite.reentrant else None]))) if suite.defines: f.writeln(4*' '+'.define_names = __test__%s__define_names,' % suite.name) @@ -434,7 +428,9 @@ def compile(**args): # write any internal tests for suite in suites: for case in suite.cases: - if case.in_ == args.get('source'): + if (case.in_ is not None + and os.path.normpath(case.in_) + == os.path.normpath(args['source'])): # write defines, but note we need to undef any # new defines since we're in someone else's file if suite.defines: @@ -475,15 +471,27 @@ def compile(**args): def runner(**args): cmd = args['runner'].copy() - # TODO multiple paths? - if 'test_paths' in args: - cmd.extend(args.get('test_paths')) + cmd.extend(args.get('test_ids')) + # run under some external command? + cmd[:0] = args.get('exec', []) + + # run under valgrind? + if args.get('valgrind'): + cmd[:0] = filter(None, [ + 'valgrind', + '--leak-check=full', + '--track-origins=yes', + '--error-exitcode=4', + '-q']) + + # filter tests? if args.get('normal'): cmd.append('-n') if args.get('reentrant'): cmd.append('-r') - if args.get('valgrind'): cmd.append('-V') if args.get('geometry'): cmd.append('-G%s' % args.get('geometry')) + + # defines? if args.get('define'): for define in args.get('define'): cmd.append('-D%s' % define) @@ -522,8 +530,7 @@ def find_cases(runner_, **args): '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' '[^\s]+\s+(?P\d+)/(?P\d+)') # skip the first line - next(proc.stdout) - for line in proc.stdout: + for line in it.islice(proc.stdout, 1, None): m = pattern.match(line) if m: filtered = int(m.group('filtered')) @@ -558,7 +565,6 @@ def find_paths(runner_, **args): pattern = re.compile( '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' '(?P[^:]+):(?P\d+)') - # skip the first line for line in proc.stdout: m = pattern.match(line) if m: @@ -586,7 +592,6 @@ def find_defines(runner_, **args): pattern = re.compile( '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' '(?P(?:\w+=\w+\s*)+)') - # skip the first line for line in proc.stdout: m = pattern.match(line) if m: @@ -614,7 +619,6 @@ def run_stage(name, runner_, **args): expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( find_cases(runner_, **args)) - # TODO valgrind/gdb/exec passed_suite_perms = co.defaultdict(lambda: 0) passed_case_perms = co.defaultdict(lambda: 0) passed_perms = 0 @@ -627,7 +631,6 @@ def run_stage(name, runner_, **args): '|' '(?P[^:]+):(?P\d+):(?Passert):' ' *(?P.*)' ')$') locals = th.local() - # TODO use process group instead of this set? children = set() def run_runner(runner_): @@ -714,17 +717,23 @@ def run_stage(name, runner_, **args): nonlocal failures nonlocal locals - while (start or 0) < total_perms: + start = start or 0 + step = step or 1 + while start < total_perms: runner_ = runner.copy() if start is not None: runner_.append('--start=%d' % start) if step is not None: runner_.append('--step=%d' % step) + if args.get('isolate') or args.get('valgrind'): + runner_.append('--stop=%d' % (start+step)) try: # run the tests locals.seen_perms = 0 run_runner(runner_) + assert locals.seen_perms > 0 + start += locals.seen_perms*step except TestFailure as failure: # race condition for multiple failures? @@ -735,14 +744,13 @@ def run_stage(name, runner_, **args): if args.get('keep_going') and not killed: # resume after failed test - start = (start or 0) + locals.seen_perms*(step or 1) + assert locals.seen_perms > 0 + start += locals.seen_perms*step continue else: # stop other tests for child in children.copy(): child.kill() - - break # parallel jobs? @@ -808,7 +816,7 @@ def run(**args): start = time.time() runner_ = runner(**args) - print('using runner `%s`' + print('using runner: %s' % ' '.join(shlex.quote(c) for c in runner_)) expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( find_cases(runner_, **args)) @@ -823,15 +831,19 @@ def run(**args): passed = 0 failures = [] for type, by in it.product( - ['normal', 'reentrant', 'valgrind'], + ['normal', 'reentrant'], expected_case_perms.keys() if args.get('by_cases') else expected_suite_perms.keys() if args.get('by_suites') else [None]): + # rebuild runner for each stage to override test identifier if needed + stage_runner = runner(**args | { + 'test_ids': [by] if by is not None else args.get('test_ids', []), + 'normal': type == 'normal', + 'reentrant': type == 'reentrant'}) + # spawn jobs for stage expected_, passed_, failures_, killed = run_stage( - '%s %s' % (type, by or 'tests'), - runner_ + ['--%s' % type] + ([by] if by is not None else []), - **args) + '%s %s' % (type, by or 'tests'), stage_runner, **args) expected += expected_ passed += passed_ failures.extend(failures_) @@ -879,6 +891,40 @@ def run(**args): print(line) print() + # drop into gdb? + if failures and (args.get('gdb') + or args.get('gdb_case') + or args.get('gdb_main')): + failure = failures[0] + runner_ = runner(**args | {'test_ids': [failure.id]}) + + if args.get('gdb_main'): + cmd = ['gdb', + '-ex', 'break main', + '-ex', 'run', + '--args'] + runner_ + elif args.get('gdb_case'): + path, lineno = runner_paths[testcase(failure.id)] + cmd = ['gdb', + '-ex', 'break %s:%d' % (path, lineno), + '-ex', 'run', + '--args'] + runner_ + elif failure.assert_ is not None: + cmd = ['gdb', + '-ex', 'run', + '-ex', 'frame function raise', + '-ex', 'up 2', + '--args'] + runner_ + else: + cmd = ['gdb', + '-ex', 'run', + '--args'] + runner_ + + # exec gdb interactively + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + os.execvp(cmd[0], cmd) + return 1 if failures else 0 @@ -903,10 +949,11 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( description="Build and run tests.", conflict_handler='resolve') - # TODO document test case/perm specifier - parser.add_argument('test_paths', nargs='*', + parser.add_argument('test_ids', nargs='*', help="Description of testis to run. May be a directory, path, or \ - test identifier. Defaults to %r." % TEST_PATHS) + test identifier. Test identifiers are of the form \ + ##, but suffixes can be \ + dropped to run any matching tests. Defaults to %r." % TEST_PATHS) parser.add_argument('-v', '--verbose', action='store_true', help="Output commands that run behind the scenes.") # test flags @@ -933,8 +980,6 @@ if __name__ == "__main__": help="Filter for normal tests. Can be combined.") test_parser.add_argument('-r', '--reentrant', action='store_true', help="Filter for reentrant tests. Can be combined.") - test_parser.add_argument('-V', '--valgrind', action='store_true', - help="Filter for Valgrind tests. Can be combined.") test_parser.add_argument('-d', '--disk', help="Use this file as the disk.") test_parser.add_argument('-t', '--trace', @@ -949,10 +994,25 @@ if __name__ == "__main__": help="Number of parallel runners to run.") test_parser.add_argument('-k', '--keep-going', action='store_true', help="Don't stop on first error.") + test_parser.add_argument('-i', '--isolate', action='store_true', + help="Run each test permutation in a separate process.") test_parser.add_argument('-b', '--by-suites', action='store_true', help="Step through tests by suite.") test_parser.add_argument('-B', '--by-cases', action='store_true', help="Step through tests by case.") + test_parser.add_argument('--gdb', action='store_true', + help="Drop into gdb on test failure.") + test_parser.add_argument('--gdb-case', action='store_true', + help="Drop into gdb on test failure but stop at the beginning \ + of the failing test case.") + test_parser.add_argument('--gdb-main', action='store_true', + help="Drop into gdb on test failure but stop at the beginning \ + of main.") + test_parser.add_argument('--valgrind', action='store_true', + help="Run under Valgrind to find memory errors. Implicitly sets \ + --isolate.") + test_parser.add_argument('--exec', default=[], type=lambda e: e.split(), + help="Run under another executable.") # compilation flags comp_parser = parser.add_argument_group('compilation options') comp_parser.add_argument('-c', '--compile', action='store_true', From 0781f50edb0433e44fa498d8e8ca41d7c4948b77 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 14 May 2022 02:55:17 -0500 Subject: [PATCH 13/81] Ported tests to new framework This mostly required names for each test case, declarations of previously-implicit variables since the new test framework is more conservative with what it declares (the small extra effort to add declarations is well worth the simplicity and improved readability), and tweaks to work with not-really-constant defines. Also renamed test_ -> test, replacing the old ./scripts/test.py, unfortunately git seems to have had a hard time with this. --- Makefile | 36 +- runners/test_runner.c | 22 +- runners/test_runner.h | 6 +- scripts/test.py | 1735 +++++++++++++++++++--------------- scripts/test_.py | 1027 -------------------- tests/test_alloc.toml | 292 +++--- tests/test_attrs.toml | 50 +- tests/test_badblocks.toml | 129 +-- tests/test_dirs.toml | 267 ++++-- tests/test_entries.toml | 87 +- tests/test_evil.toml | 116 ++- tests/test_exhaustion.toml | 204 ++-- tests/test_files.toml | 174 ++-- tests/test_interspersed.toml | 72 +- tests/test_move.toml | 458 +++++---- tests/test_orphans.toml | 61 +- tests/test_paths.toml | 121 ++- tests/test_relocations.toml | 162 ++-- tests/test_seek.toml | 97 +- tests/test_superblocks.toml | 86 +- tests/test_truncate.toml | 154 +-- 21 files changed, 2538 insertions(+), 2818 deletions(-) delete mode 100755 scripts/test_.py diff --git a/Makefile b/Makefile index fc164516..9cc37706 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ DEP := $(SRC:%.c=$(BUILDDIR)%.d) ASM := $(SRC:%.c=$(BUILDDIR)%.s) CGI := $(SRC:%.c=$(BUILDDIR)%.ci) -TESTS ?= $(wildcard tests_/*.toml) +TESTS ?= $(wildcard tests/*.toml) TEST_SRC ?= $(SRC) \ $(filter-out $(wildcard bd/*.*.c),$(wildcard bd/*.c)) \ runners/test_runner.c @@ -53,10 +53,11 @@ override CFLAGS += -g3 override CFLAGS += -I. override CFLAGS += -std=c99 -Wall -pedantic override CFLAGS += -Wextra -Wshadow -Wjump-misses-init -Wundef +override CFLAGS += -ftrack-macro-expansion=0 -override TESTFLAGS_ += -b +override TESTFLAGS += -b # forward -j flag -override TESTFLAGS_ += $(filter -j%,$(MAKEFLAGS)) +override TESTFLAGS += $(filter -j%,$(MAKEFLAGS)) ifdef VERBOSE override TESTFLAGS += -v override CALLSFLAGS += -v @@ -65,15 +66,14 @@ override DATAFLAGS += -v override STACKFLAGS += -v override STRUCTSFLAGS += -v override COVERAGEFLAGS += -v -override TESTFLAGS_ += -v -override TESTCFLAGS_ += -v +override TESTFLAGS += -v +override TESTCFLAGS += -v endif ifdef EXEC -override TESTFLAGS_ += --exec="$(EXEC)" +override TESTFLAGS += --exec="$(EXEC)" endif ifdef COVERAGE -override TESTFLAGS += --coverage -override TESTFLAGS_ += --coverage +override TESTFLAGS += --coverage endif ifdef BUILDDIR override TESTFLAGS += --build-dir="$(BUILDDIR:/=)" @@ -112,23 +112,16 @@ tags: calls: $(CGI) ./scripts/calls.py $^ $(CALLSFLAGS) -.PHONY: test -test: - ./scripts/test.py $(TESTFLAGS) -.SECONDEXPANSION: -test%: tests/test$$(firstword $$(subst \#, ,%)).toml - ./scripts/test.py $@ $(TESTFLAGS) - .PHONY: test_runner test_runner: $(BUILDDIR)runners/test_runner -.PHONY: test_ -test_: test_runner - ./scripts/test_.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS_) +.PHONY: test +test: test_runner + ./scripts/test.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS) .PHONY: test_list test_list: test_runner - ./scripts/test_.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS_) -l + ./scripts/test.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS) -l .PHONY: code code: $(OBJ) @@ -199,10 +192,10 @@ $(BUILDDIR)%.a.c: $(BUILDDIR)%.c ./scripts/explode_asserts.py $< -o $@ $(BUILDDIR)%.t.c: %.toml - ./scripts/test_.py -c $< $(TESTCFLAGS_) -o $@ + ./scripts/test.py -c $< $(TESTCFLAGS) -o $@ $(BUILDDIR)%.t.c: %.c $(TESTS) - ./scripts/test_.py -c $(TESTS) -s $< $(TESTCFLAGS_) -o $@ + ./scripts/test.py -c $(TESTS) -s $< $(TESTCFLAGS) -o $@ # clean everything .PHONY: clean @@ -215,7 +208,6 @@ clean: rm -f $(CGI) rm -f $(DEP) rm -f $(ASM) - rm -f $(BUILDDIR)tests/*.toml.* rm -f $(TEST_TSRC) rm -f $(TEST_TASRC) rm -f $(TEST_TAOBJ) diff --git a/runners/test_runner.c b/runners/test_runner.c index 1b3ea5c1..21ff1c00 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -10,17 +10,17 @@ // test geometries struct test_geometry { const char *name; - uintmax_t defines[TEST_GEOMETRY_DEFINE_COUNT]; + intmax_t defines[TEST_GEOMETRY_DEFINE_COUNT]; }; const struct test_geometry test_geometries[TEST_GEOMETRY_COUNT] = TEST_GEOMETRIES; // test define lookup and management -const uintmax_t *test_override_defines; -uintmax_t (*const *test_case_defines)(void); -const uintmax_t *test_geometry_defines; -const uintmax_t test_default_defines[TEST_PREDEFINE_COUNT] +const intmax_t *test_override_defines; +intmax_t (*const *test_case_defines)(void); +const intmax_t *test_geometry_defines; +const intmax_t test_default_defines[TEST_PREDEFINE_COUNT] = TEST_DEFAULTS; uint8_t test_override_predefine_map[TEST_PREDEFINE_COUNT]; @@ -37,7 +37,7 @@ const char *const *test_define_names; size_t test_define_count; -uintmax_t test_predefine(size_t define) { +intmax_t test_predefine(size_t define) { if (test_override_defines && test_override_predefine_map[define] != 0xff) { return test_override_defines[test_override_predefine_map[define]]; @@ -52,7 +52,7 @@ uintmax_t test_predefine(size_t define) { } } -uintmax_t test_define(size_t define) { +intmax_t test_define(size_t define) { if (test_override_defines && test_override_define_map[define] != 0xff) { return test_override_defines[test_override_define_map[define]]; @@ -73,7 +73,7 @@ static void test_define_geometry(const struct test_geometry *geometry) { static void test_define_overrides( const char *const *override_names, - const uintmax_t *override_defines, + const intmax_t *override_defines, size_t override_count) { test_override_defines = override_defines; test_override_names = override_names; @@ -583,7 +583,7 @@ int main(int argc, char **argv) { void (*op)(void) = run; static const char **override_names = NULL; - static uintmax_t *override_defines = NULL; + static intmax_t *override_defines = NULL; static size_t override_count = 0; static size_t override_cap = 0; @@ -685,10 +685,10 @@ int main(int argc, char **argv) { override_names = realloc(override_names, override_cap * sizeof(const char *)); override_defines = realloc(override_defines, override_cap - * sizeof(uintmax_t)); + * sizeof(intmax_t)); } - // parse into string key/uintmax_t value, cannibalizing the + // parse into string key/intmax_t value, cannibalizing the // arg in the process char *sep = strchr(optarg, '='); char *parsed = NULL; diff --git a/runners/test_runner.h b/runners/test_runner.h index 22cbcaa4..e0336379 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -19,7 +19,7 @@ struct test_case { test_types_t types; size_t permutations; - uintmax_t (*const *const *defines)(void); + intmax_t (*const *const *defines)(void); bool (*filter)(void); void (*run)(struct lfs_config *cfg); @@ -43,8 +43,8 @@ extern const size_t test_suite_count; // access generated test defines -uintmax_t test_predefine(size_t define); -uintmax_t test_define(size_t define); +intmax_t test_predefine(size_t define); +intmax_t test_define(size_t define); // a few preconfigured defines that control how tests run #define READ_SIZE test_predefine(0) diff --git a/scripts/test.py b/scripts/test.py index 92a13b1d..4110bbae 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -1,291 +1,666 @@ #!/usr/bin/env python3 - -# This script manages littlefs tests, which are configured with -# .toml files stored in the tests directory. +# +# Script to compile and runs tests. # -import toml -import glob -import re -import os -import io -import itertools as it -import collections.abc as abc -import subprocess as sp -import base64 -import sys -import copy -import shlex -import pty +import collections as co import errno +import glob +import itertools as it +import math as m +import os +import pty +import re +import shlex +import shutil import signal +import subprocess as sp +import threading as th +import time +import toml -TEST_PATHS = 'tests' -RULES = """ -# add block devices to sources -TESTSRC ?= $(SRC) $(wildcard bd/*.c) -define FLATTEN -%(path)s%%$(subst /,.,$(target)): $(target) - ./scripts/explode_asserts.py $$< -o $$@ -endef -$(foreach target,$(TESTSRC),$(eval $(FLATTEN))) +TEST_PATHS = ['tests'] +RUNNER_PATH = './runners/test_runner' --include %(path)s*.d -.SECONDARY: - -%(path)s.test: %(path)s.test.o \\ - $(foreach t,$(subst /,.,$(TESTSRC:.c=.o)),%(path)s.$t) - $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ - -# needed in case builddir is different -%(path)s%%.o: %(path)s%%.c - $(CC) -c -MMD $(CFLAGS) $< -o $@ -""" -COVERAGE_RULES = """ -%(path)s.test: override CFLAGS += -fprofile-arcs -ftest-coverage - -# delete lingering coverage -%(path)s.test: | %(path)s.info.clean -.PHONY: %(path)s.info.clean -%(path)s.info.clean: - rm -f %(path)s*.gcda - -# accumulate coverage info -.PHONY: %(path)s.info -%(path)s.info: - $(strip $(LCOV) -c \\ - $(addprefix -d ,$(wildcard %(path)s*.gcda)) \\ - --rc 'geninfo_adjust_src_path=$(shell pwd)' \\ - -o $@) - $(LCOV) -e $@ $(addprefix /,$(SRC)) -o $@ -ifdef COVERAGETARGET - $(strip $(LCOV) -a $@ \\ - $(addprefix -a ,$(wildcard $(COVERAGETARGET))) \\ - -o $(COVERAGETARGET)) -endif -""" -GLOBALS = """ -//////////////// AUTOGENERATED TEST //////////////// -#include "lfs.h" +SUITE_PROLOGUE = """ +#include "runners/test_runner.h" #include "bd/lfs_testbd.h" #include -extern const char *lfs_testbd_path; -extern uint32_t lfs_testbd_cycles; """ -DEFINES = { - 'LFS_READ_SIZE': 16, - 'LFS_PROG_SIZE': 'LFS_READ_SIZE', - 'LFS_BLOCK_SIZE': 512, - 'LFS_BLOCK_COUNT': 1024, - 'LFS_BLOCK_CYCLES': -1, - 'LFS_CACHE_SIZE': '(64 % LFS_PROG_SIZE == 0 ? 64 : LFS_PROG_SIZE)', - 'LFS_LOOKAHEAD_SIZE': 16, - 'LFS_ERASE_VALUE': 0xff, - 'LFS_ERASE_CYCLES': 0, - 'LFS_BADBLOCK_BEHAVIOR': 'LFS_TESTBD_BADBLOCK_PROGERROR', -} -PROLOGUE = """ - // prologue - __attribute__((unused)) lfs_t lfs; - __attribute__((unused)) lfs_testbd_t bd; - __attribute__((unused)) lfs_file_t file; - __attribute__((unused)) lfs_dir_t dir; - __attribute__((unused)) struct lfs_info info; - __attribute__((unused)) char path[1024]; - __attribute__((unused)) uint8_t buffer[1024]; - __attribute__((unused)) lfs_size_t size; - __attribute__((unused)) int err; - - __attribute__((unused)) const struct lfs_config cfg = { - .context = &bd, - .read = lfs_testbd_read, - .prog = lfs_testbd_prog, - .erase = lfs_testbd_erase, - .sync = lfs_testbd_sync, - .read_size = LFS_READ_SIZE, - .prog_size = LFS_PROG_SIZE, - .block_size = LFS_BLOCK_SIZE, - .block_count = LFS_BLOCK_COUNT, - .block_cycles = LFS_BLOCK_CYCLES, - .cache_size = LFS_CACHE_SIZE, - .lookahead_size = LFS_LOOKAHEAD_SIZE, - }; - - __attribute__((unused)) const struct lfs_testbd_config bdcfg = { - .erase_value = LFS_ERASE_VALUE, - .erase_cycles = LFS_ERASE_CYCLES, - .badblock_behavior = LFS_BADBLOCK_BEHAVIOR, - .power_cycles = lfs_testbd_cycles, - }; - - lfs_testbd_createcfg(&cfg, lfs_testbd_path, &bdcfg) => 0; +CASE_PROLOGUE = """ """ -EPILOGUE = """ - // epilogue - lfs_testbd_destroy(&cfg) => 0; +CASE_EPILOGUE = """ """ -PASS = '\033[32m✓\033[0m' -FAIL = '\033[31m✗\033[0m' -class TestFailure(Exception): - def __init__(self, case, returncode=None, stdout=None, assert_=None): - self.case = case - self.returncode = returncode - self.stdout = stdout - self.assert_ = assert_ + +def testpath(path): + path, *_ = path.split('#', 1) + return path + +def testsuite(path): + suite = testpath(path) + suite = os.path.basename(suite) + if suite.endswith('.toml'): + suite = suite[:-len('.toml')] + return suite + +def testcase(path): + _, case, *_ = path.split('#', 2) + return '%s#%s' % (testsuite(path), case) + +# TODO move this out in other files +def openio(path, mode='r'): + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) class TestCase: - def __init__(self, config, filter=filter, - suite=None, caseno=None, lineno=None, **_): - self.config = config - self.filter = filter - self.suite = suite - self.caseno = caseno - self.lineno = lineno + # create a TestCase object from a config + def __init__(self, config, args={}): + self.name = config.pop('name') + self.path = config.pop('path') + self.suite = config.pop('suite') + self.lineno = config.pop('lineno', None) + self.if_ = config.pop('if', None) + if isinstance(self.if_, bool): + self.if_ = 'true' if self.if_ else 'false' + self.code = config.pop('code') + self.code_lineno = config.pop('code_lineno', None) + self.in_ = config.pop('in', + config.pop('suite_in', None)) - self.code = config['code'] - self.code_lineno = config['code_lineno'] - self.defines = config.get('define', {}) - self.if_ = config.get('if', None) - self.in_ = config.get('in', None) + self.normal = config.pop('normal', + config.pop('suite_normal', True)) + self.reentrant = config.pop('reentrant', + config.pop('suite_reentrant', False)) - self.result = None + # figure out defines and build possible permutations + self.defines = set() + self.permutations = [] + + suite_defines = config.pop('suite_defines', {}) + if not isinstance(suite_defines, list): + suite_defines = [suite_defines] + defines = config.pop('defines', {}) + if not isinstance(defines, list): + defines = [defines] + + # build possible permutations + for suite_defines_ in suite_defines: + self.defines |= suite_defines_.keys() + for defines_ in defines: + self.defines |= defines_.keys() + self.permutations.extend(map(dict, it.product(*( + [(k, v) for v in (vs if isinstance(vs, list) else [vs])] + for k, vs in sorted( + (suite_defines_ | defines_).items()))))) + + for k in config.keys(): + print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r' + % (self.id(), k), + file=sys.stderr) + + def id(self): + return '%s#%s' % (self.suite, self.name) + + +class TestSuite: + # create a TestSuite object from a toml file + def __init__(self, path, args={}): + self.name = testsuite(path) + self.path = testpath(path) + + # load toml file and parse test cases + with open(self.path) as f: + # load tests + config = toml.load(f) + + # find line numbers + f.seek(0) + case_linenos = [] + code_linenos = [] + for i, line in enumerate(f): + match = re.match( + '(?P\[\s*cases\s*\.\s*(?P\w+)\s*\])' + '|' '(?Pcode\s*=)', + line) + if match and match.group('case'): + case_linenos.append((i+1, match.group('name'))) + elif match and match.group('code'): + code_linenos.append(i+2) + + # sort in case toml parsing did not retain order + case_linenos.sort() + + cases = config.pop('cases') + for (lineno, name), (nlineno, _) in it.zip_longest( + case_linenos, case_linenos[1:], + fillvalue=(float('inf'), None)): + code_lineno = min( + (l for l in code_linenos if l >= lineno and l < nlineno), + default=None) + cases[name]['lineno'] = lineno + cases[name]['code_lineno'] = code_lineno + + self.if_ = config.pop('if', None) + if isinstance(self.if_, bool): + self.if_ = 'true' if self.if_ else 'false' + + self.code = config.pop('code', None) + self.code_lineno = min( + (l for l in code_linenos + if not case_linenos or l < case_linenos[0][0]), + default=None) + + # a couple of these we just forward to all cases + defines = config.pop('defines', {}) + in_ = config.pop('in', None) + normal = config.pop('normal', True) + reentrant = config.pop('reentrant', False) + + self.cases = [] + for name, case in sorted(cases.items(), + key=lambda c: c[1].get('lineno')): + self.cases.append(TestCase(config={ + 'name': name, + 'path': path + (':%d' % case['lineno'] + if 'lineno' in case else ''), + 'suite': self.name, + 'suite_defines': defines, + 'suite_in': in_, + 'suite_normal': normal, + 'suite_reentrant': reentrant, + **case})) + + # combine per-case defines + self.defines = set.union(*( + set(case.defines) for case in self.cases)) + + # combine other per-case things + self.normal = any(case.normal for case in self.cases) + self.reentrant = any(case.reentrant for case in self.cases) + + for k in config.keys(): + print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r' + % (self.id(), k), + file=sys.stderr) + + def id(self): + return self.name + + + +def compile(**args): + # find .toml files + paths = [] + for path in args.get('test_ids', TEST_PATHS): + if os.path.isdir(path): + path = path + '/*.toml' + + for path in glob.glob(path): + paths.append(path) + + if not paths: + print('no test suites found in %r?' % args['test_ids']) + sys.exit(-1) + + if not args.get('source'): + if len(paths) > 1: + print('more than one test suite for compilation? (%r)' + % args['test_ids']) + sys.exit(-1) + + # load our suite + suite = TestSuite(paths[0]) + else: + # load all suites + suites = [TestSuite(path) for path in paths] + suites.sort(key=lambda s: s.name) + + # write generated test source + if 'output' in args: + with openio(args['output'], 'w') as f: + _write = f.write + def write(s): + f.lineno += s.count('\n') + _write(s) + def writeln(s=''): + f.lineno += s.count('\n') + 1 + _write(s) + _write('\n') + f.lineno = 1 + f.write = write + f.writeln = writeln + + # redirect littlefs tracing + f.writeln('#define LFS_TRACE_(fmt, ...) do { \\') + f.writeln(8*' '+'extern FILE *test_trace; \\') + f.writeln(8*' '+'if (test_trace) { \\') + f.writeln(12*' '+'fprintf(test_trace, ' + '"%s:%d:trace: " fmt "%s\\n", \\') + f.writeln(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\') + f.writeln(8*' '+'} \\') + f.writeln(4*' '+'} while (0)') + f.writeln('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")') + f.writeln('#define LFS_TESTBD_TRACE(...) ' + 'LFS_TRACE_(__VA_ARGS__, "")') + f.writeln() + + # write out generated functions, this can end up in different + # files depending on the "in" attribute + # + # note it's up to the specific generated file to declare + # the test defines + def write_case_functions(f, suite, case): + # create case define functions + if case.defines: + # deduplicate defines by value to try to reduce the + # number of functions we generate + define_cbs = {} + for i, defines in enumerate(case.permutations): + for k, v in sorted(defines.items()): + if v not in define_cbs: + name = ('__test__%s__%s__%s__%d' + % (suite.name, case.name, k, i)) + define_cbs[v] = name + f.writeln('intmax_t %s(void) {' % name) + f.writeln(4*' '+'return %s;' % v) + f.writeln('}') + f.writeln() + f.writeln('intmax_t (*const *const ' + '__test__%s__%s__defines[])(void) = {' + % (suite.name, case.name)) + for defines in case.permutations: + f.writeln(4*' '+'(intmax_t (*const[])(void)){') + for define in sorted(suite.defines): + f.writeln(8*' '+'%s,' % ( + define_cbs[defines[define]] + if define in defines + else 'NULL')) + f.writeln(4*' '+'},') + f.writeln('};') + f.writeln() + + # create case filter function + if suite.if_ is not None or case.if_ is not None: + f.writeln('bool __test__%s__%s__filter(void) {' + % (suite.name, case.name)) + f.writeln(4*' '+'return %s;' + % ' && '.join('(%s)' % if_ + for if_ in [suite.if_, case.if_] + if if_ is not None)) + f.writeln('}') + f.writeln() + + # create case run function + f.writeln('void __test__%s__%s__run(' + '__attribute__((unused)) struct lfs_config *cfg) {' + % (suite.name, case.name)) + if CASE_PROLOGUE.strip(): + f.writeln(4*' '+'%s' + % CASE_PROLOGUE.strip().replace('\n', '\n'+4*' ')) + f.writeln() + f.writeln(4*' '+'// test case %s' % case.id()) + if case.code_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (case.code_lineno, suite.path)) + f.write(case.code) + if case.code_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (f.lineno+1, args['output'])) + if CASE_EPILOGUE.strip(): + f.writeln() + f.writeln(4*' '+'%s' + % CASE_EPILOGUE.strip().replace('\n', '\n'+4*' ')) + f.writeln('}') + f.writeln() + + if not args.get('source'): + # write test suite prologue + f.writeln('%s' % SUITE_PROLOGUE.strip()) + f.writeln() + if suite.code is not None: + if suite.code_lineno is not None: + f.writeln('#line %d "%s"' + % (suite.code_lineno, suite.path)) + f.write(suite.code) + if suite.code_lineno is not None: + f.writeln('#line %d "%s"' + % (f.lineno+1, args['output'])) + f.writeln() + + if suite.defines: + for i, define in enumerate(sorted(suite.defines)): + f.writeln('#ifndef %s' % define) + f.writeln('#define %-24s test_define(%d)' + % (define, i)) + f.writeln('#endif') + f.writeln() + + for case in suite.cases: + # create case functions + if case.in_ is None: + write_case_functions(f, suite, case) + else: + if case.defines: + f.writeln('extern intmax_t (*const *const ' + '__test__%s__%s__defines[])(void);' + % (suite.name, case.name)) + if suite.if_ is not None or case.if_ is not None: + f.writeln('extern bool __test__%s__%s__filter(' + 'void);' + % (suite.name, case.name)) + f.writeln('extern void __test__%s__%s__run(' + 'struct lfs_config *cfg);' + % (suite.name, case.name)) + f.writeln() + + # create case struct + f.writeln('const struct test_case __test__%s__%s__case = {' + % (suite.name, case.name)) + f.writeln(4*' '+'.id = "%s",' % case.id()) + f.writeln(4*' '+'.name = "%s",' % case.name) + f.writeln(4*' '+'.path = "%s",' % case.path) + f.writeln(4*' '+'.types = %s,' + % ' | '.join(filter(None, [ + 'TEST_NORMAL' if case.normal else None, + 'TEST_REENTRANT' if case.reentrant else None]))) + f.writeln(4*' '+'.permutations = %d,' + % len(case.permutations)) + if case.defines: + f.writeln(4*' '+'.defines = __test__%s__%s__defines,' + % (suite.name, case.name)) + if suite.if_ is not None or case.if_ is not None: + f.writeln(4*' '+'.filter = __test__%s__%s__filter,' + % (suite.name, case.name)) + f.writeln(4*' '+'.run = __test__%s__%s__run,' + % (suite.name, case.name)) + f.writeln('};') + f.writeln() + + # create suite define names + if suite.defines: + f.writeln('const char *const __test__%s__define_names[] = {' + % suite.name) + for k in sorted(suite.defines): + f.writeln(4*' '+'"%s",' % k) + f.writeln('};') + f.writeln() + + # create suite struct + f.writeln('const struct test_suite __test__%s__suite = {' + % suite.name) + f.writeln(4*' '+'.id = "%s",' % suite.id()) + f.writeln(4*' '+'.name = "%s",' % suite.name) + f.writeln(4*' '+'.path = "%s",' % suite.path) + f.writeln(4*' '+'.types = %s,' + % ' | '.join(filter(None, [ + 'TEST_NORMAL' if suite.normal else None, + 'TEST_REENTRANT' if suite.reentrant else None]))) + if suite.defines: + f.writeln(4*' '+'.define_names = __test__%s__define_names,' + % suite.name) + f.writeln(4*' '+'.define_count = %d,' % len(suite.defines)) + f.writeln(4*' '+'.cases = (const struct test_case *const []){') + for case in suite.cases: + f.writeln(8*' '+'&__test__%s__%s__case,' + % (suite.name, case.name)) + f.writeln(4*' '+'},') + f.writeln(4*' '+'.case_count = %d,' % len(suite.cases)) + f.writeln('};') + f.writeln() - def __str__(self): - if hasattr(self, 'permno'): - if any(k not in self.case.defines for k in self.defines): - return '%s#%d#%d (%s)' % ( - self.suite.name, self.caseno, self.permno, ', '.join( - '%s=%s' % (k, v) for k, v in self.defines.items() - if k not in self.case.defines)) else: - return '%s#%d#%d' % ( - self.suite.name, self.caseno, self.permno) - else: - return '%s#%d' % ( - self.suite.name, self.caseno) + # copy source + f.writeln('#line 1 "%s"' % args['source']) + with open(args['source']) as sf: + shutil.copyfileobj(sf, f) + f.writeln() - def permute(self, class_=None, defines={}, permno=None, **_): - ncase = (class_ or type(self))(self.config) - for k, v in self.__dict__.items(): - setattr(ncase, k, v) - ncase.case = self - ncase.perms = [ncase] - ncase.permno = permno - ncase.defines = defines - return ncase + f.write(SUITE_PROLOGUE) + f.writeln() - def build(self, f, **_): - # prologue - for k, v in sorted(self.defines.items()): - if k not in self.suite.defines: - f.write('#define %s %s\n' % (k, v)) + # write any internal tests + for suite in suites: + for case in suite.cases: + if (case.in_ is not None + and os.path.normpath(case.in_) + == os.path.normpath(args['source'])): + # write defines, but note we need to undef any + # new defines since we're in someone else's file + if suite.defines: + for i, define in enumerate( + sorted(suite.defines)): + f.writeln('#ifndef %s' % define) + f.writeln('#define %-24s test_define(%d)' + % (define, i)) + f.writeln('#define __TEST__%s__NEEDS_UNDEF' + % define) + f.writeln('#endif') + f.writeln() - f.write('void test_case%d(%s) {' % (self.caseno, ','.join( - '\n'+8*' '+'__attribute__((unused)) intmax_t %s' % k - for k in sorted(self.perms[0].defines) - if k not in self.defines))) + write_case_functions(f, suite, case) - f.write(PROLOGUE) - f.write('\n') - f.write(4*' '+'// test case %d\n' % self.caseno) - f.write(4*' '+'#line %d "%s"\n' % (self.code_lineno, self.suite.path)) + if suite.defines: + for define in sorted(suite.defines): + f.writeln('#ifdef __TEST__%s__NEEDS_UNDEF' + % define) + f.writeln('#undef __TEST__%s__NEEDS_UNDEF' + % define) + f.writeln('#undef %s' % define) + f.writeln('#endif') + f.writeln() - # test case goes here - f.write(self.code) + # add suite info to test_runner.c + if args['source'] == 'runners/test_runner.c': + f.writeln() + for suite in suites: + f.writeln('extern const struct test_suite ' + '__test__%s__suite;' % suite.name) + f.writeln('const struct test_suite *test_suites[] = {') + for suite in suites: + f.writeln(4*' '+'&__test__%s__suite,' % suite.name) + f.writeln('};') + f.writeln('const size_t test_suite_count = %d;' + % len(suites)) - # epilogue - f.write(EPILOGUE) - f.write('}\n') +def runner(**args): + cmd = args['runner'].copy() + cmd.extend(args.get('test_ids')) - for k, v in sorted(self.defines.items()): - if k not in self.suite.defines: - f.write('#undef %s\n' % k) + # run under some external command? + cmd[:0] = args.get('exec', []) - def shouldtest(self, **args): - if (self.filter is not None and - len(self.filter) >= 1 and - self.filter[0] != self.caseno): - return False - elif (self.filter is not None and - len(self.filter) >= 2 and - self.filter[1] != self.permno): - return False - elif args.get('no_internal') and self.in_ is not None: - return False - elif self.if_ is not None: - if_ = self.if_ - while True: - for k, v in sorted(self.defines.items(), - key=lambda x: len(x[0]), reverse=True): - if k in if_: - if_ = if_.replace(k, '(%s)' % v) - break - else: - break - if_ = ( - re.sub('(\&\&|\?)', ' and ', - re.sub('(\|\||:)', ' or ', - re.sub('!(?!=)', ' not ', if_)))) - return eval(if_) - else: - return True + # run under valgrind? + if args.get('valgrind'): + cmd[:0] = filter(None, [ + 'valgrind', + '--leak-check=full', + '--track-origins=yes', + '--error-exitcode=4', + '-q']) - def test(self, exec=[], persist=False, cycles=None, - gdb=False, failure=None, disk=None, **args): - # build command - cmd = exec + ['./%s.test' % self.suite.path, - repr(self.caseno), repr(self.permno)] + # filter tests? + if args.get('normal'): cmd.append('-n') + if args.get('reentrant'): cmd.append('-r') + if args.get('geometry'): + cmd.append('-G%s' % args.get('geometry')) - # persist disk or keep in RAM for speed? - if persist: - if not disk: - disk = self.suite.path + '.disk' - if persist != 'noerase': - try: - with open(disk, 'w') as f: - f.truncate(0) - if args.get('verbose'): - print('truncate --size=0', disk) - except FileNotFoundError: - pass + # defines? + if args.get('define'): + for define in args.get('define'): + cmd.append('-D%s' % define) - cmd.append(disk) + return cmd - # simulate power-loss after n cycles? - if cycles: - cmd.append(str(cycles)) +def list_(**args): + cmd = runner(**args) + if args.get('summary'): cmd.append('--summary') + if args.get('list_suites'): cmd.append('--list-suites') + if args.get('list_cases'): cmd.append('--list-cases') + if args.get('list_paths'): cmd.append('--list-paths') + if args.get('list_defines'): cmd.append('--list-defines') + if args.get('list_geometries'): cmd.append('--list-geometries') - # failed? drop into debugger? - if gdb and failure: - ncmd = ['gdb'] - if gdb == 'assert': - ncmd.extend(['-ex', 'r']) - if failure.assert_: - ncmd.extend(['-ex', 'up 2']) - elif gdb == 'main': - ncmd.extend([ - '-ex', 'b %s:%d' % (self.suite.path, self.code_lineno), - '-ex', 'r']) - ncmd.extend(['--args'] + cmd) + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + sys.exit(sp.call(cmd)) - if args.get('verbose'): - print(' '.join(shlex.quote(c) for c in ncmd)) - signal.signal(signal.SIGINT, signal.SIG_IGN) - sys.exit(sp.call(ncmd)) - # run test case! - mpty, spty = pty.openpty() +def find_cases(runner_, **args): + # query from runner + cmd = runner_ + ['--list-cases'] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace') + expected_suite_perms = co.defaultdict(lambda: 0) + expected_case_perms = co.defaultdict(lambda: 0) + expected_perms = 0 + total_perms = 0 + pattern = re.compile( + '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' + '[^\s]+\s+(?P\d+)/(?P\d+)') + # skip the first line + for line in it.islice(proc.stdout, 1, None): + m = pattern.match(line) + if m: + filtered = int(m.group('filtered')) + expected_suite_perms[m.group('suite')] += filtered + expected_case_perms[m.group('id')] += filtered + expected_perms += filtered + total_perms += int(m.group('perms')) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return ( + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) + +def find_paths(runner_, **args): + # query from runner + cmd = runner_ + ['--list-paths'] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace') + paths = co.OrderedDict() + pattern = re.compile( + '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' + '(?P[^:]+):(?P\d+)') + for line in proc.stdout: + m = pattern.match(line) + if m: + paths[m.group('id')] = (m.group('path'), int(m.group('lineno'))) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return paths + +def find_defines(runner_, **args): + # query from runner + cmd = runner_ + ['--list-defines'] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace') + defines = co.OrderedDict() + pattern = re.compile( + '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' + '(?P(?:\w+=\w+\s*)+)') + for line in proc.stdout: + m = pattern.match(line) + if m: + defines[m.group('id')] = {k: v + for k, v in re.findall('(\w+)=(\w+)', m.group('defines'))} + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return defines + + +class TestFailure(Exception): + def __init__(self, id, returncode, output, assert_=None): + self.id = id + self.returncode = returncode + self.output = output + self.assert_ = assert_ + +def run_stage(name, runner_, **args): + # get expected suite/case/perm counts + expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( + find_cases(runner_, **args)) + + passed_suite_perms = co.defaultdict(lambda: 0) + passed_case_perms = co.defaultdict(lambda: 0) + passed_perms = 0 + failures = [] + killed = False + + pattern = re.compile('^(?:' + '(?Prunning|finished|skipped) ' + '(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)' + '|' '(?P[^:]+):(?P\d+):(?Passert):' + ' *(?P.*)' ')$') + locals = th.local() + children = set() + + def run_runner(runner_): + nonlocal passed_suite_perms + nonlocal passed_case_perms + nonlocal passed_perms + nonlocal locals + + # run the tests! + cmd = runner_.copy() + if args.get('disk'): + cmd.append('--disk=%s' % args['disk']) + if args.get('trace'): + cmd.append('--trace=%s' % args['trace']) if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) + mpty, spty = pty.openpty() proc = sp.Popen(cmd, stdout=spty, stderr=spty) os.close(spty) + children.add(proc) mpty = os.fdopen(mpty, 'r', 1) - stdout = [] - assert_ = None + if args.get('output'): + output = openio(args['output'], 'w') + + last_id = None + last_output = [] + last_assert = None try: while True: + # parse a line for state changes try: line = mpty.readline() except OSError as e: @@ -293,568 +668,360 @@ class TestCase: break raise if not line: - break; - stdout.append(line) - if args.get('verbose'): + break + last_output.append(line) + if args.get('output'): + output.write(line) + elif args.get('verbose'): sys.stdout.write(line) - # intercept asserts - m = re.match( - '^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$' - .format('(?:\033\[[\d;]*.| )*', 'assert'), - line) - if m and assert_ is None: - try: - with open(m.group(1)) as f: - lineno = int(m.group(2)) - line = (next(it.islice(f, lineno-1, None)) - .strip('\n')) - assert_ = { - 'path': m.group(1), - 'line': line, - 'lineno': lineno, - 'message': m.group(3)} - except: - pass + + m = pattern.match(line) + if m: + op = m.group('op') or m.group('op_') + if op == 'running': + locals.seen_perms += 1 + last_id = m.group('id') + last_output = [] + last_assert = None + elif op == 'finished': + passed_suite_perms[m.group('suite')] += 1 + passed_case_perms[m.group('case')] += 1 + passed_perms += 1 + elif op == 'skipped': + locals.seen_perms += 1 + elif op == 'assert': + last_assert = ( + m.group('path'), + int(m.group('lineno')), + m.group('message')) + # go ahead and kill the process, aborting takes a while + if args.get('keep_going'): + proc.kill() except KeyboardInterrupt: - raise TestFailure(self, 1, stdout, None) + raise TestFailure(last_id, 1, last_output) + finally: + children.remove(proc) + mpty.close() + if args.get('output'): + output.close() + proc.wait() - - # did we pass? if proc.returncode != 0: - raise TestFailure(self, proc.returncode, stdout, assert_) - else: - return PASS + raise TestFailure( + last_id, + proc.returncode, + last_output, + last_assert) -class ValgrindTestCase(TestCase): - def __init__(self, config, **args): - self.leaky = config.get('leaky', False) - super().__init__(config, **args) + def run_job(runner, start=None, step=None): + nonlocal failures + nonlocal locals - def shouldtest(self, **args): - return not self.leaky and super().shouldtest(**args) + start = start or 0 + step = step or 1 + while start < total_perms: + runner_ = runner.copy() + if start is not None: + runner_.append('--start=%d' % start) + if step is not None: + runner_.append('--step=%d' % step) + if args.get('isolate') or args.get('valgrind'): + runner_.append('--stop=%d' % (start+step)) - def test(self, exec=[], **args): - verbose = args.get('verbose') - uninit = (self.defines.get('LFS_ERASE_VALUE', None) == -1) - exec = [ - 'valgrind', - '--leak-check=full', - ] + (['--undef-value-errors=no'] if uninit else []) + [ - ] + (['--track-origins=yes'] if not uninit else []) + [ - '--error-exitcode=4', - '--error-limit=no', - ] + (['--num-callers=1'] if not verbose else []) + [ - '-q'] + exec - return super().test(exec=exec, **args) - -class ReentrantTestCase(TestCase): - def __init__(self, config, **args): - self.reentrant = config.get('reentrant', False) - super().__init__(config, **args) - - def shouldtest(self, **args): - return self.reentrant and super().shouldtest(**args) - - def test(self, persist=False, gdb=False, failure=None, **args): - for cycles in it.count(1): - # clear disk first? - if cycles == 1 and persist != 'noerase': - persist = 'erase' - else: - persist = 'noerase' - - # exact cycle we should drop into debugger? - if gdb and failure and failure.cycleno == cycles: - return super().test(gdb=gdb, persist=persist, cycles=cycles, - failure=failure, **args) - - # run tests, but kill the program after prog/erase has - # been hit n cycles. We exit with a special return code if the - # program has not finished, since this isn't a test failure. try: - return super().test(persist=persist, cycles=cycles, **args) - except TestFailure as nfailure: - if nfailure.returncode == 33: + # run the tests + locals.seen_perms = 0 + run_runner(runner_) + assert locals.seen_perms > 0 + start += locals.seen_perms*step + + except TestFailure as failure: + # race condition for multiple failures? + if failures and not args.get('keep_going'): + break + + failures.append(failure) + + if args.get('keep_going') and not killed: + # resume after failed test + assert locals.seen_perms > 0 + start += locals.seen_perms*step continue else: - nfailure.cycleno = cycles - raise + # stop other tests + for child in children.copy(): + child.kill() + -class TestSuite: - def __init__(self, path, classes=[TestCase], defines={}, - filter=None, **args): - self.name = os.path.basename(path) - if self.name.endswith('.toml'): - self.name = self.name[:-len('.toml')] - if args.get('build_dir'): - self.toml = path - self.path = args['build_dir'] + '/' + path - else: - self.toml = path - self.path = path - self.classes = classes - self.defines = defines.copy() - self.filter = filter + # parallel jobs? + runners = [] + if 'jobs' in args: + for job in range(args['jobs']): + runners.append(th.Thread( + target=run_job, args=(runner_, job, args['jobs']))) + else: + runners.append(th.Thread( + target=run_job, args=(runner_, None, None))) - with open(self.toml) as f: - # load tests - config = toml.load(f) + for r in runners: + r.start() - # find line numbers - f.seek(0) - linenos = [] - code_linenos = [] - for i, line in enumerate(f): - if re.match(r'\[\[\s*case\s*\]\]', line): - linenos.append(i+1) - if re.match(r'code\s*=\s*(\'\'\'|""")', line): - code_linenos.append(i+2) - - code_linenos.reverse() - - # grab global config - for k, v in config.get('define', {}).items(): - if k not in self.defines: - self.defines[k] = v - self.code = config.get('code', None) - if self.code is not None: - self.code_lineno = code_linenos.pop() - - # create initial test cases - self.cases = [] - for i, (case, lineno) in enumerate(zip(config['case'], linenos)): - # code lineno? - if 'code' in case: - case['code_lineno'] = code_linenos.pop() - # merge conditions if necessary - if 'if' in config and 'if' in case: - case['if'] = '(%s) && (%s)' % (config['if'], case['if']) - elif 'if' in config: - case['if'] = config['if'] - # initialize test case - self.cases.append(TestCase(case, filter=filter, - suite=self, caseno=i+1, lineno=lineno, **args)) - - def __str__(self): - return self.name - - def __lt__(self, other): - return self.name < other.name - - def permute(self, **args): - for case in self.cases: - # lets find all parameterized definitions, in one of [args.D, - # suite.defines, case.defines, DEFINES]. Note that each of these - # can be either a dict of defines, or a list of dicts, expressing - # an initial set of permutations. - pending = [{}] - for inits in [self.defines, case.defines, DEFINES]: - if not isinstance(inits, list): - inits = [inits] - - npending = [] - for init, pinit in it.product(inits, pending): - ninit = pinit.copy() - for k, v in init.items(): - if k not in ninit: - try: - ninit[k] = eval(v) - except: - ninit[k] = v - npending.append(ninit) - - pending = npending - - # expand permutations - pending = list(reversed(pending)) - expanded = [] - while pending: - perm = pending.pop() - for k, v in sorted(perm.items()): - if not isinstance(v, str) and isinstance(v, abc.Iterable): - for nv in reversed(v): - nperm = perm.copy() - nperm[k] = nv - pending.append(nperm) - break - else: - expanded.append(perm) - - # generate permutations - case.perms = [] - for i, (class_, defines) in enumerate( - it.product(self.classes, expanded)): - case.perms.append(case.permute( - class_, defines, permno=i+1, **args)) - - # also track non-unique defines - case.defines = {} - for k, v in case.perms[0].defines.items(): - if all(perm.defines[k] == v for perm in case.perms): - case.defines[k] = v - - # track all perms and non-unique defines - self.perms = [] - for case in self.cases: - self.perms.extend(case.perms) - - self.defines = {} - for k, v in self.perms[0].defines.items(): - if all(perm.defines.get(k, None) == v for perm in self.perms): - self.defines[k] = v - - return self.perms - - def build(self, **args): - # build test files - tf = open(self.path + '.test.tc', 'w') - tf.write(GLOBALS) - if self.code is not None: - tf.write('#line %d "%s"\n' % (self.code_lineno, self.path)) - tf.write(self.code) - - tfs = {None: tf} - for case in self.cases: - if case.in_ not in tfs: - tfs[case.in_] = open(self.path+'.'+ - re.sub('(\.c)?$', '.tc', case.in_.replace('/', '.')), 'w') - tfs[case.in_].write('#line 1 "%s"\n' % case.in_) - with open(case.in_) as f: - for line in f: - tfs[case.in_].write(line) - tfs[case.in_].write('\n') - tfs[case.in_].write(GLOBALS) - - tfs[case.in_].write('\n') - case.build(tfs[case.in_], **args) - - tf.write('\n') - tf.write('const char *lfs_testbd_path;\n') - tf.write('uint32_t lfs_testbd_cycles;\n') - tf.write('int main(int argc, char **argv) {\n') - tf.write(4*' '+'int case_ = (argc > 1) ? atoi(argv[1]) : 0;\n') - tf.write(4*' '+'int perm = (argc > 2) ? atoi(argv[2]) : 0;\n') - tf.write(4*' '+'lfs_testbd_path = (argc > 3) ? argv[3] : NULL;\n') - tf.write(4*' '+'lfs_testbd_cycles = (argc > 4) ? atoi(argv[4]) : 0;\n') - for perm in self.perms: - # test declaration - tf.write(4*' '+'extern void test_case%d(%s);\n' % ( - perm.caseno, ', '.join( - 'intmax_t %s' % k for k in sorted(perm.defines) - if k not in perm.case.defines))) - # test call - tf.write(4*' '+ - 'if (argc < 3 || (case_ == %d && perm == %d)) {' - ' test_case%d(%s); ' - '}\n' % (perm.caseno, perm.permno, perm.caseno, ', '.join( - str(v) for k, v in sorted(perm.defines.items()) - if k not in perm.case.defines))) - tf.write('}\n') - - for tf in tfs.values(): - tf.close() - - # write makefiles - with open(self.path + '.mk', 'w') as mk: - mk.write(RULES.replace(4*' ', '\t') % dict(path=self.path)) - mk.write('\n') - - # add coverage hooks? - if args.get('coverage'): - mk.write(COVERAGE_RULES.replace(4*' ', '\t') % dict( - path=self.path)) - mk.write('\n') - - # add truly global defines globally - for k, v in sorted(self.defines.items()): - mk.write('%s.test: override CFLAGS += -D%s=%r\n' - % (self.path, k, v)) - - for path in tfs: - if path is None: - mk.write('%s: %s | %s\n' % ( - self.path+'.test.c', - self.toml, - self.path+'.test.tc')) - else: - mk.write('%s: %s %s | %s\n' % ( - self.path+'.'+path.replace('/', '.'), - self.toml, - path, - self.path+'.'+re.sub('(\.c)?$', '.tc', - path.replace('/', '.')))) - mk.write('\t./scripts/explode_asserts.py $| -o $@\n') - - self.makefile = self.path + '.mk' - self.target = self.path + '.test' - return self.makefile, self.target - - def test(self, **args): - # run test suite! - if not args.get('verbose', True): - sys.stdout.write(self.name + ' ') - sys.stdout.flush() - for perm in self.perms: - if not perm.shouldtest(**args): - continue - - try: - result = perm.test(**args) - except TestFailure as failure: - perm.result = failure - if not args.get('verbose', True): - sys.stdout.write(FAIL) - sys.stdout.flush() - if not args.get('keep_going'): - if not args.get('verbose', True): - sys.stdout.write('\n') - raise - else: - perm.result = PASS - if not args.get('verbose', True): - sys.stdout.write(PASS) - sys.stdout.flush() - - if not args.get('verbose', True): - sys.stdout.write('\n') - -def main(**args): - # figure out explicit defines - defines = {} - for define in args['D']: - k, v, *_ = define.split('=', 2) + [''] - defines[k] = v - - # and what class of TestCase to run - classes = [] - if args.get('normal'): - classes.append(TestCase) - if args.get('reentrant'): - classes.append(ReentrantTestCase) - if args.get('valgrind'): - classes.append(ValgrindTestCase) - if not classes: - classes = [TestCase] - - suites = [] - for testpath in args['test_paths']: - # optionally specified test case/perm - testpath, *filter = testpath.split('#') - filter = [int(f) for f in filter] - - # figure out the suite's toml file - if os.path.isdir(testpath): - testpath = testpath + '/*.toml' - elif os.path.isfile(testpath): - testpath = testpath - elif testpath.endswith('.toml'): - testpath = TEST_PATHS + '/' + testpath - else: - testpath = TEST_PATHS + '/' + testpath + '.toml' - - # find tests - for path in glob.glob(testpath): - suites.append(TestSuite(path, classes, defines, filter, **args)) - - # sort for reproducibility - suites = sorted(suites) - - # generate permutations - for suite in suites: - suite.permute(**args) - - # build tests in parallel - print('====== building ======') - makefiles = [] - targets = [] - for suite in suites: - makefile, target = suite.build(**args) - makefiles.append(makefile) - targets.append(target) - - cmd = (['make', '-f', 'Makefile'] + - list(it.chain.from_iterable(['-f', m] for m in makefiles)) + - [target for target in targets]) - mpty, spty = pty.openpty() - if args.get('verbose'): - print(' '.join(shlex.quote(c) for c in cmd)) - proc = sp.Popen(cmd, stdout=spty, stderr=spty) - os.close(spty) - mpty = os.fdopen(mpty, 'r', 1) - stdout = [] - while True: - try: - line = mpty.readline() - except OSError as e: - if e.errno == errno.EIO: - break - raise - if not line: - break; - stdout.append(line) - if args.get('verbose'): - sys.stdout.write(line) - # intercept warnings - m = re.match( - '^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$' - .format('(?:\033\[[\d;]*.| )*', 'warning'), - line) - if m and not args.get('verbose'): - try: - with open(m.group(1)) as f: - lineno = int(m.group(2)) - line = next(it.islice(f, lineno-1, None)).strip('\n') - sys.stdout.write( - "\033[01m{path}:{lineno}:\033[01;35mwarning:\033[m " - "{message}\n{line}\n\n".format( - path=m.group(1), line=line, lineno=lineno, - message=m.group(3))) - except: - pass - proc.wait() - if proc.returncode != 0: - if not args.get('verbose'): - for line in stdout: - sys.stdout.write(line) - sys.exit(-1) - - print('built %d test suites, %d test cases, %d permutations' % ( - len(suites), - sum(len(suite.cases) for suite in suites), - sum(len(suite.perms) for suite in suites))) - - total = 0 - for suite in suites: - for perm in suite.perms: - total += perm.shouldtest(**args) - if total != sum(len(suite.perms) for suite in suites): - print('filtered down to %d permutations' % total) - - # only requested to build? - if args.get('build'): - return 0 - - print('====== testing ======') + needs_newline = False try: - for suite in suites: - suite.test(**args) - except TestFailure: - pass + while any(r.is_alive() for r in runners): + time.sleep(0.01) - print('====== results ======') + if not args.get('verbose'): + sys.stdout.write('\r\x1b[K' + 'running \x1b[%dm%s:\x1b[m %s ' + % (32 if not failures else 31, + name, + ', '.join(filter(None, [ + '%d/%d suites' % ( + sum(passed_suite_perms[k] == v + for k, v in expected_suite_perms.items()), + len(expected_suite_perms)) + if (not args.get('by_suites') + and not args.get('by_cases')) else None, + '%d/%d cases' % ( + sum(passed_case_perms[k] == v + for k, v in expected_case_perms.items()), + len(expected_case_perms)) + if not args.get('by_cases') else None, + '%d/%d perms' % (passed_perms, expected_perms), + '\x1b[31m%d/%d failures\x1b[m' + % (len(failures), expected_perms) + if failures else None])))) + sys.stdout.flush() + needs_newline = True + except KeyboardInterrupt: + # this is handled by the runner threads, we just + # need to not abort here + killed = True + finally: + if needs_newline: + print() + + for r in runners: + r.join() + + return ( + expected_perms, + passed_perms, + failures, + killed) + + +def run(**args): + start = time.time() + + runner_ = runner(**args) + print('using runner: %s' + % ' '.join(shlex.quote(c) for c in runner_)) + expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( + find_cases(runner_, **args)) + print('found %d suites, %d cases, %d/%d permutations' + % (len(expected_suite_perms), + len(expected_case_perms), + expected_perms, + total_perms)) + print() + + expected = 0 passed = 0 - failed = 0 - for suite in suites: - for perm in suite.perms: - if perm.result == PASS: - passed += 1 - elif isinstance(perm.result, TestFailure): - sys.stdout.write( - "\033[01m{path}:{lineno}:\033[01;31mfailure:\033[m " - "{perm} failed\n".format( - perm=perm, path=perm.suite.path, lineno=perm.lineno, - returncode=perm.result.returncode or 0)) - if perm.result.stdout: - if perm.result.assert_: - stdout = perm.result.stdout[:-1] - else: - stdout = perm.result.stdout - for line in stdout[-5:]: - sys.stdout.write(line) - if perm.result.assert_: - sys.stdout.write( - "\033[01m{path}:{lineno}:\033[01;31massert:\033[m " - "{message}\n{line}\n".format( - **perm.result.assert_)) - sys.stdout.write('\n') - failed += 1 + failures = [] + for type, by in it.product( + ['normal', 'reentrant'], + expected_case_perms.keys() if args.get('by_cases') + else expected_suite_perms.keys() if args.get('by_suites') + else [None]): + # rebuild runner for each stage to override test identifier if needed + stage_runner = runner(**args | { + 'test_ids': [by] if by is not None else args.get('test_ids', []), + 'normal': type == 'normal', + 'reentrant': type == 'reentrant'}) - if args.get('coverage'): - # collect coverage info - # why -j1? lcov doesn't work in parallel because of gcov limitations - cmd = (['make', '-j1', '-f', 'Makefile'] + - list(it.chain.from_iterable(['-f', m] for m in makefiles)) + - (['COVERAGETARGET=%s' % args['coverage']] - if isinstance(args['coverage'], str) else []) + - [suite.path + '.info' for suite in suites - if any(perm.result == PASS for perm in suite.perms)]) + # spawn jobs for stage + expected_, passed_, failures_, killed = run_stage( + '%s %s' % (type, by or 'tests'), stage_runner, **args) + expected += expected_ + passed += passed_ + failures.extend(failures_) + if (failures and not args.get('keep_going')) or killed: + break + + # show summary + print() + print('\x1b[%dmdone:\x1b[m %d/%d passed, %d/%d failed, in %.2fs' + % (32 if not failures else 31, + passed, expected, len(failures), expected, + time.time()-start)) + print() + + # print each failure + if failures: + # get some extra info from runner + runner_paths = find_paths(runner_, **args) + runner_defines = find_defines(runner_, **args) + + for failure in failures: + # show summary of failure + path, lineno = runner_paths[testcase(failure.id)] + defines = runner_defines[failure.id] + + print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s%s failed' + % (path, lineno, failure.id, + ' (%s)' % ', '.join( + '%s=%s' % (k, v) for k, v in defines.items()) + if defines else '')) + + if failure.output: + output = failure.output + if failure.assert_ is not None: + output = output[:-1] + for line in output[-5:]: + sys.stdout.write(line) + + if failure.assert_ is not None: + path, lineno, message = failure.assert_ + print('\x1b[01m%s:%d:\x1b[01;31massert:\x1b[m %s' + % (path, lineno, message)) + with open(path) as f: + line = next(it.islice(f, lineno-1, None)).strip('\n') + print(line) + print() + + # drop into gdb? + if failures and (args.get('gdb') + or args.get('gdb_case') + or args.get('gdb_main')): + failure = failures[0] + runner_ = runner(**args | {'test_ids': [failure.id]}) + + if args.get('gdb_main'): + cmd = ['gdb', + '-ex', 'break main', + '-ex', 'run', + '--args'] + runner_ + elif args.get('gdb_case'): + path, lineno = runner_paths[testcase(failure.id)] + cmd = ['gdb', + '-ex', 'break %s:%d' % (path, lineno), + '-ex', 'run', + '--args'] + runner_ + elif failure.assert_ is not None: + cmd = ['gdb', + '-ex', 'run', + '-ex', 'frame function raise', + '-ex', 'up 2', + '--args'] + runner_ + else: + cmd = ['gdb', + '-ex', 'run', + '--args'] + runner_ + + # exec gdb interactively if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) - proc = sp.Popen(cmd, - stdout=sp.PIPE if not args.get('verbose') else None, - stderr=sp.STDOUT if not args.get('verbose') else None, - universal_newlines=True) - stdout = [] - for line in proc.stdout: - stdout.append(line) - proc.wait() - if proc.returncode != 0: - if not args.get('verbose'): - for line in stdout: - sys.stdout.write(line) - sys.exit(-1) + os.execvp(cmd[0], cmd) - if args.get('gdb'): - failure = None - for suite in suites: - for perm in suite.perms: - if isinstance(perm.result, TestFailure): - failure = perm.result - if failure is not None: - print('======= gdb ======') - # drop into gdb - failure.case.test(failure=failure, **args) - sys.exit(0) + return 1 if failures else 0 + + +def main(**args): + if args.get('compile'): + compile(**args) + elif (args.get('summary') + or args.get('list_suites') + or args.get('list_cases') + or args.get('list_paths') + or args.get('list_defines') + or args.get('list_geometries') + or args.get('list_defaults')): + list_(**args) + else: + run(**args) - print('tests passed %d/%d (%.1f%%)' % (passed, total, - 100*(passed/total if total else 1.0))) - print('tests failed %d/%d (%.1f%%)' % (failed, total, - 100*(failed/total if total else 1.0))) - return 1 if failed > 0 else 0 if __name__ == "__main__": import argparse + import sys parser = argparse.ArgumentParser( - description="Run parameterized tests in various configurations.") - parser.add_argument('test_paths', nargs='*', default=[TEST_PATHS], - help="Description of test(s) to run. By default, this is all tests \ - found in the \"{0}\" directory. Here, you can specify a different \ - directory of tests, a specific file, a suite by name, and even \ - specific test cases and permutations. For example \ - \"test_dirs#1\" or \"{0}/test_dirs.toml#1#1\".".format(TEST_PATHS)) - parser.add_argument('-D', action='append', default=[], - help="Overriding parameter definitions.") + description="Build and run tests.", + conflict_handler='resolve') + parser.add_argument('test_ids', nargs='*', + help="Description of testis to run. May be a directory, path, or \ + test identifier. Test identifiers are of the form \ + ##, but suffixes can be \ + dropped to run any matching tests. Defaults to %r." % TEST_PATHS) parser.add_argument('-v', '--verbose', action='store_true', - help="Output everything that is happening.") - parser.add_argument('-k', '--keep-going', action='store_true', - help="Run all tests instead of stopping on first error. Useful for CI.") - parser.add_argument('-p', '--persist', choices=['erase', 'noerase'], - nargs='?', const='erase', - help="Store disk image in a file.") - parser.add_argument('-b', '--build', action='store_true', - help="Only build the tests, do not execute.") - parser.add_argument('-g', '--gdb', choices=['init', 'main', 'assert'], - nargs='?', const='assert', + help="Output commands that run behind the scenes.") + # test flags + test_parser = parser.add_argument_group('test options') + test_parser.add_argument('-Y', '--summary', action='store_true', + help="Show quick summary.") + test_parser.add_argument('-l', '--list-suites', action='store_true', + help="List test suites.") + test_parser.add_argument('-L', '--list-cases', action='store_true', + help="List test cases.") + test_parser.add_argument('--list-paths', action='store_true', + help="List the path for each test case.") + test_parser.add_argument('--list-defines', action='store_true', + help="List the defines for each test permutation.") + test_parser.add_argument('--list-geometries', action='store_true', + help="List the disk geometries used for testing.") + test_parser.add_argument('--list-defaults', action='store_true', + help="List the default defines in this test-runner.") + test_parser.add_argument('-D', '--define', action='append', + help="Override a test define.") + test_parser.add_argument('-G', '--geometry', + help="Filter by geometry.") + test_parser.add_argument('-n', '--normal', action='store_true', + help="Filter for normal tests. Can be combined.") + test_parser.add_argument('-r', '--reentrant', action='store_true', + help="Filter for reentrant tests. Can be combined.") + test_parser.add_argument('-d', '--disk', + help="Use this file as the disk.") + test_parser.add_argument('-t', '--trace', + help="Redirect trace output to this file.") + test_parser.add_argument('-o', '--output', + help="Redirect stdout and stderr to this file.") + test_parser.add_argument('--runner', default=[RUNNER_PATH], + type=lambda x: x.split(), + help="Path to runner, defaults to %r" % RUNNER_PATH) + test_parser.add_argument('-j', '--jobs', nargs='?', type=int, + const=len(os.sched_getaffinity(0)), + help="Number of parallel runners to run.") + test_parser.add_argument('-k', '--keep-going', action='store_true', + help="Don't stop on first error.") + test_parser.add_argument('-i', '--isolate', action='store_true', + help="Run each test permutation in a separate process.") + test_parser.add_argument('-b', '--by-suites', action='store_true', + help="Step through tests by suite.") + test_parser.add_argument('-B', '--by-cases', action='store_true', + help="Step through tests by case.") + test_parser.add_argument('--gdb', action='store_true', help="Drop into gdb on test failure.") - parser.add_argument('--no-internal', action='store_true', - help="Don't run tests that require internal knowledge.") - parser.add_argument('-n', '--normal', action='store_true', - help="Run tests normally.") - parser.add_argument('-r', '--reentrant', action='store_true', - help="Run reentrant tests with simulated power-loss.") - parser.add_argument('--valgrind', action='store_true', - help="Run non-leaky tests under valgrind to check for memory leaks.") - parser.add_argument('--exec', default=[], type=lambda e: e.split(), - help="Run tests with another executable prefixed on the command line.") - parser.add_argument('--disk', - help="Specify a file to use for persistent/reentrant tests.") - parser.add_argument('--coverage', type=lambda x: x if x else True, - nargs='?', const='', - help="Collect coverage information during testing. This uses lcov/gcov \ - to accumulate coverage information into *.info files. May also \ - a path to a *.info file to accumulate coverage info into.") - parser.add_argument('--build-dir', - help="Build relative to the specified directory instead of the \ - current directory.") - - sys.exit(main(**vars(parser.parse_args()))) + test_parser.add_argument('--gdb-case', action='store_true', + help="Drop into gdb on test failure but stop at the beginning \ + of the failing test case.") + test_parser.add_argument('--gdb-main', action='store_true', + help="Drop into gdb on test failure but stop at the beginning \ + of main.") + test_parser.add_argument('--valgrind', action='store_true', + help="Run under Valgrind to find memory errors. Implicitly sets \ + --isolate.") + test_parser.add_argument('--exec', default=[], type=lambda e: e.split(), + help="Run under another executable.") + # compilation flags + comp_parser = parser.add_argument_group('compilation options') + comp_parser.add_argument('-c', '--compile', action='store_true', + help="Compile a test suite or source file.") + comp_parser.add_argument('-s', '--source', + help="Source file to compile, possibly injecting internal tests.") + comp_parser.add_argument('-o', '--output', + help="Output file.") + # TODO apply this to other scripts? + sys.exit(main(**{k: v + for k, v in vars(parser.parse_args()).items() + if v is not None})) diff --git a/scripts/test_.py b/scripts/test_.py deleted file mode 100755 index 02147ef2..00000000 --- a/scripts/test_.py +++ /dev/null @@ -1,1027 +0,0 @@ -#!/usr/bin/env python3 -# -# Script to compile and runs tests. -# - -import collections as co -import errno -import glob -import itertools as it -import math as m -import os -import pty -import re -import shlex -import shutil -import signal -import subprocess as sp -import threading as th -import time -import toml - - -TEST_PATHS = ['tests_'] -RUNNER_PATH = './runners/test_runner' - -SUITE_PROLOGUE = """ -#include "runners/test_runner.h" -#include "bd/lfs_testbd.h" -#include -""" -CASE_PROLOGUE = """ -""" -CASE_EPILOGUE = """ -""" - - -def testpath(path): - path, *_ = path.split('#', 1) - return path - -def testsuite(path): - suite = testpath(path) - suite = os.path.basename(suite) - if suite.endswith('.toml'): - suite = suite[:-len('.toml')] - return suite - -def testcase(path): - _, case, *_ = path.split('#', 2) - return '%s#%s' % (testsuite(path), case) - -# TODO move this out in other files -def openio(path, mode='r'): - if path == '-': - if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), 'r') - else: - return os.fdopen(os.dup(sys.stdout.fileno()), 'w') - else: - return open(path, mode) - -class TestCase: - # create a TestCase object from a config - def __init__(self, config, args={}): - self.name = config.pop('name') - self.path = config.pop('path') - self.suite = config.pop('suite') - self.lineno = config.pop('lineno', None) - self.if_ = config.pop('if', None) - if isinstance(self.if_, bool): - self.if_ = 'true' if self.if_ else 'false' - self.code = config.pop('code') - self.code_lineno = config.pop('code_lineno', None) - self.in_ = config.pop('in', - config.pop('suite_in', None)) - - self.normal = config.pop('normal', - config.pop('suite_normal', True)) - self.reentrant = config.pop('reentrant', - config.pop('suite_reentrant', False)) - - # figure out defines and build possible permutations - self.defines = set() - self.permutations = [] - - suite_defines = config.pop('suite_defines', {}) - if not isinstance(suite_defines, list): - suite_defines = [suite_defines] - defines = config.pop('defines', {}) - if not isinstance(defines, list): - defines = [defines] - - # build possible permutations - for suite_defines_ in suite_defines: - self.defines |= suite_defines_.keys() - for defines_ in defines: - self.defines |= defines_.keys() - self.permutations.extend(map(dict, it.product(*( - [(k, v) for v in (vs if isinstance(vs, list) else [vs])] - for k, vs in sorted( - (suite_defines_ | defines_).items()))))) - - for k in config.keys(): - print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r' - % (self.id(), k), - file=sys.stderr) - - def id(self): - return '%s#%s' % (self.suite, self.name) - - -class TestSuite: - # create a TestSuite object from a toml file - def __init__(self, path, args={}): - self.name = testsuite(path) - self.path = testpath(path) - - # load toml file and parse test cases - with open(self.path) as f: - # load tests - config = toml.load(f) - - # find line numbers - f.seek(0) - case_linenos = [] - code_linenos = [] - for i, line in enumerate(f): - match = re.match( - '(?P\[\s*cases\s*\.\s*(?P\w+)\s*\])' - '|' '(?Pcode\s*=)', - line) - if match and match.group('case'): - case_linenos.append((i+1, match.group('name'))) - elif match and match.group('code'): - code_linenos.append(i+2) - - # sort in case toml parsing did not retain order - case_linenos.sort() - - cases = config.pop('cases') - for (lineno, name), (nlineno, _) in it.zip_longest( - case_linenos, case_linenos[1:], - fillvalue=(float('inf'), None)): - code_lineno = min( - (l for l in code_linenos if l >= lineno and l < nlineno), - default=None) - cases[name]['lineno'] = lineno - cases[name]['code_lineno'] = code_lineno - - self.if_ = config.pop('if', None) - if isinstance(self.if_, bool): - self.if_ = 'true' if self.if_ else 'false' - - self.code = config.pop('code', None) - self.code_lineno = min( - (l for l in code_linenos - if not case_linenos or l < case_linenos[0][0]), - default=None) - - # a couple of these we just forward to all cases - defines = config.pop('defines', {}) - in_ = config.pop('in', None) - normal = config.pop('normal', True) - reentrant = config.pop('reentrant', False) - - self.cases = [] - for name, case in sorted(cases.items(), - key=lambda c: c[1].get('lineno')): - self.cases.append(TestCase(config={ - 'name': name, - 'path': path + (':%d' % case['lineno'] - if 'lineno' in case else ''), - 'suite': self.name, - 'suite_defines': defines, - 'suite_in': in_, - 'suite_normal': normal, - 'suite_reentrant': reentrant, - **case})) - - # combine per-case defines - self.defines = set.union(*( - set(case.defines) for case in self.cases)) - - # combine other per-case things - self.normal = any(case.normal for case in self.cases) - self.reentrant = any(case.reentrant for case in self.cases) - - for k in config.keys(): - print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r' - % (self.id(), k), - file=sys.stderr) - - def id(self): - return self.name - - - -def compile(**args): - # find .toml files - paths = [] - for path in args.get('test_ids', TEST_PATHS): - if os.path.isdir(path): - path = path + '/*.toml' - - for path in glob.glob(path): - paths.append(path) - - if not paths: - print('no test suites found in %r?' % args['test_ids']) - sys.exit(-1) - - if not args.get('source'): - if len(paths) > 1: - print('more than one test suite for compilation? (%r)' - % args['test_ids']) - sys.exit(-1) - - # load our suite - suite = TestSuite(paths[0]) - else: - # load all suites - suites = [TestSuite(path) for path in paths] - suites.sort(key=lambda s: s.name) - - # write generated test source - if 'output' in args: - with openio(args['output'], 'w') as f: - _write = f.write - def write(s): - f.lineno += s.count('\n') - _write(s) - def writeln(s=''): - f.lineno += s.count('\n') + 1 - _write(s) - _write('\n') - f.lineno = 1 - f.write = write - f.writeln = writeln - - # redirect littlefs tracing - f.writeln('#define LFS_TRACE_(fmt, ...) do { \\') - f.writeln(8*' '+'extern FILE *test_trace; \\') - f.writeln(8*' '+'if (test_trace) { \\') - f.writeln(12*' '+'fprintf(test_trace, ' - '"%s:%d:trace: " fmt "%s\\n", \\') - f.writeln(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\') - f.writeln(8*' '+'} \\') - f.writeln(4*' '+'} while (0)') - f.writeln('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")') - f.writeln('#define LFS_TESTBD_TRACE(...) ' - 'LFS_TRACE_(__VA_ARGS__, "")') - f.writeln() - - # write out generated functions, this can end up in different - # files depending on the "in" attribute - # - # note it's up to the specific generated file to declare - # the test defines - def write_case_functions(f, suite, case): - # create case define functions - if case.defines: - # deduplicate defines by value to try to reduce the - # number of functions we generate - define_cbs = {} - for i, defines in enumerate(case.permutations): - for k, v in sorted(defines.items()): - if v not in define_cbs: - name = ('__test__%s__%s__%s__%d' - % (suite.name, case.name, k, i)) - define_cbs[v] = name - f.writeln('uintmax_t %s(void) {' % name) - f.writeln(4*' '+'return %s;' % v) - f.writeln('}') - f.writeln() - f.writeln('uintmax_t (*const *const ' - '__test__%s__%s__defines[])(void) = {' - % (suite.name, case.name)) - for defines in case.permutations: - f.writeln(4*' '+'(uintmax_t (*const[])(void)){') - for define in sorted(suite.defines): - f.writeln(8*' '+'%s,' % ( - define_cbs[defines[define]] - if define in defines - else 'NULL')) - f.writeln(4*' '+'},') - f.writeln('};') - f.writeln() - - # create case filter function - if suite.if_ is not None or case.if_ is not None: - f.writeln('bool __test__%s__%s__filter(void) {' - % (suite.name, case.name)) - f.writeln(4*' '+'return %s;' - % ' && '.join('(%s)' % if_ - for if_ in [suite.if_, case.if_] - if if_ is not None)) - f.writeln('}') - f.writeln() - - # create case run function - f.writeln('void __test__%s__%s__run(' - '__attribute__((unused)) struct lfs_config *cfg) {' - % (suite.name, case.name)) - if CASE_PROLOGUE.strip(): - f.writeln(4*' '+'%s' - % CASE_PROLOGUE.strip().replace('\n', '\n'+4*' ')) - f.writeln() - f.writeln(4*' '+'// test case %s' % case.id()) - if case.code_lineno is not None: - f.writeln(4*' '+'#line %d "%s"' - % (case.code_lineno, suite.path)) - f.write(case.code) - if case.code_lineno is not None: - f.writeln(4*' '+'#line %d "%s"' - % (f.lineno+1, args['output'])) - if CASE_EPILOGUE.strip(): - f.writeln() - f.writeln(4*' '+'%s' - % CASE_EPILOGUE.strip().replace('\n', '\n'+4*' ')) - f.writeln('}') - f.writeln() - - if not args.get('source'): - # write test suite prologue - f.writeln('%s' % SUITE_PROLOGUE.strip()) - f.writeln() - if suite.code is not None: - if suite.code_lineno is not None: - f.writeln('#line %d "%s"' - % (suite.code_lineno, suite.path)) - f.write(suite.code) - if suite.code_lineno is not None: - f.writeln('#line %d "%s"' - % (f.lineno+1, args['output'])) - f.writeln() - - if suite.defines: - for i, define in enumerate(sorted(suite.defines)): - f.writeln('#ifndef %s' % define) - f.writeln('#define %-24s test_define(%d)' - % (define, i)) - f.writeln('#endif') - f.writeln() - - for case in suite.cases: - # create case functions - if case.in_ is None: - write_case_functions(f, suite, case) - else: - if case.defines: - f.writeln('extern uintmax_t (*const *const ' - '__test__%s__%s__defines[])(void);' - % (suite.name, case.name)) - if suite.if_ is not None or case.if_ is not None: - f.writeln('extern bool __test__%s__%s__filter(' - 'void);' - % (suite.name, case.name)) - f.writeln('extern void __test__%s__%s__run(' - 'struct lfs_config *cfg);' - % (suite.name, case.name)) - f.writeln() - - # create case struct - f.writeln('const struct test_case __test__%s__%s__case = {' - % (suite.name, case.name)) - f.writeln(4*' '+'.id = "%s",' % case.id()) - f.writeln(4*' '+'.name = "%s",' % case.name) - f.writeln(4*' '+'.path = "%s",' % case.path) - f.writeln(4*' '+'.types = %s,' - % ' | '.join(filter(None, [ - 'TEST_NORMAL' if case.normal else None, - 'TEST_REENTRANT' if case.reentrant else None]))) - f.writeln(4*' '+'.permutations = %d,' - % len(case.permutations)) - if case.defines: - f.writeln(4*' '+'.defines = __test__%s__%s__defines,' - % (suite.name, case.name)) - if suite.if_ is not None or case.if_ is not None: - f.writeln(4*' '+'.filter = __test__%s__%s__filter,' - % (suite.name, case.name)) - f.writeln(4*' '+'.run = __test__%s__%s__run,' - % (suite.name, case.name)) - f.writeln('};') - f.writeln() - - # create suite define names - if suite.defines: - f.writeln('const char *const __test__%s__define_names[] = {' - % suite.name) - for k in sorted(suite.defines): - f.writeln(4*' '+'"%s",' % k) - f.writeln('};') - f.writeln() - - # create suite struct - f.writeln('const struct test_suite __test__%s__suite = {' - % suite.name) - f.writeln(4*' '+'.id = "%s",' % suite.id()) - f.writeln(4*' '+'.name = "%s",' % suite.name) - f.writeln(4*' '+'.path = "%s",' % suite.path) - f.writeln(4*' '+'.types = %s,' - % ' | '.join(filter(None, [ - 'TEST_NORMAL' if suite.normal else None, - 'TEST_REENTRANT' if suite.reentrant else None]))) - if suite.defines: - f.writeln(4*' '+'.define_names = __test__%s__define_names,' - % suite.name) - f.writeln(4*' '+'.define_count = %d,' % len(suite.defines)) - f.writeln(4*' '+'.cases = (const struct test_case *const []){') - for case in suite.cases: - f.writeln(8*' '+'&__test__%s__%s__case,' - % (suite.name, case.name)) - f.writeln(4*' '+'},') - f.writeln(4*' '+'.case_count = %d,' % len(suite.cases)) - f.writeln('};') - f.writeln() - - else: - # copy source - f.writeln('#line 1 "%s"' % args['source']) - with open(args['source']) as sf: - shutil.copyfileobj(sf, f) - f.writeln() - - f.write(SUITE_PROLOGUE) - f.writeln() - - # write any internal tests - for suite in suites: - for case in suite.cases: - if (case.in_ is not None - and os.path.normpath(case.in_) - == os.path.normpath(args['source'])): - # write defines, but note we need to undef any - # new defines since we're in someone else's file - if suite.defines: - for i, define in enumerate( - sorted(suite.defines)): - f.writeln('#ifndef %s' % define) - f.writeln('#define %-24s test_define(%d)' - % (define, i)) - f.writeln('#define __TEST__%s__NEEDS_UNDEF' - % define) - f.writeln('#endif') - f.writeln() - - write_case_functions(f, suite, case) - - if suite.defines: - for define in sorted(suite.defines): - f.writeln('#ifdef __TEST__%s__NEEDS_UNDEF' - % define) - f.writeln('#undef __TEST__%s__NEEDS_UNDEF' - % define) - f.writeln('#undef %s' % define) - f.writeln('#endif') - f.writeln() - - # add suite info to test_runner.c - if args['source'] == 'runners/test_runner.c': - f.writeln() - for suite in suites: - f.writeln('extern const struct test_suite ' - '__test__%s__suite;' % suite.name) - f.writeln('const struct test_suite *test_suites[] = {') - for suite in suites: - f.writeln(4*' '+'&__test__%s__suite,' % suite.name) - f.writeln('};') - f.writeln('const size_t test_suite_count = %d;' - % len(suites)) - -def runner(**args): - cmd = args['runner'].copy() - cmd.extend(args.get('test_ids')) - - # run under some external command? - cmd[:0] = args.get('exec', []) - - # run under valgrind? - if args.get('valgrind'): - cmd[:0] = filter(None, [ - 'valgrind', - '--leak-check=full', - '--track-origins=yes', - '--error-exitcode=4', - '-q']) - - # filter tests? - if args.get('normal'): cmd.append('-n') - if args.get('reentrant'): cmd.append('-r') - if args.get('geometry'): - cmd.append('-G%s' % args.get('geometry')) - - # defines? - if args.get('define'): - for define in args.get('define'): - cmd.append('-D%s' % define) - - return cmd - -def list_(**args): - cmd = runner(**args) - if args.get('summary'): cmd.append('--summary') - if args.get('list_suites'): cmd.append('--list-suites') - if args.get('list_cases'): cmd.append('--list-cases') - if args.get('list_paths'): cmd.append('--list-paths') - if args.get('list_defines'): cmd.append('--list-defines') - if args.get('list_geometries'): cmd.append('--list-geometries') - - if args.get('verbose'): - print(' '.join(shlex.quote(c) for c in cmd)) - sys.exit(sp.call(cmd)) - - -def find_cases(runner_, **args): - # query from runner - cmd = runner_ + ['--list-cases'] - if args.get('verbose'): - print(' '.join(shlex.quote(c) for c in cmd)) - proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=sp.PIPE if not args.get('verbose') else None, - universal_newlines=True, - errors='replace') - expected_suite_perms = co.defaultdict(lambda: 0) - expected_case_perms = co.defaultdict(lambda: 0) - expected_perms = 0 - total_perms = 0 - pattern = re.compile( - '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' - '[^\s]+\s+(?P\d+)/(?P\d+)') - # skip the first line - for line in it.islice(proc.stdout, 1, None): - m = pattern.match(line) - if m: - filtered = int(m.group('filtered')) - expected_suite_perms[m.group('suite')] += filtered - expected_case_perms[m.group('id')] += filtered - expected_perms += filtered - total_perms += int(m.group('perms')) - proc.wait() - if proc.returncode != 0: - if not args.get('verbose'): - for line in proc.stderr: - sys.stdout.write(line) - sys.exit(-1) - - return ( - expected_suite_perms, - expected_case_perms, - expected_perms, - total_perms) - -def find_paths(runner_, **args): - # query from runner - cmd = runner_ + ['--list-paths'] - if args.get('verbose'): - print(' '.join(shlex.quote(c) for c in cmd)) - proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=sp.PIPE if not args.get('verbose') else None, - universal_newlines=True, - errors='replace') - paths = co.OrderedDict() - pattern = re.compile( - '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' - '(?P[^:]+):(?P\d+)') - for line in proc.stdout: - m = pattern.match(line) - if m: - paths[m.group('id')] = (m.group('path'), int(m.group('lineno'))) - proc.wait() - if proc.returncode != 0: - if not args.get('verbose'): - for line in proc.stderr: - sys.stdout.write(line) - sys.exit(-1) - - return paths - -def find_defines(runner_, **args): - # query from runner - cmd = runner_ + ['--list-defines'] - if args.get('verbose'): - print(' '.join(shlex.quote(c) for c in cmd)) - proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=sp.PIPE if not args.get('verbose') else None, - universal_newlines=True, - errors='replace') - defines = co.OrderedDict() - pattern = re.compile( - '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' - '(?P(?:\w+=\w+\s*)+)') - for line in proc.stdout: - m = pattern.match(line) - if m: - defines[m.group('id')] = {k: v - for k, v in re.findall('(\w+)=(\w+)', m.group('defines'))} - proc.wait() - if proc.returncode != 0: - if not args.get('verbose'): - for line in proc.stderr: - sys.stdout.write(line) - sys.exit(-1) - - return defines - - -class TestFailure(Exception): - def __init__(self, id, returncode, output, assert_=None): - self.id = id - self.returncode = returncode - self.output = output - self.assert_ = assert_ - -def run_stage(name, runner_, **args): - # get expected suite/case/perm counts - expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( - find_cases(runner_, **args)) - - passed_suite_perms = co.defaultdict(lambda: 0) - passed_case_perms = co.defaultdict(lambda: 0) - passed_perms = 0 - failures = [] - killed = False - - pattern = re.compile('^(?:' - '(?Prunning|finished|skipped) ' - '(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)' - '|' '(?P[^:]+):(?P\d+):(?Passert):' - ' *(?P.*)' ')$') - locals = th.local() - children = set() - - def run_runner(runner_): - nonlocal passed_suite_perms - nonlocal passed_case_perms - nonlocal passed_perms - nonlocal locals - - # run the tests! - cmd = runner_.copy() - if args.get('disk'): - cmd.append('--disk=%s' % args['disk']) - if args.get('trace'): - cmd.append('--trace=%s' % args['trace']) - if args.get('verbose'): - print(' '.join(shlex.quote(c) for c in cmd)) - mpty, spty = pty.openpty() - proc = sp.Popen(cmd, stdout=spty, stderr=spty) - os.close(spty) - children.add(proc) - mpty = os.fdopen(mpty, 'r', 1) - if args.get('output'): - output = openio(args['output'], 'w') - - last_id = None - last_output = [] - last_assert = None - try: - while True: - # parse a line for state changes - try: - line = mpty.readline() - except OSError as e: - if e.errno == errno.EIO: - break - raise - if not line: - break - last_output.append(line) - if args.get('output'): - output.write(line) - elif args.get('verbose'): - sys.stdout.write(line) - - m = pattern.match(line) - if m: - op = m.group('op') or m.group('op_') - if op == 'running': - locals.seen_perms += 1 - last_id = m.group('id') - last_output = [] - last_assert = None - elif op == 'finished': - passed_suite_perms[m.group('suite')] += 1 - passed_case_perms[m.group('case')] += 1 - passed_perms += 1 - elif op == 'skipped': - locals.seen_perms += 1 - elif op == 'assert': - last_assert = ( - m.group('path'), - int(m.group('lineno')), - m.group('message')) - # go ahead and kill the process, aborting takes a while - if args.get('keep_going'): - proc.kill() - except KeyboardInterrupt: - raise TestFailure(last_id, 1, last_output) - finally: - children.remove(proc) - mpty.close() - if args.get('output'): - output.close() - - proc.wait() - if proc.returncode != 0: - raise TestFailure( - last_id, - proc.returncode, - last_output, - last_assert) - - def run_job(runner, start=None, step=None): - nonlocal failures - nonlocal locals - - start = start or 0 - step = step or 1 - while start < total_perms: - runner_ = runner.copy() - if start is not None: - runner_.append('--start=%d' % start) - if step is not None: - runner_.append('--step=%d' % step) - if args.get('isolate') or args.get('valgrind'): - runner_.append('--stop=%d' % (start+step)) - - try: - # run the tests - locals.seen_perms = 0 - run_runner(runner_) - assert locals.seen_perms > 0 - start += locals.seen_perms*step - - except TestFailure as failure: - # race condition for multiple failures? - if failures and not args.get('keep_going'): - break - - failures.append(failure) - - if args.get('keep_going') and not killed: - # resume after failed test - assert locals.seen_perms > 0 - start += locals.seen_perms*step - continue - else: - # stop other tests - for child in children.copy(): - child.kill() - - - # parallel jobs? - runners = [] - if 'jobs' in args: - for job in range(args['jobs']): - runners.append(th.Thread( - target=run_job, args=(runner_, job, args['jobs']))) - else: - runners.append(th.Thread( - target=run_job, args=(runner_, None, None))) - - for r in runners: - r.start() - - needs_newline = False - try: - while any(r.is_alive() for r in runners): - time.sleep(0.01) - - if not args.get('verbose'): - sys.stdout.write('\r\x1b[K' - 'running \x1b[%dm%s:\x1b[m %s ' - % (32 if not failures else 31, - name, - ', '.join(filter(None, [ - '%d/%d suites' % ( - sum(passed_suite_perms[k] == v - for k, v in expected_suite_perms.items()), - len(expected_suite_perms)) - if (not args.get('by_suites') - and not args.get('by_cases')) else None, - '%d/%d cases' % ( - sum(passed_case_perms[k] == v - for k, v in expected_case_perms.items()), - len(expected_case_perms)) - if not args.get('by_cases') else None, - '%d/%d perms' % (passed_perms, expected_perms), - '\x1b[31m%d/%d failures\x1b[m' - % (len(failures), expected_perms) - if failures else None])))) - sys.stdout.flush() - needs_newline = True - except KeyboardInterrupt: - # this is handled by the runner threads, we just - # need to not abort here - killed = True - finally: - if needs_newline: - print() - - for r in runners: - r.join() - - return ( - expected_perms, - passed_perms, - failures, - killed) - - -def run(**args): - start = time.time() - - runner_ = runner(**args) - print('using runner: %s' - % ' '.join(shlex.quote(c) for c in runner_)) - expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( - find_cases(runner_, **args)) - print('found %d suites, %d cases, %d/%d permutations' - % (len(expected_suite_perms), - len(expected_case_perms), - expected_perms, - total_perms)) - print() - - expected = 0 - passed = 0 - failures = [] - for type, by in it.product( - ['normal', 'reentrant'], - expected_case_perms.keys() if args.get('by_cases') - else expected_suite_perms.keys() if args.get('by_suites') - else [None]): - # rebuild runner for each stage to override test identifier if needed - stage_runner = runner(**args | { - 'test_ids': [by] if by is not None else args.get('test_ids', []), - 'normal': type == 'normal', - 'reentrant': type == 'reentrant'}) - - # spawn jobs for stage - expected_, passed_, failures_, killed = run_stage( - '%s %s' % (type, by or 'tests'), stage_runner, **args) - expected += expected_ - passed += passed_ - failures.extend(failures_) - if (failures and not args.get('keep_going')) or killed: - break - - # show summary - print() - print('\x1b[%dmdone:\x1b[m %d/%d passed, %d/%d failed, in %.2fs' - % (32 if not failures else 31, - passed, expected, len(failures), expected, - time.time()-start)) - print() - - # print each failure - if failures: - # get some extra info from runner - runner_paths = find_paths(runner_, **args) - runner_defines = find_defines(runner_, **args) - - for failure in failures: - # show summary of failure - path, lineno = runner_paths[testcase(failure.id)] - defines = runner_defines[failure.id] - - print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s%s failed' - % (path, lineno, failure.id, - ' (%s)' % ', '.join( - '%s=%s' % (k, v) for k, v in defines.items()) - if defines else '')) - - if failure.output: - output = failure.output - if failure.assert_ is not None: - output = output[:-1] - for line in output[-5:]: - sys.stdout.write(line) - - if failure.assert_ is not None: - path, lineno, message = failure.assert_ - print('\x1b[01m%s:%d:\x1b[01;31massert:\x1b[m %s' - % (path, lineno, message)) - with open(path) as f: - line = next(it.islice(f, lineno-1, None)).strip('\n') - print(line) - print() - - # drop into gdb? - if failures and (args.get('gdb') - or args.get('gdb_case') - or args.get('gdb_main')): - failure = failures[0] - runner_ = runner(**args | {'test_ids': [failure.id]}) - - if args.get('gdb_main'): - cmd = ['gdb', - '-ex', 'break main', - '-ex', 'run', - '--args'] + runner_ - elif args.get('gdb_case'): - path, lineno = runner_paths[testcase(failure.id)] - cmd = ['gdb', - '-ex', 'break %s:%d' % (path, lineno), - '-ex', 'run', - '--args'] + runner_ - elif failure.assert_ is not None: - cmd = ['gdb', - '-ex', 'run', - '-ex', 'frame function raise', - '-ex', 'up 2', - '--args'] + runner_ - else: - cmd = ['gdb', - '-ex', 'run', - '--args'] + runner_ - - # exec gdb interactively - if args.get('verbose'): - print(' '.join(shlex.quote(c) for c in cmd)) - os.execvp(cmd[0], cmd) - - return 1 if failures else 0 - - -def main(**args): - if args.get('compile'): - compile(**args) - elif (args.get('summary') - or args.get('list_suites') - or args.get('list_cases') - or args.get('list_paths') - or args.get('list_defines') - or args.get('list_geometries') - or args.get('list_defaults')): - list_(**args) - else: - run(**args) - - -if __name__ == "__main__": - import argparse - import sys - parser = argparse.ArgumentParser( - description="Build and run tests.", - conflict_handler='resolve') - parser.add_argument('test_ids', nargs='*', - help="Description of testis to run. May be a directory, path, or \ - test identifier. Test identifiers are of the form \ - ##, but suffixes can be \ - dropped to run any matching tests. Defaults to %r." % TEST_PATHS) - parser.add_argument('-v', '--verbose', action='store_true', - help="Output commands that run behind the scenes.") - # test flags - test_parser = parser.add_argument_group('test options') - test_parser.add_argument('-Y', '--summary', action='store_true', - help="Show quick summary.") - test_parser.add_argument('-l', '--list-suites', action='store_true', - help="List test suites.") - test_parser.add_argument('-L', '--list-cases', action='store_true', - help="List test cases.") - test_parser.add_argument('--list-paths', action='store_true', - help="List the path for each test case.") - test_parser.add_argument('--list-defines', action='store_true', - help="List the defines for each test permutation.") - test_parser.add_argument('--list-geometries', action='store_true', - help="List the disk geometries used for testing.") - test_parser.add_argument('--list-defaults', action='store_true', - help="List the default defines in this test-runner.") - test_parser.add_argument('-D', '--define', action='append', - help="Override a test define.") - test_parser.add_argument('-G', '--geometry', - help="Filter by geometry.") - test_parser.add_argument('-n', '--normal', action='store_true', - help="Filter for normal tests. Can be combined.") - test_parser.add_argument('-r', '--reentrant', action='store_true', - help="Filter for reentrant tests. Can be combined.") - test_parser.add_argument('-d', '--disk', - help="Use this file as the disk.") - test_parser.add_argument('-t', '--trace', - help="Redirect trace output to this file.") - test_parser.add_argument('-o', '--output', - help="Redirect stdout and stderr to this file.") - test_parser.add_argument('--runner', default=[RUNNER_PATH], - type=lambda x: x.split(), - help="Path to runner, defaults to %r" % RUNNER_PATH) - test_parser.add_argument('-j', '--jobs', nargs='?', type=int, - const=len(os.sched_getaffinity(0)), - help="Number of parallel runners to run.") - test_parser.add_argument('-k', '--keep-going', action='store_true', - help="Don't stop on first error.") - test_parser.add_argument('-i', '--isolate', action='store_true', - help="Run each test permutation in a separate process.") - test_parser.add_argument('-b', '--by-suites', action='store_true', - help="Step through tests by suite.") - test_parser.add_argument('-B', '--by-cases', action='store_true', - help="Step through tests by case.") - test_parser.add_argument('--gdb', action='store_true', - help="Drop into gdb on test failure.") - test_parser.add_argument('--gdb-case', action='store_true', - help="Drop into gdb on test failure but stop at the beginning \ - of the failing test case.") - test_parser.add_argument('--gdb-main', action='store_true', - help="Drop into gdb on test failure but stop at the beginning \ - of main.") - test_parser.add_argument('--valgrind', action='store_true', - help="Run under Valgrind to find memory errors. Implicitly sets \ - --isolate.") - test_parser.add_argument('--exec', default=[], type=lambda e: e.split(), - help="Run under another executable.") - # compilation flags - comp_parser = parser.add_argument_group('compilation options') - comp_parser.add_argument('-c', '--compile', action='store_true', - help="Compile a test suite or source file.") - comp_parser.add_argument('-s', '--source', - help="Source file to compile, possibly injecting internal tests.") - comp_parser.add_argument('-o', '--output', - help="Output file.") - # TODO apply this to other scripts? - sys.exit(main(**{k: v - for k, v in vars(parser.parse_args()).items() - if v is not None})) diff --git a/tests/test_alloc.toml b/tests/test_alloc.toml index fa92da51..4e43db33 100644 --- a/tests/test_alloc.toml +++ b/tests/test_alloc.toml @@ -1,27 +1,30 @@ # allocator tests # note for these to work there are a number constraints on the device geometry -if = 'LFS_BLOCK_CYCLES == -1' +if = 'BLOCK_CYCLES == -1' -[[case]] # parallel allocation test -define.FILES = 3 -define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-6)) / FILES)' +# parallel allocation test +[cases.parallel_allocation] +defines.FILES = 3 +defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / FILES)' code = ''' - const char *names[FILES] = {"bacon", "eggs", "pancakes"}; + const char *names[] = {"bacon", "eggs", "pancakes"}; lfs_file_t files[FILES]; - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "breakfast") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int n = 0; n < FILES; n++) { + char path[1024]; sprintf(path, "breakfast/%s", names[n]); lfs_file_open(&lfs, &files[n], path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0; } for (int n = 0; n < FILES; n++) { - size = strlen(names[n]); + size_t size = strlen(names[n]); for (lfs_size_t i = 0; i < SIZE; i += size) { lfs_file_write(&lfs, &files[n], names[n], size) => size; } @@ -31,12 +34,15 @@ code = ''' } lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int n = 0; n < FILES; n++) { + char path[1024]; sprintf(path, "breakfast/%s", names[n]); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_RDONLY) => 0; - size = strlen(names[n]); + size_t size = strlen(names[n]); for (lfs_size_t i = 0; i < SIZE; i += size) { + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, size) => size; assert(memcmp(buffer, names[n], size) == 0); } @@ -45,23 +51,28 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # serial allocation test -define.FILES = 3 -define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-6)) / FILES)' +# serial allocation test +[cases.serial_allocation] +defines.FILES = 3 +defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / FILES)' code = ''' - const char *names[FILES] = {"bacon", "eggs", "pancakes"}; + const char *names[] = {"bacon", "eggs", "pancakes"}; - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "breakfast") => 0; lfs_unmount(&lfs) => 0; for (int n = 0; n < FILES; n++) { - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + char path[1024]; sprintf(path, "breakfast/%s", names[n]); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0; - size = strlen(names[n]); + size_t size = strlen(names[n]); + uint8_t buffer[1024]; memcpy(buffer, names[n], size); for (int i = 0; i < SIZE; i += size) { lfs_file_write(&lfs, &file, buffer, size) => size; @@ -70,12 +81,15 @@ code = ''' lfs_unmount(&lfs) => 0; } - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int n = 0; n < FILES; n++) { + char path[1024]; sprintf(path, "breakfast/%s", names[n]); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_RDONLY) => 0; - size = strlen(names[n]); + size_t size = strlen(names[n]); for (int i = 0; i < SIZE; i += size) { + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, size) => size; assert(memcmp(buffer, names[n], size) == 0); } @@ -84,29 +98,32 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # parallel allocation reuse test -define.FILES = 3 -define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-6)) / FILES)' -define.CYCLES = [1, 10] +# parallel allocation reuse test +[cases.parallel_allocation_reuse] +defines.FILES = 3 +defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / FILES)' +defines.CYCLES = [1, 10] code = ''' - const char *names[FILES] = {"bacon", "eggs", "pancakes"}; + const char *names[] = {"bacon", "eggs", "pancakes"}; lfs_file_t files[FILES]; - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; for (int c = 0; c < CYCLES; c++) { - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "breakfast") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int n = 0; n < FILES; n++) { + char path[1024]; sprintf(path, "breakfast/%s", names[n]); lfs_file_open(&lfs, &files[n], path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0; } for (int n = 0; n < FILES; n++) { - size = strlen(names[n]); + size_t size = strlen(names[n]); for (int i = 0; i < SIZE; i += size) { lfs_file_write(&lfs, &files[n], names[n], size) => size; } @@ -116,12 +133,15 @@ code = ''' } lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int n = 0; n < FILES; n++) { + char path[1024]; sprintf(path, "breakfast/%s", names[n]); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_RDONLY) => 0; - size = strlen(names[n]); + size_t size = strlen(names[n]); for (int i = 0; i < SIZE; i += size) { + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, size) => size; assert(memcmp(buffer, names[n], size) == 0); } @@ -129,8 +149,9 @@ code = ''' } lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int n = 0; n < FILES; n++) { + char path[1024]; sprintf(path, "breakfast/%s", names[n]); lfs_remove(&lfs, path) => 0; } @@ -139,26 +160,31 @@ code = ''' } ''' -[[case]] # serial allocation reuse test -define.FILES = 3 -define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-6)) / FILES)' -define.CYCLES = [1, 10] +# serial allocation reuse test +[cases.serial_allocation_reuse] +defines.FILES = 3 +defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / FILES)' +defines.CYCLES = [1, 10] code = ''' - const char *names[FILES] = {"bacon", "eggs", "pancakes"}; + const char *names[] = {"bacon", "eggs", "pancakes"}; - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; for (int c = 0; c < CYCLES; c++) { - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "breakfast") => 0; lfs_unmount(&lfs) => 0; for (int n = 0; n < FILES; n++) { - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + char path[1024]; sprintf(path, "breakfast/%s", names[n]); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0; - size = strlen(names[n]); + size_t size = strlen(names[n]); + uint8_t buffer[1024]; memcpy(buffer, names[n], size); for (int i = 0; i < SIZE; i += size) { lfs_file_write(&lfs, &file, buffer, size) => size; @@ -167,12 +193,15 @@ code = ''' lfs_unmount(&lfs) => 0; } - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int n = 0; n < FILES; n++) { + char path[1024]; sprintf(path, "breakfast/%s", names[n]); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_RDONLY) => 0; - size = strlen(names[n]); + size_t size = strlen(names[n]); for (int i = 0; i < SIZE; i += size) { + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, size) => size; assert(memcmp(buffer, names[n], size) == 0); } @@ -180,8 +209,9 @@ code = ''' } lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int n = 0; n < FILES; n++) { + char path[1024]; sprintf(path, "breakfast/%s", names[n]); lfs_remove(&lfs, path) => 0; } @@ -190,12 +220,16 @@ code = ''' } ''' -[[case]] # exhaustion test +# exhaustion test +[cases.exhaustion] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); - size = strlen("exhaustion"); + size_t size = strlen("exhaustion"); + uint8_t buffer[1024]; memcpy(buffer, "exhaustion", size); lfs_file_write(&lfs, &file, buffer, size) => size; lfs_file_sync(&lfs, &file) => 0; @@ -216,7 +250,7 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "exhaustion", LFS_O_RDONLY); size = strlen("exhaustion"); lfs_file_size(&lfs, &file) => size; @@ -226,14 +260,18 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # exhaustion wraparound test -define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-4)) / 3)' +# exhaustion wraparound test +[cases.exhaustion_wraparound] +defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-4)) / 3)' code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "padding", LFS_O_WRONLY | LFS_O_CREAT); - size = strlen("buffering"); + size_t size = strlen("buffering"); + uint8_t buffer[1024]; memcpy(buffer, "buffering", size); for (int i = 0; i < SIZE; i += size) { lfs_file_write(&lfs, &file, buffer, size) => size; @@ -263,7 +301,7 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "exhaustion", LFS_O_RDONLY); size = strlen("exhaustion"); lfs_file_size(&lfs, &file) => size; @@ -274,17 +312,22 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # dir exhaustion test +# dir exhaustion test +[cases.dir_exhaustion] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // find out max file size lfs_mkdir(&lfs, "exhaustiondir") => 0; - size = strlen("blahblahblahblah"); + size_t size = strlen("blahblahblahblah"); + uint8_t buffer[1024]; memcpy(buffer, "blahblahblahblah", size); + lfs_file_t file; lfs_file_open(&lfs, &file, "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); int count = 0; + int err; while (true) { err = lfs_file_write(&lfs, &file, buffer, size); if (err < 0) { @@ -323,17 +366,21 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # what if we have a bad block during an allocation scan? +# what if we have a bad block during an allocation scan? +[cases.bad_block_allocation] in = "lfs.c" -define.LFS_ERASE_CYCLES = 0xffffffff -define.LFS_BADBLOCK_BEHAVIOR = 'LFS_TESTBD_BADBLOCK_READERROR' +defines.ERASE_CYCLES = 0xffffffff +defines.BADBLOCK_BEHAVIOR = 'LFS_TESTBD_BADBLOCK_READERROR' code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // first fill to exhaustion to find available space + lfs_file_t file; lfs_file_open(&lfs, &file, "pacman", LFS_O_WRONLY | LFS_O_CREAT) => 0; + uint8_t buffer[1024]; strcpy((char*)buffer, "waka"); - size = strlen("waka"); + size_t size = strlen("waka"); lfs_size_t filesize = 0; while (true) { lfs_ssize_t res = lfs_file_write(&lfs, &file, buffer, size); @@ -345,7 +392,7 @@ code = ''' } lfs_file_close(&lfs, &file) => 0; // now fill all but a couple of blocks of the filesystem with data - filesize -= 3*LFS_BLOCK_SIZE; + filesize -= 3*BLOCK_SIZE; lfs_file_open(&lfs, &file, "pacman", LFS_O_WRONLY | LFS_O_CREAT) => 0; strcpy((char*)buffer, "waka"); size = strlen("waka"); @@ -358,11 +405,11 @@ code = ''' lfs_unmount(&lfs) => 0; // remount to force an alloc scan - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // but mark the head of our file as a "bad block", this is force our // scan to bail early - lfs_testbd_setwear(&cfg, fileblock, 0xffffffff) => 0; + lfs_testbd_setwear(cfg, fileblock, 0xffffffff) => 0; lfs_file_open(&lfs, &file, "ghost", LFS_O_WRONLY | LFS_O_CREAT) => 0; strcpy((char*)buffer, "chomp"); size = strlen("chomp"); @@ -377,7 +424,7 @@ code = ''' // now reverse the "bad block" and try to write the file again until we // run out of space - lfs_testbd_setwear(&cfg, fileblock, 0) => 0; + lfs_testbd_setwear(cfg, fileblock, 0) => 0; lfs_file_open(&lfs, &file, "ghost", LFS_O_WRONLY | LFS_O_CREAT) => 0; strcpy((char*)buffer, "chomp"); size = strlen("chomp"); @@ -393,7 +440,7 @@ code = ''' lfs_unmount(&lfs) => 0; // check that the disk isn't hurt - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "pacman", LFS_O_RDONLY) => 0; strcpy((char*)buffer, "waka"); size = strlen("waka"); @@ -411,24 +458,29 @@ code = ''' # on the geometry of the block device. But they are valuable. Eventually they # should be removed and replaced with generalized tests. -[[case]] # chained dir exhaustion test -define.LFS_BLOCK_SIZE = 512 -define.LFS_BLOCK_COUNT = 1024 -if = 'LFS_BLOCK_SIZE == 512 && LFS_BLOCK_COUNT == 1024' +# chained dir exhaustion test +[cases.chained_dir_exhaustion] +if = 'BLOCK_SIZE == 512' +defines.BLOCK_COUNT = 1024 code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // find out max file size lfs_mkdir(&lfs, "exhaustiondir") => 0; for (int i = 0; i < 10; i++) { + char path[1024]; sprintf(path, "dirwithanexhaustivelylongnameforpadding%d", i); lfs_mkdir(&lfs, path) => 0; } - size = strlen("blahblahblahblah"); + size_t size = strlen("blahblahblahblah"); + uint8_t buffer[1024]; memcpy(buffer, "blahblahblahblah", size); + lfs_file_t file; lfs_file_open(&lfs, &file, "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); int count = 0; + int err; while (true) { err = lfs_file_write(&lfs, &file, buffer, size); if (err < 0) { @@ -443,6 +495,7 @@ code = ''' lfs_remove(&lfs, "exhaustion") => 0; lfs_remove(&lfs, "exhaustiondir") => 0; for (int i = 0; i < 10; i++) { + char path[1024]; sprintf(path, "dirwithanexhaustivelylongnameforpadding%d", i); lfs_remove(&lfs, path) => 0; } @@ -455,6 +508,7 @@ code = ''' lfs_file_sync(&lfs, &file) => 0; for (int i = 0; i < 10; i++) { + char path[1024]; sprintf(path, "dirwithanexhaustivelylongnameforpadding%d", i); lfs_mkdir(&lfs, path) => 0; } @@ -482,27 +536,31 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # split dir test -define.LFS_BLOCK_SIZE = 512 -define.LFS_BLOCK_COUNT = 1024 -if = 'LFS_BLOCK_SIZE == 512 && LFS_BLOCK_COUNT == 1024' +# split dir test +[cases.split_dir] +if = 'BLOCK_SIZE == 512' +defines.BLOCK_COUNT = 1024 code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // create one block hole for half a directory + lfs_file_t file; lfs_file_open(&lfs, &file, "bump", LFS_O_WRONLY | LFS_O_CREAT) => 0; - for (lfs_size_t i = 0; i < cfg.block_size; i += 2) { + for (lfs_size_t i = 0; i < cfg->block_size; i += 2) { + uint8_t buffer[1024]; memcpy(&buffer[i], "hi", 2); } - lfs_file_write(&lfs, &file, buffer, cfg.block_size) => cfg.block_size; + uint8_t buffer[1024]; + lfs_file_write(&lfs, &file, buffer, cfg->block_size) => cfg->block_size; lfs_file_close(&lfs, &file) => 0; lfs_file_open(&lfs, &file, "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); - size = strlen("blahblahblahblah"); + size_t size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < (cfg.block_count-4)*(cfg.block_size-8); + i < (cfg->block_count-4)*(cfg->block_size-8); i += size) { lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -510,7 +568,7 @@ code = ''' // remount to force reset of lookahead lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // open hole lfs_remove(&lfs, "bump") => 0; @@ -518,30 +576,33 @@ code = ''' lfs_mkdir(&lfs, "splitdir") => 0; lfs_file_open(&lfs, &file, "splitdir/bump", LFS_O_WRONLY | LFS_O_CREAT) => 0; - for (lfs_size_t i = 0; i < cfg.block_size; i += 2) { + for (lfs_size_t i = 0; i < cfg->block_size; i += 2) { memcpy(&buffer[i], "hi", 2); } - lfs_file_write(&lfs, &file, buffer, 2*cfg.block_size) => LFS_ERR_NOSPC; + lfs_file_write(&lfs, &file, buffer, 2*cfg->block_size) => LFS_ERR_NOSPC; lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; ''' -[[case]] # outdated lookahead test -define.LFS_BLOCK_SIZE = 512 -define.LFS_BLOCK_COUNT = 1024 -if = 'LFS_BLOCK_SIZE == 512 && LFS_BLOCK_COUNT == 1024' +# outdated lookahead test +[cases.outdated_lookahead] +if = 'BLOCK_SIZE == 512' +defines.BLOCK_COUNT = 1024 code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // fill completely with two files + lfs_file_t file; lfs_file_open(&lfs, &file, "exhaustion1", LFS_O_WRONLY | LFS_O_CREAT) => 0; - size = strlen("blahblahblahblah"); + size_t size = strlen("blahblahblahblah"); + uint8_t buffer[1024]; memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < ((cfg.block_count-2)/2)*(cfg.block_size-8); + i < ((cfg->block_count-2)/2)*(cfg->block_size-8); i += size) { lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -552,7 +613,7 @@ code = ''' size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < ((cfg.block_count-2+1)/2)*(cfg.block_size-8); + i < ((cfg->block_count-2+1)/2)*(cfg->block_size-8); i += size) { lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -560,7 +621,7 @@ code = ''' // remount to force reset of lookahead lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // rewrite one file lfs_file_open(&lfs, &file, "exhaustion1", @@ -569,7 +630,7 @@ code = ''' size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < ((cfg.block_count-2)/2)*(cfg.block_size-8); + i < ((cfg->block_count-2)/2)*(cfg->block_size-8); i += size) { lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -583,7 +644,7 @@ code = ''' size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < ((cfg.block_count-2+1)/2)*(cfg.block_size-8); + i < ((cfg->block_count-2+1)/2)*(cfg->block_size-8); i += size) { lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -592,21 +653,24 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # outdated lookahead and split dir test -define.LFS_BLOCK_SIZE = 512 -define.LFS_BLOCK_COUNT = 1024 -if = 'LFS_BLOCK_SIZE == 512 && LFS_BLOCK_COUNT == 1024' +# outdated lookahead and split dir test +[cases.outdated_lookahead_split_dir] +if = 'BLOCK_SIZE == 512' +defines.BLOCK_COUNT = 1024 code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // fill completely with two files + lfs_file_t file; lfs_file_open(&lfs, &file, "exhaustion1", LFS_O_WRONLY | LFS_O_CREAT) => 0; - size = strlen("blahblahblahblah"); + size_t size = strlen("blahblahblahblah"); + uint8_t buffer[1024]; memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < ((cfg.block_count-2)/2)*(cfg.block_size-8); + i < ((cfg->block_count-2)/2)*(cfg->block_size-8); i += size) { lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -617,7 +681,7 @@ code = ''' size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < ((cfg.block_count-2+1)/2)*(cfg.block_size-8); + i < ((cfg->block_count-2+1)/2)*(cfg->block_size-8); i += size) { lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -625,7 +689,7 @@ code = ''' // remount to force reset of lookahead lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // rewrite one file with a hole of one block lfs_file_open(&lfs, &file, "exhaustion1", @@ -634,7 +698,7 @@ code = ''' size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < ((cfg.block_count-2)/2 - 1)*(cfg.block_size-8); + i < ((cfg->block_count-2)/2 - 1)*(cfg->block_size-8); i += size) { lfs_file_write(&lfs, &file, buffer, size) => size; } diff --git a/tests/test_attrs.toml b/tests/test_attrs.toml index db8d0c7e..719ea682 100644 --- a/tests/test_attrs.toml +++ b/tests/test_attrs.toml @@ -1,14 +1,17 @@ -[[case]] # set/get attribute +[cases.get_set_attrs] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "hello") => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "hello/hello", LFS_O_WRONLY | LFS_O_CREAT) => 0; lfs_file_write(&lfs, &file, "hello", strlen("hello")) => strlen("hello"); lfs_file_close(&lfs, &file); lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + uint8_t buffer[1024]; memset(buffer, 0, sizeof(buffer)); lfs_setattr(&lfs, "hello", 'A', "aaaa", 4) => 0; lfs_setattr(&lfs, "hello", 'B', "bbbbbb", 6) => 0; @@ -60,7 +63,7 @@ code = ''' lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; memset(buffer, 0, sizeof(buffer)); lfs_getattr(&lfs, "hello", 'A', buffer, 4) => 4; lfs_getattr(&lfs, "hello", 'B', buffer+4, 9) => 9; @@ -76,17 +79,20 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # set/get root attribute +[cases.get_set_root_attrs] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "hello") => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "hello/hello", LFS_O_WRONLY | LFS_O_CREAT) => 0; lfs_file_write(&lfs, &file, "hello", strlen("hello")) => strlen("hello"); lfs_file_close(&lfs, &file); lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + uint8_t buffer[1024]; memset(buffer, 0, sizeof(buffer)); lfs_setattr(&lfs, "/", 'A', "aaaa", 4) => 0; lfs_setattr(&lfs, "/", 'B', "bbbbbb", 6) => 0; @@ -137,7 +143,7 @@ code = ''' lfs_getattr(&lfs, "/", 'C', buffer+10, 5) => 5; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; memset(buffer, 0, sizeof(buffer)); lfs_getattr(&lfs, "/", 'A', buffer, 4) => 4; lfs_getattr(&lfs, "/", 'B', buffer+4, 9) => 9; @@ -153,17 +159,20 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # set/get file attribute +[cases.get_set_file_attrs] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "hello") => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "hello/hello", LFS_O_WRONLY | LFS_O_CREAT) => 0; lfs_file_write(&lfs, &file, "hello", strlen("hello")) => strlen("hello"); lfs_file_close(&lfs, &file); lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + uint8_t buffer[1024]; memset(buffer, 0, sizeof(buffer)); struct lfs_attr attrs1[] = { {'A', buffer, 4}, @@ -238,7 +247,7 @@ code = ''' lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; memset(buffer, 0, sizeof(buffer)); struct lfs_attr attrs3[] = { {'A', buffer, 4}, @@ -260,20 +269,23 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # deferred file attributes +[cases.deferred_file_attrs] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "hello") => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "hello/hello", LFS_O_WRONLY | LFS_O_CREAT) => 0; lfs_file_write(&lfs, &file, "hello", strlen("hello")) => strlen("hello"); lfs_file_close(&lfs, &file); lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_setattr(&lfs, "hello/hello", 'B', "fffffffff", 9) => 0; lfs_setattr(&lfs, "hello/hello", 'C', "ccccc", 5) => 0; + uint8_t buffer[1024]; memset(buffer, 0, sizeof(buffer)); struct lfs_attr attrs1[] = { {'B', "gggg", 4}, diff --git a/tests/test_badblocks.toml b/tests/test_badblocks.toml index 06967a67..c5cab47c 100644 --- a/tests/test_badblocks.toml +++ b/tests/test_badblocks.toml @@ -1,28 +1,30 @@ # bad blocks with block cycles should be tested in test_relocations -if = 'LFS_BLOCK_CYCLES == -1' +if = '(int32_t)BLOCK_CYCLES == -1' -[[case]] # single bad blocks -define.LFS_BLOCK_COUNT = 256 # small bd so test runs faster -define.LFS_ERASE_CYCLES = 0xffffffff -define.LFS_ERASE_VALUE = [0x00, 0xff, -1] -define.LFS_BADBLOCK_BEHAVIOR = [ +[cases.single_bad_blocks] +defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_CYCLES = 0xffffffff +defines.ERASE_VALUE = [0x00, 0xff, -1] +defines.BADBLOCK_BEHAVIOR = [ 'LFS_TESTBD_BADBLOCK_PROGERROR', 'LFS_TESTBD_BADBLOCK_ERASEERROR', 'LFS_TESTBD_BADBLOCK_READERROR', 'LFS_TESTBD_BADBLOCK_PROGNOOP', 'LFS_TESTBD_BADBLOCK_ERASENOOP', ] -define.NAMEMULT = 64 -define.FILEMULT = 1 +defines.NAMEMULT = 64 +defines.FILEMULT = 1 code = ''' - for (lfs_block_t badblock = 2; badblock < LFS_BLOCK_COUNT; badblock++) { - lfs_testbd_setwear(&cfg, badblock-1, 0) => 0; - lfs_testbd_setwear(&cfg, badblock, 0xffffffff) => 0; - - lfs_format(&lfs, &cfg) => 0; + for (lfs_block_t badblock = 2; badblock < BLOCK_COUNT; badblock++) { + lfs_testbd_setwear(cfg, badblock-1, 0) => 0; + lfs_testbd_setwear(cfg, badblock, 0xffffffff) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + lfs_mount(&lfs, cfg) => 0; for (int i = 1; i < 10; i++) { + uint8_t buffer[1024]; for (int j = 0; j < NAMEMULT; j++) { buffer[j] = '0'+i; } @@ -34,10 +36,11 @@ code = ''' buffer[j+NAMEMULT+1] = '0'+i; } buffer[2*NAMEMULT+1] = '\0'; + lfs_file_t file; lfs_file_open(&lfs, &file, (char*)buffer, LFS_O_WRONLY | LFS_O_CREAT) => 0; - size = NAMEMULT; + lfs_size_t size = NAMEMULT; for (int j = 0; j < i*FILEMULT; j++) { lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -46,12 +49,14 @@ code = ''' } lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 1; i < 10; i++) { + uint8_t buffer[1024]; for (int j = 0; j < NAMEMULT; j++) { buffer[j] = '0'+i; } buffer[NAMEMULT] = '\0'; + struct lfs_info info; lfs_stat(&lfs, (char*)buffer, &info) => 0; info.type => LFS_TYPE_DIR; @@ -60,9 +65,10 @@ code = ''' buffer[j+NAMEMULT+1] = '0'+i; } buffer[2*NAMEMULT+1] = '\0'; + lfs_file_t file; lfs_file_open(&lfs, &file, (char*)buffer, LFS_O_RDONLY) => 0; - size = NAMEMULT; + int size = NAMEMULT; for (int j = 0; j < i*FILEMULT; j++) { uint8_t rbuffer[1024]; lfs_file_read(&lfs, &file, rbuffer, size) => size; @@ -75,28 +81,30 @@ code = ''' } ''' -[[case]] # region corruption (causes cascading failures) -define.LFS_BLOCK_COUNT = 256 # small bd so test runs faster -define.LFS_ERASE_CYCLES = 0xffffffff -define.LFS_ERASE_VALUE = [0x00, 0xff, -1] -define.LFS_BADBLOCK_BEHAVIOR = [ +[cases.region_corruption] # (causes cascading failures) +defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_CYCLES = 0xffffffff +defines.ERASE_VALUE = [0x00, 0xff, -1] +defines.BADBLOCK_BEHAVIOR = [ 'LFS_TESTBD_BADBLOCK_PROGERROR', 'LFS_TESTBD_BADBLOCK_ERASEERROR', 'LFS_TESTBD_BADBLOCK_READERROR', 'LFS_TESTBD_BADBLOCK_PROGNOOP', 'LFS_TESTBD_BADBLOCK_ERASENOOP', ] -define.NAMEMULT = 64 -define.FILEMULT = 1 +defines.NAMEMULT = 64 +defines.FILEMULT = 1 code = ''' - for (lfs_block_t i = 0; i < (LFS_BLOCK_COUNT-2)/2; i++) { - lfs_testbd_setwear(&cfg, i+2, 0xffffffff) => 0; + for (lfs_block_t i = 0; i < (BLOCK_COUNT-2)/2; i++) { + lfs_testbd_setwear(cfg, i+2, 0xffffffff) => 0; } - - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + lfs_mount(&lfs, cfg) => 0; for (int i = 1; i < 10; i++) { + uint8_t buffer[1024]; for (int j = 0; j < NAMEMULT; j++) { buffer[j] = '0'+i; } @@ -108,10 +116,11 @@ code = ''' buffer[j+NAMEMULT+1] = '0'+i; } buffer[2*NAMEMULT+1] = '\0'; + lfs_file_t file; lfs_file_open(&lfs, &file, (char*)buffer, LFS_O_WRONLY | LFS_O_CREAT) => 0; - size = NAMEMULT; + lfs_size_t size = NAMEMULT; for (int j = 0; j < i*FILEMULT; j++) { lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -120,12 +129,14 @@ code = ''' } lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 1; i < 10; i++) { + uint8_t buffer[1024]; for (int j = 0; j < NAMEMULT; j++) { buffer[j] = '0'+i; } buffer[NAMEMULT] = '\0'; + struct lfs_info info; lfs_stat(&lfs, (char*)buffer, &info) => 0; info.type => LFS_TYPE_DIR; @@ -134,9 +145,10 @@ code = ''' buffer[j+NAMEMULT+1] = '0'+i; } buffer[2*NAMEMULT+1] = '\0'; + lfs_file_t file; lfs_file_open(&lfs, &file, (char*)buffer, LFS_O_RDONLY) => 0; - size = NAMEMULT; + lfs_size_t size = NAMEMULT; for (int j = 0; j < i*FILEMULT; j++) { uint8_t rbuffer[1024]; lfs_file_read(&lfs, &file, rbuffer, size) => size; @@ -148,28 +160,30 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # alternating corruption (causes cascading failures) -define.LFS_BLOCK_COUNT = 256 # small bd so test runs faster -define.LFS_ERASE_CYCLES = 0xffffffff -define.LFS_ERASE_VALUE = [0x00, 0xff, -1] -define.LFS_BADBLOCK_BEHAVIOR = [ +[cases.alternating_corruption] # (causes cascading failures) +defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_CYCLES = 0xffffffff +defines.ERASE_VALUE = [0x00, 0xff, -1] +defines.BADBLOCK_BEHAVIOR = [ 'LFS_TESTBD_BADBLOCK_PROGERROR', 'LFS_TESTBD_BADBLOCK_ERASEERROR', 'LFS_TESTBD_BADBLOCK_READERROR', 'LFS_TESTBD_BADBLOCK_PROGNOOP', 'LFS_TESTBD_BADBLOCK_ERASENOOP', ] -define.NAMEMULT = 64 -define.FILEMULT = 1 +defines.NAMEMULT = 64 +defines.FILEMULT = 1 code = ''' - for (lfs_block_t i = 0; i < (LFS_BLOCK_COUNT-2)/2; i++) { - lfs_testbd_setwear(&cfg, (2*i) + 2, 0xffffffff) => 0; + for (lfs_block_t i = 0; i < (BLOCK_COUNT-2)/2; i++) { + lfs_testbd_setwear(cfg, (2*i) + 2, 0xffffffff) => 0; } - - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + lfs_mount(&lfs, cfg) => 0; for (int i = 1; i < 10; i++) { + uint8_t buffer[1024]; for (int j = 0; j < NAMEMULT; j++) { buffer[j] = '0'+i; } @@ -181,10 +195,11 @@ code = ''' buffer[j+NAMEMULT+1] = '0'+i; } buffer[2*NAMEMULT+1] = '\0'; + lfs_file_t file; lfs_file_open(&lfs, &file, (char*)buffer, LFS_O_WRONLY | LFS_O_CREAT) => 0; - size = NAMEMULT; + lfs_size_t size = NAMEMULT; for (int j = 0; j < i*FILEMULT; j++) { lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -193,12 +208,14 @@ code = ''' } lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 1; i < 10; i++) { + uint8_t buffer[1024]; for (int j = 0; j < NAMEMULT; j++) { buffer[j] = '0'+i; } buffer[NAMEMULT] = '\0'; + struct lfs_info info; lfs_stat(&lfs, (char*)buffer, &info) => 0; info.type => LFS_TYPE_DIR; @@ -207,9 +224,10 @@ code = ''' buffer[j+NAMEMULT+1] = '0'+i; } buffer[2*NAMEMULT+1] = '\0'; + lfs_file_t file; lfs_file_open(&lfs, &file, (char*)buffer, LFS_O_RDONLY) => 0; - size = NAMEMULT; + lfs_size_t size = NAMEMULT; for (int j = 0; j < i*FILEMULT; j++) { uint8_t rbuffer[1024]; lfs_file_read(&lfs, &file, rbuffer, size) => size; @@ -222,10 +240,10 @@ code = ''' ''' # other corner cases -[[case]] # bad superblocks (corrupt 1 or 0) -define.LFS_ERASE_CYCLES = 0xffffffff -define.LFS_ERASE_VALUE = [0x00, 0xff, -1] -define.LFS_BADBLOCK_BEHAVIOR = [ +[cases.bad_superblocks] # (corrupt 1 or 0) +defines.ERASE_CYCLES = 0xffffffff +defines.ERASE_VALUE = [0x00, 0xff, -1] +defines.BADBLOCK_BEHAVIOR = [ 'LFS_TESTBD_BADBLOCK_PROGERROR', 'LFS_TESTBD_BADBLOCK_ERASEERROR', 'LFS_TESTBD_BADBLOCK_READERROR', @@ -233,9 +251,10 @@ define.LFS_BADBLOCK_BEHAVIOR = [ 'LFS_TESTBD_BADBLOCK_ERASENOOP', ] code = ''' - lfs_testbd_setwear(&cfg, 0, 0xffffffff) => 0; - lfs_testbd_setwear(&cfg, 1, 0xffffffff) => 0; + lfs_testbd_setwear(cfg, 0, 0xffffffff) => 0; + lfs_testbd_setwear(cfg, 1, 0xffffffff) => 0; - lfs_format(&lfs, &cfg) => LFS_ERR_NOSPC; - lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT; + lfs_t lfs; + lfs_format(&lfs, cfg) => LFS_ERR_NOSPC; + lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT; ''' diff --git a/tests/test_dirs.toml b/tests/test_dirs.toml index 270f4f8e..60346c0c 100644 --- a/tests/test_dirs.toml +++ b/tests/test_dirs.toml @@ -1,8 +1,11 @@ -[[case]] # root +[cases.root] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, ".") == 0); @@ -14,20 +17,25 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # many directory creation -define.N = 'range(0, 100, 3)' +[cases.many_dir_creation] +defines.N = [3,6,9,12,21,33,57,66,72,93,99] +if = 'N < BLOCK_COUNT/2' code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "dir%03d", i); lfs_mkdir(&lfs, path) => 0; } lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, ".") == 0); @@ -35,6 +43,7 @@ code = ''' assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, "..") == 0); for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "dir%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); @@ -45,20 +54,25 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # many directory removal -define.N = 'range(3, 100, 11)' +[cases.many_dir_removal] +defines.N = [3,6,9,12,21,33,57,66,72,93,99] +if = 'N < BLOCK_COUNT/2' code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "removeme%03d", i); lfs_mkdir(&lfs, path) => 0; } lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, ".") == 0); @@ -66,6 +80,7 @@ code = ''' assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, "..") == 0); for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "removeme%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); @@ -75,14 +90,15 @@ code = ''' lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs); - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "removeme%03d", i); lfs_remove(&lfs, path) => 0; } lfs_unmount(&lfs); - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "/") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); @@ -95,20 +111,25 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # many directory rename -define.N = 'range(3, 100, 11)' +[cases.many_dir_rename] +defines.N = [3,6,9,12,21,33,57,66,72,93,99] +if = 'N < BLOCK_COUNT/2' code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "test%03d", i); lfs_mkdir(&lfs, path) => 0; } lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, ".") == 0); @@ -116,6 +137,7 @@ code = ''' assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, "..") == 0); for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "test%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); @@ -125,7 +147,7 @@ code = ''' lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs); - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 0; i < N; i++) { char oldpath[128]; char newpath[128]; @@ -135,7 +157,7 @@ code = ''' } lfs_unmount(&lfs); - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "/") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); @@ -144,6 +166,7 @@ code = ''' assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, "..") == 0); for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "tedd%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); @@ -154,29 +177,34 @@ code = ''' lfs_unmount(&lfs); ''' -[[case]] # reentrant many directory creation/rename/removal -define.N = [5, 11] +[cases.reentrant_many_dir] +defines.N = [5, 11] reentrant = true code = ''' - err = lfs_mount(&lfs, &cfg); + lfs_t lfs; + int err = lfs_mount(&lfs, cfg); if (err) { - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; } for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "hi%03d", i); err = lfs_mkdir(&lfs, path); assert(err == 0 || err == LFS_ERR_EXIST); } for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "hello%03d", i); err = lfs_remove(&lfs, path); assert(err == 0 || err == LFS_ERR_NOENT); } + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, ".") == 0); @@ -184,6 +212,7 @@ code = ''' assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, "..") == 0); for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "hi%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); @@ -209,6 +238,7 @@ code = ''' assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, "..") == 0); for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "hello%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); @@ -218,6 +248,7 @@ code = ''' lfs_dir_close(&lfs, &dir) => 0; for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "hello%03d", i); lfs_remove(&lfs, path) => 0; } @@ -234,22 +265,28 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # file creation -define.N = 'range(3, 100, 11)' +[cases.file_creation] +defines.N = [3,6,9,12,21,33,57,66,72,93,99] +if = 'N < BLOCK_COUNT/2' code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "file%03d", i); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; lfs_file_close(&lfs, &file) => 0; } lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, ".") == 0); @@ -257,6 +294,7 @@ code = ''' assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, "..") == 0); for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "file%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_REG); @@ -267,22 +305,28 @@ code = ''' lfs_unmount(&lfs); ''' -[[case]] # file removal -define.N = 'range(0, 100, 3)' +[cases.file_removal] +defines.N = [3,6,9,12,21,33,57,66,72,93,99] +if = 'N < BLOCK_COUNT/2' code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "removeme%03d", i); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; lfs_file_close(&lfs, &file) => 0; } lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, ".") == 0); @@ -290,6 +334,7 @@ code = ''' assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, "..") == 0); for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "removeme%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_REG); @@ -299,14 +344,15 @@ code = ''' lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs); - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "removeme%03d", i); lfs_remove(&lfs, path) => 0; } lfs_unmount(&lfs); - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "/") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); @@ -319,22 +365,28 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # file rename -define.N = 'range(0, 100, 3)' +[cases.file_rename] +defines.N = [3,6,9,12,21,33,57,66,72,93,99] +if = 'N < BLOCK_COUNT/2' code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "test%03d", i); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; lfs_file_close(&lfs, &file) => 0; } lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, ".") == 0); @@ -342,6 +394,7 @@ code = ''' assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, "..") == 0); for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "test%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_REG); @@ -351,7 +404,7 @@ code = ''' lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs); - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 0; i < N; i++) { char oldpath[128]; char newpath[128]; @@ -361,7 +414,7 @@ code = ''' } lfs_unmount(&lfs); - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "/") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); @@ -370,6 +423,7 @@ code = ''' assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, "..") == 0); for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "tedd%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_REG); @@ -380,29 +434,36 @@ code = ''' lfs_unmount(&lfs); ''' -[[case]] # reentrant file creation/rename/removal -define.N = [5, 25] +[cases.reentrant_files] +defines.N = [5, 25] +if = 'N < BLOCK_COUNT/2' reentrant = true code = ''' - err = lfs_mount(&lfs, &cfg); + lfs_t lfs; + int err = lfs_mount(&lfs, cfg); if (err) { - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; } for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "hi%03d", i); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_CREAT | LFS_O_WRONLY) => 0; lfs_file_close(&lfs, &file) => 0; } for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "hello%03d", i); err = lfs_remove(&lfs, path); assert(err == 0 || err == LFS_ERR_NOENT); } + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, ".") == 0); @@ -410,6 +471,7 @@ code = ''' assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, "..") == 0); for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "hi%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_REG); @@ -435,6 +497,7 @@ code = ''' assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, "..") == 0); for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "hello%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_REG); @@ -444,6 +507,7 @@ code = ''' lfs_dir_close(&lfs, &dir) => 0; for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "hello%03d", i); lfs_remove(&lfs, path) => 0; } @@ -460,24 +524,28 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # nested directories +[cases.nested_dirs] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "potato") => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "burito", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "potato/baked") => 0; lfs_mkdir(&lfs, "potato/sweet") => 0; lfs_mkdir(&lfs, "potato/fried") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "potato") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); info.type => LFS_TYPE_DIR; @@ -498,21 +566,21 @@ code = ''' lfs_unmount(&lfs) => 0; // try removing? - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_remove(&lfs, "potato") => LFS_ERR_NOTEMPTY; lfs_unmount(&lfs) => 0; // try renaming? - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_rename(&lfs, "potato", "coldpotato") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_rename(&lfs, "coldpotato", "warmpotato") => 0; lfs_rename(&lfs, "warmpotato", "hotpotato") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_remove(&lfs, "potato") => LFS_ERR_NOENT; lfs_remove(&lfs, "coldpotato") => LFS_ERR_NOENT; lfs_remove(&lfs, "warmpotato") => LFS_ERR_NOENT; @@ -520,7 +588,7 @@ code = ''' lfs_unmount(&lfs) => 0; // try cross-directory renaming - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "coldpotato") => 0; lfs_rename(&lfs, "hotpotato/baked", "coldpotato/baked") => 0; lfs_rename(&lfs, "coldpotato", "hotpotato") => LFS_ERR_NOTEMPTY; @@ -536,7 +604,7 @@ code = ''' lfs_remove(&lfs, "hotpotato") => LFS_ERR_NOTEMPTY; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "hotpotato") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -558,7 +626,7 @@ code = ''' lfs_unmount(&lfs) => 0; // final remove - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_remove(&lfs, "hotpotato") => LFS_ERR_NOTEMPTY; lfs_remove(&lfs, "hotpotato/baked") => 0; lfs_remove(&lfs, "hotpotato") => LFS_ERR_NOTEMPTY; @@ -568,7 +636,7 @@ code = ''' lfs_remove(&lfs, "hotpotato") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "/") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -584,17 +652,22 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # recursive remove -define.N = [10, 100] +[cases.recursive_remove] +defines.N = [10, 100] +if = 'N < BLOCK_COUNT/2' code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "prickly-pear") => 0; for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "prickly-pear/cactus%03d", i); lfs_mkdir(&lfs, path) => 0; } + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "prickly-pear") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, ".") == 0); @@ -602,6 +675,7 @@ code = ''' assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, "..") == 0); for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "cactus%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); @@ -611,7 +685,7 @@ code = ''' lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs); - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_remove(&lfs, "prickly-pear") => LFS_ERR_NOTEMPTY; lfs_dir_open(&lfs, &dir, "prickly-pear") => 0; @@ -622,6 +696,7 @@ code = ''' assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, "..") == 0); for (int i = 0; i < N; i++) { + char path[1024]; sprintf(path, "cactus%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); @@ -636,22 +711,24 @@ code = ''' lfs_remove(&lfs, "prickly-pear") => LFS_ERR_NOENT; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_remove(&lfs, "prickly-pear") => LFS_ERR_NOENT; lfs_unmount(&lfs) => 0; ''' -[[case]] # other error cases +[cases.other_errors] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "potato") => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "burito", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "potato") => LFS_ERR_EXIST; lfs_mkdir(&lfs, "burito") => LFS_ERR_EXIST; @@ -659,6 +736,7 @@ code = ''' LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => LFS_ERR_EXIST; lfs_file_open(&lfs, &file, "potato", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => LFS_ERR_EXIST; + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "tomato") => LFS_ERR_NOENT; lfs_dir_open(&lfs, &dir, "burito") => LFS_ERR_NOTDIR; lfs_file_open(&lfs, &file, "tomato", LFS_O_RDONLY) => LFS_ERR_NOENT; @@ -678,6 +756,7 @@ code = ''' // check that errors did not corrupt directory lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); assert(strcmp(info.name, ".") == 0); @@ -696,7 +775,7 @@ code = ''' lfs_unmount(&lfs) => 0; // or on disk - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "/") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(info.type == LFS_TYPE_DIR); @@ -715,21 +794,26 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # directory seek -define.COUNT = [4, 128, 132] +[cases.directory_seek] +defines.COUNT = [4, 128, 132] +if = 'COUNT < BLOCK_COUNT/2' code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "hello") => 0; for (int i = 0; i < COUNT; i++) { + char path[1024]; sprintf(path, "hello/kitty%03d", i); lfs_mkdir(&lfs, path) => 0; } lfs_unmount(&lfs) => 0; for (int j = 2; j < COUNT; j++) { - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "hello") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); assert(info.type == LFS_TYPE_DIR); @@ -739,6 +823,7 @@ code = ''' lfs_soff_t pos; for (int i = 0; i < j; i++) { + char path[1024]; sprintf(path, "kitty%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, path) == 0); @@ -748,13 +833,14 @@ code = ''' } lfs_dir_seek(&lfs, &dir, pos) => 0; + char path[1024]; sprintf(path, "kitty%03d", j); lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, path) == 0); assert(info.type == LFS_TYPE_DIR); lfs_dir_rewind(&lfs, &dir) => 0; - sprintf(path, "kitty%03d", 0); + sprintf(path, "kitty%03u", 0); lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); assert(info.type == LFS_TYPE_DIR); @@ -776,20 +862,25 @@ code = ''' } ''' -[[case]] # root seek -define.COUNT = [4, 128, 132] +[cases.root_seek] +defines.COUNT = [4, 128, 132] +if = 'COUNT < BLOCK_COUNT/2' code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 0; i < COUNT; i++) { + char path[1024]; sprintf(path, "hi%03d", i); lfs_mkdir(&lfs, path) => 0; } lfs_unmount(&lfs) => 0; for (int j = 2; j < COUNT; j++) { - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); assert(info.type == LFS_TYPE_DIR); @@ -799,6 +890,7 @@ code = ''' lfs_soff_t pos; for (int i = 0; i < j; i++) { + char path[1024]; sprintf(path, "hi%03d", i); lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, path) == 0); @@ -808,13 +900,14 @@ code = ''' } lfs_dir_seek(&lfs, &dir, pos) => 0; + char path[1024]; sprintf(path, "hi%03d", j); lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, path) == 0); assert(info.type == LFS_TYPE_DIR); lfs_dir_rewind(&lfs, &dir) => 0; - sprintf(path, "hi%03d", 0); + sprintf(path, "hi%03u", 0); lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); assert(info.type == LFS_TYPE_DIR); diff --git a/tests/test_entries.toml b/tests/test_entries.toml index 81e175f5..6c1f1d7f 100644 --- a/tests/test_entries.toml +++ b/tests/test_entries.toml @@ -2,19 +2,23 @@ # Note that these tests are intended for 512 byte inline sizes. They should # still pass with other inline sizes but wouldn't be testing anything. -define.LFS_CACHE_SIZE = 512 -if = 'LFS_CACHE_SIZE % LFS_PROG_SIZE == 0 && LFS_CACHE_SIZE == 512' +defines.CACHE_SIZE = 512 +if = 'CACHE_SIZE % PROG_SIZE == 0 && CACHE_SIZE == 512' -[[case]] # entry grow test +[cases.entry_grow] code = ''' uint8_t wbuffer[1024]; uint8_t rbuffer[1024]; - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // write hi0 20 + char path[1024]; + lfs_size_t size; sprintf(path, "hi0"); size = 20; + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; memset(wbuffer, 'c', size); @@ -94,16 +98,20 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # entry shrink test +[cases.entry_shrink] code = ''' uint8_t wbuffer[1024]; uint8_t rbuffer[1024]; - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // write hi0 20 + char path[1024]; + lfs_size_t size; sprintf(path, "hi0"); size = 20; + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; memset(wbuffer, 'c', size); @@ -183,16 +191,20 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # entry spill test +[cases.entry_spill] code = ''' uint8_t wbuffer[1024]; uint8_t rbuffer[1024]; - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // write hi0 200 + char path[1024]; + lfs_size_t size; sprintf(path, "hi0"); size = 200; + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; memset(wbuffer, 'c', size); @@ -256,16 +268,20 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # entry push spill test +[cases.entry_push_spill] code = ''' uint8_t wbuffer[1024]; uint8_t rbuffer[1024]; - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // write hi0 200 + char path[1024]; + lfs_size_t size; sprintf(path, "hi0"); size = 200; + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; memset(wbuffer, 'c', size); @@ -345,16 +361,20 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # entry push spill two test +[cases.entry_push_spill_two] code = ''' uint8_t wbuffer[1024]; uint8_t rbuffer[1024]; - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // write hi0 200 + char path[1024]; + lfs_size_t size; sprintf(path, "hi0"); size = 200; + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; memset(wbuffer, 'c', size); @@ -449,16 +469,20 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # entry drop test +[cases.entry_drop] code = ''' uint8_t wbuffer[1024]; uint8_t rbuffer[1024]; - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // write hi0 200 + char path[1024]; + lfs_size_t size; sprintf(path, "hi0"); size = 200; + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; memset(wbuffer, 'c', size); @@ -491,6 +515,7 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_remove(&lfs, "hi1") => 0; + struct lfs_info info; lfs_stat(&lfs, "hi1", &info) => LFS_ERR_NOENT; // read hi0 200 sprintf(path, "hi0"); size = 200; @@ -547,15 +572,18 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # create too big +[cases.create_too_big] code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + char path[1024]; memset(path, 'm', 200); path[200] = '\0'; - size = 400; + lfs_size_t size = 400; + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; uint8_t wbuffer[1024]; @@ -572,15 +600,18 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # resize too big +[cases.resize_too_big] code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + char path[1024]; memset(path, 'm', 200); path[200] = '\0'; - size = 40; + lfs_size_t size = 40; + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; uint8_t wbuffer[1024]; diff --git a/tests/test_evil.toml b/tests/test_evil.toml index 920d3a0e..78a5034d 100644 --- a/tests/test_evil.toml +++ b/tests/test_evil.toml @@ -3,16 +3,17 @@ # invalid pointer tests (outside of block_count) -[[case]] # invalid tail-pointer test -define.TAIL_TYPE = ['LFS_TYPE_HARDTAIL', 'LFS_TYPE_SOFTTAIL'] -define.INVALSET = [0x3, 0x1, 0x2] +[cases.invalid_tail_pointer] +defines.TAIL_TYPE = ['LFS_TYPE_HARDTAIL', 'LFS_TYPE_SOFTTAIL'] +defines.INVALSET = [0x3, 0x1, 0x2] in = "lfs.c" code = ''' // create littlefs - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; // change tail-pointer to invalid pointers - lfs_init(&lfs, &cfg) => 0; + lfs_init(&lfs, cfg) => 0; lfs_mdir_t mdir; lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0; lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS( @@ -23,25 +24,27 @@ code = ''' lfs_deinit(&lfs) => 0; // test that mount fails gracefully - lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT; + lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT; ''' -[[case]] # invalid dir pointer test -define.INVALSET = [0x3, 0x1, 0x2] +[cases.invalid_dir_pointer] +defines.INVALSET = [0x3, 0x1, 0x2] in = "lfs.c" code = ''' // create littlefs - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; // make a dir - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "dir_here") => 0; lfs_unmount(&lfs) => 0; // change the dir pointer to be invalid - lfs_init(&lfs, &cfg) => 0; + lfs_init(&lfs, cfg) => 0; lfs_mdir_t mdir; lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0; // make sure id 1 == our directory + uint8_t buffer[1024]; lfs_dir_get(&lfs, &mdir, LFS_MKTAG(0x700, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_NAME, 1, strlen("dir_here")), buffer) @@ -57,14 +60,17 @@ code = ''' // test that accessing our bad dir fails, note there's a number // of ways to access the dir, some can fail, but some don't - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + struct lfs_info info; lfs_stat(&lfs, "dir_here", &info) => 0; assert(strcmp(info.name, "dir_here") == 0); assert(info.type == LFS_TYPE_DIR); + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "dir_here") => LFS_ERR_CORRUPT; lfs_stat(&lfs, "dir_here/file_here", &info) => LFS_ERR_CORRUPT; lfs_dir_open(&lfs, &dir, "dir_here/dir_here") => LFS_ERR_CORRUPT; + lfs_file_t file; lfs_file_open(&lfs, &file, "dir_here/file_here", LFS_O_RDONLY) => LFS_ERR_CORRUPT; lfs_file_open(&lfs, &file, "dir_here/file_here", @@ -72,24 +78,27 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # invalid file pointer test +[cases.invalid_file_pointer] in = "lfs.c" -define.SIZE = [10, 1000, 100000] # faked file size +defines.SIZE = [10, 1000, 100000] # faked file size code = ''' // create littlefs - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; // make a file - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "file_here", LFS_O_WRONLY | LFS_O_CREAT) => 0; lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; // change the file pointer to be invalid - lfs_init(&lfs, &cfg) => 0; + lfs_init(&lfs, cfg) => 0; lfs_mdir_t mdir; lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0; // make sure id 1 == our file + uint8_t buffer[1024]; lfs_dir_get(&lfs, &mdir, LFS_MKTAG(0x700, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_NAME, 1, strlen("file_here")), buffer) @@ -103,7 +112,8 @@ code = ''' // test that accessing our bad file fails, note there's a number // of ways to access the dir, some can fail, but some don't - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + struct lfs_info info; lfs_stat(&lfs, "file_here", &info) => 0; assert(strcmp(info.name, "file_here") == 0); assert(info.type == LFS_TYPE_REG); @@ -114,20 +124,22 @@ code = ''' lfs_file_close(&lfs, &file) => 0; // any allocs that traverse CTZ must unfortunately must fail - if (SIZE > 2*LFS_BLOCK_SIZE) { + if (SIZE > 2*BLOCK_SIZE) { lfs_mkdir(&lfs, "dir_here") => LFS_ERR_CORRUPT; } lfs_unmount(&lfs) => 0; ''' -[[case]] # invalid pointer in CTZ skip-list test -define.SIZE = ['2*LFS_BLOCK_SIZE', '3*LFS_BLOCK_SIZE', '4*LFS_BLOCK_SIZE'] +[cases.invalid_ctz_pointer] # invalid pointer in CTZ skip-list test +defines.SIZE = ['2*BLOCK_SIZE', '3*BLOCK_SIZE', '4*BLOCK_SIZE'] in = "lfs.c" code = ''' // create littlefs - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; // make a file - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "file_here", LFS_O_WRONLY | LFS_O_CREAT) => 0; for (int i = 0; i < SIZE; i++) { @@ -137,10 +149,11 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; // change pointer in CTZ skip-list to be invalid - lfs_init(&lfs, &cfg) => 0; + lfs_init(&lfs, cfg) => 0; lfs_mdir_t mdir; lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0; // make sure id 1 == our file and get our CTZ structure + uint8_t buffer[4*BLOCK_SIZE]; lfs_dir_get(&lfs, &mdir, LFS_MKTAG(0x700, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_NAME, 1, strlen("file_here")), buffer) @@ -153,18 +166,19 @@ code = ''' => LFS_MKTAG(LFS_TYPE_CTZSTRUCT, 1, sizeof(struct lfs_ctz)); lfs_ctz_fromle32(&ctz); // rewrite block to contain bad pointer - uint8_t bbuffer[LFS_BLOCK_SIZE]; - cfg.read(&cfg, ctz.head, 0, bbuffer, LFS_BLOCK_SIZE) => 0; + uint8_t bbuffer[BLOCK_SIZE]; + cfg->read(cfg, ctz.head, 0, bbuffer, BLOCK_SIZE) => 0; uint32_t bad = lfs_tole32(0xcccccccc); memcpy(&bbuffer[0], &bad, sizeof(bad)); memcpy(&bbuffer[4], &bad, sizeof(bad)); - cfg.erase(&cfg, ctz.head) => 0; - cfg.prog(&cfg, ctz.head, 0, bbuffer, LFS_BLOCK_SIZE) => 0; + cfg->erase(cfg, ctz.head) => 0; + cfg->prog(cfg, ctz.head, 0, bbuffer, BLOCK_SIZE) => 0; lfs_deinit(&lfs) => 0; // test that accessing our bad file fails, note there's a number // of ways to access the dir, some can fail, but some don't - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + struct lfs_info info; lfs_stat(&lfs, "file_here", &info) => 0; assert(strcmp(info.name, "file_here") == 0); assert(info.type == LFS_TYPE_REG); @@ -175,22 +189,23 @@ code = ''' lfs_file_close(&lfs, &file) => 0; // any allocs that traverse CTZ must unfortunately must fail - if (SIZE > 2*LFS_BLOCK_SIZE) { + if (SIZE > 2*BLOCK_SIZE) { lfs_mkdir(&lfs, "dir_here") => LFS_ERR_CORRUPT; } lfs_unmount(&lfs) => 0; ''' -[[case]] # invalid gstate pointer -define.INVALSET = [0x3, 0x1, 0x2] +[cases.invalid_gstate_pointer] +defines.INVALSET = [0x3, 0x1, 0x2] in = "lfs.c" code = ''' // create littlefs - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; // create an invalid gstate - lfs_init(&lfs, &cfg) => 0; + lfs_init(&lfs, cfg) => 0; lfs_mdir_t mdir; lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0; lfs_fs_prepmove(&lfs, 1, (lfs_block_t [2]){ @@ -202,21 +217,22 @@ code = ''' // test that mount fails gracefully // mount may not fail, but our first alloc should fail when // we try to fix the gstate - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "should_fail") => LFS_ERR_CORRUPT; lfs_unmount(&lfs) => 0; ''' # cycle detection/recovery tests -[[case]] # metadata-pair threaded-list loop test +[cases.mdir_loop] # metadata-pair threaded-list loop test in = "lfs.c" code = ''' // create littlefs - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; // change tail-pointer to point to ourself - lfs_init(&lfs, &cfg) => 0; + lfs_init(&lfs, cfg) => 0; lfs_mdir_t mdir; lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0; lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS( @@ -225,20 +241,21 @@ code = ''' lfs_deinit(&lfs) => 0; // test that mount fails gracefully - lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT; + lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT; ''' -[[case]] # metadata-pair threaded-list 2-length loop test +[cases.mdir_loop_2] # metadata-pair threaded-list 2-length loop test in = "lfs.c" code = ''' // create littlefs with child dir - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "child") => 0; lfs_unmount(&lfs) => 0; // find child - lfs_init(&lfs, &cfg) => 0; + lfs_init(&lfs, cfg) => 0; lfs_mdir_t mdir; lfs_block_t pair[2]; lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0; @@ -255,20 +272,21 @@ code = ''' lfs_deinit(&lfs) => 0; // test that mount fails gracefully - lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT; + lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT; ''' -[[case]] # metadata-pair threaded-list 1-length child loop test +[cases.mdir_loop_child] # metadata-pair threaded-list 1-length child loop test in = "lfs.c" code = ''' // create littlefs with child dir - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "child") => 0; lfs_unmount(&lfs) => 0; // find child - lfs_init(&lfs, &cfg) => 0; + lfs_init(&lfs, cfg) => 0; lfs_mdir_t mdir; lfs_block_t pair[2]; lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0; @@ -284,5 +302,5 @@ code = ''' lfs_deinit(&lfs) => 0; // test that mount fails gracefully - lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT; + lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT; ''' diff --git a/tests/test_exhaustion.toml b/tests/test_exhaustion.toml index 569611c5..1914d628 100644 --- a/tests/test_exhaustion.toml +++ b/tests/test_exhaustion.toml @@ -1,30 +1,34 @@ -[[case]] # test running a filesystem to exhaustion -define.LFS_ERASE_CYCLES = 10 -define.LFS_BLOCK_COUNT = 256 # small bd so test runs faster -define.LFS_BLOCK_CYCLES = 'LFS_ERASE_CYCLES / 2' -define.LFS_BADBLOCK_BEHAVIOR = [ +# test running a filesystem to exhaustion +[cases.exhaustion] +defines.ERASE_CYCLES = 10 +defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' +defines.BADBLOCK_BEHAVIOR = [ 'LFS_TESTBD_BADBLOCK_PROGERROR', 'LFS_TESTBD_BADBLOCK_ERASEERROR', 'LFS_TESTBD_BADBLOCK_READERROR', 'LFS_TESTBD_BADBLOCK_PROGNOOP', 'LFS_TESTBD_BADBLOCK_ERASENOOP', ] -define.FILES = 10 +defines.FILES = 10 code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "roadrunner") => 0; lfs_unmount(&lfs) => 0; uint32_t cycle = 0; while (true) { - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (uint32_t i = 0; i < FILES; i++) { // chose name, roughly random seed, and random 2^n size + char path[1024]; sprintf(path, "roadrunner/test%d", i); srand(cycle * i); - size = 1 << ((rand() % 10)+2); + lfs_size_t size = 1 << ((rand() % 10)+2); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; @@ -33,14 +37,14 @@ code = ''' lfs_ssize_t res = lfs_file_write(&lfs, &file, &c, 1); assert(res == 1 || res == LFS_ERR_NOSPC); if (res == LFS_ERR_NOSPC) { - err = lfs_file_close(&lfs, &file); + int err = lfs_file_close(&lfs, &file); assert(err == 0 || err == LFS_ERR_NOSPC); lfs_unmount(&lfs) => 0; goto exhausted; } } - err = lfs_file_close(&lfs, &file); + int err = lfs_file_close(&lfs, &file); assert(err == 0 || err == LFS_ERR_NOSPC); if (err == LFS_ERR_NOSPC) { lfs_unmount(&lfs) => 0; @@ -50,10 +54,12 @@ code = ''' for (uint32_t i = 0; i < FILES; i++) { // check for errors + char path[1024]; sprintf(path, "roadrunner/test%d", i); srand(cycle * i); - size = 1 << ((rand() % 10)+2); + lfs_size_t size = 1 << ((rand() % 10)+2); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_RDONLY) => 0; for (lfs_size_t j = 0; j < size; j++) { char c = 'a' + (rand() % 26); @@ -71,10 +77,12 @@ code = ''' exhausted: // should still be readable - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (uint32_t i = 0; i < FILES; i++) { // check for errors + char path[1024]; sprintf(path, "roadrunner/test%d", i); + struct lfs_info info; lfs_stat(&lfs, path, &info) => 0; } lfs_unmount(&lfs) => 0; @@ -82,31 +90,35 @@ exhausted: LFS_WARN("completed %d cycles", cycle); ''' -[[case]] # test running a filesystem to exhaustion - # which also requires expanding superblocks -define.LFS_ERASE_CYCLES = 10 -define.LFS_BLOCK_COUNT = 256 # small bd so test runs faster -define.LFS_BLOCK_CYCLES = 'LFS_ERASE_CYCLES / 2' -define.LFS_BADBLOCK_BEHAVIOR = [ +# test running a filesystem to exhaustion +# which also requires expanding superblocks +[cases.exhaustion_superblocks] +defines.ERASE_CYCLES = 10 +defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' +defines.BADBLOCK_BEHAVIOR = [ 'LFS_TESTBD_BADBLOCK_PROGERROR', 'LFS_TESTBD_BADBLOCK_ERASEERROR', 'LFS_TESTBD_BADBLOCK_READERROR', 'LFS_TESTBD_BADBLOCK_PROGNOOP', 'LFS_TESTBD_BADBLOCK_ERASENOOP', ] -define.FILES = 10 +defines.FILES = 10 code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; uint32_t cycle = 0; while (true) { - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (uint32_t i = 0; i < FILES; i++) { // chose name, roughly random seed, and random 2^n size + char path[1024]; sprintf(path, "test%d", i); srand(cycle * i); - size = 1 << ((rand() % 10)+2); + lfs_size_t size = 1 << ((rand() % 10)+2); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; @@ -115,14 +127,14 @@ code = ''' lfs_ssize_t res = lfs_file_write(&lfs, &file, &c, 1); assert(res == 1 || res == LFS_ERR_NOSPC); if (res == LFS_ERR_NOSPC) { - err = lfs_file_close(&lfs, &file); + int err = lfs_file_close(&lfs, &file); assert(err == 0 || err == LFS_ERR_NOSPC); lfs_unmount(&lfs) => 0; goto exhausted; } } - err = lfs_file_close(&lfs, &file); + int err = lfs_file_close(&lfs, &file); assert(err == 0 || err == LFS_ERR_NOSPC); if (err == LFS_ERR_NOSPC) { lfs_unmount(&lfs) => 0; @@ -132,10 +144,12 @@ code = ''' for (uint32_t i = 0; i < FILES; i++) { // check for errors + char path[1024]; sprintf(path, "test%d", i); srand(cycle * i); - size = 1 << ((rand() % 10)+2); + lfs_size_t size = 1 << ((rand() % 10)+2); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_RDONLY) => 0; for (lfs_size_t j = 0; j < size; j++) { char c = 'a' + (rand() % 26); @@ -153,9 +167,11 @@ code = ''' exhausted: // should still be readable - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (uint32_t i = 0; i < FILES; i++) { // check for errors + char path[1024]; + struct lfs_info info; sprintf(path, "test%d", i); lfs_stat(&lfs, path, &info) => 0; } @@ -169,35 +185,39 @@ exhausted: # into increasing the block devices lifetime. This is something we can actually # check for. -[[case]] # wear-level test running a filesystem to exhaustion -define.LFS_ERASE_CYCLES = 20 -define.LFS_BLOCK_COUNT = 256 # small bd so test runs faster -define.LFS_BLOCK_CYCLES = 'LFS_ERASE_CYCLES / 2' -define.FILES = 10 +# wear-level test running a filesystem to exhaustion +[cases.wear_leveling_exhaustion] +defines.ERASE_CYCLES = 20 +defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' +defines.FILES = 10 code = ''' uint32_t run_cycles[2]; - const uint32_t run_block_count[2] = {LFS_BLOCK_COUNT/2, LFS_BLOCK_COUNT}; + const uint32_t run_block_count[2] = {BLOCK_COUNT/2, BLOCK_COUNT}; for (int run = 0; run < 2; run++) { - for (lfs_block_t b = 0; b < LFS_BLOCK_COUNT; b++) { - lfs_testbd_setwear(&cfg, b, - (b < run_block_count[run]) ? 0 : LFS_ERASE_CYCLES) => 0; + for (lfs_block_t b = 0; b < BLOCK_COUNT; b++) { + lfs_testbd_setwear(cfg, b, + (b < run_block_count[run]) ? 0 : ERASE_CYCLES) => 0; } - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "roadrunner") => 0; lfs_unmount(&lfs) => 0; uint32_t cycle = 0; while (true) { - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (uint32_t i = 0; i < FILES; i++) { // chose name, roughly random seed, and random 2^n size + char path[1024]; sprintf(path, "roadrunner/test%d", i); srand(cycle * i); - size = 1 << ((rand() % 10)+2); + lfs_size_t size = 1 << ((rand() % 10)+2); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; @@ -206,14 +226,14 @@ code = ''' lfs_ssize_t res = lfs_file_write(&lfs, &file, &c, 1); assert(res == 1 || res == LFS_ERR_NOSPC); if (res == LFS_ERR_NOSPC) { - err = lfs_file_close(&lfs, &file); + int err = lfs_file_close(&lfs, &file); assert(err == 0 || err == LFS_ERR_NOSPC); lfs_unmount(&lfs) => 0; goto exhausted; } } - err = lfs_file_close(&lfs, &file); + int err = lfs_file_close(&lfs, &file); assert(err == 0 || err == LFS_ERR_NOSPC); if (err == LFS_ERR_NOSPC) { lfs_unmount(&lfs) => 0; @@ -223,10 +243,12 @@ code = ''' for (uint32_t i = 0; i < FILES; i++) { // check for errors + char path[1024]; sprintf(path, "roadrunner/test%d", i); srand(cycle * i); - size = 1 << ((rand() % 10)+2); + lfs_size_t size = 1 << ((rand() % 10)+2); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_RDONLY) => 0; for (lfs_size_t j = 0; j < size; j++) { char c = 'a' + (rand() % 26); @@ -244,9 +266,11 @@ code = ''' exhausted: // should still be readable - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (uint32_t i = 0; i < FILES; i++) { // check for errors + char path[1024]; + struct lfs_info info; sprintf(path, "roadrunner/test%d", i); lfs_stat(&lfs, path, &info) => 0; } @@ -261,32 +285,36 @@ exhausted: LFS_ASSERT(run_cycles[1]*110/100 > 2*run_cycles[0]); ''' -[[case]] # wear-level test + expanding superblock -define.LFS_ERASE_CYCLES = 20 -define.LFS_BLOCK_COUNT = 256 # small bd so test runs faster -define.LFS_BLOCK_CYCLES = 'LFS_ERASE_CYCLES / 2' -define.FILES = 10 +# wear-level test + expanding superblock +[cases.wear_leveling_exhaustion_superblocks] +defines.ERASE_CYCLES = 20 +defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' +defines.FILES = 10 code = ''' uint32_t run_cycles[2]; - const uint32_t run_block_count[2] = {LFS_BLOCK_COUNT/2, LFS_BLOCK_COUNT}; + const uint32_t run_block_count[2] = {BLOCK_COUNT/2, BLOCK_COUNT}; for (int run = 0; run < 2; run++) { - for (lfs_block_t b = 0; b < LFS_BLOCK_COUNT; b++) { - lfs_testbd_setwear(&cfg, b, - (b < run_block_count[run]) ? 0 : LFS_ERASE_CYCLES) => 0; + for (lfs_block_t b = 0; b < BLOCK_COUNT; b++) { + lfs_testbd_setwear(cfg, b, + (b < run_block_count[run]) ? 0 : ERASE_CYCLES) => 0; } - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; uint32_t cycle = 0; while (true) { - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (uint32_t i = 0; i < FILES; i++) { // chose name, roughly random seed, and random 2^n size + char path[1024]; sprintf(path, "test%d", i); srand(cycle * i); - size = 1 << ((rand() % 10)+2); + lfs_size_t size = 1 << ((rand() % 10)+2); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; @@ -295,14 +323,14 @@ code = ''' lfs_ssize_t res = lfs_file_write(&lfs, &file, &c, 1); assert(res == 1 || res == LFS_ERR_NOSPC); if (res == LFS_ERR_NOSPC) { - err = lfs_file_close(&lfs, &file); + int err = lfs_file_close(&lfs, &file); assert(err == 0 || err == LFS_ERR_NOSPC); lfs_unmount(&lfs) => 0; goto exhausted; } } - err = lfs_file_close(&lfs, &file); + int err = lfs_file_close(&lfs, &file); assert(err == 0 || err == LFS_ERR_NOSPC); if (err == LFS_ERR_NOSPC) { lfs_unmount(&lfs) => 0; @@ -312,10 +340,12 @@ code = ''' for (uint32_t i = 0; i < FILES; i++) { // check for errors + char path[1024]; sprintf(path, "test%d", i); srand(cycle * i); - size = 1 << ((rand() % 10)+2); + lfs_size_t size = 1 << ((rand() % 10)+2); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_RDONLY) => 0; for (lfs_size_t j = 0; j < size; j++) { char c = 'a' + (rand() % 26); @@ -333,9 +363,11 @@ code = ''' exhausted: // should still be readable - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (uint32_t i = 0; i < FILES; i++) { // check for errors + char path[1024]; + struct lfs_info info; sprintf(path, "test%d", i); lfs_stat(&lfs, path, &info) => 0; } @@ -350,28 +382,32 @@ exhausted: LFS_ASSERT(run_cycles[1]*110/100 > 2*run_cycles[0]); ''' -[[case]] # test that we wear blocks roughly evenly -define.LFS_ERASE_CYCLES = 0xffffffff -define.LFS_BLOCK_COUNT = 256 # small bd so test runs faster -define.LFS_BLOCK_CYCLES = [5, 4, 3, 2, 1] -define.CYCLES = 100 -define.FILES = 10 -if = 'LFS_BLOCK_CYCLES < CYCLES/10' +# test that we wear blocks roughly evenly +[cases.wear_leveling_distribution] +defines.ERASE_CYCLES = 0xffffffff +defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.BLOCK_CYCLES = [5, 4, 3, 2, 1] +defines.CYCLES = 100 +defines.FILES = 10 +if = 'BLOCK_CYCLES < CYCLES/10' code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "roadrunner") => 0; lfs_unmount(&lfs) => 0; uint32_t cycle = 0; while (cycle < CYCLES) { - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (uint32_t i = 0; i < FILES; i++) { // chose name, roughly random seed, and random 2^n size + char path[1024]; sprintf(path, "roadrunner/test%d", i); srand(cycle * i); - size = 1 << 4; //((rand() % 10)+2); + lfs_size_t size = 1 << 4; //((rand() % 10)+2); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; @@ -380,14 +416,14 @@ code = ''' lfs_ssize_t res = lfs_file_write(&lfs, &file, &c, 1); assert(res == 1 || res == LFS_ERR_NOSPC); if (res == LFS_ERR_NOSPC) { - err = lfs_file_close(&lfs, &file); + int err = lfs_file_close(&lfs, &file); assert(err == 0 || err == LFS_ERR_NOSPC); lfs_unmount(&lfs) => 0; goto exhausted; } } - err = lfs_file_close(&lfs, &file); + int err = lfs_file_close(&lfs, &file); assert(err == 0 || err == LFS_ERR_NOSPC); if (err == LFS_ERR_NOSPC) { lfs_unmount(&lfs) => 0; @@ -397,10 +433,12 @@ code = ''' for (uint32_t i = 0; i < FILES; i++) { // check for errors + char path[1024]; sprintf(path, "roadrunner/test%d", i); srand(cycle * i); - size = 1 << 4; //((rand() % 10)+2); + lfs_size_t size = 1 << 4; //((rand() % 10)+2); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_RDONLY) => 0; for (lfs_size_t j = 0; j < size; j++) { char c = 'a' + (rand() % 26); @@ -418,9 +456,11 @@ code = ''' exhausted: // should still be readable - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (uint32_t i = 0; i < FILES; i++) { // check for errors + char path[1024]; + struct lfs_info info; sprintf(path, "roadrunner/test%d", i); lfs_stat(&lfs, path, &info) => 0; } @@ -433,8 +473,8 @@ exhausted: lfs_testbd_wear_t totalwear = 0; lfs_testbd_wear_t maxwear = 0; // skip 0 and 1 as superblock movement is intentionally avoided - for (lfs_block_t b = 2; b < LFS_BLOCK_COUNT; b++) { - lfs_testbd_wear_t wear = lfs_testbd_getwear(&cfg, b); + for (lfs_block_t b = 2; b < BLOCK_COUNT; b++) { + lfs_testbd_wear_t wear = lfs_testbd_getwear(cfg, b); printf("%08x: wear %d\n", b, wear); assert(wear >= 0); if (wear < minwear) { @@ -445,15 +485,15 @@ exhausted: } totalwear += wear; } - lfs_testbd_wear_t avgwear = totalwear / LFS_BLOCK_COUNT; + lfs_testbd_wear_t avgwear = totalwear / BLOCK_COUNT; LFS_WARN("max wear: %d cycles", maxwear); - LFS_WARN("avg wear: %d cycles", totalwear / LFS_BLOCK_COUNT); + LFS_WARN("avg wear: %d cycles", totalwear / (int)BLOCK_COUNT); LFS_WARN("min wear: %d cycles", minwear); // find standard deviation^2 lfs_testbd_wear_t dev2 = 0; - for (lfs_block_t b = 2; b < LFS_BLOCK_COUNT; b++) { - lfs_testbd_wear_t wear = lfs_testbd_getwear(&cfg, b); + for (lfs_block_t b = 2; b < BLOCK_COUNT; b++) { + lfs_testbd_wear_t wear = lfs_testbd_getwear(cfg, b); assert(wear >= 0); lfs_testbd_swear_t diff = wear - avgwear; dev2 += diff*diff; diff --git a/tests/test_files.toml b/tests/test_files.toml index 54630546..026e47ff 100644 --- a/tests/test_files.toml +++ b/tests/test_files.toml @@ -1,17 +1,20 @@ -[[case]] # simple file test +[cases.simple_file] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "hello", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; - size = strlen("Hello World!")+1; + lfs_size_t size = strlen("Hello World!")+1; + uint8_t buffer[1024]; strcpy((char*)buffer, "Hello World!"); lfs_file_write(&lfs, &file, buffer, size) => size; lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "hello", LFS_O_RDONLY) => 0; lfs_file_read(&lfs, &file, buffer, size) => size; assert(strcmp((char*)buffer, "Hello World!") == 0); @@ -19,17 +22,20 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # larger files -define.SIZE = [32, 8192, 262144, 0, 7, 8193] -define.CHUNKSIZE = [31, 16, 33, 1, 1023] +[cases.large_files] +defines.SIZE = [32, 8192, 262144, 0, 7, 8193] +defines.CHUNKSIZE = [31, 16, 33, 1, 1023] code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; // write - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "avacado", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; srand(1); + uint8_t buffer[1024]; for (lfs_size_t i = 0; i < SIZE; i += CHUNKSIZE) { lfs_size_t chunk = lfs_min(CHUNKSIZE, SIZE-i); for (lfs_size_t b = 0; b < chunk; b++) { @@ -41,7 +47,7 @@ code = ''' lfs_unmount(&lfs) => 0; // read - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "avacado", LFS_O_RDONLY) => 0; lfs_file_size(&lfs, &file) => SIZE; srand(1); @@ -57,15 +63,18 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # rewriting files -define.SIZE1 = [32, 8192, 131072, 0, 7, 8193] -define.SIZE2 = [32, 8192, 131072, 0, 7, 8193] -define.CHUNKSIZE = [31, 16, 1] +[cases.rewriting_files] +defines.SIZE1 = [32, 8192, 131072, 0, 7, 8193] +defines.SIZE2 = [32, 8192, 131072, 0, 7, 8193] +defines.CHUNKSIZE = [31, 16, 1] code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; // write - lfs_mount(&lfs, &cfg) => 1; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; + uint8_t buffer[1024]; lfs_file_open(&lfs, &file, "avacado", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; srand(1); @@ -80,7 +89,7 @@ code = ''' lfs_unmount(&lfs) => 0; // read - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "avacado", LFS_O_RDONLY) => 0; lfs_file_size(&lfs, &file) => SIZE1; srand(1); @@ -96,7 +105,7 @@ code = ''' lfs_unmount(&lfs) => 0; // rewrite - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "avacado", LFS_O_WRONLY) => 0; srand(2); for (lfs_size_t i = 0; i < SIZE2; i += CHUNKSIZE) { @@ -110,7 +119,7 @@ code = ''' lfs_unmount(&lfs) => 0; // read - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "avacado", LFS_O_RDONLY) => 0; lfs_file_size(&lfs, &file) => lfs_max(SIZE1, SIZE2); srand(2); @@ -139,15 +148,18 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # appending files -define.SIZE1 = [32, 8192, 131072, 0, 7, 8193] -define.SIZE2 = [32, 8192, 131072, 0, 7, 8193] -define.CHUNKSIZE = [31, 16, 1] +[cases.appending_files] +defines.SIZE1 = [32, 8192, 131072, 0, 7, 8193] +defines.SIZE2 = [32, 8192, 131072, 0, 7, 8193] +defines.CHUNKSIZE = [31, 16, 1] code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; // write - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; + uint8_t buffer[1024]; lfs_file_open(&lfs, &file, "avacado", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; srand(1); @@ -162,7 +174,7 @@ code = ''' lfs_unmount(&lfs) => 0; // read - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "avacado", LFS_O_RDONLY) => 0; lfs_file_size(&lfs, &file) => SIZE1; srand(1); @@ -178,7 +190,7 @@ code = ''' lfs_unmount(&lfs) => 0; // append - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "avacado", LFS_O_WRONLY | LFS_O_APPEND) => 0; srand(2); for (lfs_size_t i = 0; i < SIZE2; i += CHUNKSIZE) { @@ -192,7 +204,7 @@ code = ''' lfs_unmount(&lfs) => 0; // read - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "avacado", LFS_O_RDONLY) => 0; lfs_file_size(&lfs, &file) => SIZE1 + SIZE2; srand(1); @@ -216,15 +228,18 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # truncating files -define.SIZE1 = [32, 8192, 131072, 0, 7, 8193] -define.SIZE2 = [32, 8192, 131072, 0, 7, 8193] -define.CHUNKSIZE = [31, 16, 1] +[cases.truncating_files] +defines.SIZE1 = [32, 8192, 131072, 0, 7, 8193] +defines.SIZE2 = [32, 8192, 131072, 0, 7, 8193] +defines.CHUNKSIZE = [31, 16, 1] code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; // write - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; + uint8_t buffer[1024]; lfs_file_open(&lfs, &file, "avacado", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; srand(1); @@ -239,7 +254,7 @@ code = ''' lfs_unmount(&lfs) => 0; // read - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "avacado", LFS_O_RDONLY) => 0; lfs_file_size(&lfs, &file) => SIZE1; srand(1); @@ -255,7 +270,7 @@ code = ''' lfs_unmount(&lfs) => 0; // truncate - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "avacado", LFS_O_WRONLY | LFS_O_TRUNC) => 0; srand(2); for (lfs_size_t i = 0; i < SIZE2; i += CHUNKSIZE) { @@ -269,7 +284,7 @@ code = ''' lfs_unmount(&lfs) => 0; // read - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "avacado", LFS_O_RDONLY) => 0; lfs_file_size(&lfs, &file) => SIZE2; srand(2); @@ -285,22 +300,25 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # reentrant file writing -define.SIZE = [32, 0, 7, 2049] -define.CHUNKSIZE = [31, 16, 65] +[cases.reentrant_file_writing] +defines.SIZE = [32, 0, 7, 2049] +defines.CHUNKSIZE = [31, 16, 65] reentrant = true code = ''' - err = lfs_mount(&lfs, &cfg); + lfs_t lfs; + int err = lfs_mount(&lfs, cfg); if (err) { - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; } + lfs_file_t file; + uint8_t buffer[1024]; err = lfs_file_open(&lfs, &file, "avacado", LFS_O_RDONLY); assert(err == LFS_ERR_NOENT || err == 0); if (err == 0) { // can only be 0 (new file) or full size - size = lfs_file_size(&lfs, &file); + lfs_size_t size = lfs_file_size(&lfs, &file); assert(size == 0 || size == SIZE); lfs_file_close(&lfs, &file) => 0; } @@ -333,8 +351,8 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # reentrant file writing with syncs -define = [ +[cases.reentrant_file_writing_sync] +defines = [ # append (O(n)) {MODE='LFS_O_APPEND', SIZE=[32, 0, 7, 2049], CHUNKSIZE=[31, 16, 65]}, # truncate (O(n^2)) @@ -344,17 +362,20 @@ define = [ ] reentrant = true code = ''' - err = lfs_mount(&lfs, &cfg); + lfs_t lfs; + int err = lfs_mount(&lfs, cfg); if (err) { - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; } + lfs_file_t file; + uint8_t buffer[1024]; err = lfs_file_open(&lfs, &file, "avacado", LFS_O_RDONLY); assert(err == LFS_ERR_NOENT || err == 0); if (err == 0) { // with syncs we could be any size, but it at least must be valid data - size = lfs_file_size(&lfs, &file); + lfs_size_t size = lfs_file_size(&lfs, &file); assert(size <= SIZE); srand(1); for (lfs_size_t i = 0; i < size; i += CHUNKSIZE) { @@ -370,7 +391,7 @@ code = ''' // write lfs_file_open(&lfs, &file, "avacado", LFS_O_WRONLY | LFS_O_CREAT | MODE) => 0; - size = lfs_file_size(&lfs, &file); + lfs_size_t size = lfs_file_size(&lfs, &file); assert(size <= SIZE); srand(1); lfs_size_t skip = (MODE == LFS_O_APPEND) ? size : 0; @@ -403,19 +424,22 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # many files -define.N = 300 +[cases.many_files] +defines.N = 300 code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; // create N files of 7 bytes - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 0; i < N; i++) { + lfs_file_t file; + char path[1024]; sprintf(path, "file_%03d", i); lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; char wbuffer[1024]; - size = 7; - snprintf(wbuffer, size, "Hi %03d", i); + lfs_size_t size = 7; + sprintf(wbuffer, "Hi %03d", i); lfs_file_write(&lfs, &file, wbuffer, size) => size; lfs_file_close(&lfs, &file) => 0; @@ -428,25 +452,28 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # many files with power cycle -define.N = 300 +[cases.many_files_power_cycle] +defines.N = 300 code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; // create N files of 7 bytes - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 0; i < N; i++) { + lfs_file_t file; + char path[1024]; sprintf(path, "file_%03d", i); lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; char wbuffer[1024]; - size = 7; - snprintf(wbuffer, size, "Hi %03d", i); + lfs_size_t size = 7; + sprintf(wbuffer, "Hi %03d", i); lfs_file_write(&lfs, &file, wbuffer, size) => size; lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; char rbuffer[1024]; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, path, LFS_O_RDONLY) => 0; lfs_file_read(&lfs, &file, rbuffer, size) => size; assert(strcmp(rbuffer, wbuffer) == 0); @@ -455,22 +482,25 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # many files with power loss -define.N = 300 +[cases.many_files_power_loss] +defines.N = 300 reentrant = true code = ''' - err = lfs_mount(&lfs, &cfg); + lfs_t lfs; + int err = lfs_mount(&lfs, cfg); if (err) { - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; } // create N files of 7 bytes for (int i = 0; i < N; i++) { + lfs_file_t file; + char path[1024]; sprintf(path, "file_%03d", i); err = lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT); char wbuffer[1024]; - size = 7; - snprintf(wbuffer, size, "Hi %03d", i); + lfs_size_t size = 7; + sprintf(wbuffer, "Hi %03d", i); if ((lfs_size_t)lfs_file_size(&lfs, &file) != size) { lfs_file_write(&lfs, &file, wbuffer, size) => size; } diff --git a/tests/test_interspersed.toml b/tests/test_interspersed.toml index 87a05780..92d96d83 100644 --- a/tests/test_interspersed.toml +++ b/tests/test_interspersed.toml @@ -1,13 +1,15 @@ -[[case]] # interspersed file test -define.SIZE = [10, 100] -define.FILES = [4, 10, 26] +[cases.interspersed_files] +defines.SIZE = [10, 100] +defines.FILES = [4, 10, 26] code = ''' + lfs_t lfs; lfs_file_t files[FILES]; const char alphas[] = "abcdefghijklmnopqrstuvwxyz"; - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int j = 0; j < FILES; j++) { + char path[1024]; sprintf(path, "%c", alphas[j]); lfs_file_open(&lfs, &files[j], path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; @@ -23,7 +25,9 @@ code = ''' lfs_file_close(&lfs, &files[j]); } + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); assert(info.type == LFS_TYPE_DIR); @@ -31,6 +35,7 @@ code = ''' assert(strcmp(info.name, "..") == 0); assert(info.type == LFS_TYPE_DIR); for (int j = 0; j < FILES; j++) { + char path[1024]; sprintf(path, "%c", alphas[j]); lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, path) == 0); @@ -41,12 +46,14 @@ code = ''' lfs_dir_close(&lfs, &dir) => 0; for (int j = 0; j < FILES; j++) { + char path[1024]; sprintf(path, "%c", alphas[j]); lfs_file_open(&lfs, &files[j], path, LFS_O_RDONLY) => 0; } for (int i = 0; i < 10; i++) { for (int j = 0; j < FILES; j++) { + uint8_t buffer[1024]; lfs_file_read(&lfs, &files[j], buffer, 1) => 1; assert(buffer[0] == alphas[j]); } @@ -59,15 +66,18 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # interspersed remove file test -define.SIZE = [10, 100] -define.FILES = [4, 10, 26] +[cases.interspersed_remove_files] +defines.SIZE = [10, 100] +defines.FILES = [4, 10, 26] code = ''' + lfs_t lfs; const char alphas[] = "abcdefghijklmnopqrstuvwxyz"; - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int j = 0; j < FILES; j++) { + char path[1024]; sprintf(path, "%c", alphas[j]); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; for (int i = 0; i < SIZE; i++) { @@ -77,18 +87,22 @@ code = ''' } lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "zzz", LFS_O_WRONLY | LFS_O_CREAT) => 0; for (int j = 0; j < FILES; j++) { lfs_file_write(&lfs, &file, (const void*)"~", 1) => 1; lfs_file_sync(&lfs, &file) => 0; + char path[1024]; sprintf(path, "%c", alphas[j]); lfs_remove(&lfs, path) => 0; } lfs_file_close(&lfs, &file); + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); assert(info.type == LFS_TYPE_DIR); @@ -104,6 +118,7 @@ code = ''' lfs_file_open(&lfs, &file, "zzz", LFS_O_RDONLY) => 0; for (int i = 0; i < FILES; i++) { + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, 1) => 1; assert(buffer[0] == '~'); } @@ -112,11 +127,12 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # remove inconveniently test -define.SIZE = [10, 100] +[cases.remove_inconveniently] +defines.SIZE = [10, 100] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_t files[3]; lfs_file_open(&lfs, &files[0], "e", LFS_O_WRONLY | LFS_O_CREAT) => 0; lfs_file_open(&lfs, &files[1], "f", LFS_O_WRONLY | LFS_O_CREAT) => 0; @@ -140,7 +156,9 @@ code = ''' lfs_file_close(&lfs, &files[1]); lfs_file_close(&lfs, &files[2]); + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); assert(info.type == LFS_TYPE_DIR); @@ -161,6 +179,7 @@ code = ''' lfs_file_open(&lfs, &files[0], "e", LFS_O_RDONLY) => 0; lfs_file_open(&lfs, &files[1], "g", LFS_O_RDONLY) => 0; for (int i = 0; i < SIZE; i++) { + uint8_t buffer[1024]; lfs_file_read(&lfs, &files[0], buffer, 1) => 1; assert(buffer[0] == 'e'); lfs_file_read(&lfs, &files[1], buffer, 1) => 1; @@ -172,21 +191,23 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # reentrant interspersed file test -define.SIZE = [10, 100] -define.FILES = [4, 10, 26] +[cases.reentrant_interspersed_files] +defines.SIZE = [10, 100] +defines.FILES = [4, 10, 26] reentrant = true code = ''' + lfs_t lfs; lfs_file_t files[FILES]; const char alphas[] = "abcdefghijklmnopqrstuvwxyz"; - err = lfs_mount(&lfs, &cfg); + int err = lfs_mount(&lfs, cfg); if (err) { - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; } for (int j = 0; j < FILES; j++) { + char path[1024]; sprintf(path, "%c", alphas[j]); lfs_file_open(&lfs, &files[j], path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0; @@ -194,8 +215,8 @@ code = ''' for (int i = 0; i < SIZE; i++) { for (int j = 0; j < FILES; j++) { - size = lfs_file_size(&lfs, &files[j]); - assert((int)size >= 0); + lfs_ssize_t size = lfs_file_size(&lfs, &files[j]); + assert(size >= 0); if ((int)size <= i) { lfs_file_write(&lfs, &files[j], &alphas[j], 1) => 1; lfs_file_sync(&lfs, &files[j]) => 0; @@ -207,7 +228,9 @@ code = ''' lfs_file_close(&lfs, &files[j]); } + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); assert(info.type == LFS_TYPE_DIR); @@ -215,6 +238,7 @@ code = ''' assert(strcmp(info.name, "..") == 0); assert(info.type == LFS_TYPE_DIR); for (int j = 0; j < FILES; j++) { + char path[1024]; sprintf(path, "%c", alphas[j]); lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, path) == 0); @@ -225,12 +249,14 @@ code = ''' lfs_dir_close(&lfs, &dir) => 0; for (int j = 0; j < FILES; j++) { + char path[1024]; sprintf(path, "%c", alphas[j]); lfs_file_open(&lfs, &files[j], path, LFS_O_RDONLY) => 0; } for (int i = 0; i < 10; i++) { for (int j = 0; j < FILES; j++) { + uint8_t buffer[1024]; lfs_file_read(&lfs, &files[j], buffer, 1) => 1; assert(buffer[0] == alphas[j]); } diff --git a/tests/test_move.toml b/tests/test_move.toml index bb3b713f..f1825e48 100644 --- a/tests/test_move.toml +++ b/tests/test_move.toml @@ -1,11 +1,13 @@ -[[case]] # move file +[cases.move_file] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "a") => 0; lfs_mkdir(&lfs, "b") => 0; lfs_mkdir(&lfs, "c") => 0; lfs_mkdir(&lfs, "d") => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "a/hello", LFS_O_CREAT | LFS_O_WRONLY) => 0; lfs_file_write(&lfs, &file, "hola\n", 5) => 5; lfs_file_write(&lfs, &file, "bonjour\n", 8) => 8; @@ -13,11 +15,13 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_rename(&lfs, "a/hello", "c/hello") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -44,6 +48,7 @@ code = ''' lfs_file_open(&lfs, &file, "a/hello", LFS_O_RDONLY) => LFS_ERR_NOENT; lfs_file_open(&lfs, &file, "b/hello", LFS_O_RDONLY) => LFS_ERR_NOENT; lfs_file_open(&lfs, &file, "c/hello", LFS_O_RDONLY) => 0; + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, 5) => 5; memcmp(buffer, "hola\n", 5) => 0; lfs_file_read(&lfs, &file, buffer, 8) => 8; @@ -55,31 +60,35 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # noop move, yes this is legal +[cases.nop_move] # yes this is legal code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "hi") => 0; lfs_rename(&lfs, "hi", "hi") => 0; lfs_mkdir(&lfs, "hi/hi") => 0; lfs_rename(&lfs, "hi/hi", "hi/hi") => 0; lfs_mkdir(&lfs, "hi/hi/hi") => 0; lfs_rename(&lfs, "hi/hi/hi", "hi/hi/hi") => 0; + struct lfs_info info; lfs_stat(&lfs, "hi/hi/hi", &info) => 0; assert(strcmp(info.name, "hi") == 0); assert(info.type == LFS_TYPE_DIR); lfs_unmount(&lfs) => 0; ''' -[[case]] # move file corrupt source +[cases.move_file_corrupt_source] in = "lfs.c" code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "a") => 0; lfs_mkdir(&lfs, "b") => 0; lfs_mkdir(&lfs, "c") => 0; lfs_mkdir(&lfs, "d") => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "a/hello", LFS_O_CREAT | LFS_O_WRONLY) => 0; lfs_file_write(&lfs, &file, "hola\n", 5) => 5; lfs_file_write(&lfs, &file, "bonjour\n", 8) => 8; @@ -87,28 +96,30 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_rename(&lfs, "a/hello", "c/hello") => 0; lfs_unmount(&lfs) => 0; // corrupt the source - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_block_t block = dir.m.pair[0]; lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs) => 0; - uint8_t bbuffer[LFS_BLOCK_SIZE]; - cfg.read(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - int off = LFS_BLOCK_SIZE-1; - while (off >= 0 && bbuffer[off] == LFS_ERASE_VALUE) { + uint8_t buffer[BLOCK_SIZE]; + cfg->read(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + int off = BLOCK_SIZE-1; + while (off >= 0 && buffer[off] == ERASE_VALUE) { off -= 1; } - memset(&bbuffer[off-3], LFS_BLOCK_SIZE, 3); - cfg.erase(&cfg, block) => 0; - cfg.prog(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - cfg.sync(&cfg) => 0; + memset(&buffer[off-3], BLOCK_SIZE, 3); + cfg->erase(cfg, block) => 0; + cfg->prog(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + cfg->sync(cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -146,16 +157,19 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # move file corrupt source and dest +# move file corrupt source and dest +[cases.move_file_corrupt_source_dest] in = "lfs.c" -if = 'LFS_PROG_SIZE <= 0x3fe' # only works with one crc per commit +if = 'PROG_SIZE <= 0x3fe' # only works with one crc per commit code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "a") => 0; lfs_mkdir(&lfs, "b") => 0; lfs_mkdir(&lfs, "c") => 0; lfs_mkdir(&lfs, "d") => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "a/hello", LFS_O_CREAT | LFS_O_WRONLY) => 0; lfs_file_write(&lfs, &file, "hola\n", 5) => 5; lfs_file_write(&lfs, &file, "bonjour\n", 8) => 8; @@ -163,44 +177,46 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_rename(&lfs, "a/hello", "c/hello") => 0; lfs_unmount(&lfs) => 0; // corrupt the source - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_block_t block = dir.m.pair[0]; lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs) => 0; - uint8_t bbuffer[LFS_BLOCK_SIZE]; - cfg.read(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - int off = LFS_BLOCK_SIZE-1; - while (off >= 0 && bbuffer[off] == LFS_ERASE_VALUE) { + uint8_t buffer[BLOCK_SIZE]; + cfg->read(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + int off = BLOCK_SIZE-1; + while (off >= 0 && buffer[off] == ERASE_VALUE) { off -= 1; } - memset(&bbuffer[off-3], LFS_BLOCK_SIZE, 3); - cfg.erase(&cfg, block) => 0; - cfg.prog(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - cfg.sync(&cfg) => 0; + memset(&buffer[off-3], BLOCK_SIZE, 3); + cfg->erase(cfg, block) => 0; + cfg->prog(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + cfg->sync(cfg) => 0; // corrupt the destination - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "c") => 0; block = dir.m.pair[0]; lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs) => 0; - cfg.read(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - off = LFS_BLOCK_SIZE-1; - while (off >= 0 && bbuffer[off] == LFS_ERASE_VALUE) { + cfg->read(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + off = BLOCK_SIZE-1; + while (off >= 0 && buffer[off] == ERASE_VALUE) { off -= 1; } - memset(&bbuffer[off-3], LFS_BLOCK_SIZE, 3); - cfg.erase(&cfg, block) => 0; - cfg.prog(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - cfg.sync(&cfg) => 0; + memset(&buffer[off-3], BLOCK_SIZE, 3); + cfg->erase(cfg, block) => 0; + cfg->prog(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + cfg->sync(cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -238,16 +254,18 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # move file after corrupt +[cases.move_file_after_corrupt] in = "lfs.c" -if = 'LFS_PROG_SIZE <= 0x3fe' # only works with one crc per commit +if = 'PROG_SIZE <= 0x3fe' # only works with one crc per commit code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "a") => 0; lfs_mkdir(&lfs, "b") => 0; lfs_mkdir(&lfs, "c") => 0; lfs_mkdir(&lfs, "d") => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "a/hello", LFS_O_CREAT | LFS_O_WRONLY) => 0; lfs_file_write(&lfs, &file, "hola\n", 5) => 5; lfs_file_write(&lfs, &file, "bonjour\n", 8) => 8; @@ -255,49 +273,51 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_rename(&lfs, "a/hello", "c/hello") => 0; lfs_unmount(&lfs) => 0; // corrupt the source - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_block_t block = dir.m.pair[0]; lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs) => 0; - uint8_t bbuffer[LFS_BLOCK_SIZE]; - cfg.read(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - int off = LFS_BLOCK_SIZE-1; - while (off >= 0 && bbuffer[off] == LFS_ERASE_VALUE) { + uint8_t buffer[BLOCK_SIZE]; + cfg->read(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + int off = BLOCK_SIZE-1; + while (off >= 0 && buffer[off] == ERASE_VALUE) { off -= 1; } - memset(&bbuffer[off-3], LFS_BLOCK_SIZE, 3); - cfg.erase(&cfg, block) => 0; - cfg.prog(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - cfg.sync(&cfg) => 0; + memset(&buffer[off-3], BLOCK_SIZE, 3); + cfg->erase(cfg, block) => 0; + cfg->prog(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + cfg->sync(cfg) => 0; // corrupt the destination - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "c") => 0; block = dir.m.pair[0]; lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs) => 0; - cfg.read(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - off = LFS_BLOCK_SIZE-1; - while (off >= 0 && bbuffer[off] == LFS_ERASE_VALUE) { + cfg->read(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + off = BLOCK_SIZE-1; + while (off >= 0 && buffer[off] == ERASE_VALUE) { off -= 1; } - memset(&bbuffer[off-3], LFS_BLOCK_SIZE, 3); - cfg.erase(&cfg, block) => 0; - cfg.prog(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - cfg.sync(&cfg) => 0; + memset(&buffer[off-3], BLOCK_SIZE, 3); + cfg->erase(cfg, block) => 0; + cfg->prog(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + cfg->sync(cfg) => 0; // continue move - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_rename(&lfs, "a/hello", "c/hello") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -335,13 +355,14 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # simple reentrant move file +[cases.reentrant_move_file] reentrant = true code = ''' - err = lfs_mount(&lfs, &cfg); + lfs_t lfs; + int err = lfs_mount(&lfs, cfg); if (err) { - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; } err = lfs_mkdir(&lfs, "a"); assert(!err || err == LFS_ERR_EXIST); @@ -354,9 +375,10 @@ code = ''' lfs_unmount(&lfs) => 0; while (true) { - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // there should never exist _2_ hello files int count = 0; + struct lfs_info info; if (lfs_stat(&lfs, "a/hello", &info) == 0) { assert(strcmp(info.name, "hello") == 0); assert(info.type == LFS_TYPE_REG); @@ -384,7 +406,7 @@ code = ''' assert(count <= 1); lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; if (lfs_stat(&lfs, "a/hello", &info) == 0 && info.size > 0) { lfs_rename(&lfs, "a/hello", "b/hello") => 0; } else if (lfs_stat(&lfs, "b/hello", &info) == 0) { @@ -397,6 +419,7 @@ code = ''' break; } else { // create file + lfs_file_t file; lfs_file_open(&lfs, &file, "a/hello", LFS_O_WRONLY | LFS_O_CREAT) => 0; lfs_file_write(&lfs, &file, "hola\n", 5) => 5; @@ -407,7 +430,9 @@ code = ''' lfs_unmount(&lfs) => 0; } - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -431,10 +456,12 @@ code = ''' lfs_dir_read(&lfs, &dir, &info) => 0; lfs_dir_close(&lfs, &dir) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "a/hello", LFS_O_RDONLY) => LFS_ERR_NOENT; lfs_file_open(&lfs, &file, "b/hello", LFS_O_RDONLY) => LFS_ERR_NOENT; lfs_file_open(&lfs, &file, "c/hello", LFS_O_RDONLY) => LFS_ERR_NOENT; lfs_file_open(&lfs, &file, "d/hello", LFS_O_RDONLY) => 0; + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, 5) => 5; memcmp(buffer, "hola\n", 5) => 0; lfs_file_read(&lfs, &file, buffer, 8) => 8; @@ -445,10 +472,11 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # move dir +[cases.move_dir] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "a") => 0; lfs_mkdir(&lfs, "b") => 0; lfs_mkdir(&lfs, "c") => 0; @@ -459,11 +487,13 @@ code = ''' lfs_mkdir(&lfs, "a/hi/ohayo") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_rename(&lfs, "a/hi", "c/hi") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -510,11 +540,12 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # move dir corrupt source +[cases.move_dir_corrupt_source] in = "lfs.c" code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "a") => 0; lfs_mkdir(&lfs, "b") => 0; lfs_mkdir(&lfs, "c") => 0; @@ -525,28 +556,30 @@ code = ''' lfs_mkdir(&lfs, "a/hi/ohayo") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_rename(&lfs, "a/hi", "c/hi") => 0; lfs_unmount(&lfs) => 0; // corrupt the source - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_block_t block = dir.m.pair[0]; lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs) => 0; - uint8_t bbuffer[LFS_BLOCK_SIZE]; - cfg.read(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - int off = LFS_BLOCK_SIZE-1; - while (off >= 0 && bbuffer[off] == LFS_ERASE_VALUE) { + uint8_t buffer[BLOCK_SIZE]; + cfg->read(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + int off = BLOCK_SIZE-1; + while (off >= 0 && buffer[off] == ERASE_VALUE) { off -= 1; } - memset(&bbuffer[off-3], LFS_BLOCK_SIZE, 3); - cfg.erase(&cfg, block) => 0; - cfg.prog(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - cfg.sync(&cfg) => 0; + memset(&buffer[off-3], BLOCK_SIZE, 3); + cfg->erase(cfg, block) => 0; + cfg->prog(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + cfg->sync(cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -593,12 +626,13 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # move dir corrupt source and dest +[cases.move_dir_corrupt_source_dest] in = "lfs.c" -if = 'LFS_PROG_SIZE <= 0x3fe' # only works with one crc per commit +if = 'PROG_SIZE <= 0x3fe' # only works with one crc per commit code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "a") => 0; lfs_mkdir(&lfs, "b") => 0; lfs_mkdir(&lfs, "c") => 0; @@ -609,44 +643,46 @@ code = ''' lfs_mkdir(&lfs, "a/hi/ohayo") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_rename(&lfs, "a/hi", "c/hi") => 0; lfs_unmount(&lfs) => 0; // corrupt the source - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_block_t block = dir.m.pair[0]; lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs) => 0; - uint8_t bbuffer[LFS_BLOCK_SIZE]; - cfg.read(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - int off = LFS_BLOCK_SIZE-1; - while (off >= 0 && bbuffer[off] == LFS_ERASE_VALUE) { + uint8_t buffer[BLOCK_SIZE]; + cfg->read(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + int off = BLOCK_SIZE-1; + while (off >= 0 && buffer[off] == ERASE_VALUE) { off -= 1; } - memset(&bbuffer[off-3], LFS_BLOCK_SIZE, 3); - cfg.erase(&cfg, block) => 0; - cfg.prog(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - cfg.sync(&cfg) => 0; + memset(&buffer[off-3], BLOCK_SIZE, 3); + cfg->erase(cfg, block) => 0; + cfg->prog(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + cfg->sync(cfg) => 0; // corrupt the destination - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "c") => 0; block = dir.m.pair[0]; lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs) => 0; - cfg.read(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - off = LFS_BLOCK_SIZE-1; - while (off >= 0 && bbuffer[off] == LFS_ERASE_VALUE) { + cfg->read(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + off = BLOCK_SIZE-1; + while (off >= 0 && buffer[off] == ERASE_VALUE) { off -= 1; } - memset(&bbuffer[off-3], LFS_BLOCK_SIZE, 3); - cfg.erase(&cfg, block) => 0; - cfg.prog(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - cfg.sync(&cfg) => 0; + memset(&buffer[off-3], BLOCK_SIZE, 3); + cfg->erase(cfg, block) => 0; + cfg->prog(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + cfg->sync(cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -693,12 +729,13 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # move dir after corrupt +[cases.move_dir_after_corrupt] in = "lfs.c" -if = 'LFS_PROG_SIZE <= 0x3fe' # only works with one crc per commit +if = 'PROG_SIZE <= 0x3fe' # only works with one crc per commit code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "a") => 0; lfs_mkdir(&lfs, "b") => 0; lfs_mkdir(&lfs, "c") => 0; @@ -709,49 +746,51 @@ code = ''' lfs_mkdir(&lfs, "a/hi/ohayo") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_rename(&lfs, "a/hi", "c/hi") => 0; lfs_unmount(&lfs) => 0; // corrupt the source - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_block_t block = dir.m.pair[0]; lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs) => 0; - uint8_t bbuffer[LFS_BLOCK_SIZE]; - cfg.read(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - int off = LFS_BLOCK_SIZE-1; - while (off >= 0 && bbuffer[off] == LFS_ERASE_VALUE) { + uint8_t buffer[BLOCK_SIZE]; + cfg->read(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + int off = BLOCK_SIZE-1; + while (off >= 0 && buffer[off] == ERASE_VALUE) { off -= 1; } - memset(&bbuffer[off-3], LFS_BLOCK_SIZE, 3); - cfg.erase(&cfg, block) => 0; - cfg.prog(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - cfg.sync(&cfg) => 0; + memset(&buffer[off-3], BLOCK_SIZE, 3); + cfg->erase(cfg, block) => 0; + cfg->prog(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + cfg->sync(cfg) => 0; // corrupt the destination - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "c") => 0; block = dir.m.pair[0]; lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs) => 0; - cfg.read(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - off = LFS_BLOCK_SIZE-1; - while (off >= 0 && bbuffer[off] == LFS_ERASE_VALUE) { + cfg->read(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + off = BLOCK_SIZE-1; + while (off >= 0 && buffer[off] == ERASE_VALUE) { off -= 1; } - memset(&bbuffer[off-3], LFS_BLOCK_SIZE, 3); - cfg.erase(&cfg, block) => 0; - cfg.prog(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - cfg.sync(&cfg) => 0; + memset(&buffer[off-3], BLOCK_SIZE, 3); + cfg->erase(cfg, block) => 0; + cfg->prog(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + cfg->sync(cfg) => 0; // continue move - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_rename(&lfs, "a/hi", "c/hi") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -798,13 +837,14 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # simple reentrant move dir +[cases.reentrant_move_dir] reentrant = true code = ''' - err = lfs_mount(&lfs, &cfg); + lfs_t lfs; + int err = lfs_mount(&lfs, cfg); if (err) { - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; } err = lfs_mkdir(&lfs, "a"); assert(!err || err == LFS_ERR_EXIST); @@ -817,9 +857,10 @@ code = ''' lfs_unmount(&lfs) => 0; while (true) { - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // there should never exist _2_ hi directories int count = 0; + struct lfs_info info; if (lfs_stat(&lfs, "a/hi", &info) == 0) { assert(strcmp(info.name, "hi") == 0); assert(info.type == LFS_TYPE_DIR); @@ -843,7 +884,7 @@ code = ''' assert(count <= 1); lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; if (lfs_stat(&lfs, "a/hi", &info) == 0) { lfs_rename(&lfs, "a/hi", "b/hi") => 0; } else if (lfs_stat(&lfs, "b/hi", &info) == 0) { @@ -868,7 +909,9 @@ code = ''' lfs_unmount(&lfs) => 0; } - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "a") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -915,14 +958,16 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # move state stealing +[cases.move_state_stealing] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "a") => 0; lfs_mkdir(&lfs, "b") => 0; lfs_mkdir(&lfs, "c") => 0; lfs_mkdir(&lfs, "d") => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "a/hello", LFS_O_CREAT | LFS_O_WRONLY) => 0; lfs_file_write(&lfs, &file, "hola\n", 5) => 5; lfs_file_write(&lfs, &file, "bonjour\n", 8) => 8; @@ -930,21 +975,22 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_rename(&lfs, "a/hello", "b/hello") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_rename(&lfs, "b/hello", "c/hello") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_rename(&lfs, "c/hello", "d/hello") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "a/hello", LFS_O_RDONLY) => LFS_ERR_NOENT; lfs_file_open(&lfs, &file, "b/hello", LFS_O_RDONLY) => LFS_ERR_NOENT; lfs_file_open(&lfs, &file, "c/hello", LFS_O_RDONLY) => LFS_ERR_NOENT; lfs_file_open(&lfs, &file, "d/hello", LFS_O_RDONLY) => 0; + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, 5) => 5; memcmp(buffer, "hola\n", 5) => 0; lfs_file_read(&lfs, &file, buffer, 8) => 8; @@ -954,12 +1000,13 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_remove(&lfs, "b") => 0; lfs_remove(&lfs, "c") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + struct lfs_info info; lfs_stat(&lfs, "a", &info) => 0; lfs_stat(&lfs, "b", &info) => LFS_ERR_NOENT; lfs_stat(&lfs, "c", &info) => LFS_ERR_NOENT; @@ -979,12 +1026,16 @@ code = ''' ''' # Other specific corner cases -[[case]] # create + delete in same commit with neighbors + +# create + delete in same commit with neighbors +[cases.create_delete_same] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // littlefs keeps files sorted, so we know the order these will be in + lfs_file_t file; lfs_file_open(&lfs, &file, "/1.move_me", LFS_O_WRONLY | LFS_O_CREAT) => 0; lfs_file_close(&lfs, &file) => 0; @@ -1024,6 +1075,8 @@ code = ''' lfs_file_close(&lfs, &files[2]) => 0; // check that nothing was corrupted + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "/") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -1051,6 +1104,7 @@ code = ''' lfs_dir_close(&lfs, &dir) => 0; lfs_file_open(&lfs, &file, "/0.before", LFS_O_RDONLY) => 0; + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, 7) => 7; assert(strcmp((char*)buffer, "test.4") == 0); lfs_file_close(&lfs, &file) => 0; @@ -1124,13 +1178,15 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -# Other specific corner cases -[[case]] # create + delete + delete in same commit with neighbors +# create + delete + delete in same commit with neighbors +[cases.create_delete_delete_same] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // littlefs keeps files sorted, so we know the order these will be in + lfs_file_t file; lfs_file_open(&lfs, &file, "/1.move_me", LFS_O_WRONLY | LFS_O_CREAT) => 0; lfs_file_close(&lfs, &file) => 0; @@ -1175,6 +1231,8 @@ code = ''' lfs_file_close(&lfs, &files[2]) => 0; // check that nothing was corrupted + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "/") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -1202,6 +1260,7 @@ code = ''' lfs_dir_close(&lfs, &dir) => 0; lfs_file_open(&lfs, &file, "/0.before", LFS_O_RDONLY) => 0; + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, 7) => 7; assert(strcmp((char*)buffer, "test.4") == 0); lfs_file_close(&lfs, &file) => 0; @@ -1281,14 +1340,17 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # create + delete in different dirs with neighbors +# create + delete in different dirs with neighbors +[cases.create_delete_different] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // littlefs keeps files sorted, so we know the order these will be in lfs_mkdir(&lfs, "/dir.1") => 0; lfs_mkdir(&lfs, "/dir.2") => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "/dir.1/1.move_me", LFS_O_WRONLY | LFS_O_CREAT) => 0; lfs_file_close(&lfs, &file) => 0; @@ -1340,6 +1402,8 @@ code = ''' lfs_file_close(&lfs, &files[3]) => 0; // check that nothing was corrupted + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "/") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -1397,6 +1461,7 @@ code = ''' lfs_dir_close(&lfs, &dir) => 0; lfs_file_open(&lfs, &file, "/dir.1/0.before", LFS_O_RDONLY) => 0; + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, 7) => 7; assert(strcmp((char*)buffer, "test.5") == 0); lfs_file_close(&lfs, &file) => 0; @@ -1518,17 +1583,20 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # move fix in relocation +# move fix in relocation +[cases.move_fix_relocation] in = "lfs.c" -define.RELOCATIONS = 'range(0x3+1)' -define.LFS_ERASE_CYCLES = 0xffffffff +defines.RELOCATIONS = [0x0, 0x1, 0x2, 0x3] +defines.ERASE_CYCLES = 0xffffffff code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "/parent") => 0; lfs_mkdir(&lfs, "/parent/child") => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "/parent/1.move_me", LFS_O_WRONLY | LFS_O_CREAT) => 0; lfs_file_write(&lfs, &file, "move me", @@ -1568,15 +1636,17 @@ code = ''' // force specific directories to relocate if (RELOCATIONS & 0x1) { + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/parent"); - lfs_testbd_setwear(&cfg, dir.m.pair[0], 0xffffffff) => 0; - lfs_testbd_setwear(&cfg, dir.m.pair[1], 0xffffffff) => 0; + lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; + lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; lfs_dir_close(&lfs, &dir) => 0; } if (RELOCATIONS & 0x2) { + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/parent/child"); - lfs_testbd_setwear(&cfg, dir.m.pair[0], 0xffffffff) => 0; - lfs_testbd_setwear(&cfg, dir.m.pair[1], 0xffffffff) => 0; + lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; + lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; lfs_dir_close(&lfs, &dir) => 0; } @@ -1593,6 +1663,8 @@ code = ''' lfs_file_close(&lfs, &files[3]) => 0; // check that nothing was corrupted + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "/parent") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -1637,6 +1709,7 @@ code = ''' lfs_dir_close(&lfs, &dir) => 0; lfs_file_open(&lfs, &file, "/parent/0.before", LFS_O_RDONLY) => 0; + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, 7) => 7; assert(strcmp((char*)buffer, "test.5") == 0); lfs_file_close(&lfs, &file) => 0; @@ -1655,18 +1728,21 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # move fix in relocation with predecessor +# move fix in relocation with predecessor +[cases.move_fix_relocation_predecessor] in = "lfs.c" -define.RELOCATIONS = 'range(0x7+1)' -define.LFS_ERASE_CYCLES = 0xffffffff +defines.RELOCATIONS = [0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7] +defines.ERASE_CYCLES = 0xffffffff code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "/parent") => 0; lfs_mkdir(&lfs, "/parent/child") => 0; lfs_mkdir(&lfs, "/parent/sibling") => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "/parent/sibling/1.move_me", LFS_O_WRONLY | LFS_O_CREAT) => 0; lfs_file_write(&lfs, &file, "move me", @@ -1706,21 +1782,24 @@ code = ''' // force specific directories to relocate if (RELOCATIONS & 0x1) { + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/parent"); - lfs_testbd_setwear(&cfg, dir.m.pair[0], 0xffffffff) => 0; - lfs_testbd_setwear(&cfg, dir.m.pair[1], 0xffffffff) => 0; + lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; + lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; lfs_dir_close(&lfs, &dir) => 0; } if (RELOCATIONS & 0x2) { + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/parent/sibling"); - lfs_testbd_setwear(&cfg, dir.m.pair[0], 0xffffffff) => 0; - lfs_testbd_setwear(&cfg, dir.m.pair[1], 0xffffffff) => 0; + lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; + lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; lfs_dir_close(&lfs, &dir) => 0; } if (RELOCATIONS & 0x4) { + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/parent/child"); - lfs_testbd_setwear(&cfg, dir.m.pair[0], 0xffffffff) => 0; - lfs_testbd_setwear(&cfg, dir.m.pair[1], 0xffffffff) => 0; + lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; + lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; lfs_dir_close(&lfs, &dir) => 0; } @@ -1739,6 +1818,8 @@ code = ''' lfs_file_close(&lfs, &files[3]) => 0; // check that nothing was corrupted + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "/parent") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; assert(strcmp(info.name, ".") == 0); @@ -1796,6 +1877,7 @@ code = ''' lfs_dir_close(&lfs, &dir) => 0; lfs_file_open(&lfs, &file, "/parent/sibling/0.before", LFS_O_RDONLY) => 0; + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, 7) => 7; assert(strcmp((char*)buffer, "test.5") == 0); lfs_file_close(&lfs, &file) => 0; diff --git a/tests/test_orphans.toml b/tests/test_orphans.toml index 241e273e..fd9b521c 100644 --- a/tests/test_orphans.toml +++ b/tests/test_orphans.toml @@ -1,9 +1,10 @@ -[[case]] # orphan test +[cases.orphan] in = "lfs.c" -if = 'LFS_PROG_SIZE <= 0x3fe' # only works with one crc per commit +if = 'PROG_SIZE <= 0x3fe' # only works with one crc per commit code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "parent") => 0; lfs_mkdir(&lfs, "parent/orphan") => 0; lfs_mkdir(&lfs, "parent/child") => 0; @@ -13,29 +14,31 @@ code = ''' // corrupt the child's most recent commit, this should be the update // to the linked-list entry, which should orphan the orphan. Note this // makes a lot of assumptions about the remove operation. - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "parent/child") => 0; lfs_block_t block = dir.m.pair[0]; lfs_dir_close(&lfs, &dir) => 0; lfs_unmount(&lfs) => 0; - uint8_t bbuffer[LFS_BLOCK_SIZE]; - cfg.read(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - int off = LFS_BLOCK_SIZE-1; - while (off >= 0 && bbuffer[off] == LFS_ERASE_VALUE) { + uint8_t buffer[BLOCK_SIZE]; + cfg->read(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + int off = BLOCK_SIZE-1; + while (off >= 0 && buffer[off] == ERASE_VALUE) { off -= 1; } - memset(&bbuffer[off-3], LFS_BLOCK_SIZE, 3); - cfg.erase(&cfg, block) => 0; - cfg.prog(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0; - cfg.sync(&cfg) => 0; + memset(&buffer[off-3], BLOCK_SIZE, 3); + cfg->erase(cfg, block) => 0; + cfg->prog(cfg, block, 0, buffer, BLOCK_SIZE) => 0; + cfg->sync(cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + struct lfs_info info; lfs_stat(&lfs, "parent/orphan", &info) => LFS_ERR_NOENT; lfs_stat(&lfs, "parent/child", &info) => 0; lfs_fs_size(&lfs) => 8; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_stat(&lfs, "parent/orphan", &info) => LFS_ERR_NOENT; lfs_stat(&lfs, "parent/child", &info) => 0; lfs_fs_size(&lfs) => 8; @@ -48,7 +51,7 @@ code = ''' lfs_fs_size(&lfs) => 8; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_stat(&lfs, "parent/orphan", &info) => LFS_ERR_NOENT; lfs_stat(&lfs, "parent/child", &info) => 0; lfs_stat(&lfs, "parent/otherchild", &info) => 0; @@ -56,43 +59,48 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # reentrant testing for orphans, basically just spam mkdir/remove +# reentrant testing for orphans, basically just spam mkdir/remove +[cases.reentrant_orphan] reentrant = true # TODO fix this case, caused by non-DAG trees -if = '!(DEPTH == 3 && LFS_CACHE_SIZE != 64)' -define = [ +if = '!(DEPTH == 3 && CACHE_SIZE != 64)' +defines = [ {FILES=6, DEPTH=1, CYCLES=20}, {FILES=26, DEPTH=1, CYCLES=20}, {FILES=3, DEPTH=3, CYCLES=20}, ] code = ''' - err = lfs_mount(&lfs, &cfg); + lfs_t lfs; + int err = lfs_mount(&lfs, cfg); if (err) { - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; } srand(1); const char alpha[] = "abcdefghijklmnopqrstuvwxyz"; - for (int i = 0; i < CYCLES; i++) { + for (unsigned i = 0; i < CYCLES; i++) { // create random path char full_path[256]; - for (int d = 0; d < DEPTH; d++) { + for (unsigned d = 0; d < DEPTH; d++) { sprintf(&full_path[2*d], "/%c", alpha[rand() % FILES]); } // if it does not exist, we create it, else we destroy + struct lfs_info info; int res = lfs_stat(&lfs, full_path, &info); if (res == LFS_ERR_NOENT) { // create each directory in turn, ignore if dir already exists - for (int d = 0; d < DEPTH; d++) { + for (unsigned d = 0; d < DEPTH; d++) { + char path[1024]; strcpy(path, full_path); path[2*d+2] = '\0'; err = lfs_mkdir(&lfs, path); assert(!err || err == LFS_ERR_EXIST); } - for (int d = 0; d < DEPTH; d++) { + for (unsigned d = 0; d < DEPTH; d++) { + char path[1024]; strcpy(path, full_path); path[2*d+2] = '\0'; lfs_stat(&lfs, path, &info) => 0; @@ -106,6 +114,7 @@ code = ''' // try to delete path in reverse order, ignore if dir is not empty for (int d = DEPTH-1; d >= 0; d--) { + char path[1024]; strcpy(path, full_path); path[2*d+2] = '\0'; err = lfs_remove(&lfs, path); diff --git a/tests/test_paths.toml b/tests/test_paths.toml index a7474c0b..310364d8 100644 --- a/tests/test_paths.toml +++ b/tests/test_paths.toml @@ -1,13 +1,16 @@ -[[case]] # simple path test +# simple path test +[cases.path] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "tea") => 0; lfs_mkdir(&lfs, "tea/hottea") => 0; lfs_mkdir(&lfs, "tea/warmtea") => 0; lfs_mkdir(&lfs, "tea/coldtea") => 0; + struct lfs_info info; lfs_stat(&lfs, "tea/hottea", &info) => 0; assert(strcmp(info.name, "hottea") == 0); lfs_stat(&lfs, "/tea/hottea", &info) => 0; @@ -21,15 +24,18 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # redundant slashes +# redundant slashes +[cases.redundant_slashes] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "tea") => 0; lfs_mkdir(&lfs, "tea/hottea") => 0; lfs_mkdir(&lfs, "tea/warmtea") => 0; lfs_mkdir(&lfs, "tea/coldtea") => 0; + struct lfs_info info; lfs_stat(&lfs, "/tea/hottea", &info) => 0; assert(strcmp(info.name, "hottea") == 0); lfs_stat(&lfs, "//tea//hottea", &info) => 0; @@ -45,15 +51,18 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # dot path test +# dot path test +[cases.dot_path] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "tea") => 0; lfs_mkdir(&lfs, "tea/hottea") => 0; lfs_mkdir(&lfs, "tea/warmtea") => 0; lfs_mkdir(&lfs, "tea/coldtea") => 0; + struct lfs_info info; lfs_stat(&lfs, "./tea/hottea", &info) => 0; assert(strcmp(info.name, "hottea") == 0); lfs_stat(&lfs, "/./tea/hottea", &info) => 0; @@ -71,10 +80,12 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # dot dot path test +# dot dot path test +[cases.dot_dot_path] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "tea") => 0; lfs_mkdir(&lfs, "tea/hottea") => 0; lfs_mkdir(&lfs, "tea/warmtea") => 0; @@ -84,6 +95,7 @@ code = ''' lfs_mkdir(&lfs, "coffee/warmcoffee") => 0; lfs_mkdir(&lfs, "coffee/coldcoffee") => 0; + struct lfs_info info; lfs_stat(&lfs, "coffee/../tea/hottea", &info) => 0; assert(strcmp(info.name, "hottea") == 0); lfs_stat(&lfs, "tea/coldtea/../hottea", &info) => 0; @@ -101,15 +113,18 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # trailing dot path test +# trailing dot path test +[cases.trailing_dot_path] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "tea") => 0; lfs_mkdir(&lfs, "tea/hottea") => 0; lfs_mkdir(&lfs, "tea/warmtea") => 0; lfs_mkdir(&lfs, "tea/coldtea") => 0; + struct lfs_info info; lfs_stat(&lfs, "tea/hottea/", &info) => 0; assert(strcmp(info.name, "hottea") == 0); lfs_stat(&lfs, "tea/hottea/.", &info) => 0; @@ -123,11 +138,14 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # leading dot path test +# leading dot path test +[cases.leading_dot_path] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, ".milk") => 0; + struct lfs_info info; lfs_stat(&lfs, ".milk", &info) => 0; strcmp(info.name, ".milk") => 0; lfs_stat(&lfs, "tea/.././.milk", &info) => 0; @@ -135,10 +153,12 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # root dot dot path test +# root dot dot path test +[cases.root_dot_dot_path] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "tea") => 0; lfs_mkdir(&lfs, "tea/hottea") => 0; lfs_mkdir(&lfs, "tea/warmtea") => 0; @@ -148,6 +168,7 @@ code = ''' lfs_mkdir(&lfs, "coffee/warmcoffee") => 0; lfs_mkdir(&lfs, "coffee/coldcoffee") => 0; + struct lfs_info info; lfs_stat(&lfs, "coffee/../../../../../../tea/hottea", &info) => 0; strcmp(info.name, "hottea") => 0; @@ -159,10 +180,13 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # invalid path tests +# invalid path tests +[cases.invalid_path] code = ''' - lfs_format(&lfs, &cfg); - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg); + lfs_mount(&lfs, cfg) => 0; + struct lfs_info info; lfs_stat(&lfs, "dirt", &info) => LFS_ERR_NOENT; lfs_stat(&lfs, "dirt/ground", &info) => LFS_ERR_NOENT; lfs_stat(&lfs, "dirt/ground/earth", &info) => LFS_ERR_NOENT; @@ -172,6 +196,7 @@ code = ''' lfs_remove(&lfs, "dirt/ground/earth") => LFS_ERR_NOENT; lfs_mkdir(&lfs, "dirt/ground") => LFS_ERR_NOENT; + lfs_file_t file; lfs_file_open(&lfs, &file, "dirt/ground", LFS_O_WRONLY | LFS_O_CREAT) => LFS_ERR_NOENT; lfs_mkdir(&lfs, "dirt/ground/earth") => LFS_ERR_NOENT; @@ -180,15 +205,19 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # root operations +# root operations +[cases.root] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + struct lfs_info info; lfs_stat(&lfs, "/", &info) => 0; assert(strcmp(info.name, "/") == 0); assert(info.type == LFS_TYPE_DIR); lfs_mkdir(&lfs, "/") => LFS_ERR_EXIST; + lfs_file_t file; lfs_file_open(&lfs, &file, "/", LFS_O_WRONLY | LFS_O_CREAT) => LFS_ERR_ISDIR; @@ -196,10 +225,13 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # root representations +# root representations +[cases.root_reprs] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + struct lfs_info info; lfs_stat(&lfs, "/", &info) => 0; assert(strcmp(info.name, "/") == 0); assert(info.type == LFS_TYPE_DIR); @@ -221,10 +253,13 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # superblock conflict test +# superblock conflict test +[cases.superblock_conflict] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + struct lfs_info info; lfs_stat(&lfs, "littlefs", &info) => LFS_ERR_NOENT; lfs_remove(&lfs, "littlefs") => LFS_ERR_NOENT; @@ -237,18 +272,22 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # max path test +# max path test +[cases.max_path] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "coffee") => 0; lfs_mkdir(&lfs, "coffee/hotcoffee") => 0; lfs_mkdir(&lfs, "coffee/warmcoffee") => 0; lfs_mkdir(&lfs, "coffee/coldcoffee") => 0; + char path[1024]; memset(path, 'w', LFS_NAME_MAX+1); path[LFS_NAME_MAX+1] = '\0'; lfs_mkdir(&lfs, path) => LFS_ERR_NAMETOOLONG; + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT) => LFS_ERR_NAMETOOLONG; @@ -261,19 +300,23 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # really big path test +# really big path test +[cases.really_big_path] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_mkdir(&lfs, "coffee") => 0; lfs_mkdir(&lfs, "coffee/hotcoffee") => 0; lfs_mkdir(&lfs, "coffee/warmcoffee") => 0; lfs_mkdir(&lfs, "coffee/coldcoffee") => 0; + char path[1024]; memset(path, 'w', LFS_NAME_MAX); path[LFS_NAME_MAX] = '\0'; lfs_mkdir(&lfs, path) => 0; lfs_remove(&lfs, path) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT) => 0; lfs_file_close(&lfs, &file) => 0; diff --git a/tests/test_relocations.toml b/tests/test_relocations.toml index 71b10475..f177c730 100644 --- a/tests/test_relocations.toml +++ b/tests/test_relocations.toml @@ -1,15 +1,18 @@ # specific corner cases worth explicitly testing for -[[case]] # dangling split dir test -define.ITERATIONS = 20 -define.COUNT = 10 -define.LFS_BLOCK_CYCLES = [8, 1] +[cases.dangling_split_dir] +defines.ITERATIONS = 20 +defines.COUNT = 10 +defines.BLOCK_CYCLES = [8, 1] code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; // fill up filesystem so only ~16 blocks are left - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "padding", LFS_O_CREAT | LFS_O_WRONLY) => 0; + uint8_t buffer[512]; memset(buffer, 0, 512); - while (LFS_BLOCK_COUNT - lfs_fs_size(&lfs) > 16) { + while (BLOCK_COUNT - lfs_fs_size(&lfs) > 16) { lfs_file_write(&lfs, &file, buffer, 512) => 512; } lfs_file_close(&lfs, &file) => 0; @@ -17,18 +20,22 @@ code = ''' lfs_mkdir(&lfs, "child") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; - for (int j = 0; j < ITERATIONS; j++) { - for (int i = 0; i < COUNT; i++) { + lfs_mount(&lfs, cfg) => 0; + for (unsigned j = 0; j < ITERATIONS; j++) { + for (unsigned i = 0; i < COUNT; i++) { + char path[1024]; sprintf(path, "child/test%03d_loooooooooooooooooong_name", i); lfs_file_open(&lfs, &file, path, LFS_O_CREAT | LFS_O_WRONLY) => 0; lfs_file_close(&lfs, &file) => 0; } + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "child") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; lfs_dir_read(&lfs, &dir, &info) => 1; - for (int i = 0; i < COUNT; i++) { + for (unsigned i = 0; i < COUNT; i++) { + char path[1024]; sprintf(path, "test%03d_loooooooooooooooooong_name", i); lfs_dir_read(&lfs, &dir, &info) => 1; strcmp(info.name, path) => 0; @@ -36,46 +43,54 @@ code = ''' lfs_dir_read(&lfs, &dir, &info) => 0; lfs_dir_close(&lfs, &dir) => 0; - if (j == ITERATIONS-1) { + if (j == (unsigned)ITERATIONS-1) { break; } - for (int i = 0; i < COUNT; i++) { + for (unsigned i = 0; i < COUNT; i++) { + char path[1024]; sprintf(path, "child/test%03d_loooooooooooooooooong_name", i); lfs_remove(&lfs, path) => 0; } } lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "child") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; lfs_dir_read(&lfs, &dir, &info) => 1; - for (int i = 0; i < COUNT; i++) { + for (unsigned i = 0; i < COUNT; i++) { + char path[1024]; sprintf(path, "test%03d_loooooooooooooooooong_name", i); lfs_dir_read(&lfs, &dir, &info) => 1; strcmp(info.name, path) => 0; } lfs_dir_read(&lfs, &dir, &info) => 0; lfs_dir_close(&lfs, &dir) => 0; - for (int i = 0; i < COUNT; i++) { + for (unsigned i = 0; i < COUNT; i++) { + char path[1024]; sprintf(path, "child/test%03d_loooooooooooooooooong_name", i); lfs_remove(&lfs, path) => 0; } lfs_unmount(&lfs) => 0; ''' -[[case]] # outdated head test -define.ITERATIONS = 20 -define.COUNT = 10 -define.LFS_BLOCK_CYCLES = [8, 1] +[cases.outdated_head] +defines.ITERATIONS = 20 +defines.COUNT = 10 +defines.BLOCK_CYCLES = [8, 1] code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; // fill up filesystem so only ~16 blocks are left - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "padding", LFS_O_CREAT | LFS_O_WRONLY) => 0; + uint8_t buffer[512]; memset(buffer, 0, 512); - while (LFS_BLOCK_COUNT - lfs_fs_size(&lfs) > 16) { + while (BLOCK_COUNT - lfs_fs_size(&lfs) > 16) { lfs_file_write(&lfs, &file, buffer, 512) => 512; } lfs_file_close(&lfs, &file) => 0; @@ -83,18 +98,22 @@ code = ''' lfs_mkdir(&lfs, "child") => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; - for (int j = 0; j < ITERATIONS; j++) { - for (int i = 0; i < COUNT; i++) { + lfs_mount(&lfs, cfg) => 0; + for (unsigned j = 0; j < ITERATIONS; j++) { + for (unsigned i = 0; i < COUNT; i++) { + char path[1024]; sprintf(path, "child/test%03d_loooooooooooooooooong_name", i); lfs_file_open(&lfs, &file, path, LFS_O_CREAT | LFS_O_WRONLY) => 0; lfs_file_close(&lfs, &file) => 0; } + lfs_dir_t dir; + struct lfs_info info; lfs_dir_open(&lfs, &dir, "child") => 0; lfs_dir_read(&lfs, &dir, &info) => 1; lfs_dir_read(&lfs, &dir, &info) => 1; - for (int i = 0; i < COUNT; i++) { + for (unsigned i = 0; i < COUNT; i++) { + char path[1024]; sprintf(path, "test%03d_loooooooooooooooooong_name", i); lfs_dir_read(&lfs, &dir, &info) => 1; strcmp(info.name, path) => 0; @@ -110,7 +129,8 @@ code = ''' lfs_dir_rewind(&lfs, &dir) => 0; lfs_dir_read(&lfs, &dir, &info) => 1; lfs_dir_read(&lfs, &dir, &info) => 1; - for (int i = 0; i < COUNT; i++) { + for (unsigned i = 0; i < COUNT; i++) { + char path[1024]; sprintf(path, "test%03d_loooooooooooooooooong_name", i); lfs_dir_read(&lfs, &dir, &info) => 1; strcmp(info.name, path) => 0; @@ -126,7 +146,8 @@ code = ''' lfs_dir_rewind(&lfs, &dir) => 0; lfs_dir_read(&lfs, &dir, &info) => 1; lfs_dir_read(&lfs, &dir, &info) => 1; - for (int i = 0; i < COUNT; i++) { + for (unsigned i = 0; i < COUNT; i++) { + char path[1024]; sprintf(path, "test%03d_loooooooooooooooooong_name", i); lfs_dir_read(&lfs, &dir, &info) => 1; strcmp(info.name, path) => 0; @@ -135,7 +156,8 @@ code = ''' lfs_dir_read(&lfs, &dir, &info) => 0; lfs_dir_close(&lfs, &dir) => 0; - for (int i = 0; i < COUNT; i++) { + for (unsigned i = 0; i < COUNT; i++) { + char path[1024]; sprintf(path, "child/test%03d_loooooooooooooooooong_name", i); lfs_remove(&lfs, path) => 0; } @@ -143,45 +165,50 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # reentrant testing for relocations, this is the same as the - # orphan testing, except here we also set block_cycles so that - # almost every tree operation needs a relocation +# reentrant testing for relocations, this is the same as the +# orphan testing, except here we also set block_cycles so that +# almost every tree operation needs a relocation +[cases.reentrant_relocations] reentrant = true # TODO fix this case, caused by non-DAG trees -if = '!(DEPTH == 3 && LFS_CACHE_SIZE != 64)' -define = [ - {FILES=6, DEPTH=1, CYCLES=20, LFS_BLOCK_CYCLES=1}, - {FILES=26, DEPTH=1, CYCLES=20, LFS_BLOCK_CYCLES=1}, - {FILES=3, DEPTH=3, CYCLES=20, LFS_BLOCK_CYCLES=1}, +if = '!(DEPTH == 3 && CACHE_SIZE != 64)' +defines = [ + {FILES=6, DEPTH=1, CYCLES=20, BLOCK_CYCLES=1}, + {FILES=26, DEPTH=1, CYCLES=20, BLOCK_CYCLES=1}, + {FILES=3, DEPTH=3, CYCLES=20, BLOCK_CYCLES=1}, ] code = ''' - err = lfs_mount(&lfs, &cfg); + lfs_t lfs; + int err = lfs_mount(&lfs, cfg); if (err) { - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; } srand(1); const char alpha[] = "abcdefghijklmnopqrstuvwxyz"; - for (int i = 0; i < CYCLES; i++) { + for (unsigned i = 0; i < CYCLES; i++) { // create random path char full_path[256]; - for (int d = 0; d < DEPTH; d++) { + for (unsigned d = 0; d < DEPTH; d++) { sprintf(&full_path[2*d], "/%c", alpha[rand() % FILES]); } // if it does not exist, we create it, else we destroy + struct lfs_info info; int res = lfs_stat(&lfs, full_path, &info); if (res == LFS_ERR_NOENT) { // create each directory in turn, ignore if dir already exists - for (int d = 0; d < DEPTH; d++) { + for (unsigned d = 0; d < DEPTH; d++) { + char path[1024]; strcpy(path, full_path); path[2*d+2] = '\0'; err = lfs_mkdir(&lfs, path); assert(!err || err == LFS_ERR_EXIST); } - for (int d = 0; d < DEPTH; d++) { + for (unsigned d = 0; d < DEPTH; d++) { + char path[1024]; strcpy(path, full_path); path[2*d+2] = '\0'; lfs_stat(&lfs, path, &info) => 0; @@ -194,7 +221,8 @@ code = ''' assert(info.type == LFS_TYPE_DIR); // try to delete path in reverse order, ignore if dir is not empty - for (int d = DEPTH-1; d >= 0; d--) { + for (unsigned d = DEPTH-1; d+1 > 0; d--) { + char path[1024]; strcpy(path, full_path); path[2*d+2] = '\0'; err = lfs_remove(&lfs, path); @@ -207,44 +235,49 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # reentrant testing for relocations, but now with random renames! +# reentrant testing for relocations, but now with random renames! +[cases.reentrant_relocations_renames] reentrant = true # TODO fix this case, caused by non-DAG trees -if = '!(DEPTH == 3 && LFS_CACHE_SIZE != 64)' -define = [ - {FILES=6, DEPTH=1, CYCLES=20, LFS_BLOCK_CYCLES=1}, - {FILES=26, DEPTH=1, CYCLES=20, LFS_BLOCK_CYCLES=1}, - {FILES=3, DEPTH=3, CYCLES=20, LFS_BLOCK_CYCLES=1}, +if = '!(DEPTH == 3 && CACHE_SIZE != 64)' +defines = [ + {FILES=6, DEPTH=1, CYCLES=20, BLOCK_CYCLES=1}, + {FILES=26, DEPTH=1, CYCLES=20, BLOCK_CYCLES=1}, + {FILES=3, DEPTH=3, CYCLES=20, BLOCK_CYCLES=1}, ] code = ''' - err = lfs_mount(&lfs, &cfg); + lfs_t lfs; + int err = lfs_mount(&lfs, cfg); if (err) { - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; } srand(1); const char alpha[] = "abcdefghijklmnopqrstuvwxyz"; - for (int i = 0; i < CYCLES; i++) { + for (unsigned i = 0; i < CYCLES; i++) { // create random path char full_path[256]; - for (int d = 0; d < DEPTH; d++) { + for (unsigned d = 0; d < DEPTH; d++) { sprintf(&full_path[2*d], "/%c", alpha[rand() % FILES]); } // if it does not exist, we create it, else we destroy + struct lfs_info info; int res = lfs_stat(&lfs, full_path, &info); assert(!res || res == LFS_ERR_NOENT); if (res == LFS_ERR_NOENT) { // create each directory in turn, ignore if dir already exists - for (int d = 0; d < DEPTH; d++) { + for (unsigned d = 0; d < DEPTH; d++) { + char path[1024]; strcpy(path, full_path); path[2*d+2] = '\0'; err = lfs_mkdir(&lfs, path); assert(!err || err == LFS_ERR_EXIST); } - for (int d = 0; d < DEPTH; d++) { + for (unsigned d = 0; d < DEPTH; d++) { + char path[1024]; strcpy(path, full_path); path[2*d+2] = '\0'; lfs_stat(&lfs, path, &info) => 0; @@ -257,7 +290,7 @@ code = ''' // create new random path char new_path[256]; - for (int d = 0; d < DEPTH; d++) { + for (unsigned d = 0; d < DEPTH; d++) { sprintf(&new_path[2*d], "/%c", alpha[rand() % FILES]); } @@ -266,7 +299,8 @@ code = ''' assert(!res || res == LFS_ERR_NOENT); if (res == LFS_ERR_NOENT) { // stop once some dir is renamed - for (int d = 0; d < DEPTH; d++) { + for (unsigned d = 0; d < DEPTH; d++) { + char path[1024]; strcpy(&path[2*d], &full_path[2*d]); path[2*d+2] = '\0'; strcpy(&path[128+2*d], &new_path[2*d]); @@ -278,7 +312,8 @@ code = ''' } } - for (int d = 0; d < DEPTH; d++) { + for (unsigned d = 0; d < DEPTH; d++) { + char path[1024]; strcpy(path, new_path); path[2*d+2] = '\0'; lfs_stat(&lfs, path, &info) => 0; @@ -290,7 +325,8 @@ code = ''' } else { // try to delete path in reverse order, // ignore if dir is not empty - for (int d = DEPTH-1; d >= 0; d--) { + for (unsigned d = DEPTH-1; d+1 > 0; d--) { + char path[1024]; strcpy(path, full_path); path[2*d+2] = '\0'; err = lfs_remove(&lfs, path); diff --git a/tests/test_seek.toml b/tests/test_seek.toml index 79d7728a..383c1ba1 100644 --- a/tests/test_seek.toml +++ b/tests/test_seek.toml @@ -1,6 +1,7 @@ -[[case]] # simple file seek -define = [ +# simple file seek +[cases.seek] +defines = [ {COUNT=132, SKIP=4}, {COUNT=132, SKIP=128}, {COUNT=200, SKIP=10}, @@ -9,11 +10,14 @@ define = [ {COUNT=4, SKIP=2}, ] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "kitty", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0; - size = strlen("kittycatcat"); + size_t size = strlen("kittycatcat"); + uint8_t buffer[1024]; memcpy(buffer, "kittycatcat", size); for (int j = 0; j < COUNT; j++) { lfs_file_write(&lfs, &file, buffer, size); @@ -21,7 +25,7 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "kitty", LFS_O_RDONLY) => 0; lfs_soff_t pos = -1; @@ -68,8 +72,9 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # simple file seek and write -define = [ +# simple file seek and write +[cases.seek_write] +defines = [ {COUNT=132, SKIP=4}, {COUNT=132, SKIP=128}, {COUNT=200, SKIP=10}, @@ -78,11 +83,14 @@ define = [ {COUNT=4, SKIP=2}, ] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "kitty", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0; - size = strlen("kittycatcat"); + size_t size = strlen("kittycatcat"); + uint8_t buffer[1024]; memcpy(buffer, "kittycatcat", size); for (int j = 0; j < COUNT; j++) { lfs_file_write(&lfs, &file, buffer, size); @@ -90,7 +98,7 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "kitty", LFS_O_RDWR) => 0; lfs_soff_t pos = -1; @@ -129,15 +137,18 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # boundary seek and writes -define.COUNT = 132 -define.OFFSETS = '"{512, 1020, 513, 1021, 511, 1019, 1441}"' +# boundary seek and writes +[cases.boundary_seek_write] +defines.COUNT = 132 code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "kitty", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0; - size = strlen("kittycatcat"); + size_t size = strlen("kittycatcat"); + uint8_t buffer[1024]; memcpy(buffer, "kittycatcat", size); for (int j = 0; j < COUNT; j++) { lfs_file_write(&lfs, &file, buffer, size); @@ -145,11 +156,11 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "kitty", LFS_O_RDWR) => 0; size = strlen("hedgehoghog"); - const lfs_soff_t offsets[] = OFFSETS; + const lfs_soff_t offsets[] = {512, 1020, 513, 1021, 511, 1019, 1441}; for (unsigned i = 0; i < sizeof(offsets) / sizeof(offsets[0]); i++) { lfs_soff_t off = offsets[i]; @@ -183,8 +194,9 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # out of bounds seek -define = [ +# out of bounds seek +[cases.out_of_bounds_seek] +defines = [ {COUNT=132, SKIP=4}, {COUNT=132, SKIP=128}, {COUNT=200, SKIP=10}, @@ -193,18 +205,21 @@ define = [ {COUNT=4, SKIP=3}, ] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "kitty", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0; - size = strlen("kittycatcat"); + size_t size = strlen("kittycatcat"); + uint8_t buffer[1024]; memcpy(buffer, "kittycatcat", size); for (int j = 0; j < COUNT; j++) { lfs_file_write(&lfs, &file, buffer, size); } lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "kitty", LFS_O_RDWR) => 0; size = strlen("kittycatcat"); @@ -238,16 +253,20 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # inline write and seek -define.SIZE = [2, 4, 128, 132] +# inline write and seek +[cases.inline_write_seek] +defines.SIZE = [2, 4, 128, 132] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "tinykitty", LFS_O_RDWR | LFS_O_CREAT) => 0; int j = 0; int k = 0; + uint8_t buffer[1024]; memcpy(buffer, "abcdefghijklmnopqrstuvwxyz", 26); for (unsigned i = 0; i < SIZE; i++) { lfs_file_write(&lfs, &file, &buffer[j++ % 26], 1) => 1; @@ -305,16 +324,20 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # file seek and write with power-loss +# file seek and write with power-loss +[cases.reentrant_seek_write] # must be power-of-2 for quadratic probing to be exhaustive -define.COUNT = [4, 64, 128] +defines.COUNT = [4, 64, 128] reentrant = true code = ''' - err = lfs_mount(&lfs, &cfg); + lfs_t lfs; + int err = lfs_mount(&lfs, cfg); if (err) { - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; } + lfs_file_t file; + uint8_t buffer[1024]; err = lfs_file_open(&lfs, &file, "kitty", LFS_O_RDONLY); assert(!err || err == LFS_ERR_NOENT); if (!err) { @@ -334,14 +357,14 @@ code = ''' if (lfs_file_size(&lfs, &file) == 0) { for (int j = 0; j < COUNT; j++) { strcpy((char*)buffer, "kittycatcat"); - size = strlen((char*)buffer); + size_t size = strlen((char*)buffer); lfs_file_write(&lfs, &file, buffer, size) => size; } } lfs_file_close(&lfs, &file) => 0; strcpy((char*)buffer, "doggodogdog"); - size = strlen((char*)buffer); + size_t size = strlen((char*)buffer); lfs_file_open(&lfs, &file, "kitty", LFS_O_RDWR) => 0; lfs_file_size(&lfs, &file) => COUNT*size; diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index 407c8454..d511675f 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -1,41 +1,53 @@ -[[case]] # simple formatting test +# simple formatting test +[cases.format] code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; ''' -[[case]] # mount/unmount +# mount/unmount +[cases.mount] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_unmount(&lfs) => 0; ''' -[[case]] # reentrant format +# reentrant format +[cases.reentrant_format] reentrant = true code = ''' - err = lfs_mount(&lfs, &cfg); + lfs_t lfs; + int err = lfs_mount(&lfs, cfg); if (err) { - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; } lfs_unmount(&lfs) => 0; ''' -[[case]] # invalid mount +# invalid mount +[cases.invalid_mount] code = ''' - lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT; + lfs_t lfs; + lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT; ''' -[[case]] # expanding superblock -define.LFS_BLOCK_CYCLES = [32, 33, 1] -define.N = [10, 100, 1000] +# expanding superblock +[cases.expanding_superblock] +defines.LFS_BLOCK_CYCLES = [32, 33, 1] +defines.N = [10, 100, 1000] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (int i = 0; i < N; i++) { + lfs_file_t file; lfs_file_open(&lfs, &file, "dummy", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; lfs_file_close(&lfs, &file) => 0; + struct lfs_info info; lfs_stat(&lfs, "dummy", &info) => 0; assert(strcmp(info.name, "dummy") == 0); assert(info.type == LFS_TYPE_REG); @@ -44,25 +56,30 @@ code = ''' lfs_unmount(&lfs) => 0; // one last check after power-cycle - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "dummy", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; lfs_file_close(&lfs, &file) => 0; + struct lfs_info info; lfs_stat(&lfs, "dummy", &info) => 0; assert(strcmp(info.name, "dummy") == 0); assert(info.type == LFS_TYPE_REG); lfs_unmount(&lfs) => 0; ''' -[[case]] # expanding superblock with power cycle -define.LFS_BLOCK_CYCLES = [32, 33, 1] -define.N = [10, 100, 1000] +# expanding superblock with power cycle +[cases.expanding_superblock_power_cycle] +defines.LFS_BLOCK_CYCLES = [32, 33, 1] +defines.N = [10, 100, 1000] code = ''' - lfs_format(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; for (int i = 0; i < N; i++) { - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; // remove lingering dummy? - err = lfs_stat(&lfs, "dummy", &info); + struct lfs_info info; + int err = lfs_stat(&lfs, "dummy", &info); assert(err == 0 || (err == LFS_ERR_NOENT && i == 0)); if (!err) { assert(strcmp(info.name, "dummy") == 0); @@ -70,6 +87,7 @@ code = ''' lfs_remove(&lfs, "dummy") => 0; } + lfs_file_t file; lfs_file_open(&lfs, &file, "dummy", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; lfs_file_close(&lfs, &file) => 0; @@ -80,26 +98,30 @@ code = ''' } // one last check after power-cycle - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + struct lfs_info info; lfs_stat(&lfs, "dummy", &info) => 0; assert(strcmp(info.name, "dummy") == 0); assert(info.type == LFS_TYPE_REG); lfs_unmount(&lfs) => 0; ''' -[[case]] # reentrant expanding superblock -define.LFS_BLOCK_CYCLES = [2, 1] -define.N = 24 +# reentrant expanding superblock +[cases.reentrant_expanding_superblock] +defines.LFS_BLOCK_CYCLES = [2, 1] +defines.N = 24 reentrant = true code = ''' - err = lfs_mount(&lfs, &cfg); + lfs_t lfs; + int err = lfs_mount(&lfs, cfg); if (err) { - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; } for (int i = 0; i < N; i++) { // remove lingering dummy? + struct lfs_info info; err = lfs_stat(&lfs, "dummy", &info); assert(err == 0 || (err == LFS_ERR_NOENT && i == 0)); if (!err) { @@ -108,6 +130,7 @@ code = ''' lfs_remove(&lfs, "dummy") => 0; } + lfs_file_t file; lfs_file_open(&lfs, &file, "dummy", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; lfs_file_close(&lfs, &file) => 0; @@ -119,7 +142,8 @@ code = ''' lfs_unmount(&lfs) => 0; // one last check after power-cycle - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + struct lfs_info info; lfs_stat(&lfs, "dummy", &info) => 0; assert(strcmp(info.name, "dummy") == 0); assert(info.type == LFS_TYPE_REG); diff --git a/tests/test_truncate.toml b/tests/test_truncate.toml index 850d7aae..fc83ce37 100644 --- a/tests/test_truncate.toml +++ b/tests/test_truncate.toml @@ -1,14 +1,18 @@ -[[case]] # simple truncate -define.MEDIUMSIZE = [32, 2048] -define.LARGESIZE = 8192 +# simple truncate +[cases.truncate] +defines.MEDIUMSIZE = [32, 2048] +defines.LARGESIZE = 8192 code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "baldynoop", LFS_O_WRONLY | LFS_O_CREAT) => 0; + uint8_t buffer[1024]; strcpy((char*)buffer, "hair"); - size = strlen((char*)buffer); + size_t size = strlen((char*)buffer); for (lfs_off_t j = 0; j < LARGESIZE; j += size) { lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -17,7 +21,7 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "baldynoop", LFS_O_RDWR) => 0; lfs_file_size(&lfs, &file) => LARGESIZE; @@ -27,7 +31,7 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "baldynoop", LFS_O_RDONLY) => 0; lfs_file_size(&lfs, &file) => MEDIUMSIZE; @@ -42,17 +46,21 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # truncate and read -define.MEDIUMSIZE = [32, 2048] -define.LARGESIZE = 8192 +# truncate and read +[cases.truncate_read] +defines.MEDIUMSIZE = [32, 2048] +defines.LARGESIZE = 8192 code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "baldyread", LFS_O_WRONLY | LFS_O_CREAT) => 0; + uint8_t buffer[1024]; strcpy((char*)buffer, "hair"); - size = strlen((char*)buffer); + size_t size = strlen((char*)buffer); for (lfs_off_t j = 0; j < LARGESIZE; j += size) { lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -61,7 +69,7 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "baldyread", LFS_O_RDWR) => 0; lfs_file_size(&lfs, &file) => LARGESIZE; @@ -78,7 +86,7 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "baldyread", LFS_O_RDONLY) => 0; lfs_file_size(&lfs, &file) => MEDIUMSIZE; @@ -93,14 +101,18 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # write, truncate, and read +# write, truncate, and read +[cases.write_truncate_read] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "sequence", LFS_O_RDWR | LFS_O_CREAT | LFS_O_TRUNC) => 0; - size = lfs_min(lfs.cfg->cache_size, sizeof(buffer)/2); + uint8_t buffer[1024]; + size_t size = lfs_min(lfs.cfg->cache_size, sizeof(buffer)/2); lfs_size_t qsize = size / 4; uint8_t *wb = buffer; uint8_t *rb = buffer + size; @@ -145,17 +157,21 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # truncate and write -define.MEDIUMSIZE = [32, 2048] -define.LARGESIZE = 8192 +# truncate and write +[cases.truncate_write] +defines.MEDIUMSIZE = [32, 2048] +defines.LARGESIZE = 8192 code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "baldywrite", LFS_O_WRONLY | LFS_O_CREAT) => 0; + uint8_t buffer[1024]; strcpy((char*)buffer, "hair"); - size = strlen((char*)buffer); + size_t size = strlen((char*)buffer); for (lfs_off_t j = 0; j < LARGESIZE; j += size) { lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -164,7 +180,7 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "baldywrite", LFS_O_RDWR) => 0; lfs_file_size(&lfs, &file) => LARGESIZE; @@ -181,7 +197,7 @@ code = ''' lfs_file_close(&lfs, &file) => 0; lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "baldywrite", LFS_O_RDONLY) => 0; lfs_file_size(&lfs, &file) => MEDIUMSIZE; @@ -196,26 +212,30 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # truncate write under powerloss -define.SMALLSIZE = [4, 512] -define.MEDIUMSIZE = [32, 1024] -define.LARGESIZE = 2048 +# truncate write under powerloss +[cases.reentrant_truncate_write] +defines.SMALLSIZE = [4, 512] +defines.MEDIUMSIZE = [32, 1024] +defines.LARGESIZE = 2048 reentrant = true code = ''' - err = lfs_mount(&lfs, &cfg); + lfs_t lfs; + int err = lfs_mount(&lfs, cfg); if (err) { - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; } + lfs_file_t file; err = lfs_file_open(&lfs, &file, "baldy", LFS_O_RDONLY); assert(!err || err == LFS_ERR_NOENT); if (!err) { - size = lfs_file_size(&lfs, &file); + size_t size = lfs_file_size(&lfs, &file); assert(size == 0 || - size == LARGESIZE || - size == MEDIUMSIZE || - size == SMALLSIZE); + size == (size_t)LARGESIZE || + size == (size_t)MEDIUMSIZE || + size == (size_t)SMALLSIZE); for (lfs_off_t j = 0; j < size; j += 4) { + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, 4) => 4; assert(memcmp(buffer, "hair", 4) == 0 || memcmp(buffer, "bald", 4) == 0 || @@ -227,8 +247,9 @@ code = ''' lfs_file_open(&lfs, &file, "baldy", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; lfs_file_size(&lfs, &file) => 0; + uint8_t buffer[1024]; strcpy((char*)buffer, "hair"); - size = strlen((char*)buffer); + size_t size = strlen((char*)buffer); for (lfs_off_t j = 0; j < LARGESIZE; j += size) { lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -262,12 +283,14 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # more aggressive general truncation tests -define.CONFIG = 'range(6)' -define.SMALLSIZE = 32 -define.MEDIUMSIZE = 2048 -define.LARGESIZE = 8192 +# more aggressive general truncation tests +[cases.aggressive_truncate] +defines.CONFIG = [0,1,2,3,4,5] +defines.SMALLSIZE = 32 +defines.MEDIUMSIZE = 2048 +defines.LARGESIZE = 8192 code = ''' + lfs_t lfs; #define COUNT 5 const struct { lfs_off_t startsizes[COUNT]; @@ -312,16 +335,19 @@ code = ''' const lfs_off_t *hotsizes = configs[CONFIG].hotsizes; const lfs_off_t *coldsizes = configs[CONFIG].coldsizes; - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (unsigned i = 0; i < COUNT; i++) { + char path[1024]; sprintf(path, "hairyhead%d", i); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + uint8_t buffer[1024]; strcpy((char*)buffer, "hair"); - size = strlen((char*)buffer); + size_t size = strlen((char*)buffer); for (lfs_off_t j = 0; j < startsizes[i]; j += size) { lfs_file_write(&lfs, &file, buffer, size) => size; } @@ -340,21 +366,25 @@ code = ''' lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (unsigned i = 0; i < COUNT; i++) { + char path[1024]; sprintf(path, "hairyhead%d", i); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_RDWR) => 0; lfs_file_size(&lfs, &file) => hotsizes[i]; - size = strlen("hair"); + size_t size = strlen("hair"); lfs_off_t j = 0; for (; j < startsizes[i] && j < hotsizes[i]; j += size) { + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, size) => size; memcmp(buffer, "hair", size) => 0; } for (; j < hotsizes[i]; j += size) { + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, size) => size; memcmp(buffer, "\0\0\0\0", size) => 0; } @@ -367,22 +397,26 @@ code = ''' lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; for (unsigned i = 0; i < COUNT; i++) { + char path[1024]; sprintf(path, "hairyhead%d", i); + lfs_file_t file; lfs_file_open(&lfs, &file, path, LFS_O_RDONLY) => 0; lfs_file_size(&lfs, &file) => coldsizes[i]; - size = strlen("hair"); + size_t size = strlen("hair"); lfs_off_t j = 0; for (; j < startsizes[i] && j < hotsizes[i] && j < coldsizes[i]; j += size) { + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, size) => size; memcmp(buffer, "hair", size) => 0; } for (; j < coldsizes[i]; j += size) { + uint8_t buffer[1024]; lfs_file_read(&lfs, &file, buffer, size) => size; memcmp(buffer, "\0\0\0\0", size) => 0; } @@ -393,16 +427,20 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[[case]] # noop truncate -define.MEDIUMSIZE = [32, 2048] +# noop truncate +[cases.nop_truncate] +defines.MEDIUMSIZE = [32, 2048] code = ''' - lfs_format(&lfs, &cfg) => 0; - lfs_mount(&lfs, &cfg) => 0; + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; lfs_file_open(&lfs, &file, "baldynoop", LFS_O_RDWR | LFS_O_CREAT) => 0; + uint8_t buffer[1024]; strcpy((char*)buffer, "hair"); - size = strlen((char*)buffer); + size_t size = strlen((char*)buffer); for (lfs_off_t j = 0; j < MEDIUMSIZE; j += size) { lfs_file_write(&lfs, &file, buffer, size) => size; @@ -426,7 +464,7 @@ code = ''' lfs_unmount(&lfs) => 0; // still there after reboot? - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, cfg) => 0; lfs_file_open(&lfs, &file, "baldynoop", LFS_O_RDWR) => 0; lfs_file_size(&lfs, &file) => MEDIUMSIZE; for (lfs_off_t j = 0; j < MEDIUMSIZE; j += size) { From 4a4232679791c107d0278418a820c1c3bc31cfb6 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 14 May 2022 03:37:55 -0500 Subject: [PATCH 14/81] Moved test suites into custom linker section This simplifies the interaction between code generation and the test-runner. In theory it also reduces compilation dependencies, but internal tests make this difficult. --- runners/test_runner.c | 129 ++++++++++++++++++++++-------------------- runners/test_runner.h | 5 +- scripts/test.py | 77 +++++++++---------------- 3 files changed, 96 insertions(+), 115 deletions(-) diff --git a/runners/test_runner.c b/runners/test_runner.c index 21ff1c00..cc932b73 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -7,6 +7,14 @@ #include +// test suites in a custom ld section +extern struct test_suite __start__test_suites; +extern struct test_suite __stop__test_suites; + +const struct test_suite *test_suites = &__start__test_suites; +#define TEST_SUITE_COUNT \ + ((size_t)(&__stop__test_suites - &__start__test_suites)) + // test geometries struct test_geometry { const char *name; @@ -212,24 +220,24 @@ static void summary(void) { test_types_t types = 0; size_t perms = 0; size_t filtered = 0; - for (size_t i = 0; i < test_suite_count; i++) { - if (test_suite_skip(test_suites[i])) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + if (test_suite_skip(&test_suites[i])) { continue; } - test_define_suite(test_suites[i]); + test_define_suite(&test_suites[i]); - for (size_t j = 0; j < test_suites[i]->case_count; j++) { - if (test_case_skip(test_suites[i]->cases[j])) { + for (size_t j = 0; j < test_suites[i].case_count; j++) { + if (test_case_skip(&test_suites[i].cases[j])) { continue; } - test_case_permcount(test_suites[i], test_suites[i]->cases[j], + test_case_permcount(&test_suites[i], &test_suites[i].cases[j], &perms, &filtered); } - cases += test_suites[i]->case_count; - types |= test_suites[i]->types; + cases += test_suites[i].case_count; + types |= test_suites[i].types; } char perm_buf[64]; @@ -241,28 +249,28 @@ static void summary(void) { printf("%-36s %7s %7zu %7zu %11s\n", "TOTAL", type_buf, - test_suite_count, + TEST_SUITE_COUNT, cases, perm_buf); } static void list_suites(void) { printf("%-36s %7s %7s %11s\n", "suite", "types", "cases", "perms"); - for (size_t i = 0; i < test_suite_count; i++) { - if (test_suite_skip(test_suites[i])) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + if (test_suite_skip(&test_suites[i])) { continue; } - test_define_suite(test_suites[i]); + test_define_suite(&test_suites[i]); size_t perms = 0; size_t filtered = 0; - for (size_t j = 0; j < test_suites[i]->case_count; j++) { - if (test_case_skip(test_suites[i]->cases[j])) { + for (size_t j = 0; j < test_suites[i].case_count; j++) { + if (test_case_skip(&test_suites[i].cases[j])) { continue; } - test_case_permcount(test_suites[i], test_suites[i]->cases[j], + test_case_permcount(&test_suites[i], &test_suites[i].cases[j], &perms, &filtered); } @@ -270,35 +278,35 @@ static void list_suites(void) { sprintf(perm_buf, "%zu/%zu", filtered, perms); char type_buf[64]; sprintf(type_buf, "%s%s", - (test_suites[i]->types & TEST_NORMAL) ? "n" : "", - (test_suites[i]->types & TEST_REENTRANT) ? "r" : ""); + (test_suites[i].types & TEST_NORMAL) ? "n" : "", + (test_suites[i].types & TEST_REENTRANT) ? "r" : ""); printf("%-36s %7s %7zu %11s\n", - test_suites[i]->id, + test_suites[i].id, type_buf, - test_suites[i]->case_count, + test_suites[i].case_count, perm_buf); } } static void list_cases(void) { printf("%-36s %7s %11s\n", "case", "types", "perms"); - for (size_t i = 0; i < test_suite_count; i++) { - if (test_suite_skip(test_suites[i])) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + if (test_suite_skip(&test_suites[i])) { continue; } - test_define_suite(test_suites[i]); + test_define_suite(&test_suites[i]); - for (size_t j = 0; j < test_suites[i]->case_count; j++) { - if (test_case_skip(test_suites[i]->cases[j])) { + for (size_t j = 0; j < test_suites[i].case_count; j++) { + if (test_case_skip(&test_suites[i].cases[j])) { continue; } size_t perms = 0; size_t filtered = 0; - test_case_permcount(test_suites[i], test_suites[i]->cases[j], + test_case_permcount(&test_suites[i], &test_suites[i].cases[j], &perms, &filtered); - test_types_t types = test_suites[i]->cases[j]->types; + test_types_t types = test_suites[i].cases[j].types; char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", filtered, perms); @@ -307,7 +315,7 @@ static void list_cases(void) { (types & TEST_NORMAL) ? "n" : "", (types & TEST_REENTRANT) ? "r" : ""); printf("%-36s %7s %11s\n", - test_suites[i]->cases[j]->id, + test_suites[i].cases[j].id, type_buf, perm_buf); } @@ -315,39 +323,39 @@ static void list_cases(void) { } static void list_paths(void) { - for (size_t i = 0; i < test_suite_count; i++) { - if (test_suite_skip(test_suites[i])) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + if (test_suite_skip(&test_suites[i])) { continue; } - for (size_t j = 0; j < test_suites[i]->case_count; j++) { - if (test_case_skip(test_suites[i]->cases[j])) { + for (size_t j = 0; j < test_suites[i].case_count; j++) { + if (test_case_skip(&test_suites[i].cases[j])) { continue; } printf("%-36s %-36s\n", - test_suites[i]->cases[j]->id, - test_suites[i]->cases[j]->path); + test_suites[i].cases[j].id, + test_suites[i].cases[j].path); } } } static void list_defines(void) { - for (size_t i = 0; i < test_suite_count; i++) { - if (test_suite_skip(test_suites[i])) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + if (test_suite_skip(&test_suites[i])) { continue; } - test_define_suite(test_suites[i]); + test_define_suite(&test_suites[i]); - for (size_t j = 0; j < test_suites[i]->case_count; j++) { - if (test_case_skip(test_suites[i]->cases[j])) { + for (size_t j = 0; j < test_suites[i].case_count; j++) { + if (test_case_skip(&test_suites[i].cases[j])) { continue; } for (size_t perm = 0; perm < TEST_GEOMETRY_COUNT - * test_suites[i]->cases[j]->permutations; + * test_suites[i].cases[j].permutations; perm++) { if (test_perm_skip(perm)) { continue; @@ -356,25 +364,24 @@ static void list_defines(void) { // setup defines size_t case_perm = perm / TEST_GEOMETRY_COUNT; size_t geom_perm = perm % TEST_GEOMETRY_COUNT; - test_define_perm(test_suites[i], - test_suites[i]->cases[j], case_perm); + test_define_perm(&test_suites[i], + &test_suites[i].cases[j], case_perm); test_define_geometry(&test_geometries[geom_perm]); // print the case char id_buf[256]; - sprintf(id_buf, "%s#%zu", test_suites[i]->cases[j]->id, perm); + sprintf(id_buf, "%s#%zu", test_suites[i].cases[j].id, perm); printf("%-36s ", id_buf); // special case for the current geometry printf("GEOMETRY=%s ", test_geometries[geom_perm].name); // print each define - for (size_t k = 0; k < test_suites[i]->define_count; k++) { - if (test_suites[i]->cases[j]->defines - && test_suites[i]->cases[j] - ->defines[case_perm][k]) { + for (size_t k = 0; k < test_suites[i].define_count; k++) { + if (test_suites[i].cases[j].defines + && test_suites[i].cases[j].defines[case_perm][k]) { printf("%s=%jd ", - test_suites[i]->define_names[k], + test_suites[i].define_names[k], test_define(k)); } } @@ -419,21 +426,21 @@ static void list_defaults(void) { static void run(void) { size_t step = 0; - for (size_t i = 0; i < test_suite_count; i++) { - if (test_suite_skip(test_suites[i])) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + if (test_suite_skip(&test_suites[i])) { continue; } - test_define_suite(test_suites[i]); + test_define_suite(&test_suites[i]); - for (size_t j = 0; j < test_suites[i]->case_count; j++) { - if (test_case_skip(test_suites[i]->cases[j])) { + for (size_t j = 0; j < test_suites[i].case_count; j++) { + if (test_case_skip(&test_suites[i].cases[j])) { continue; } for (size_t perm = 0; perm < TEST_GEOMETRY_COUNT - * test_suites[i]->cases[j]->permutations; + * test_suites[i].cases[j].permutations; perm++) { if (test_perm_skip(perm)) { continue; @@ -447,15 +454,15 @@ static void run(void) { // setup defines size_t case_perm = perm / TEST_GEOMETRY_COUNT; size_t geom_perm = perm % TEST_GEOMETRY_COUNT; - test_define_perm(test_suites[i], - test_suites[i]->cases[j], case_perm); + test_define_perm(&test_suites[i], + &test_suites[i].cases[j], case_perm); test_define_geometry(&test_geometries[geom_perm]); // filter? - if (test_suites[i]->cases[j]->filter) { - if (!test_suites[i]->cases[j]->filter()) { + if (test_suites[i].cases[j].filter) { + if (!test_suites[i].cases[j].filter()) { printf("skipped %s#%zu\n", - test_suites[i]->cases[j]->id, + test_suites[i].cases[j].id, perm); continue; } @@ -494,11 +501,11 @@ static void run(void) { } // run the test - printf("running %s#%zu\n", test_suites[i]->cases[j]->id, perm); + printf("running %s#%zu\n", test_suites[i].cases[j].id, perm); - test_suites[i]->cases[j]->run(&cfg); + test_suites[i].cases[j].run(&cfg); - printf("finished %s#%zu\n", test_suites[i]->cases[j]->id, perm); + printf("finished %s#%zu\n", test_suites[i].cases[j].id, perm); // cleanup err = lfs_testbd_destroy(&cfg); diff --git a/runners/test_runner.h b/runners/test_runner.h index e0336379..64ad15d7 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -34,13 +34,10 @@ struct test_suite { const char *const *define_names; size_t define_count; - const struct test_case *const *cases; + const struct test_case *cases; size_t case_count; }; -extern const struct test_suite *test_suites[]; -extern const size_t test_suite_count; - // access generated test defines intmax_t test_predefine(size_t define); diff --git a/scripts/test.py b/scripts/test.py index 4110bbae..048b1811 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -342,8 +342,8 @@ def compile(**args): f.writeln('#endif') f.writeln() + # create case functions for case in suite.cases: - # create case functions if case.in_ is None: write_case_functions(f, suite, case) else: @@ -360,39 +360,8 @@ def compile(**args): % (suite.name, case.name)) f.writeln() - # create case struct - f.writeln('const struct test_case __test__%s__%s__case = {' - % (suite.name, case.name)) - f.writeln(4*' '+'.id = "%s",' % case.id()) - f.writeln(4*' '+'.name = "%s",' % case.name) - f.writeln(4*' '+'.path = "%s",' % case.path) - f.writeln(4*' '+'.types = %s,' - % ' | '.join(filter(None, [ - 'TEST_NORMAL' if case.normal else None, - 'TEST_REENTRANT' if case.reentrant else None]))) - f.writeln(4*' '+'.permutations = %d,' - % len(case.permutations)) - if case.defines: - f.writeln(4*' '+'.defines = __test__%s__%s__defines,' - % (suite.name, case.name)) - if suite.if_ is not None or case.if_ is not None: - f.writeln(4*' '+'.filter = __test__%s__%s__filter,' - % (suite.name, case.name)) - f.writeln(4*' '+'.run = __test__%s__%s__run,' - % (suite.name, case.name)) - f.writeln('};') - f.writeln() - - # create suite define names - if suite.defines: - f.writeln('const char *const __test__%s__define_names[] = {' - % suite.name) - for k in sorted(suite.defines): - f.writeln(4*' '+'"%s",' % k) - f.writeln('};') - f.writeln() - # create suite struct + f.writeln('__attribute__((section("_test_suites")))') f.writeln('const struct test_suite __test__%s__suite = {' % suite.name) f.writeln(4*' '+'.id = "%s",' % suite.id()) @@ -403,13 +372,34 @@ def compile(**args): 'TEST_NORMAL' if suite.normal else None, 'TEST_REENTRANT' if suite.reentrant else None]))) if suite.defines: - f.writeln(4*' '+'.define_names = __test__%s__define_names,' - % suite.name) + # create suite define names + f.writeln(4*' '+'.define_names = (const char *const[]){') + for k in sorted(suite.defines): + f.writeln(8*' '+'"%s",' % k) + f.writeln(4*' '+'},') f.writeln(4*' '+'.define_count = %d,' % len(suite.defines)) - f.writeln(4*' '+'.cases = (const struct test_case *const []){') + f.writeln(4*' '+'.cases = (const struct test_case[]){') for case in suite.cases: - f.writeln(8*' '+'&__test__%s__%s__case,' + # create case structs + f.writeln(8*' '+'{') + f.writeln(12*' '+'.id = "%s",' % case.id()) + f.writeln(12*' '+'.name = "%s",' % case.name) + f.writeln(12*' '+'.path = "%s",' % case.path) + f.writeln(12*' '+'.types = %s,' + % ' | '.join(filter(None, [ + 'TEST_NORMAL' if case.normal else None, + 'TEST_REENTRANT' if case.reentrant else None]))) + f.writeln(12*' '+'.permutations = %d,' + % len(case.permutations)) + if case.defines: + f.writeln(12*' '+'.defines = __test__%s__%s__defines,' + % (suite.name, case.name)) + if suite.if_ is not None or case.if_ is not None: + f.writeln(12*' '+'.filter = __test__%s__%s__filter,' + % (suite.name, case.name)) + f.writeln(12*' '+'.run = __test__%s__%s__run,' % (suite.name, case.name)) + f.writeln(8*' '+'},') f.writeln(4*' '+'},') f.writeln(4*' '+'.case_count = %d,' % len(suite.cases)) f.writeln('};') @@ -456,19 +446,6 @@ def compile(**args): f.writeln('#endif') f.writeln() - # add suite info to test_runner.c - if args['source'] == 'runners/test_runner.c': - f.writeln() - for suite in suites: - f.writeln('extern const struct test_suite ' - '__test__%s__suite;' % suite.name) - f.writeln('const struct test_suite *test_suites[] = {') - for suite in suites: - f.writeln(4*' '+'&__test__%s__suite,' % suite.name) - f.writeln('};') - f.writeln('const size_t test_suite_count = %d;' - % len(suites)) - def runner(**args): cmd = args['runner'].copy() cmd.extend(args.get('test_ids')) From 161611566221cd7a147c45d070f7d1ca4914ba43 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 14 May 2022 23:37:19 -0500 Subject: [PATCH 15/81] Fix test.py hang on ctrl-C, cleanup TODOs A small mistake in test.py's control flow meant the failing test job would succesfully kill all other test jobs, but then humorously start up a new process to continue testing. --- scripts/code.py | 22 ++++++++++++---------- scripts/coverage.py | 22 ++++++++++++---------- scripts/data.py | 22 ++++++++++++---------- scripts/stack.py | 22 ++++++++++++---------- scripts/structs.py | 22 ++++++++++++---------- scripts/summary.py | 20 +++++++++++--------- scripts/test.py | 3 +-- 7 files changed, 72 insertions(+), 61 deletions(-) diff --git a/scripts/code.py b/scripts/code.py index b394e9cd..66502fa9 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -17,6 +17,15 @@ import collections as co OBJ_PATHS = ['*.o'] +def openio(path, mode='r'): + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) + def collect(paths, **args): results = co.defaultdict(lambda: 0) pattern = re.compile( @@ -64,15 +73,6 @@ def collect(paths, **args): return flat_results def main(**args): - def openio(path, mode='r'): - if path == '-': - if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), 'r') - else: - return os.fdopen(os.dup(sys.stdout.fileno()), 'w') - else: - return open(path, mode) - # find sizes if not args.get('use', None): # find .o files @@ -281,4 +281,6 @@ if __name__ == "__main__": parser.add_argument('--build-dir', help="Specify the relative build directory. Used to map object files \ to the correct source files.") - sys.exit(main(**vars(parser.parse_args()))) + sys.exit(main(**{k: v + for k, v in vars(parser.parse_args()).items() + if v is not None})) diff --git a/scripts/coverage.py b/scripts/coverage.py index b3a90ed2..e4b754af 100755 --- a/scripts/coverage.py +++ b/scripts/coverage.py @@ -12,6 +12,15 @@ import bisect as b INFO_PATHS = ['tests/*.toml.info'] +def openio(path, mode='r'): + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) + def collect(paths, **args): file = None funcs = [] @@ -66,15 +75,6 @@ def collect(paths, **args): def main(**args): - def openio(path, mode='r'): - if path == '-': - if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), 'r') - else: - return os.fdopen(os.dup(sys.stdout.fileno()), 'w') - else: - return open(path, mode) - # find coverage if not args.get('use'): # find *.info files @@ -320,4 +320,6 @@ if __name__ == "__main__": parser.add_argument('--build-dir', help="Specify the relative build directory. Used to map object files \ to the correct source files.") - sys.exit(main(**vars(parser.parse_args()))) + sys.exit(main(**{k: v + for k, v in vars(parser.parse_args()).items() + if v is not None})) diff --git a/scripts/data.py b/scripts/data.py index 4b8e00da..efef4ff1 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -17,6 +17,15 @@ import collections as co OBJ_PATHS = ['*.o'] +def openio(path, mode='r'): + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) + def collect(paths, **args): results = co.defaultdict(lambda: 0) pattern = re.compile( @@ -63,15 +72,6 @@ def collect(paths, **args): return flat_results def main(**args): - def openio(path, mode='r'): - if path == '-': - if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), 'r') - else: - return os.fdopen(os.dup(sys.stdout.fileno()), 'w') - else: - return open(path, mode) - # find sizes if not args.get('use', None): # find .o files @@ -280,4 +280,6 @@ if __name__ == "__main__": parser.add_argument('--build-dir', help="Specify the relative build directory. Used to map object files \ to the correct source files.") - sys.exit(main(**vars(parser.parse_args()))) + sys.exit(main(**{k: v + for k, v in vars(parser.parse_args()).items() + if v is not None})) diff --git a/scripts/stack.py b/scripts/stack.py index 0c652d8d..6cdfc46f 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -15,6 +15,15 @@ import math as m CI_PATHS = ['*.ci'] +def openio(path, mode='r'): + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) + def collect(paths, **args): # parse the vcg format k_pattern = re.compile('([a-z]+)\s*:', re.DOTALL) @@ -116,15 +125,6 @@ def collect(paths, **args): return flat_results def main(**args): - def openio(path, mode='r'): - if path == '-': - if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), 'r') - else: - return os.fdopen(os.dup(sys.stdout.fileno()), 'w') - else: - return open(path, mode) - # find sizes if not args.get('use', None): # find .ci files @@ -427,4 +427,6 @@ if __name__ == "__main__": parser.add_argument('--build-dir', help="Specify the relative build directory. Used to map object files \ to the correct source files.") - sys.exit(main(**vars(parser.parse_args()))) + sys.exit(main(**{k: v + for k, v in vars(parser.parse_args()).items() + if v is not None})) diff --git a/scripts/structs.py b/scripts/structs.py index e8d7193e..de266b87 100755 --- a/scripts/structs.py +++ b/scripts/structs.py @@ -15,6 +15,15 @@ import collections as co OBJ_PATHS = ['*.o'] +def openio(path, mode='r'): + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) + def collect(paths, **args): decl_pattern = re.compile( '^\s+(?P[0-9]+)' @@ -115,15 +124,6 @@ def collect(paths, **args): def main(**args): - def openio(path, mode='r'): - if path == '-': - if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), 'r') - else: - return os.fdopen(os.dup(sys.stdout.fileno()), 'w') - else: - return open(path, mode) - # find sizes if not args.get('use', None): # find .o files @@ -328,4 +328,6 @@ if __name__ == "__main__": parser.add_argument('--build-dir', help="Specify the relative build directory. Used to map object files \ to the correct source files.") - sys.exit(main(**vars(parser.parse_args()))) + sys.exit(main(**{k: v + for k, v in vars(parser.parse_args()).items() + if v is not None})) diff --git a/scripts/summary.py b/scripts/summary.py index 7ce769bf..2aab274d 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -57,16 +57,16 @@ FIELDS = [ ] -def main(**args): - def openio(path, mode='r'): - if path == '-': - if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), 'r') - else: - return os.fdopen(os.dup(sys.stdout.fileno()), 'w') +def openio(path, mode='r'): + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') else: - return open(path, mode) + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) +def main(**args): # find results results = co.defaultdict(lambda: {}) for path in args.get('csv_paths', '-'): @@ -276,4 +276,6 @@ if __name__ == "__main__": help="Show file-level calls.") parser.add_argument('-Y', '--summary', action='store_true', help="Only show the totals.") - sys.exit(main(**vars(parser.parse_args()))) + sys.exit(main(**{k: v + for k, v in vars(parser.parse_args()).items() + if v is not None})) diff --git a/scripts/test.py b/scripts/test.py index 048b1811..9d94f006 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -49,7 +49,6 @@ def testcase(path): _, case, *_ = path.split('#', 2) return '%s#%s' % (testsuite(path), case) -# TODO move this out in other files def openio(path, mode='r'): if path == '-': if 'r' in mode: @@ -728,6 +727,7 @@ def run_stage(name, runner_, **args): # stop other tests for child in children.copy(): child.kill() + break # parallel jobs? @@ -998,7 +998,6 @@ if __name__ == "__main__": help="Source file to compile, possibly injecting internal tests.") comp_parser.add_argument('-o', '--output', help="Output file.") - # TODO apply this to other scripts? sys.exit(main(**{k: v for k, v in vars(parser.parse_args()).items() if v is not None})) From 2b11f2b426deb4df8f63abfbcf2a9e96ca60a5cc Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 15 May 2022 16:18:36 -0500 Subject: [PATCH 16/81] Tweaked generation of .cgi files, error code for recursion in stack.py GCC is a bit annoying here, it can't generate .cgi files without generating the related .o files, though I suppose the alternative risks duplicating a large amount of compilation work (littlefs is really a small project). Previously we rebuilt the .o files anytime we needed .cgi files (callgraph info used for stack.py). This changes it so we always built .cgi files as a side-effect of compilation. This is similar to the .d file generation, though may be annoying if the system cc doesn't support --callgraph-info. --- Makefile | 19 ++++--------------- scripts/stack.py | 7 +++++++ 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 9cc37706..9d5900a9 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,6 @@ override TESTFLAGS += -b override TESTFLAGS += $(filter -j%,$(MAKEFLAGS)) ifdef VERBOSE override TESTFLAGS += -v -override CALLSFLAGS += -v override CODEFLAGS += -v override DATAFLAGS += -v override STACKFLAGS += -v @@ -77,7 +76,6 @@ override TESTFLAGS += --coverage endif ifdef BUILDDIR override TESTFLAGS += --build-dir="$(BUILDDIR:/=)" -override CALLSFLAGS += --build-dir="$(BUILDDIR:/=)" override CODEFLAGS += --build-dir="$(BUILDDIR:/=)" override DATAFLAGS += --build-dir="$(BUILDDIR:/=)" override STACKFLAGS += --build-dir="$(BUILDDIR:/=)" @@ -108,10 +106,6 @@ size: $(OBJ) tags: $(CTAGS) --totals --c-types=+p $(shell find -H -name '*.h') $(SRC) -.PHONY: calls -calls: $(CGI) - ./scripts/calls.py $^ $(CALLSFLAGS) - .PHONY: test_runner test_runner: $(BUILDDIR)runners/test_runner @@ -172,19 +166,14 @@ $(BUILDDIR)lfs.csv: $(OBJ) $(CGI) $(BUILDDIR)runners/test_runner: $(TEST_TAOBJ) $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ -$(BUILDDIR)%.o: %.c - $(CC) -c -MMD $(CFLAGS) $< -o $@ +# our main build rule generates .o, .d, and .ci files, the latter +# used for stack analysis +$(BUILDDIR)%.o $(BUILDDIR)%.ci: %.c + $(CC) -c -MMD -fcallgraph-info=su $(CFLAGS) $< -o $(BUILDDIR)$*.o $(BUILDDIR)%.s: %.c $(CC) -S $(CFLAGS) $< -o $@ -# gcc depends on the output file for intermediate file names, so -# we can't omit to .o output. We also need to serialize with the -# normal .o rule because otherwise we can end up with multiprocess -# problems with two instances of gcc modifying the same .o -$(BUILDDIR)%.ci: %.c | $(BUILDDIR)%.o - $(CC) -c -MMD -fcallgraph-info=su $(CFLAGS) $< -o $| - $(BUILDDIR)%.a.c: %.c ./scripts/explode_asserts.py $< -o $@ diff --git a/scripts/stack.py b/scripts/stack.py index 6cdfc46f..b0894b84 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -384,6 +384,11 @@ def main(**args): print_entries(by='name') print_totals() + # catch recursion + if args.get('error_on_recursion') and any( + m.isinf(limit) for _, _, _, limit, _ in results): + sys.exit(2) + if __name__ == "__main__": import argparse @@ -424,6 +429,8 @@ if __name__ == "__main__": help="Show file-level calls.") parser.add_argument('-Y', '--summary', action='store_true', help="Only show the total stack size.") + parser.add_argument('-e', '--error-on-recursion', action='store_true', + help="Error if any functions are recursive.") parser.add_argument('--build-dir', help="Specify the relative build directory. Used to map object files \ to the correct source files.") From 4a7e94fb1572f3b2cc376ae2b8a1dfe88f828b6d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 15 May 2022 23:03:58 -0500 Subject: [PATCH 17/81] Reimplemented coverage.py, using only gcov and with line+branch coverage This also adds coverage support to the new test framework, which due to reduction in scope, no longer needs aggregation and can be much simpler. Really all we need to do is pass --coverage to GCC, which builds its .gcda files during testing in a multi-process-safe manner. The addition of branch coverage leverages information that was available in both lcov and gcov. This was made easier with the addition of the --json-format to gcov in GCC 9.0, however the lax backwards compatibility for gcov's intermediary options is a bit concerning. Hopefully --json-format sticks around for a while. --- Makefile | 23 ++- scripts/coverage.py | 490 +++++++++++++++++++++++++------------------- scripts/stack.py | 8 +- scripts/test.py | 4 +- 4 files changed, 300 insertions(+), 225 deletions(-) diff --git a/Makefile b/Makefile index 9d5900a9..2e161b22 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,7 @@ OBJ := $(SRC:%.c=$(BUILDDIR)%.o) DEP := $(SRC:%.c=$(BUILDDIR)%.d) ASM := $(SRC:%.c=$(BUILDDIR)%.s) CGI := $(SRC:%.c=$(BUILDDIR)%.ci) +TAGCDA := $(SRC:%.c=$(BUILDDIR)%.t.a.gcda) TESTS ?= $(wildcard tests/*.toml) TEST_SRC ?= $(SRC) \ @@ -40,6 +41,8 @@ TEST_TSRC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) $(TEST_SRC:%.c=$(BUILDDIR)%.t.c) TEST_TASRC := $(TEST_TSRC:%.t.c=%.t.a.c) TEST_TAOBJ := $(TEST_TASRC:%.t.a.c=%.t.a.o) TEST_TADEP := $(TEST_TASRC:%.t.a.c=%.t.a.d) +TEST_TAGCNO := $(TEST_TASRC:%.t.a.c=%.t.a.gcno) +TEST_TAGCDA := $(TEST_TASRC:%.t.a.c=%.t.a.gcda) ifdef DEBUG override CFLAGS += -O0 @@ -106,15 +109,17 @@ size: $(OBJ) tags: $(CTAGS) --totals --c-types=+p $(shell find -H -name '*.h') $(SRC) -.PHONY: test_runner -test_runner: $(BUILDDIR)runners/test_runner +.PHONY: test-runner +test-runner: override CFLAGS+=--coverage +test-runner: $(BUILDDIR)runners/test_runner .PHONY: test -test: test_runner +test: test-runner + rm -f $(TEST_TAGCDA) ./scripts/test.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS) -.PHONY: test_list -test_list: test_runner +.PHONY: test-list +test-list: test-runner ./scripts/test.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS) -l .PHONY: code @@ -134,8 +139,8 @@ structs: $(OBJ) ./scripts/structs.py $^ -S $(STRUCTSFLAGS) .PHONY: coverage -coverage: - ./scripts/coverage.py $(BUILDDIR)tests/*.toml.info -s $(COVERAGEFLAGS) +coverage: $(TAGCDA) + ./scripts/coverage.py $^ -s $(COVERAGEFLAGS) .PHONY: summary summary: $(BUILDDIR)lfs.csv @@ -194,10 +199,12 @@ clean: rm -f $(BUILDDIR)lfs.csv rm -f $(BUILDDIR)runners/test_runner rm -f $(OBJ) - rm -f $(CGI) rm -f $(DEP) rm -f $(ASM) + rm -f $(CGI) rm -f $(TEST_TSRC) rm -f $(TEST_TASRC) rm -f $(TEST_TAOBJ) rm -f $(TEST_TADEP) + rm -f $(TEST_TAGCNO) + rm -f $(TEST_TAGCDA) diff --git a/scripts/coverage.py b/scripts/coverage.py index e4b754af..fbaf9f8a 100755 --- a/scripts/coverage.py +++ b/scripts/coverage.py @@ -1,16 +1,25 @@ #!/usr/bin/env python3 # -# Parse and report coverage info from .info files generated by lcov +# Script to find test coverage. Basically just a big wrapper around gcov with +# some extra conveniences for comparing builds. Heavily inspired by Linux's +# Bloat-O-Meter. # -import os -import glob -import csv -import re + import collections as co -import bisect as b +import csv +import glob +import itertools as it +import json +import os +import re +import shlex +import subprocess as sp + +# TODO use explode_asserts to avoid counting assert branches? +# TODO use dwarf=info to find functions for inline functions? -INFO_PATHS = ['tests/*.toml.info'] +GCDA_PATHS = ['*.gcda'] def openio(path, mode='r'): if path == '-': @@ -21,114 +30,214 @@ def openio(path, mode='r'): else: return open(path, mode) -def collect(paths, **args): - file = None - funcs = [] - lines = co.defaultdict(lambda: 0) - pattern = re.compile( - '^(?PSF:/?(?P.*))$' - '|^(?PFN:(?P[0-9]*),(?P.*))$' - '|^(?PDA:(?P[0-9]*),(?P[0-9]*))$') - for path in paths: - with open(path) as f: - for line in f: - m = pattern.match(line) - if m and m.group('file'): - file = m.group('file_name') - elif m and file and m.group('func'): - funcs.append((file, int(m.group('func_lineno')), - m.group('func_name'))) - elif m and file and m.group('line'): - lines[(file, int(m.group('line_lineno')))] += ( - int(m.group('line_hits'))) +class CoverageResult(co.namedtuple('CoverageResult', + 'line_hits,line_count,branch_hits,branch_count')): + __slots__ = () + def __new__(cls, line_hits=0, line_count=0, branch_hits=0, branch_count=0): + return super().__new__(cls, + int(line_hits), + int(line_count), + int(branch_hits), + int(branch_count)) - # map line numbers to functions - funcs.sort() - def func_from_lineno(file, lineno): - i = b.bisect(funcs, (file, lineno)) - if i and funcs[i-1][0] == file: - return funcs[i-1][2] + def __add__(self, other): + return self.__class__( + self.line_hits + other.line_hits, + self.line_count + other.line_count, + self.branch_hits + other.branch_hits, + self.branch_count + other.branch_count) + + def __sub__(self, other): + return CoverageDiff(other, self) + + def key(self, **args): + line_ratio = (self.line_hits/self.line_count + if self.line_count else -1) + branch_ratio = (self.branch_hits/self.branch_count + if self.branch_count else -1) + + if args.get('line_sort'): + return (-line_ratio, -branch_ratio) + elif args.get('reverse_line_sort'): + return (+line_ratio, +branch_ratio) + elif args.get('branch_sort'): + return (-branch_ratio, -line_ratio) + elif args.get('reverse_branch_sort'): + return (+branch_ratio, +line_ratio) else: return None - # reduce to function info - reduced_funcs = co.defaultdict(lambda: (0, 0)) - for (file, line_lineno), line_hits in lines.items(): - func = func_from_lineno(file, line_lineno) - if not func: - continue - hits, count = reduced_funcs[(file, func)] - reduced_funcs[(file, func)] = (hits + (line_hits > 0), count + 1) + _header = '%19s %19s' % ('hits/line', 'hits/branch') + def __str__(self): + return '%11s %7s %11s %7s' % ( + '%d/%d' % (self.line_hits, self.line_count) + if self.line_count else '-', + '%.1f%%' % (100*self.line_hits/self.line_count) + if self.line_count else '-', + '%d/%d' % (self.branch_hits, self.branch_count) + if self.branch_count else '-', + '%.1f%%' % (100*self.branch_hits/self.branch_count) + if self.branch_count else '-') - results = [] - for (file, func), (hits, count) in reduced_funcs.items(): - # discard internal/testing functions (test_* injected with - # internal testing) - if not args.get('everything'): - if func.startswith('__') or func.startswith('test_'): +class CoverageDiff(co.namedtuple('CoverageDiff', 'old,new')): + __slots__ = () + + def line_hits_diff(self): + return self.new.line_hits - self.old.line_hits + + def line_count_diff(self): + return self.new.line_count - self.old.line_count + + def line_ratio(self): + return ((self.new.line_hits/self.new.line_count + if self.new.line_count else 1.0) + - (self.old.line_hits / self.old.line_count + if self.old.line_count else 1.0)) + + def branch_hits_diff(self): + return self.new.branch_hits - self.old.branch_hits + + def branch_count_diff(self): + return self.new.branch_count - self.old.branch_count + + def branch_ratio(self): + return ((self.new.branch_hits/self.new.branch_count + if self.new.branch_count else 1.0) + - (self.old.branch_hits / self.old.branch_count + if self.old.branch_count else 1.0)) + + def key(self, **args): + new_key = self.new.key(**args) + line_ratio = self.line_ratio() + branch_ratio = self.branch_ratio() + if new_key is not None: + return new_key + else: + return (-line_ratio, -branch_ratio) + + def __bool__(self): + return bool(self.line_ratio() or self.branch_ratio()) + + _header = '%23s %23s %23s' % ('old', 'new', 'diff') + def __str__(self): + line_ratio = self.line_ratio() + branch_ratio = self.branch_ratio() + return '%11s %11s %11s %11s %11s %11s%-10s%s' % ( + '%d/%d' % (self.old.line_hits, self.old.line_count) + if self.old.line_count else '-', + '%d/%d' % (self.old.branch_hits, self.old.branch_count) + if self.old.branch_count else '-', + '%d/%d' % (self.new.line_hits, self.new.line_count) + if self.new.line_count else '-', + '%d/%d' % (self.new.branch_hits, self.new.branch_count) + if self.new.branch_count else '-', + '%+d/%+d' % (self.line_hits_diff(), self.line_count_diff()), + '%+d/%+d' % (self.branch_hits_diff(), self.branch_count_diff()), + ' (%+.1f%%)' % (100*line_ratio) if line_ratio else '', + ' (%+.1f%%)' % (100*branch_ratio) if branch_ratio else '') + + +def collect(paths, **args): + results = {} + for path in paths: + # map to source file + src_path = re.sub('\.t\.a\.gcda$', '.c', path) + # TODO test this + if args.get('build_dir'): + src_path = re.sub('%s/*' % re.escape(args['build_dir']), '', + src_path) + + # get coverage info through gcov's json output + # note, gcov-tool may contain extra args + cmd = args['gcov_tool'] + ['-b', '-t', '--json-format', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace') + data = json.load(proc.stdout) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + # collect line/branch coverage + for file in data['files']: + if file['file'] != src_path: continue - # discard .8449 suffixes created by optimizer - func = re.sub('\.[0-9]+', '', func) - results.append((file, func, hits, count)) - return results + for line in file['lines']: + func = line.get('function_name', '(inlined)') + # discard internal function (this includes injected test cases) + if not args.get('everything'): + if func.startswith('__'): + continue + results[(src_path, func, line['line_number'])] = ( + line['count'], + CoverageResult( + line_hits=1 if line['count'] > 0 else 0, + line_count=1, + branch_hits=sum( + 1 if branch['count'] > 0 else 0 + for branch in line['branches']), + branch_count=len(line['branches']))) + + # merge into functions, since this is what other scripts use + func_results = co.defaultdict(lambda: CoverageResult()) + for (file, func, _), (_, result) in results.items(): + func_results[(file, func)] += result + + return func_results, results def main(**args): - # find coverage - if not args.get('use'): - # find *.info files + # find sizes + if not args.get('use', None): + # find .gcda files paths = [] - for path in args['info_paths']: + for path in args['gcda_paths']: if os.path.isdir(path): - path = path + '/*.gcov' + path = path + '/*.gcda' for path in glob.glob(path): paths.append(path) if not paths: - print('no .info files found in %r?' % args['info_paths']) + print('no .gcda files found in %r?' % args['gcda_paths']) sys.exit(-1) - results = collect(paths, **args) + # TODO consistent behavior between this and stack.py for deps? + results, line_results = collect(paths, **args) else: with openio(args['use']) as f: r = csv.DictReader(f) - results = [ - ( result['file'], - result['name'], - int(result['coverage_hits']), - int(result['coverage_count'])) + results = { + (result['file'], result['name']): CoverageResult(**{ + k: v for k, v in result.items() + if k in CoverageResult._fields}) for result in r - if result.get('coverage_hits') not in {None, ''} - if result.get('coverage_count') not in {None, ''}] - - total_hits, total_count = 0, 0 - for _, _, hits, count in results: - total_hits += hits - total_count += count + if all(result.get(f) not in {None, ''} + for f in CoverageResult._fields)} # find previous results? if args.get('diff'): try: with openio(args['diff']) as f: r = csv.DictReader(f) - prev_results = [ - ( result['file'], - result['name'], - int(result['coverage_hits']), - int(result['coverage_count'])) + prev_results = { + (result['file'], result['name']): CoverageResult(**{ + k: v for k, v in result.items() + if k in CoverageResult._fields}) for result in r - if result.get('coverage_hits') not in {None, ''} - if result.get('coverage_count') not in {None, ''}] + if all(result.get(f) not in {None, ''} + for f in CoverageResult._fields)} except FileNotFoundError: prev_results = [] - prev_total_hits, prev_total_count = 0, 0 - for _, _, hits, count in prev_results: - prev_total_hits += hits - prev_total_count += count - # write results to CSV if args.get('output'): merged_results = co.defaultdict(lambda: {}) @@ -142,163 +251,113 @@ def main(**args): for result in r: file = result.pop('file', '') func = result.pop('name', '') - result.pop('coverage_hits', None) - result.pop('coverage_count', None) + for f in CoverageResult._fields: + result.pop(f, None) merged_results[(file, func)] = result other_fields = result.keys() except FileNotFoundError: pass - for file, func, hits, count in results: - merged_results[(file, func)]['coverage_hits'] = hits - merged_results[(file, func)]['coverage_count'] = count + for (file, func), result in results.items(): + for f in CoverageResult._fields: + merged_results[(file, func)][f] = getattr(result, f) with openio(args['output'], 'w') as f: - w = csv.DictWriter(f, ['file', 'name', *other_fields, 'coverage_hits', 'coverage_count']) + w = csv.DictWriter(f, ['file', 'name', + *other_fields, *CoverageResult._fields]) w.writeheader() for (file, func), result in sorted(merged_results.items()): w.writerow({'file': file, 'name': func, **result}) # print results - def dedup_entries(results, by='name'): - entries = co.defaultdict(lambda: (0, 0)) - for file, func, hits, count in results: - entry = (file if by == 'file' else func) - entry_hits, entry_count = entries[entry] - entries[entry] = (entry_hits + hits, entry_count + count) - return entries - - def diff_entries(olds, news): - diff = co.defaultdict(lambda: (0, 0, 0, 0, 0, 0, 0)) - for name, (new_hits, new_count) in news.items(): - diff[name] = ( - 0, 0, - new_hits, new_count, - new_hits, new_count, - (new_hits/new_count if new_count else 1.0) - 1.0) - for name, (old_hits, old_count) in olds.items(): - _, _, new_hits, new_count, _, _, _ = diff[name] - diff[name] = ( - old_hits, old_count, - new_hits, new_count, - new_hits-old_hits, new_count-old_count, - ((new_hits/new_count if new_count else 1.0) - - (old_hits/old_count if old_count else 1.0))) - return diff - - def sorted_entries(entries): - if args.get('coverage_sort'): - return sorted(entries, key=lambda x: (-(x[1][0]/x[1][1] if x[1][1] else -1), x)) - elif args.get('reverse_coverage_sort'): - return sorted(entries, key=lambda x: (+(x[1][0]/x[1][1] if x[1][1] else -1), x)) + def print_header(by): + if by == 'total': + entry = lambda k: 'TOTAL' + elif by == 'file': + entry = lambda k: k[0] else: - return sorted(entries) - - def sorted_diff_entries(entries): - if args.get('coverage_sort'): - return sorted(entries, key=lambda x: (-(x[1][2]/x[1][3] if x[1][3] else -1), x)) - elif args.get('reverse_coverage_sort'): - return sorted(entries, key=lambda x: (+(x[1][2]/x[1][3] if x[1][3] else -1), x)) - else: - return sorted(entries, key=lambda x: (-x[1][6], x)) - - def print_header(by=''): - if not args.get('diff'): - print('%-36s %19s' % (by, 'hits/line')) - else: - print('%-36s %19s %19s %11s' % (by, 'old', 'new', 'diff')) - - def print_entry(name, hits, count): - print("%-36s %11s %7s" % (name, - '%d/%d' % (hits, count) - if count else '-', - '%.1f%%' % (100*hits/count) - if count else '-')) - - def print_diff_entry(name, - old_hits, old_count, - new_hits, new_count, - diff_hits, diff_count, - ratio): - print("%-36s %11s %7s %11s %7s %11s%s" % (name, - '%d/%d' % (old_hits, old_count) - if old_count else '-', - '%.1f%%' % (100*old_hits/old_count) - if old_count else '-', - '%d/%d' % (new_hits, new_count) - if new_count else '-', - '%.1f%%' % (100*new_hits/new_count) - if new_count else '-', - '%+d/%+d' % (diff_hits, diff_count), - ' (%+.1f%%)' % (100*ratio) if ratio else '')) - - def print_entries(by='name'): - entries = dedup_entries(results, by=by) + entry = lambda k: k[1] if not args.get('diff'): - print_header(by=by) - for name, (hits, count) in sorted_entries(entries.items()): - print_entry(name, hits, count) + print('%-36s %s' % (by, CoverageResult._header)) else: - prev_entries = dedup_entries(prev_results, by=by) - diff = diff_entries(prev_entries, entries) - print_header(by='%s (%d added, %d removed)' % (by, - sum(1 for _, old, _, _, _, _, _ in diff.values() if not old), - sum(1 for _, _, _, new, _, _, _ in diff.values() if not new))) - for name, ( - old_hits, old_count, - new_hits, new_count, - diff_hits, diff_count, ratio) in sorted_diff_entries( - diff.items()): - if ratio or args.get('all'): - print_diff_entry(name, - old_hits, old_count, - new_hits, new_count, - diff_hits, diff_count, - ratio) + old = {entry(k) for k in results.keys()} + new = {entry(k) for k in prev_results.keys()} + print('%-36s %s' % ( + '%s (%d added, %d removed)' % (by, + sum(1 for k in new if k not in old), + sum(1 for k in old if k not in new)) + if by else '', + CoverageDiff._header)) - def print_totals(): - if not args.get('diff'): - print_entry('TOTAL', total_hits, total_count) + def print_entries(by): + if by == 'total': + entry = lambda k: 'TOTAL' + elif by == 'file': + entry = lambda k: k[0] else: - ratio = ((total_hits/total_count - if total_count else 1.0) - - (prev_total_hits/prev_total_count - if prev_total_count else 1.0)) - print_diff_entry('TOTAL', - prev_total_hits, prev_total_count, - total_hits, total_count, - total_hits-prev_total_hits, total_count-prev_total_count, - ratio) + entry = lambda k: k[1] + + entries = co.defaultdict(lambda: CoverageResult()) + for k, result in results.items(): + entries[entry(k)] += result + + if not args.get('diff'): + for name, result in sorted(entries.items(), + key=lambda p: (p[1].key(**args), p)): + print('%-36s %s' % (name, result)) + else: + prev_entries = co.defaultdict(lambda: CoverageResult()) + for k, result in prev_results.items(): + prev_entries[entry(k)] += result + + diff_entries = {name: entries[name] - prev_entries[name] + for name in (entries.keys() | prev_entries.keys())} + + for name, diff in sorted(diff_entries.items(), + key=lambda p: (p[1].key(**args), p)): + if diff or args.get('all'): + print('%-36s %s' % (name, diff)) if args.get('quiet'): pass elif args.get('summary'): - print_header() - print_totals() + print_header('') + print_entries('total') elif args.get('files'): - print_entries(by='file') - print_totals() + print_header('file') + print_entries('file') + print_entries('total') else: - print_entries(by='name') - print_totals() + print_header('function') + print_entries('function') + print_entries('total') + + # catch lack of coverage + if args.get('error_on_lines') and any( + r.line_hits < r.line_count for r in results.values()): + sys.exit(2) + elif args.get('error_on_branches') and any( + r.branch_hits < r.branch_count for r in results.values()): + sys.exit(3) + if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Parse and report coverage info from .info files \ - generated by lcov") - parser.add_argument('info_paths', nargs='*', default=INFO_PATHS, - help="Description of where to find *.info files. May be a directory \ - or list of paths. *.info files will be merged to show the total \ - coverage. Defaults to %r." % INFO_PATHS) + description="Find coverage info after running tests.") + parser.add_argument('gcda_paths', nargs='*', default=GCDA_PATHS, + help="Description of where to find *.gcda files. May be a directory \ + or a list of paths. Defaults to %r." % GCDA_PATHS) parser.add_argument('-v', '--verbose', action='store_true', help="Output commands that run behind the scenes.") + parser.add_argument('-q', '--quiet', action='store_true', + help="Don't show anything, useful with -o.") parser.add_argument('-o', '--output', help="Specify CSV file to store results.") parser.add_argument('-u', '--use', - help="Don't do any work, instead use this CSV file.") + help="Don't compile and find code sizes, instead use this CSV file.") parser.add_argument('-d', '--diff', help="Specify CSV file to diff code size against.") parser.add_argument('-m', '--merge', @@ -307,16 +366,25 @@ if __name__ == "__main__": help="Show all functions, not just the ones that changed.") parser.add_argument('-A', '--everything', action='store_true', help="Include builtin and libc specific symbols.") - parser.add_argument('-s', '--coverage-sort', action='store_true', - help="Sort by coverage.") - parser.add_argument('-S', '--reverse-coverage-sort', action='store_true', - help="Sort by coverage, but backwards.") + parser.add_argument('-s', '--line-sort', action='store_true', + help="Sort by line coverage.") + parser.add_argument('-S', '--reverse-line-sort', action='store_true', + help="Sort by line coverage, but backwards.") + parser.add_argument('--branch-sort', action='store_true', + help="Sort by branch coverage.") + parser.add_argument('--reverse-branch-sort', action='store_true', + help="Sort by branch coverage, but backwards.") parser.add_argument('-F', '--files', action='store_true', help="Show file-level coverage.") parser.add_argument('-Y', '--summary', action='store_true', help="Only show the total coverage.") - parser.add_argument('-q', '--quiet', action='store_true', - help="Don't show anything, useful with -o.") + parser.add_argument('-e', '--error-on-lines', action='store_true', + help="Error if any lines are not covered.") + parser.add_argument('-E', '--error-on-branches', action='store_true', + help="Error if any branches are not covered.") + parser.add_argument('--gcov-tool', default=['gcov'], + type=lambda x: x.split(), + help="Path to the gcov tool to use.") parser.add_argument('--build-dir', help="Specify the relative build directory. Used to map object files \ to the correct source files.") diff --git a/scripts/stack.py b/scripts/stack.py index b0894b84..9235b518 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -414,14 +414,14 @@ if __name__ == "__main__": help="Show all functions, not just the ones that changed.") parser.add_argument('-A', '--everything', action='store_true', help="Include builtin and libc specific symbols.") - parser.add_argument('-s', '--limit-sort', action='store_true', - help="Sort by stack limit.") - parser.add_argument('-S', '--reverse-limit-sort', action='store_true', - help="Sort by stack limit, but backwards.") parser.add_argument('--frame-sort', action='store_true', help="Sort by stack frame size.") parser.add_argument('--reverse-frame-sort', action='store_true', help="Sort by stack frame size, but backwards.") + parser.add_argument('-s', '--limit-sort', action='store_true', + help="Sort by stack limit.") + parser.add_argument('-S', '--reverse-limit-sort', action='store_true', + help="Sort by stack limit, but backwards.") parser.add_argument('-L', '--depth', default=0, type=lambda x: int(x, 0), nargs='?', const=float('inf'), help="Depth of dependencies to show.") diff --git a/scripts/test.py b/scripts/test.py index 9d94f006..281265eb 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -985,11 +985,11 @@ if __name__ == "__main__": test_parser.add_argument('--gdb-main', action='store_true', help="Drop into gdb on test failure but stop at the beginning \ of main.") + test_parser.add_argument('--exec', default=[], type=lambda e: e.split(), + help="Run under another executable.") test_parser.add_argument('--valgrind', action='store_true', help="Run under Valgrind to find memory errors. Implicitly sets \ --isolate.") - test_parser.add_argument('--exec', default=[], type=lambda e: e.split(), - help="Run under another executable.") # compilation flags comp_parser = parser.add_argument_group('compilation options') comp_parser.add_argument('-c', '--compile', action='store_true', From 5b0a6d4747a4a7eba1320f8e936a7240064595f2 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 21 May 2022 16:46:25 -0500 Subject: [PATCH 18/81] Reworked scripts to move field details into classes These scripts can't easily share the common logic, but separating field details from the print/merge/csv logic should make the common part of these scripts much easier to create/modify going forward. This also tweaked the behavior of summary.py slightly. --- .gitignore | 2 + Makefile | 6 - scripts/code.py | 256 ++++++++++---------- scripts/coverage.py | 271 +++++++++++---------- scripts/data.py | 255 ++++++++++---------- scripts/stack.py | 386 +++++++++++++++--------------- scripts/structs.py | 265 +++++++++++---------- scripts/summary.py | 559 +++++++++++++++++++++++++++----------------- 8 files changed, 1102 insertions(+), 898 deletions(-) diff --git a/.gitignore b/.gitignore index abd7eb8a..58ff85d3 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ *.csv *.t.c *.a.c +*.gcno +*.gcda # Testing things blocks/ diff --git a/Makefile b/Makefile index 2e161b22..11c6b111 100644 --- a/Makefile +++ b/Makefile @@ -74,9 +74,6 @@ endif ifdef EXEC override TESTFLAGS += --exec="$(EXEC)" endif -ifdef COVERAGE -override TESTFLAGS += --coverage -endif ifdef BUILDDIR override TESTFLAGS += --build-dir="$(BUILDDIR:/=)" override CODEFLAGS += --build-dir="$(BUILDDIR:/=)" @@ -164,9 +161,6 @@ $(BUILDDIR)lfs.csv: $(OBJ) $(CGI) ./scripts/data.py $(OBJ) -q -m $@ $(DATAFLAGS) -o $@ ./scripts/stack.py $(CGI) -q -m $@ $(STACKFLAGS) -o $@ ./scripts/structs.py $(OBJ) -q -m $@ $(STRUCTSFLAGS) -o $@ - $(if $(COVERAGE),\ - ./scripts/coverage.py $(BUILDDIR)tests/*.toml.info \ - -q -m $@ $(COVERAGEFLAGS) -o $@) $(BUILDDIR)runners/test_runner: $(TEST_TAOBJ) $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ diff --git a/scripts/code.py b/scripts/code.py index 66502fa9..27e06ebb 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -17,6 +17,61 @@ import collections as co OBJ_PATHS = ['*.o'] +class CodeResult(co.namedtuple('CodeResult', 'code_size')): + __slots__ = () + def __new__(cls, code_size=0): + return super().__new__(cls, int(code_size)) + + def __add__(self, other): + return self.__class__(self.code_size + other.code_size) + + def __sub__(self, other): + return CodeDiff(other, self) + + def __rsub__(self, other): + return self.__class__.__sub__(other, self) + + def key(self, **args): + if args.get('size_sort'): + return -self.code_size + elif args.get('reverse_size_sort'): + return +self.code_size + else: + return None + + _header = '%7s' % 'size' + def __str__(self): + return '%7d' % self.code_size + +class CodeDiff(co.namedtuple('CodeDiff', 'old,new')): + __slots__ = () + + def ratio(self): + old = self.old.code_size if self.old is not None else 0 + new = self.new.code_size if self.new is not None else 0 + return (new-old) / old if old else 1.0 + + def key(self, **args): + return ( + self.new.key(**args) if self.new is not None else 0, + -self.ratio()) + + def __bool__(self): + return bool(self.ratio()) + + _header = '%7s %7s %7s' % ('old', 'new', 'diff') + def __str__(self): + old = self.old.code_size if self.old is not None else 0 + new = self.new.code_size if self.new is not None else 0 + diff = new - old + ratio = self.ratio() + return '%7s %7s %+7d%s' % ( + old or "-", + new or "-", + diff, + ' (%+.1f%%)' % (100*ratio) if ratio else '') + + def openio(path, mode='r'): if path == '-': if 'r' in mode: @@ -27,12 +82,17 @@ def openio(path, mode='r'): return open(path, mode) def collect(paths, **args): - results = co.defaultdict(lambda: 0) + results = co.defaultdict(lambda: CodeResult()) pattern = re.compile( '^(?P[0-9a-fA-F]+)' + ' (?P[%s])' % re.escape(args['type']) + ' (?P.+?)$') for path in paths: + # map to source file + src_path = re.sub('\.o$', '.c', path) + if args.get('build_dir'): + src_path = re.sub('%s/*' % re.escape(args['build_dir']), '', + src_path) # note nm-tool may contain extra args cmd = args['nm_tool'] + ['--size-sort', path] if args.get('verbose'): @@ -45,7 +105,14 @@ def collect(paths, **args): for line in proc.stdout: m = pattern.match(line) if m: - results[(path, m.group('func'))] += int(m.group('size'), 16) + func = m.group('func') + # discard internal functions + if not args.get('everything') and func.startswith('__'): + continue + # discard .8449 suffixes created by optimizer + func = re.sub('\.[0-9]+', '', func) + results[(src_path, func)] += CodeResult( + int(m.group('size'), 16)) proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -53,24 +120,7 @@ def collect(paths, **args): sys.stdout.write(line) sys.exit(-1) - flat_results = [] - for (file, func), size in results.items(): - # map to source files - if args.get('build_dir'): - file = re.sub('%s/*' % re.escape(args['build_dir']), '', file) - # replace .o with .c, different scripts report .o/.c, we need to - # choose one if we want to deduplicate csv files - file = re.sub('\.o$', '.c', file) - # discard internal functions - if not args.get('everything'): - if func.startswith('__'): - continue - # discard .8449 suffixes created by optimizer - func = re.sub('\.[0-9]+', '', func) - - flat_results.append((file, func, size)) - - return flat_results + return results def main(**args): # find sizes @@ -92,35 +142,27 @@ def main(**args): else: with openio(args['use']) as f: r = csv.DictReader(f) - results = [ - ( result['file'], - result['name'], - int(result['code_size'])) + results = { + (result['file'], result['name']): CodeResult( + *(result[f] for f in CodeResult._fields)) for result in r - if result.get('code_size') not in {None, ''}] - - total = 0 - for _, _, size in results: - total += size + if all(result.get(f) not in {None, ''} + for f in CodeResult._fields)} # find previous results? if args.get('diff'): try: with openio(args['diff']) as f: r = csv.DictReader(f) - prev_results = [ - ( result['file'], - result['name'], - int(result['code_size'])) + prev_results = { + (result['file'], result['name']): CodeResult( + *(result[f] for f in CodeResult._fields)) for result in r - if result.get('code_size') not in {None, ''}] + if all(result.get(f) not in {None, ''} + for f in CodeResult._fields)} except FileNotFoundError: prev_results = [] - prev_total = 0 - for _, _, size in prev_results: - prev_total += size - # write results to CSV if args.get('output'): merged_results = co.defaultdict(lambda: {}) @@ -134,111 +176,87 @@ def main(**args): for result in r: file = result.pop('file', '') func = result.pop('name', '') - result.pop('code_size', None) + for f in CodeResult._fields: + result.pop(f, None) merged_results[(file, func)] = result other_fields = result.keys() except FileNotFoundError: pass - for file, func, size in results: - merged_results[(file, func)]['code_size'] = size + for (file, func), result in results.items(): + merged_results[(file, func)] |= result._asdict() with openio(args['output'], 'w') as f: - w = csv.DictWriter(f, ['file', 'name', *other_fields, 'code_size']) + w = csv.DictWriter(f, ['file', 'name', + *other_fields, *CodeResult._fields]) w.writeheader() for (file, func), result in sorted(merged_results.items()): w.writerow({'file': file, 'name': func, **result}) # print results - def dedup_entries(results, by='name'): - entries = co.defaultdict(lambda: 0) - for file, func, size in results: - entry = (file if by == 'file' else func) - entries[entry] += size - return entries - - def diff_entries(olds, news): - diff = co.defaultdict(lambda: (0, 0, 0, 0)) - for name, new in news.items(): - diff[name] = (0, new, new, 1.0) - for name, old in olds.items(): - _, new, _, _ = diff[name] - diff[name] = (old, new, new-old, (new-old)/old if old else 1.0) - return diff - - def sorted_entries(entries): - if args.get('size_sort'): - return sorted(entries, key=lambda x: (-x[1], x)) - elif args.get('reverse_size_sort'): - return sorted(entries, key=lambda x: (+x[1], x)) + def print_header(by): + if by == 'total': + entry = lambda k: 'TOTAL' + elif by == 'file': + entry = lambda k: k[0] else: - return sorted(entries) - - def sorted_diff_entries(entries): - if args.get('size_sort'): - return sorted(entries, key=lambda x: (-x[1][1], x)) - elif args.get('reverse_size_sort'): - return sorted(entries, key=lambda x: (+x[1][1], x)) - else: - return sorted(entries, key=lambda x: (-x[1][3], x)) - - def print_header(by=''): - if not args.get('diff'): - print('%-36s %7s' % (by, 'size')) - else: - print('%-36s %7s %7s %7s' % (by, 'old', 'new', 'diff')) - - def print_entry(name, size): - print("%-36s %7d" % (name, size)) - - def print_diff_entry(name, old, new, diff, ratio): - print("%-36s %7s %7s %+7d%s" % (name, - old or "-", - new or "-", - diff, - ' (%+.1f%%)' % (100*ratio) if ratio else '')) - - def print_entries(by='name'): - entries = dedup_entries(results, by=by) + entry = lambda k: k[1] if not args.get('diff'): - print_header(by=by) - for name, size in sorted_entries(entries.items()): - print_entry(name, size) + print('%-36s %s' % (by, CodeResult._header)) else: - prev_entries = dedup_entries(prev_results, by=by) - diff = diff_entries(prev_entries, entries) - print_header(by='%s (%d added, %d removed)' % (by, - sum(1 for old, _, _, _ in diff.values() if not old), - sum(1 for _, new, _, _ in diff.values() if not new))) - for name, (old, new, diff, ratio) in sorted_diff_entries( - diff.items()): - if ratio or args.get('all'): - print_diff_entry(name, old, new, diff, ratio) + old = {entry(k) for k in results.keys()} + new = {entry(k) for k in prev_results.keys()} + print('%-36s %s' % ( + '%s (%d added, %d removed)' % (by, + sum(1 for k in new if k not in old), + sum(1 for k in old if k not in new)) + if by else '', + CodeDiff._header)) - def print_totals(): - if not args.get('diff'): - print_entry('TOTAL', total) + def print_entries(by): + if by == 'total': + entry = lambda k: 'TOTAL' + elif by == 'file': + entry = lambda k: k[0] else: - ratio = (0.0 if not prev_total and not total - else 1.0 if not prev_total - else (total-prev_total)/prev_total) - print_diff_entry('TOTAL', - prev_total, total, - total-prev_total, - ratio) + entry = lambda k: k[1] + + entries = co.defaultdict(lambda: CodeResult()) + for k, result in results.items(): + entries[entry(k)] += result + + if not args.get('diff'): + for name, result in sorted(entries.items(), + key=lambda p: (p[1].key(**args), p)): + print('%-36s %s' % (name, result)) + else: + prev_entries = co.defaultdict(lambda: CodeResult()) + for k, result in prev_results.items(): + prev_entries[entry(k)] += result + + diff_entries = {name: entries.get(name) - prev_entries.get(name) + for name in (entries.keys() | prev_entries.keys())} + + for name, diff in sorted(diff_entries.items(), + key=lambda p: (p[1].key(**args), p)): + if diff or args.get('all'): + print('%-36s %s' % (name, diff)) if args.get('quiet'): pass elif args.get('summary'): - print_header() - print_totals() + print_header('') + print_entries('total') elif args.get('files'): - print_entries(by='file') - print_totals() + print_header('file') + print_entries('file') + print_entries('total') else: - print_entries(by='name') - print_totals() + print_header('function') + print_entries('function') + print_entries('total') + if __name__ == "__main__": import argparse diff --git a/scripts/coverage.py b/scripts/coverage.py index fbaf9f8a..879c4811 100755 --- a/scripts/coverage.py +++ b/scripts/coverage.py @@ -21,6 +21,140 @@ import subprocess as sp GCDA_PATHS = ['*.gcda'] +class CoverageResult(co.namedtuple('CoverageResult', + 'coverage_line_hits,coverage_line_count,' + 'coverage_branch_hits,coverage_branch_count')): + __slots__ = () + def __new__(cls, + coverage_line_hits=0, coverage_line_count=0, + coverage_branch_hits=0, coverage_branch_count=0): + return super().__new__(cls, + int(coverage_line_hits), + int(coverage_line_count), + int(coverage_branch_hits), + int(coverage_branch_count)) + + def __add__(self, other): + return self.__class__( + self.coverage_line_hits + other.coverage_line_hits, + self.coverage_line_count + other.coverage_line_count, + self.coverage_branch_hits + other.coverage_branch_hits, + self.coverage_branch_count + other.coverage_branch_count) + + def __sub__(self, other): + return CoverageDiff(other, self) + + def __rsub__(self, other): + return self.__class__.__sub__(other, self) + + def key(self, **args): + ratio_line = (self.coverage_line_hits/self.coverage_line_count + if self.coverage_line_count else -1) + ratio_branch = (self.coverage_branch_hits/self.coverage_branch_count + if self.coverage_branch_count else -1) + + if args.get('line_sort'): + return (-ratio_line, -ratio_branch) + elif args.get('reverse_line_sort'): + return (+ratio_line, +ratio_branch) + elif args.get('branch_sort'): + return (-ratio_branch, -ratio_line) + elif args.get('reverse_branch_sort'): + return (+ratio_branch, +ratio_line) + else: + return None + + _header = '%19s %19s' % ('hits/line', 'hits/branch') + def __str__(self): + line_hits = self.coverage_line_hits + line_count = self.coverage_line_count + branch_hits = self.coverage_branch_hits + branch_count = self.coverage_branch_count + return '%11s %7s %11s %7s' % ( + '%d/%d' % (line_hits, line_count) + if line_count else '-', + '%.1f%%' % (100*line_hits/line_count) + if line_count else '-', + '%d/%d' % (branch_hits, branch_count) + if branch_count else '-', + '%.1f%%' % (100*branch_hits/branch_count) + if branch_count else '-') + +class CoverageDiff(co.namedtuple('CoverageDiff', 'old,new')): + __slots__ = () + + def ratio_line(self): + old_line_hits = (self.old.coverage_line_hits + if self.old is not None else 0) + old_line_count = (self.old.coverage_line_count + if self.old is not None else 0) + new_line_hits = (self.new.coverage_line_hits + if self.new is not None else 0) + new_line_count = (self.new.coverage_line_count + if self.new is not None else 0) + return ((new_line_hits/new_line_count if new_line_count else 1.0) + - (old_line_hits/old_line_count if old_line_count else 1.0)) + + def ratio_branch(self): + old_branch_hits = (self.old.coverage_branch_hits + if self.old is not None else 0) + old_branch_count = (self.old.coverage_branch_count + if self.old is not None else 0) + new_branch_hits = (self.new.coverage_branch_hits + if self.new is not None else 0) + new_branch_count = (self.new.coverage_branch_count + if self.new is not None else 0) + return ((new_branch_hits/new_branch_count if new_branch_count else 1.0) + - (old_branch_hits/old_branch_count if old_branch_count else 1.0)) + + def key(self, **args): + return ( + self.new.key(**args) if self.new is not None else 0, + -self.ratio_line(), + -self.ratio_branch()) + + def __bool__(self): + return bool(self.ratio_line() or self.ratio_branch()) + + _header = '%23s %23s %23s' % ('old', 'new', 'diff') + def __str__(self): + old_line_hits = (self.old.coverage_line_hits + if self.old is not None else 0) + old_line_count = (self.old.coverage_line_count + if self.old is not None else 0) + old_branch_hits = (self.old.coverage_branch_hits + if self.old is not None else 0) + old_branch_count = (self.old.coverage_branch_count + if self.old is not None else 0) + new_line_hits = (self.new.coverage_line_hits + if self.new is not None else 0) + new_line_count = (self.new.coverage_line_count + if self.new is not None else 0) + new_branch_hits = (self.new.coverage_branch_hits + if self.new is not None else 0) + new_branch_count = (self.new.coverage_branch_count + if self.new is not None else 0) + diff_line_hits = new_line_hits - old_line_hits + diff_line_count = new_line_count - old_line_count + diff_branch_hits = new_branch_hits - old_branch_hits + diff_branch_count = new_branch_count - old_branch_count + ratio_line = self.ratio_line() + ratio_branch = self.ratio_branch() + return '%11s %11s %11s %11s %11s %11s%-10s%s' % ( + '%d/%d' % (old_line_hits, old_line_count) + if old_line_count else '-', + '%d/%d' % (old_branch_hits, old_branch_count) + if old_branch_count else '-', + '%d/%d' % (new_line_hits, new_line_count) + if new_line_count else '-', + '%d/%d' % (new_branch_hits, new_branch_count) + if new_branch_count else '-', + '%+d/%+d' % (diff_line_hits, diff_line_count), + '%+d/%+d' % (diff_branch_hits, diff_branch_count), + ' (%+.1f%%)' % (100*ratio_line) if ratio_line else '', + ' (%+.1f%%)' % (100*ratio_branch) if ratio_branch else '') + + def openio(path, mode='r'): if path == '-': if 'r' in mode: @@ -30,113 +164,6 @@ def openio(path, mode='r'): else: return open(path, mode) -class CoverageResult(co.namedtuple('CoverageResult', - 'line_hits,line_count,branch_hits,branch_count')): - __slots__ = () - def __new__(cls, line_hits=0, line_count=0, branch_hits=0, branch_count=0): - return super().__new__(cls, - int(line_hits), - int(line_count), - int(branch_hits), - int(branch_count)) - - def __add__(self, other): - return self.__class__( - self.line_hits + other.line_hits, - self.line_count + other.line_count, - self.branch_hits + other.branch_hits, - self.branch_count + other.branch_count) - - def __sub__(self, other): - return CoverageDiff(other, self) - - def key(self, **args): - line_ratio = (self.line_hits/self.line_count - if self.line_count else -1) - branch_ratio = (self.branch_hits/self.branch_count - if self.branch_count else -1) - - if args.get('line_sort'): - return (-line_ratio, -branch_ratio) - elif args.get('reverse_line_sort'): - return (+line_ratio, +branch_ratio) - elif args.get('branch_sort'): - return (-branch_ratio, -line_ratio) - elif args.get('reverse_branch_sort'): - return (+branch_ratio, +line_ratio) - else: - return None - - _header = '%19s %19s' % ('hits/line', 'hits/branch') - def __str__(self): - return '%11s %7s %11s %7s' % ( - '%d/%d' % (self.line_hits, self.line_count) - if self.line_count else '-', - '%.1f%%' % (100*self.line_hits/self.line_count) - if self.line_count else '-', - '%d/%d' % (self.branch_hits, self.branch_count) - if self.branch_count else '-', - '%.1f%%' % (100*self.branch_hits/self.branch_count) - if self.branch_count else '-') - -class CoverageDiff(co.namedtuple('CoverageDiff', 'old,new')): - __slots__ = () - - def line_hits_diff(self): - return self.new.line_hits - self.old.line_hits - - def line_count_diff(self): - return self.new.line_count - self.old.line_count - - def line_ratio(self): - return ((self.new.line_hits/self.new.line_count - if self.new.line_count else 1.0) - - (self.old.line_hits / self.old.line_count - if self.old.line_count else 1.0)) - - def branch_hits_diff(self): - return self.new.branch_hits - self.old.branch_hits - - def branch_count_diff(self): - return self.new.branch_count - self.old.branch_count - - def branch_ratio(self): - return ((self.new.branch_hits/self.new.branch_count - if self.new.branch_count else 1.0) - - (self.old.branch_hits / self.old.branch_count - if self.old.branch_count else 1.0)) - - def key(self, **args): - new_key = self.new.key(**args) - line_ratio = self.line_ratio() - branch_ratio = self.branch_ratio() - if new_key is not None: - return new_key - else: - return (-line_ratio, -branch_ratio) - - def __bool__(self): - return bool(self.line_ratio() or self.branch_ratio()) - - _header = '%23s %23s %23s' % ('old', 'new', 'diff') - def __str__(self): - line_ratio = self.line_ratio() - branch_ratio = self.branch_ratio() - return '%11s %11s %11s %11s %11s %11s%-10s%s' % ( - '%d/%d' % (self.old.line_hits, self.old.line_count) - if self.old.line_count else '-', - '%d/%d' % (self.old.branch_hits, self.old.branch_count) - if self.old.branch_count else '-', - '%d/%d' % (self.new.line_hits, self.new.line_count) - if self.new.line_count else '-', - '%d/%d' % (self.new.branch_hits, self.new.branch_count) - if self.new.branch_count else '-', - '%+d/%+d' % (self.line_hits_diff(), self.line_count_diff()), - '%+d/%+d' % (self.branch_hits_diff(), self.branch_count_diff()), - ' (%+.1f%%)' % (100*line_ratio) if line_ratio else '', - ' (%+.1f%%)' % (100*branch_ratio) if branch_ratio else '') - - def collect(paths, **args): results = {} for path in paths: @@ -180,12 +207,12 @@ def collect(paths, **args): results[(src_path, func, line['line_number'])] = ( line['count'], CoverageResult( - line_hits=1 if line['count'] > 0 else 0, - line_count=1, - branch_hits=sum( + coverage_line_hits=1 if line['count'] > 0 else 0, + coverage_line_count=1, + coverage_branch_hits=sum( 1 if branch['count'] > 0 else 0 for branch in line['branches']), - branch_count=len(line['branches']))) + coverage_branch_count=len(line['branches']))) # merge into functions, since this is what other scripts use func_results = co.defaultdict(lambda: CoverageResult()) @@ -210,15 +237,13 @@ def main(**args): print('no .gcda files found in %r?' % args['gcda_paths']) sys.exit(-1) - # TODO consistent behavior between this and stack.py for deps? results, line_results = collect(paths, **args) else: with openio(args['use']) as f: r = csv.DictReader(f) results = { - (result['file'], result['name']): CoverageResult(**{ - k: v for k, v in result.items() - if k in CoverageResult._fields}) + (result['file'], result['name']): CoverageResult( + *(result[f] for f in CoverageResult._fields)) for result in r if all(result.get(f) not in {None, ''} for f in CoverageResult._fields)} @@ -229,9 +254,8 @@ def main(**args): with openio(args['diff']) as f: r = csv.DictReader(f) prev_results = { - (result['file'], result['name']): CoverageResult(**{ - k: v for k, v in result.items() - if k in CoverageResult._fields}) + (result['file'], result['name']): CoverageResult( + *(result[f] for f in CoverageResult._fields)) for result in r if all(result.get(f) not in {None, ''} for f in CoverageResult._fields)} @@ -259,8 +283,7 @@ def main(**args): pass for (file, func), result in results.items(): - for f in CoverageResult._fields: - merged_results[(file, func)][f] = getattr(result, f) + merged_results[(file, func)] |= result._asdict() with openio(args['output'], 'w') as f: w = csv.DictWriter(f, ['file', 'name', @@ -311,7 +334,7 @@ def main(**args): for k, result in prev_results.items(): prev_entries[entry(k)] += result - diff_entries = {name: entries[name] - prev_entries[name] + diff_entries = {name: entries.get(name) - prev_entries.get(name) for name in (entries.keys() | prev_entries.keys())} for name, diff in sorted(diff_entries.items(), @@ -335,10 +358,12 @@ def main(**args): # catch lack of coverage if args.get('error_on_lines') and any( - r.line_hits < r.line_count for r in results.values()): + r.coverage_line_hits < r.coverage_line_count + for r in results.values()): sys.exit(2) elif args.get('error_on_branches') and any( - r.branch_hits < r.branch_count for r in results.values()): + r.coverage_branch_hits < r.coverage_branch_count + for r in results.values()): sys.exit(3) diff --git a/scripts/data.py b/scripts/data.py index efef4ff1..80b0009a 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -17,6 +17,61 @@ import collections as co OBJ_PATHS = ['*.o'] +class DataResult(co.namedtuple('DataResult', 'data_size')): + __slots__ = () + def __new__(cls, data_size=0): + return super().__new__(cls, int(data_size)) + + def __add__(self, other): + return self.__class__(self.data_size + other.data_size) + + def __sub__(self, other): + return DataDiff(other, self) + + def __rsub__(self, other): + return self.__class__.__sub__(other, self) + + def key(self, **args): + if args.get('size_sort'): + return -self.data_size + elif args.get('reverse_size_sort'): + return +self.data_size + else: + return None + + _header = '%7s' % 'size' + def __str__(self): + return '%7d' % self.data_size + +class DataDiff(co.namedtuple('DataDiff', 'old,new')): + __slots__ = () + + def ratio(self): + old = self.old.data_size if self.old is not None else 0 + new = self.new.data_size if self.new is not None else 0 + return (new-old) / old if old else 1.0 + + def key(self, **args): + return ( + self.new.key(**args) if self.new is not None else 0, + -self.ratio()) + + def __bool__(self): + return bool(self.ratio()) + + _header = '%7s %7s %7s' % ('old', 'new', 'diff') + def __str__(self): + old = self.old.data_size if self.old is not None else 0 + new = self.new.data_size if self.new is not None else 0 + diff = new - old + ratio = self.ratio() + return '%7s %7s %+7d%s' % ( + old or "-", + new or "-", + diff, + ' (%+.1f%%)' % (100*ratio) if ratio else '') + + def openio(path, mode='r'): if path == '-': if 'r' in mode: @@ -27,12 +82,17 @@ def openio(path, mode='r'): return open(path, mode) def collect(paths, **args): - results = co.defaultdict(lambda: 0) + results = co.defaultdict(lambda: DataResult()) pattern = re.compile( '^(?P[0-9a-fA-F]+)' + ' (?P[%s])' % re.escape(args['type']) + ' (?P.+?)$') for path in paths: + # map to source file + src_path = re.sub('\.o$', '.c', path) + if args.get('build_dir'): + src_path = re.sub('%s/*' % re.escape(args['build_dir']), '', + src_path) # note nm-tool may contain extra args cmd = args['nm_tool'] + ['--size-sort', path] if args.get('verbose'): @@ -45,7 +105,14 @@ def collect(paths, **args): for line in proc.stdout: m = pattern.match(line) if m: - results[(path, m.group('func'))] += int(m.group('size'), 16) + func = m.group('func') + # discard internal functions + if not args.get('everything') and func.startswith('__'): + continue + # discard .8449 suffixes created by optimizer + func = re.sub('\.[0-9]+', '', func) + results[(src_path, func)] += DataResult( + int(m.group('size'), 16)) proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -53,23 +120,7 @@ def collect(paths, **args): sys.stdout.write(line) sys.exit(-1) - flat_results = [] - for (file, func), size in results.items(): - # map to source files - if args.get('build_dir'): - file = re.sub('%s/*' % re.escape(args['build_dir']), '', file) - # replace .o with .c, different scripts report .o/.c, we need to - # choose one if we want to deduplicate csv files - file = re.sub('\.o$', '.c', file) - # discard internal functions - if not args.get('everything'): - if func.startswith('__'): - continue - # discard .8449 suffixes created by optimizer - func = re.sub('\.[0-9]+', '', func) - flat_results.append((file, func, size)) - - return flat_results + return results def main(**args): # find sizes @@ -91,35 +142,27 @@ def main(**args): else: with openio(args['use']) as f: r = csv.DictReader(f) - results = [ - ( result['file'], - result['name'], - int(result['data_size'])) + results = { + (result['file'], result['name']): DataResult( + *(result[f] for f in DataResult._fields)) for result in r - if result.get('data_size') not in {None, ''}] - - total = 0 - for _, _, size in results: - total += size + if all(result.get(f) not in {None, ''} + for f in DataResult._fields)} # find previous results? if args.get('diff'): try: with openio(args['diff']) as f: r = csv.DictReader(f) - prev_results = [ - ( result['file'], - result['name'], - int(result['data_size'])) + prev_results = { + (result['file'], result['name']): DataResult( + *(result[f] for f in DataResult._fields)) for result in r - if result.get('data_size') not in {None, ''}] + if all(result.get(f) not in {None, ''} + for f in DataResult._fields)} except FileNotFoundError: prev_results = [] - prev_total = 0 - for _, _, size in prev_results: - prev_total += size - # write results to CSV if args.get('output'): merged_results = co.defaultdict(lambda: {}) @@ -133,111 +176,87 @@ def main(**args): for result in r: file = result.pop('file', '') func = result.pop('name', '') - result.pop('data_size', None) + for f in DataResult._fields: + result.pop(f, None) merged_results[(file, func)] = result other_fields = result.keys() except FileNotFoundError: pass - for file, func, size in results: - merged_results[(file, func)]['data_size'] = size + for (file, func), result in results.items(): + merged_results[(file, func)] |= result._asdict() with openio(args['output'], 'w') as f: - w = csv.DictWriter(f, ['file', 'name', *other_fields, 'data_size']) + w = csv.DictWriter(f, ['file', 'name', + *other_fields, *DataResult._fields]) w.writeheader() for (file, func), result in sorted(merged_results.items()): w.writerow({'file': file, 'name': func, **result}) # print results - def dedup_entries(results, by='name'): - entries = co.defaultdict(lambda: 0) - for file, func, size in results: - entry = (file if by == 'file' else func) - entries[entry] += size - return entries - - def diff_entries(olds, news): - diff = co.defaultdict(lambda: (0, 0, 0, 0)) - for name, new in news.items(): - diff[name] = (0, new, new, 1.0) - for name, old in olds.items(): - _, new, _, _ = diff[name] - diff[name] = (old, new, new-old, (new-old)/old if old else 1.0) - return diff - - def sorted_entries(entries): - if args.get('size_sort'): - return sorted(entries, key=lambda x: (-x[1], x)) - elif args.get('reverse_size_sort'): - return sorted(entries, key=lambda x: (+x[1], x)) + def print_header(by): + if by == 'total': + entry = lambda k: 'TOTAL' + elif by == 'file': + entry = lambda k: k[0] else: - return sorted(entries) - - def sorted_diff_entries(entries): - if args.get('size_sort'): - return sorted(entries, key=lambda x: (-x[1][1], x)) - elif args.get('reverse_size_sort'): - return sorted(entries, key=lambda x: (+x[1][1], x)) - else: - return sorted(entries, key=lambda x: (-x[1][3], x)) - - def print_header(by=''): - if not args.get('diff'): - print('%-36s %7s' % (by, 'size')) - else: - print('%-36s %7s %7s %7s' % (by, 'old', 'new', 'diff')) - - def print_entry(name, size): - print("%-36s %7d" % (name, size)) - - def print_diff_entry(name, old, new, diff, ratio): - print("%-36s %7s %7s %+7d%s" % (name, - old or "-", - new or "-", - diff, - ' (%+.1f%%)' % (100*ratio) if ratio else '')) - - def print_entries(by='name'): - entries = dedup_entries(results, by=by) + entry = lambda k: k[1] if not args.get('diff'): - print_header(by=by) - for name, size in sorted_entries(entries.items()): - print_entry(name, size) + print('%-36s %s' % (by, DataResult._header)) else: - prev_entries = dedup_entries(prev_results, by=by) - diff = diff_entries(prev_entries, entries) - print_header(by='%s (%d added, %d removed)' % (by, - sum(1 for old, _, _, _ in diff.values() if not old), - sum(1 for _, new, _, _ in diff.values() if not new))) - for name, (old, new, diff, ratio) in sorted_diff_entries( - diff.items()): - if ratio or args.get('all'): - print_diff_entry(name, old, new, diff, ratio) + old = {entry(k) for k in results.keys()} + new = {entry(k) for k in prev_results.keys()} + print('%-36s %s' % ( + '%s (%d added, %d removed)' % (by, + sum(1 for k in new if k not in old), + sum(1 for k in old if k not in new)) + if by else '', + DataDiff._header)) - def print_totals(): - if not args.get('diff'): - print_entry('TOTAL', total) + def print_entries(by): + if by == 'total': + entry = lambda k: 'TOTAL' + elif by == 'file': + entry = lambda k: k[0] else: - ratio = (0.0 if not prev_total and not total - else 1.0 if not prev_total - else (total-prev_total)/prev_total) - print_diff_entry('TOTAL', - prev_total, total, - total-prev_total, - ratio) + entry = lambda k: k[1] + + entries = co.defaultdict(lambda: DataResult()) + for k, result in results.items(): + entries[entry(k)] += result + + if not args.get('diff'): + for name, result in sorted(entries.items(), + key=lambda p: (p[1].key(**args), p)): + print('%-36s %s' % (name, result)) + else: + prev_entries = co.defaultdict(lambda: DataResult()) + for k, result in prev_results.items(): + prev_entries[entry(k)] += result + + diff_entries = {name: entries.get(name) - prev_entries.get(name) + for name in (entries.keys() | prev_entries.keys())} + + for name, diff in sorted(diff_entries.items(), + key=lambda p: (p[1].key(**args), p)): + if diff or args.get('all'): + print('%-36s %s' % (name, diff)) if args.get('quiet'): pass elif args.get('summary'): - print_header() - print_totals() + print_header('') + print_entries('total') elif args.get('files'): - print_entries(by='file') - print_totals() + print_header('file') + print_entries('file') + print_entries('total') else: - print_entries(by='name') - print_totals() + print_header('function') + print_entries('function') + print_entries('total') + if __name__ == "__main__": import argparse diff --git a/scripts/stack.py b/scripts/stack.py index 9235b518..22169192 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -24,6 +24,90 @@ def openio(path, mode='r'): else: return open(path, mode) +class StackResult(co.namedtuple('StackResult', 'stack_frame,stack_limit')): + __slots__ = () + def __new__(cls, stack_frame=0, stack_limit=0): + return super().__new__(cls, + int(stack_frame), + float(stack_limit)) + + def __add__(self, other): + return self.__class__( + self.stack_frame + other.stack_frame, + max(self.stack_limit, other.stack_limit)) + + def __sub__(self, other): + return StackDiff(other, self) + + def __rsub__(self, other): + return self.__class__.__sub__(other, self) + + def key(self, **args): + if args.get('limit_sort'): + return -self.stack_limit + elif args.get('reverse_limit_sort'): + return +self.stack_limit + elif args.get('frame_sort'): + return -self.stack_frame + elif args.get('reverse_frame_sort'): + return +self.stack_frame + else: + return None + + _header = '%7s %7s' % ('frame', 'limit') + def __str__(self): + return '%7d %7s' % ( + self.stack_frame, + '∞' if m.isinf(self.stack_limit) else int(self.stack_limit)) + +class StackDiff(co.namedtuple('StackDiff', 'old,new')): + __slots__ = () + + def ratio(self): + old_limit = self.old.stack_limit if self.old is not None else 0 + new_limit = self.new.stack_limit if self.new is not None else 0 + return (0.0 if m.isinf(new_limit) and m.isinf(old_limit) + else +float('inf') if m.isinf(new_limit) + else -float('inf') if m.isinf(old_limit) + else 0.0 if not old_limit and not new_limit + else 1.0 if not old_limit + else (new_limit-old_limit) / old_limit) + + def key(self, **args): + return ( + self.new.key(**args) if self.new is not None else 0, + -self.ratio()) + + def __bool__(self): + return bool(self.ratio()) + + _header = '%15s %15s %15s' % ('old', 'new', 'diff') + def __str__(self): + old_frame = self.old.stack_frame if self.old is not None else 0 + old_limit = self.old.stack_limit if self.old is not None else 0 + new_frame = self.new.stack_frame if self.new is not None else 0 + new_limit = self.new.stack_limit if self.new is not None else 0 + diff_frame = new_frame - old_frame + diff_limit = (0 if m.isinf(new_limit) and m.isinf(old_limit) + else new_limit - old_limit) + ratio = self.ratio() + return '%7s %7s %7s %7s %+7d %7s%s' % ( + old_frame if self.old is not None else '-', + ('∞' if m.isinf(old_limit) else int(old_limit)) + if self.old is not None else '-', + new_frame if self.new is not None else '-', + ('∞' if m.isinf(new_limit) else int(new_limit)) + if self.new is not None else '-', + diff_frame, + '+∞' if diff_limit > 0 and m.isinf(diff_limit) + else '-∞' if diff_limit < 0 and m.isinf(diff_limit) + else '%+d' % diff_limit, + '' if not ratio + else ' (+∞%)' if ratio > 0 and m.isinf(ratio) + else ' (-∞%)' if ratio < 0 and m.isinf(ratio) + else ' (%+.1f%%)' % (100*ratio)) + + def collect(paths, **args): # parse the vcg format k_pattern = re.compile('([a-z]+)\s*:', re.DOTALL) @@ -55,7 +139,7 @@ def collect(paths, **args): return node # collect into functions - results = co.defaultdict(lambda: (None, None, 0, set())) + callgraph = co.defaultdict(lambda: (None, None, 0, set())) f_pattern = re.compile( r'([^\\]*)\\n([^:]*)[^\\]*\\n([0-9]+) bytes \((.*)\)') for path in paths: @@ -73,29 +157,29 @@ def collect(paths, **args): if not args.get('quiet') and type != 'static': print('warning: found non-static stack for %s (%s)' % (function, type)) - _, _, _, targets = results[info['title']] - results[info['title']] = ( + _, _, _, targets = callgraph[info['title']] + callgraph[info['title']] = ( file, function, int(size), targets) elif k == 'edge': info = dict(info) - _, _, _, targets = results[info['sourcename']] + _, _, _, targets = callgraph[info['sourcename']] targets.add(info['targetname']) else: continue if not args.get('everything'): - for source, (s_file, s_function, _, _) in list(results.items()): + for source, (s_file, s_function, _, _) in list(callgraph.items()): # discard internal functions if s_file.startswith('<') or s_file.startswith('/usr/include'): - del results[source] + del callgraph[source] # find maximum stack size recursively, this requires also detecting cycles # (in case of recursion) def find_limit(source, seen=None): seen = seen or set() - if source not in results: + if source not in callgraph: return 0 - _, _, frame, targets = results[source] + _, _, frame, targets = callgraph[source] limit = 0 for target in targets: @@ -107,22 +191,24 @@ def collect(paths, **args): return frame + limit - def find_deps(targets): - deps = set() + def find_calls(targets): + calls = set() for target in targets: - if target in results: - t_file, t_function, _, _ = results[target] - deps.add((t_file, t_function)) - return deps + if target in callgraph: + t_file, t_function, _, _ = callgraph[target] + calls.add((t_file, t_function)) + return calls - # flatten into a list - flat_results = [] - for source, (s_file, s_function, frame, targets) in results.items(): + # build results + results = {} + result_calls = {} + for source, (s_file, s_function, frame, targets) in callgraph.items(): limit = find_limit(source) - deps = find_deps(targets) - flat_results.append((s_file, s_function, frame, limit, deps)) + calls = find_calls(targets) + results[(s_file, s_function)] = StackResult(frame, limit) + result_calls[(s_file, s_function)] = calls - return flat_results + return results, result_calls def main(**args): # find sizes @@ -140,49 +226,33 @@ def main(**args): print('no .ci files found in %r?' % args['ci_paths']) sys.exit(-1) - results = collect(paths, **args) + results, result_calls = collect(paths, **args) else: with openio(args['use']) as f: r = csv.DictReader(f) - results = [ - ( result['file'], - result['name'], - int(result['stack_frame']), - float(result['stack_limit']), # note limit can be inf - set()) + results = { + (result['file'], result['name']): StackResult( + *(result[f] for f in StackResult._fields)) for result in r - if result.get('stack_frame') not in {None, ''} - if result.get('stack_limit') not in {None, ''}] + if all(result.get(f) not in {None, ''} + for f in StackResult._fields)} - total_frame = 0 - total_limit = 0 - for _, _, frame, limit, _ in results: - total_frame += frame - total_limit = max(total_limit, limit) + result_calls = {} # find previous results? if args.get('diff'): try: with openio(args['diff']) as f: r = csv.DictReader(f) - prev_results = [ - ( result['file'], - result['name'], - int(result['stack_frame']), - float(result['stack_limit']), - set()) + prev_results = { + (result['file'], result['name']): StackResult( + *(result[f] for f in StackResult._fields)) for result in r - if result.get('stack_frame') not in {None, ''} - if result.get('stack_limit') not in {None, ''}] + if all(result.get(f) not in {None, ''} + for f in StackResult._fields)} except FileNotFoundError: prev_results = [] - prev_total_frame = 0 - prev_total_limit = 0 - for _, _, frame, limit, _ in prev_results: - prev_total_frame += frame - prev_total_limit = max(prev_total_limit, limit) - # write results to CSV if args.get('output'): merged_results = co.defaultdict(lambda: {}) @@ -196,193 +266,113 @@ def main(**args): for result in r: file = result.pop('file', '') func = result.pop('name', '') - result.pop('stack_frame', None) - result.pop('stack_limit', None) + for f in StackResult._fields: + result.pop(f, None) merged_results[(file, func)] = result other_fields = result.keys() except FileNotFoundError: pass - for file, func, frame, limit, _ in results: - merged_results[(file, func)]['stack_frame'] = frame - merged_results[(file, func)]['stack_limit'] = limit + for (file, func), result in results.items(): + merged_results[(file, func)] |= result._asdict() with openio(args['output'], 'w') as f: - w = csv.DictWriter(f, ['file', 'name', *other_fields, 'stack_frame', 'stack_limit']) + w = csv.DictWriter(f, ['file', 'name', + *other_fields, *StackResult._fields]) w.writeheader() for (file, func), result in sorted(merged_results.items()): w.writerow({'file': file, 'name': func, **result}) # print results - def dedup_entries(results, by='name'): - entries = co.defaultdict(lambda: (0, 0, set())) - for file, func, frame, limit, deps in results: - entry = (file if by == 'file' else func) - entry_frame, entry_limit, entry_deps = entries[entry] - entries[entry] = ( - entry_frame + frame, - max(entry_limit, limit), - entry_deps | {file if by == 'file' else func - for file, func in deps}) - return entries - - def diff_entries(olds, news): - diff = co.defaultdict(lambda: (None, None, None, None, 0, 0, 0, set())) - for name, (new_frame, new_limit, deps) in news.items(): - diff[name] = ( - None, None, - new_frame, new_limit, - new_frame, new_limit, - 1.0, - deps) - for name, (old_frame, old_limit, _) in olds.items(): - _, _, new_frame, new_limit, _, _, _, deps = diff[name] - diff[name] = ( - old_frame, old_limit, - new_frame, new_limit, - (new_frame or 0) - (old_frame or 0), - 0 if m.isinf(new_limit or 0) and m.isinf(old_limit or 0) - else (new_limit or 0) - (old_limit or 0), - 0.0 if m.isinf(new_limit or 0) and m.isinf(old_limit or 0) - else +float('inf') if m.isinf(new_limit or 0) - else -float('inf') if m.isinf(old_limit or 0) - else +0.0 if not old_limit and not new_limit - else +1.0 if not old_limit - else ((new_limit or 0) - (old_limit or 0))/(old_limit or 0), - deps) - return diff - - def sorted_entries(entries): - if args.get('limit_sort'): - return sorted(entries, key=lambda x: (-x[1][1], x)) - elif args.get('reverse_limit_sort'): - return sorted(entries, key=lambda x: (+x[1][1], x)) - elif args.get('frame_sort'): - return sorted(entries, key=lambda x: (-x[1][0], x)) - elif args.get('reverse_frame_sort'): - return sorted(entries, key=lambda x: (+x[1][0], x)) + def print_header(by): + if by == 'total': + entry = lambda k: 'TOTAL' + elif by == 'file': + entry = lambda k: k[0] else: - return sorted(entries) + entry = lambda k: k[1] - def sorted_diff_entries(entries): - if args.get('limit_sort'): - return sorted(entries, key=lambda x: (-(x[1][3] or 0), x)) - elif args.get('reverse_limit_sort'): - return sorted(entries, key=lambda x: (+(x[1][3] or 0), x)) - elif args.get('frame_sort'): - return sorted(entries, key=lambda x: (-(x[1][2] or 0), x)) - elif args.get('reverse_frame_sort'): - return sorted(entries, key=lambda x: (+(x[1][2] or 0), x)) - else: - return sorted(entries, key=lambda x: (-x[1][6], x)) - - def print_header(by=''): if not args.get('diff'): - print('%-36s %7s %7s' % (by, 'frame', 'limit')) + print('%-36s %s' % (by, StackResult._header)) else: - print('%-36s %15s %15s %15s' % (by, 'old', 'new', 'diff')) + old = {entry(k) for k in results.keys()} + new = {entry(k) for k in prev_results.keys()} + print('%-36s %s' % ( + '%s (%d added, %d removed)' % (by, + sum(1 for k in new if k not in old), + sum(1 for k in old if k not in new)) + if by else '', + StackDiff._header)) - def print_entry(name, frame, limit): - print("%-36s %7d %7s" % (name, - frame, '∞' if m.isinf(limit) else int(limit))) - - def print_diff_entry(name, - old_frame, old_limit, - new_frame, new_limit, - diff_frame, diff_limit, - ratio): - print('%-36s %7s %7s %7s %7s %+7d %7s%s' % (name, - old_frame if old_frame is not None else "-", - ('∞' if m.isinf(old_limit) else int(old_limit)) - if old_limit is not None else "-", - new_frame if new_frame is not None else "-", - ('∞' if m.isinf(new_limit) else int(new_limit)) - if new_limit is not None else "-", - diff_frame, - ('+∞' if diff_limit > 0 and m.isinf(diff_limit) - else '-∞' if diff_limit < 0 and m.isinf(diff_limit) - else '%+d' % diff_limit), - '' if not ratio - else ' (+∞%)' if ratio > 0 and m.isinf(ratio) - else ' (-∞%)' if ratio < 0 and m.isinf(ratio) - else ' (%+.1f%%)' % (100*ratio))) - - def print_entries(by='name'): - # build optional tree of dependencies - def print_deps(entries, depth, print, + def print_entries(by): + # print optional tree of dependencies + def print_calls(entries, entry_calls, depth, filter=lambda _: True, prefixes=('', '', '', '')): - entries = entries if isinstance(entries, list) else list(entries) - filtered_entries = [(name, entry) - for name, entry in entries - if filter(name)] - for i, (name, entry) in enumerate(filtered_entries): + filtered_entries = { + name: result for name, result in entries.items() + if filter(name)} + for i, (name, result) in enumerate(sorted(filtered_entries.items(), + key=lambda p: (p[1].key(**args), p))): last = (i == len(filtered_entries)-1) - print(prefixes[0+last] + name, entry) + print('%-36s %s' % (prefixes[0+last] + name, result)) - if depth > 0: - deps = entry[-1] - print_deps(entries, depth-1, print, - lambda name: name in deps, + if depth > 0 and by != 'total': + calls = entry_calls.get(name, set()) + print_calls(entries, entry_calls, depth-1, + lambda name: name in calls, ( prefixes[2+last] + "|-> ", prefixes[2+last] + "'-> ", prefixes[2+last] + "| ", prefixes[2+last] + " ")) - entries = dedup_entries(results, by=by) + if by == 'total': + entry = lambda k: 'TOTAL' + elif by == 'file': + entry = lambda k: k[0] + else: + entry = lambda k: k[1] + + entries = co.defaultdict(lambda: StackResult()) + for k, result in results.items(): + entries[entry(k)] += result + + entry_calls = co.defaultdict(lambda: set()) + for k, calls in result_calls.items(): + entry_calls[entry(k)] |= {entry(c) for c in calls} if not args.get('diff'): - print_header(by=by) - print_deps( - sorted_entries(entries.items()), - args.get('depth') or 0, - lambda name, entry: print_entry(name, *entry[:-1])) + print_calls( + entries, + entry_calls, + args.get('depth', 0)) else: - prev_entries = dedup_entries(prev_results, by=by) - diff = diff_entries(prev_entries, entries) + prev_entries = co.defaultdict(lambda: StackResult()) + for k, result in prev_results.items(): + prev_entries[entry(k)] += result - print_header(by='%s (%d added, %d removed)' % (by, - sum(1 for _, old, _, _, _, _, _, _ in diff.values() if old is None), - sum(1 for _, _, _, new, _, _, _, _ in diff.values() if new is None))) - print_deps( - filter( - lambda x: x[1][6] or args.get('all'), - sorted_diff_entries(diff.items())), - args.get('depth') or 0, - lambda name, entry: print_diff_entry(name, *entry[:-1])) + diff_entries = {name: entries.get(name) - prev_entries.get(name) + for name in (entries.keys() | prev_entries.keys())} - def print_totals(): - if not args.get('diff'): - print_entry('TOTAL', total_frame, total_limit) - else: - diff_frame = total_frame - prev_total_frame - diff_limit = ( - 0 if m.isinf(total_limit or 0) and m.isinf(prev_total_limit or 0) - else (total_limit or 0) - (prev_total_limit or 0)) - ratio = ( - 0.0 if m.isinf(total_limit or 0) and m.isinf(prev_total_limit or 0) - else +float('inf') if m.isinf(total_limit or 0) - else -float('inf') if m.isinf(prev_total_limit or 0) - else 0.0 if not prev_total_limit and not total_limit - else 1.0 if not prev_total_limit - else ((total_limit or 0) - (prev_total_limit or 0))/(prev_total_limit or 0)) - print_diff_entry('TOTAL', - prev_total_frame, prev_total_limit, - total_frame, total_limit, - diff_frame, diff_limit, - ratio) + print_calls( + {name: diff for name, diff in diff_entries.items() + if diff or args.get('all')}, + entry_calls, + args.get('depth', 0)) if args.get('quiet'): pass elif args.get('summary'): - print_header() - print_totals() + print_header('') + print_entries('total') elif args.get('files'): - print_entries(by='file') - print_totals() + print_header('file') + print_entries('file') + print_entries('total') else: - print_entries(by='name') - print_totals() + print_header('function') + print_entries('function') + print_entries('total') # catch recursion if args.get('error_on_recursion') and any( diff --git a/scripts/structs.py b/scripts/structs.py index de266b87..28284fe1 100755 --- a/scripts/structs.py +++ b/scripts/structs.py @@ -24,6 +24,60 @@ def openio(path, mode='r'): else: return open(path, mode) +class StructsResult(co.namedtuple('StructsResult', 'struct_size')): + __slots__ = () + def __new__(cls, struct_size=0): + return super().__new__(cls, int(struct_size)) + + def __add__(self, other): + return self.__class__(self.struct_size + other.struct_size) + + def __sub__(self, other): + return StructsDiff(other, self) + + def __rsub__(self, other): + return self.__class__.__sub__(other, self) + + def key(self, **args): + if args.get('size_sort'): + return -self.struct_size + elif args.get('reverse_size_sort'): + return +self.struct_size + else: + return None + + _header = '%7s' % 'size' + def __str__(self): + return '%7d' % self.struct_size + +class StructsDiff(co.namedtuple('StructsDiff', 'old,new')): + __slots__ = () + + def ratio(self): + old = self.old.struct_size if self.old is not None else 0 + new = self.new.struct_size if self.new is not None else 0 + return (new-old) / old if old else 1.0 + + def key(self, **args): + return ( + self.new.key(**args) if self.new is not None else 0, + -self.ratio()) + + def __bool__(self): + return bool(self.ratio()) + + _header = '%7s %7s %7s' % ('old', 'new', 'diff') + def __str__(self): + old = self.old.struct_size if self.old is not None else 0 + new = self.new.struct_size if self.new is not None else 0 + diff = new - old + ratio = self.ratio() + return '%7s %7s %+7d%s' % ( + old or "-", + new or "-", + diff, + ' (%+.1f%%)' % (100*ratio) if ratio else '') + def collect(paths, **args): decl_pattern = re.compile( '^\s+(?P[0-9]+)' @@ -36,7 +90,7 @@ def collect(paths, **args): '|^.*DW_AT_decl_file.*:\s*(?P[0-9]+)\s*' '|^.*DW_AT_byte_size.*:\s*(?P[0-9]+)\s*)$') - results = co.defaultdict(lambda: 0) + results = {} for path in paths: # find decl, we want to filter by structs in .h files decls = {} @@ -84,8 +138,18 @@ def collect(paths, **args): if (name is not None and decl is not None and size is not None): - decl = decls.get(decl, '?') - results[(decl, name)] = size + file = decls.get(decl, '?') + # map to source file + file = re.sub('\.o$', '.c', file) + if args.get('build_dir'): + file = re.sub( + '%s/*' % re.escape(args['build_dir']), '', + file) + # only include structs declared in header files in the + # current directory, ignore internal-only structs ( + # these are represented in other measurements) + if args.get('everything') or file.endswith('.h'): + results[(file, name)] = StructsResult(size) found = (m.group('tag') == 'structure_type') name = None decl = None @@ -103,24 +167,7 @@ def collect(paths, **args): sys.stdout.write(line) sys.exit(-1) - flat_results = [] - for (file, struct), size in results.items(): - # map to source files - if args.get('build_dir'): - file = re.sub('%s/*' % re.escape(args['build_dir']), '', file) - # only include structs declared in header files in the current - # directory, ignore internal-only # structs (these are represented - # in other measurements) - if not args.get('everything'): - if not file.endswith('.h'): - continue - # replace .o with .c, different scripts report .o/.c, we need to - # choose one if we want to deduplicate csv files - file = re.sub('\.o$', '.c', file) - - flat_results.append((file, struct, size)) - - return flat_results + return results def main(**args): @@ -143,35 +190,27 @@ def main(**args): else: with openio(args['use']) as f: r = csv.DictReader(f) - results = [ - ( result['file'], - result['name'], - int(result['struct_size'])) + results = { + (result['file'], result['name']): StructsResult( + *(result[f] for f in StructsResult._fields)) for result in r - if result.get('struct_size') not in {None, ''}] - - total = 0 - for _, _, size in results: - total += size + if all(result.get(f) not in {None, ''} + for f in StructsResult._fields)} # find previous results? if args.get('diff'): try: with openio(args['diff']) as f: r = csv.DictReader(f) - prev_results = [ - ( result['file'], - result['name'], - int(result['struct_size'])) + prev_results = { + (result['file'], result['name']): StructsResult( + *(result[f] for f in StructsResult._fields)) for result in r - if result.get('struct_size') not in {None, ''}] + if all(result.get(f) not in {None, ''} + for f in StructsResult._fields)} except FileNotFoundError: prev_results = [] - prev_total = 0 - for _, _, size in prev_results: - prev_total += size - # write results to CSV if args.get('output'): merged_results = co.defaultdict(lambda: {}) @@ -184,112 +223,88 @@ def main(**args): r = csv.DictReader(f) for result in r: file = result.pop('file', '') - struct = result.pop('name', '') - result.pop('struct_size', None) - merged_results[(file, struct)] = result + func = result.pop('name', '') + for f in StructsResult._fields: + result.pop(f, None) + merged_results[(file, func)] = result other_fields = result.keys() except FileNotFoundError: pass - for file, struct, size in results: - merged_results[(file, struct)]['struct_size'] = size + for (file, func), result in results.items(): + merged_results[(file, func)] |= result._asdict() with openio(args['output'], 'w') as f: - w = csv.DictWriter(f, ['file', 'name', *other_fields, 'struct_size']) + w = csv.DictWriter(f, ['file', 'name', + *other_fields, *StructsResult._fields]) w.writeheader() - for (file, struct), result in sorted(merged_results.items()): - w.writerow({'file': file, 'name': struct, **result}) + for (file, func), result in sorted(merged_results.items()): + w.writerow({'file': file, 'name': func, **result}) # print results - def dedup_entries(results, by='name'): - entries = co.defaultdict(lambda: 0) - for file, struct, size in results: - entry = (file if by == 'file' else struct) - entries[entry] += size - return entries - - def diff_entries(olds, news): - diff = co.defaultdict(lambda: (0, 0, 0, 0)) - for name, new in news.items(): - diff[name] = (0, new, new, 1.0) - for name, old in olds.items(): - _, new, _, _ = diff[name] - diff[name] = (old, new, new-old, (new-old)/old if old else 1.0) - return diff - - def sorted_entries(entries): - if args.get('size_sort'): - return sorted(entries, key=lambda x: (-x[1], x)) - elif args.get('reverse_size_sort'): - return sorted(entries, key=lambda x: (+x[1], x)) + def print_header(by): + if by == 'total': + entry = lambda k: 'TOTAL' + elif by == 'file': + entry = lambda k: k[0] else: - return sorted(entries) - - def sorted_diff_entries(entries): - if args.get('size_sort'): - return sorted(entries, key=lambda x: (-x[1][1], x)) - elif args.get('reverse_size_sort'): - return sorted(entries, key=lambda x: (+x[1][1], x)) - else: - return sorted(entries, key=lambda x: (-x[1][3], x)) - - def print_header(by=''): - if not args.get('diff'): - print('%-36s %7s' % (by, 'size')) - else: - print('%-36s %7s %7s %7s' % (by, 'old', 'new', 'diff')) - - def print_entry(name, size): - print("%-36s %7d" % (name, size)) - - def print_diff_entry(name, old, new, diff, ratio): - print("%-36s %7s %7s %+7d%s" % (name, - old or "-", - new or "-", - diff, - ' (%+.1f%%)' % (100*ratio) if ratio else '')) - - def print_entries(by='name'): - entries = dedup_entries(results, by=by) + entry = lambda k: k[1] if not args.get('diff'): - print_header(by=by) - for name, size in sorted_entries(entries.items()): - print_entry(name, size) + print('%-36s %s' % (by, StructsResult._header)) else: - prev_entries = dedup_entries(prev_results, by=by) - diff = diff_entries(prev_entries, entries) - print_header(by='%s (%d added, %d removed)' % (by, - sum(1 for old, _, _, _ in diff.values() if not old), - sum(1 for _, new, _, _ in diff.values() if not new))) - for name, (old, new, diff, ratio) in sorted_diff_entries( - diff.items()): - if ratio or args.get('all'): - print_diff_entry(name, old, new, diff, ratio) + old = {entry(k) for k in results.keys()} + new = {entry(k) for k in prev_results.keys()} + print('%-36s %s' % ( + '%s (%d added, %d removed)' % (by, + sum(1 for k in new if k not in old), + sum(1 for k in old if k not in new)) + if by else '', + StructsDiff._header)) - def print_totals(): - if not args.get('diff'): - print_entry('TOTAL', total) + def print_entries(by): + if by == 'total': + entry = lambda k: 'TOTAL' + elif by == 'file': + entry = lambda k: k[0] else: - ratio = (0.0 if not prev_total and not total - else 1.0 if not prev_total - else (total-prev_total)/prev_total) - print_diff_entry('TOTAL', - prev_total, total, - total-prev_total, - ratio) + entry = lambda k: k[1] + + entries = co.defaultdict(lambda: StructsResult()) + for k, result in results.items(): + entries[entry(k)] += result + + if not args.get('diff'): + for name, result in sorted(entries.items(), + key=lambda p: (p[1].key(**args), p)): + print('%-36s %s' % (name, result)) + else: + prev_entries = co.defaultdict(lambda: StructsResult()) + for k, result in prev_results.items(): + prev_entries[entry(k)] += result + + diff_entries = {name: entries.get(name) - prev_entries.get(name) + for name in (entries.keys() | prev_entries.keys())} + + for name, diff in sorted(diff_entries.items(), + key=lambda p: (p[1].key(**args), p)): + if diff or args.get('all'): + print('%-36s %s' % (name, diff)) if args.get('quiet'): pass elif args.get('summary'): - print_header() - print_totals() + print_header('') + print_entries('total') elif args.get('files'): - print_entries(by='file') - print_totals() + print_header('file') + print_entries('file') + print_entries('total') else: - print_entries(by='name') - print_totals() + print_header('struct') + print_entries('struct') + print_entries('total') + if __name__ == "__main__": import argparse @@ -312,7 +327,7 @@ if __name__ == "__main__": parser.add_argument('-m', '--merge', help="Merge with an existing CSV file when writing to output.") parser.add_argument('-a', '--all', action='store_true', - help="Show all functions, not just the ones that changed.") + help="Show all structs, not just the ones that changed.") parser.add_argument('-A', '--everything', action='store_true', help="Include builtin and libc specific symbols.") parser.add_argument('-s', '--size-sort', action='store_true', diff --git a/scripts/summary.py b/scripts/summary.py index 2aab274d..c5a48f2d 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -3,58 +3,202 @@ # Script to summarize the outputs of other scripts. Operates on CSV files. # -import functools as ft import collections as co -import os import csv -import re +import functools as ft import math as m +import os +import re -# displayable fields -Field = co.namedtuple('Field', 'name,parse,acc,key,fmt,repr,null,ratio') -FIELDS = [ - # name, parse, accumulate, fmt, print, null - Field('code', - lambda r: int(r['code_size']), - sum, - lambda r: r, - '%7s', - lambda r: r, - '-', - lambda old, new: (new-old)/old), - Field('data', - lambda r: int(r['data_size']), - sum, - lambda r: r, - '%7s', - lambda r: r, - '-', - lambda old, new: (new-old)/old), - Field('stack', - lambda r: float(r['stack_limit']), - max, - lambda r: r, - '%7s', - lambda r: '∞' if m.isinf(r) else int(r), - '-', - lambda old, new: (new-old)/old), - Field('structs', - lambda r: int(r['struct_size']), - sum, - lambda r: r, - '%8s', - lambda r: r, - '-', - lambda old, new: (new-old)/old), - Field('coverage', - lambda r: (int(r['coverage_hits']), int(r['coverage_count'])), - lambda rs: ft.reduce(lambda a, b: (a[0]+b[0], a[1]+b[1]), rs), - lambda r: r[0]/r[1], - '%19s', - lambda r: '%11s %7s' % ('%d/%d' % (r[0], r[1]), '%.1f%%' % (100*r[0]/r[1])), - '%11s %7s' % ('-', '-'), - lambda old, new: ((new[0]/new[1]) - (old[0]/old[1]))) -] +# each result is a type generated by another script +RESULTS = [] +FIELDS = 'code,data,stack,structs' +def result(cls): + RESULTS.append(cls) + return cls + +@result +class CodeResult(co.namedtuple('CodeResult', 'code_size')): + __slots__ = () + def __new__(cls, code_size=0): + return super().__new__(cls, int(code_size)) + + def __add__(self, other): + return self.__class__(self.code_size + other.code_size) + + def __sub__(self, other): + old = other.code_size if other is not None else 0 + new = self.code_size if self is not None else 0 + return (new-old) / old if old else 1.0 + + def __rsub__(self, other): + return self.__class__.__sub__(other, self) + + def key(self): + return -self.code_size + + _header = '%7s' % 'code' + _nil = '%7s' % '-' + def __str__(self): + return '%7s' % self.code_size + +@result +class DataResult(co.namedtuple('DataResult', 'data_size')): + __slots__ = () + def __new__(cls, data_size=0): + return super().__new__(cls, int(data_size)) + + def __add__(self, other): + return self.__class__(self.data_size + other.data_size) + + def __sub__(self, other): + old = other.data_size if other is not None else 0 + new = self.data_size if self is not None else 0 + return (new-old) / old if old else 1.0 + + def __rsub__(self, other): + return self.__class__.__sub__(other, self) + + def key(self): + return -self.data_size + + _header = '%7s' % 'data' + _nil = '%7s' % '-' + def __str__(self): + return '%7s' % self.data_size + +@result +class StackResult(co.namedtuple('StackResult', 'stack_limit')): + __slots__ = () + def __new__(cls, stack_limit=0): + return super().__new__(cls, float(stack_limit)) + + def __add__(self, other): + return self.__class__(max(self.stack_limit, other.stack_limit)) + + def __sub__(self, other): + old_limit = other.stack_limit if other is not None else 0 + new_limit = self.stack_limit if self is not None else 0 + return (0.0 if m.isinf(new_limit) and m.isinf(old_limit) + else +float('inf') if m.isinf(new_limit) + else -float('inf') if m.isinf(old_limit) + else 0.0 if not old_limit and not new_limit + else 1.0 if not old_limit + else (new_limit-old_limit) / old_limit) + + def __rsub__(self, other): + return self.__class__.__sub__(other, self) + + def key(self): + return -self.stack_limit + + _header = '%7s' % 'stack' + _nil = '%7s' % '-' + def __str__(self): + return '%7s' % ( + '∞' if m.isinf(self.stack_limit) + else int(self.stack_limit)) + +@result +class StructsResult(co.namedtuple('StructsResult', 'struct_size')): + __slots__ = () + def __new__(cls, struct_size=0): + return super().__new__(cls, int(struct_size)) + + def __add__(self, other): + return self.__class__(self.struct_size + other.struct_size) + + def __sub__(self, other): + old = other.struct_size if other is not None else 0 + new = self.struct_size if self is not None else 0 + return (new-old) / old if old else 1.0 + + def __rsub__(self, other): + return self.__class__.__sub__(other, self) + + def key(self): + return -self.struct_size + + _header = '%7s' % 'structs' + _nil = '%7s' % '-' + def __str__(self): + return '%7s' % self.struct_size + +@result +class CoverageLineResult(co.namedtuple('CoverageResult', + 'coverage_line_hits,coverage_line_count')): + __slots__ = () + def __new__(cls, coverage_line_hits=0, coverage_line_count=0): + return super().__new__(cls, + int(coverage_line_hits), + int(coverage_line_count)) + + def __add__(self, other): + return self.__class__( + self.coverage_line_hits + other.coverage_line_hits, + self.coverage_line_count + other.coverage_line_count) + + def __sub__(self, other): + old_hits = other.coverage_line_hits if other is not None else 0 + old_count = other.coverage_line_count if other is not None else 0 + new_hits = self.coverage_line_hits if self is not None else 0 + new_count = self.coverage_line_count if self is not None else 0 + return ((new_hits/new_count if new_count else 1.0) + - (old_hits/old_count if old_count else 1.0)) + + def __rsub__(self, other): + return self.__class__.__sub__(other, self) + + def key(self): + return -(self.coverage_line_hits/self.coverage_line_count + if self.coverage_line_count else -1) + + _header = '%19s' % 'coverage/line' + _nil = '%11s %7s' % ('-', '-') + def __str__(self): + return '%11s %7s' % ( + '%d/%d' % (self.coverage_line_hits, self.coverage_line_count) + if self.coverage_line_count else '-', + '%.1f%%' % (100*self.coverage_line_hits/self.coverage_line_count) + if self.coverage_line_count else '-') + +@result +class CoverageBranchResult(co.namedtuple('CoverageResult', + 'coverage_branch_hits,coverage_branch_count')): + __slots__ = () + def __new__(cls, coverage_branch_hits=0, coverage_branch_count=0): + return super().__new__(cls, + int(coverage_branch_hits), + int(coverage_branch_count)) + + def __add__(self, other): + return self.__class__( + self.coverage_branch_hits + other.coverage_branch_hits, + self.coverage_branch_count + other.coverage_branch_count) + + def __sub__(self, other): + old_hits = other.coverage_branch_hits if other is not None else 0 + old_count = other.coverage_branch_count if other is not None else 0 + new_hits = self.coverage_branch_hits if self is not None else 0 + new_count = self.coverage_branch_count if self is not None else 0 + return ((new_hits/new_count if new_count else 1.0) + - (old_hits/old_count if old_count else 1.0)) + + def __rsub__(self, other): + return self.__class__.__sub__(other, self) + + def key(self): + return -(self.coverage_branch_hits/self.coverage_branch_count + if self.coverage_branch_count else -1) + + _header = '%19s' % 'coverage/branch' + _nil = '%11s %7s' % ('-', '-') + def __str__(self): + return '%11s %7s' % ( + '%d/%d' % (self.coverage_branch_hits, self.coverage_branch_count) + if self.coverage_branch_count else '-', + '%.1f%%' % (100*self.coverage_branch_hits/self.coverage_branch_count) + if self.coverage_branch_count else '-') def openio(path, mode='r'): @@ -76,178 +220,171 @@ def main(**args): for result in r: file = result.pop('file', '') name = result.pop('name', '') - prev = results[(file, name)] - for field in FIELDS: - try: - r = field.parse(result) - if field.name in prev: - results[(file, name)][field.name] = field.acc( - [prev[field.name], r]) - else: - results[(file, name)][field.name] = r - except (KeyError, ValueError): - pass + for Result in RESULTS: + if all(result.get(f) not in {None, ''} + for f in Result._fields): + results[(file, name)][Result.__name__] = ( + results[(file, name)].get( + Result.__name__, Result()) + + Result(*(result[f] + for f in Result._fields))) except FileNotFoundError: pass - # find fields - if args.get('all_fields'): - fields = FIELDS - elif args.get('fields') is not None: - fields_dict = {field.name: field for field in FIELDS} - fields = [fields_dict[f] for f in args['fields']] - else: - fields = [] - for field in FIELDS: - if any(field.name in result for result in results.values()): - fields.append(field) - - # find total for every field - total = {} - for result in results.values(): - for field in fields: - if field.name in result and field.name in total: - total[field.name] = field.acc( - [total[field.name], result[field.name]]) - elif field.name in result: - total[field.name] = result[field.name] - # find previous results? if args.get('diff'): prev_results = co.defaultdict(lambda: {}) - try: - with openio(args['diff']) as f: - r = csv.DictReader(f) - for result in r: - file = result.pop('file', '') - name = result.pop('name', '') - prev = prev_results[(file, name)] - for field in FIELDS: - try: - r = field.parse(result) - if field.name in prev: - prev_results[(file, name)][field.name] = field.acc( - [prev[field.name], r]) - else: - prev_results[(file, name)][field.name] = r - except (KeyError, ValueError): - pass - except FileNotFoundError: - pass + for path in args.get('csv_paths', '-'): + try: + with openio(args['diff']) as f: + r = csv.DictReader(f) + for result in r: + file = result.pop('file', '') + name = result.pop('name', '') + for Result in RESULTS: + if all(result.get(f) not in {None, ''} + for f in Result._fields): + prev_results[(file, name)][Result.__name__] = ( + prev_results[(file, name)].get( + Result.__name__, Result()) + + Result(*(result[f] + for f in Result._fields))) + except FileNotFoundError: + pass - prev_total = {} - for result in prev_results.values(): - for field in fields: - if field.name in result and field.name in prev_total: - prev_total[field.name] = field.acc( - [prev_total[field.name], result[field.name]]) - elif field.name in result: - prev_total[field.name] = result[field.name] + # filter our result types by results that are present + if 'all' in args['fields']: + filtered_results = RESULTS + else: + filtered_results = [ + Result for Result in RESULTS + if (any(f.startswith(r) + for r in args['fields'] + for f in Result._fields) + or any(Result._header.strip().startswith(r) + for r in args['fields']))] + + # figure out a sort key + if args.get('sort'): + key_Result = next( + Result for Result in RESULTS + if (any(f.startswith(args['sort']) + for f in Result._fields) + or Result._header.strip().startswith(args['sort']))) + key = lambda result: result.get(key_Result.__name__, key_Result()).key() + reverse = False + elif args.get('reverse_sort'): + key_Result = next( + Result for Result in RESULTS + if (any(f.startswith(args['reverse_sort']) + for f in Result._fields) + or Result._header.strip().startswith(args['reverse_sort']))) + key = lambda result: result.get(key_Result.__name__, key_Result()).key() + reverse = True + else: + key = lambda _: None + reverse = False + + # write merged results to CSV + if args.get('output'): + with openio(args['output'], 'w') as f: + w = csv.DictWriter(f, sum( + (Result._fields for Result in filtered_results), + ('file', 'name'))) + w.writeheader() + for (file, name), result in sorted(results.items()): + w.writerow(ft.reduce(dict.__or__, + (r._asdict() for r in result.values()), + {'file': file, 'name': name})) # print results - def dedup_entries(results, by='name'): + def print_header(by): + if by == 'total': + entry = lambda k: 'TOTAL' + elif by == 'file': + entry = lambda k: k[0] + else: + entry = lambda k: k[1] + + if not args.get('diff'): + print('%-36s %s' % (by, + ' '.join(Result._header for Result in filtered_results))) + else: + old = {entry(k) for k in results.keys()} + new = {entry(k) for k in prev_results.keys()} + print('%-36s %s' % ( + '%s (%d added, %d removed)' % (by, + sum(1 for k in new if k not in old), + sum(1 for k in old if k not in new)) + if by else '', + ' '.join('%s%-10s' % (Result._header, '') + for Result in filtered_results))) + + def print_entries(by): + if by == 'total': + entry = lambda k: 'TOTAL' + elif by == 'file': + entry = lambda k: k[0] + else: + entry = lambda k: k[1] + entries = co.defaultdict(lambda: {}) - for (file, func), result in results.items(): - entry = (file if by == 'file' else func) - prev = entries[entry] - for field in fields: - if field.name in result and field.name in prev: - entries[entry][field.name] = field.acc( - [prev[field.name], result[field.name]]) - elif field.name in result: - entries[entry][field.name] = result[field.name] - return entries - - def sorted_entries(entries): - if args.get('sort') is not None: - field = {field.name: field for field in FIELDS}[args['sort']] - return sorted(entries, key=lambda x: ( - -(field.key(x[1][field.name])) if field.name in x[1] else -1, x)) - elif args.get('reverse_sort') is not None: - field = {field.name: field for field in FIELDS}[args['reverse_sort']] - return sorted(entries, key=lambda x: ( - +(field.key(x[1][field.name])) if field.name in x[1] else -1, x)) - else: - return sorted(entries) - - def print_header(by=''): - if not args.get('diff'): - print('%-36s' % by, end='') - for field in fields: - print((' '+field.fmt) % field.name, end='') - print() - else: - print('%-36s' % by, end='') - for field in fields: - print((' '+field.fmt) % field.name, end='') - print(' %-9s' % '', end='') - print() - - def print_entry(name, result): - print('%-36s' % name, end='') - for field in fields: - r = result.get(field.name) - if r is not None: - print((' '+field.fmt) % field.repr(r), end='') - else: - print((' '+field.fmt) % '-', end='') - print() - - def print_diff_entry(name, old, new): - print('%-36s' % name, end='') - for field in fields: - n = new.get(field.name) - if n is not None: - print((' '+field.fmt) % field.repr(n), end='') - else: - print((' '+field.fmt) % '-', end='') - o = old.get(field.name) - ratio = ( - 0.0 if m.isinf(o or 0) and m.isinf(n or 0) - else +float('inf') if m.isinf(n or 0) - else -float('inf') if m.isinf(o or 0) - else 0.0 if not o and not n - else +1.0 if not o - else -1.0 if not n - else field.ratio(o, n)) - print(' %-9s' % ( - '' if not ratio - else '(+∞%)' if ratio > 0 and m.isinf(ratio) - else '(-∞%)' if ratio < 0 and m.isinf(ratio) - else '(%+.1f%%)' % (100*ratio)), end='') - print() - - def print_entries(by='name'): - entries = dedup_entries(results, by=by) + for k, result in results.items(): + entries[entry(k)] |= { + r.__class__.__name__: entries[entry(k)].get( + r.__class__.__name__, r.__class__()) + r + for r in result.values()} if not args.get('diff'): - print_header(by=by) - for name, result in sorted_entries(entries.items()): - print_entry(name, result) + for name, result in sorted(entries.items(), + key=lambda p: (key(p[1]), p), + reverse=reverse): + print('%-36s %s' % (name, ' '.join( + str(result.get(Result.__name__, Result._nil)) + for Result in filtered_results))) else: - prev_entries = dedup_entries(prev_results, by=by) - print_header(by='%s (%d added, %d removed)' % (by, - sum(1 for name in entries if name not in prev_entries), - sum(1 for name in prev_entries if name not in entries))) - for name, result in sorted_entries(entries.items()): - if args.get('all') or result != prev_entries.get(name, {}): - print_diff_entry(name, prev_entries.get(name, {}), result) + prev_entries = co.defaultdict(lambda: {}) + for k, result in prev_results.items(): + prev_entries[entry(k)] |= { + r.__class__.__name__: prev_entries[entry(k)].get( + r.__class__.__name__, r.__class__()) + r + for r in result.values()} - def print_totals(): - if not args.get('diff'): - print_entry('TOTAL', total) - else: - print_diff_entry('TOTAL', prev_total, total) + diff_entries = { + name: (prev_entries.get(name), entries.get(name)) + for name in (entries.keys() | prev_entries.keys())} - if args.get('summary'): - print_header() - print_totals() + for name, (old, new) in sorted(diff_entries.items(), + key=lambda p: (key(p[1][1]), p)): + fields = [] + changed = False + for Result in filtered_results: + o = old.get(Result.__name__) if old is not None else None + n = new.get(Result.__name__) if new is not None else None + ratio = n - o if n is not None or o is not None else 0 + changed = changed or ratio + fields.append('%s%-10s' % ( + n if n is not None else Result._nil, + '' if not ratio + else ' (+∞%)' if ratio > 0 and m.isinf(ratio) + else ' (-∞%)' if ratio < 0 and m.isinf(ratio) + else ' (%+.1f%%)' % (100*ratio))) + if changed or args.get('all'): + print('%-36s %s' % (name, ' '.join(fields))) + + if args.get('quiet'): + pass + elif args.get('summary'): + print_header('') + print_entries('total') elif args.get('files'): - print_entries(by='file') - print_totals() + print_header('file') + print_entries('file') + print_entries('total') else: - print_entries(by='name') - print_totals() + print_header('name') + print_entries('name') + print_entries('total') if __name__ == "__main__": @@ -257,17 +394,21 @@ if __name__ == "__main__": description="Summarize measurements") parser.add_argument('csv_paths', nargs='*', default='-', help="Description of where to find *.csv files. May be a directory \ - or list of paths. *.csv files will be merged to show the total \ - coverage.") + or list of paths.") + parser.add_argument('-q', '--quiet', action='store_true', + help="Don't show anything, useful with -o.") + parser.add_argument('-o', '--output', + help="Specify CSV file to store results.") parser.add_argument('-d', '--diff', help="Specify CSV file to diff against.") parser.add_argument('-a', '--all', action='store_true', help="Show all objects, not just the ones that changed.") - parser.add_argument('-e', '--all-fields', action='store_true', - help="Show all fields, even those with no results.") - parser.add_argument('-f', '--fields', type=lambda x: re.split('\s*,\s*', x), + parser.add_argument('-f', '--fields', + type=lambda x: set(re.split('\s*,\s*', x)), + default=FIELDS, help="Comma separated list of fields to print, by default all fields \ - that are found in the CSV files are printed.") + that are found in the CSV files are printed. \"all\" prints all \ + fields this script knows. Defaults to %r." % FIELDS) parser.add_argument('-s', '--sort', help="Sort by this field.") parser.add_argument('-S', '--reverse-sort', From 46cc6d44505579d137d9a51705b3807381a5d4b0 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 23 May 2022 01:35:00 -0500 Subject: [PATCH 19/81] Added support for annotated source in coverage.py On one hand this isn't very different than the source annotation in gcov, on the other hand I find it a bit more readable after a bit of experimentation. --- scripts/coverage.py | 104 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/scripts/coverage.py b/scripts/coverage.py index 879c4811..8c81e6fc 100755 --- a/scripts/coverage.py +++ b/scripts/coverage.py @@ -164,6 +164,14 @@ def openio(path, mode='r'): else: return open(path, mode) +def color(**args): + if args.get('color') == 'auto': + return sys.stdout.isatty() + elif args.get('color') == 'always': + return True + else: + return False + def collect(paths, **args): results = {} for path in paths: @@ -221,6 +229,81 @@ def collect(paths, **args): return func_results, results +def annotate(paths, results, **args): + for path in paths: + # map to source file + src_path = re.sub('\.t\.a\.gcda$', '.c', path) + # TODO test this + if args.get('build_dir'): + src_path = re.sub('%s/*' % re.escape(args['build_dir']), '', + src_path) + + # flatten to line info + line_results = {line: (hits, result) + for (_, _, line), (hits, result) in results.items()} + + # calculate spans to show + if not args.get('annotate'): + spans = [] + last = None + for line, (hits, result) in sorted(line_results.items()): + if ((args.get('lines') and hits == 0) + or (args.get('branches') + and result.coverage_branch_hits + < result.coverage_branch_count)): + if last is not None and line - last.stop <= args['context']: + last = range( + last.start, + line+1+args['context']) + else: + if last is not None: + spans.append(last) + last = range( + line-args['context'], + line+1+args['context']) + if last is not None: + spans.append(last) + + with open(src_path) as f: + skipped = False + for i, line in enumerate(f): + # skip lines not in spans? + if (not args.get('annotate') + and not any(i+1 in s for s in spans)): + skipped = True + continue + + if skipped: + skipped = False + print('%s@@ %s:%d @@%s' % ( + '\x1b[36m' if color(**args) else '', + src_path, + i+1, + '\x1b[m' if color(**args) else '')) + + # build line + if line.endswith('\n'): + line = line[:-1] + + if i+1 in line_results: + hits, result = line_results[i+1] + line = '%-*s // %d hits, %d/%d branches' % ( + args['width'], + line, + hits, + result.coverage_branch_hits, + result.coverage_branch_count) + + if color(**args): + if args.get('lines') and hits == 0: + line = '\x1b[1;31m%s\x1b[m' % line + elif (args.get('branches') and + result.coverage_branch_hits + < result.coverage_branch_count): + line = '\x1b[35m%s\x1b[m' % line + + print(line) + def main(**args): # find sizes if not args.get('use', None): @@ -246,7 +329,10 @@ def main(**args): *(result[f] for f in CoverageResult._fields)) for result in r if all(result.get(f) not in {None, ''} + for f in CoverageResult._fields)} + paths = [] + line_results = {} # find previous results? if args.get('diff'): @@ -344,6 +430,10 @@ def main(**args): if args.get('quiet'): pass + elif (args.get('annotate') + or args.get('lines') + or args.get('branches')): + annotate(paths, line_results, **args) elif args.get('summary'): print_header('') print_entries('total') @@ -403,6 +493,20 @@ if __name__ == "__main__": help="Show file-level coverage.") parser.add_argument('-Y', '--summary', action='store_true', help="Only show the total coverage.") + parser.add_argument('-p', '--annotate', action='store_true', + help="Show source files annotated with coverage info.") + parser.add_argument('-l', '--lines', action='store_true', + help="Show uncovered lines.") + parser.add_argument('-b', '--branches', action='store_true', + help="Show uncovered branches.") + parser.add_argument('-c', '--context', type=lambda x: int(x, 0), default=3, + help="Show a additional lines of context. Defaults to 3.") + parser.add_argument('-w', '--width', type=lambda x: int(x, 0), default=80, + help="Assume source is styled with this many columns. Defaults to 80.") + # TODO add this to test.py? + parser.add_argument('--color', + choices=['never', 'always', 'auto'], default='auto', + help="When to use terminal colors.") parser.add_argument('-e', '--error-on-lines', action='store_true', help="Error if any lines are not covered.") parser.add_argument('-E', '--error-on-branches', action='store_true', From 92eee8e6cd04328bcf2aaf88f17c418dd2a0545f Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 15 Aug 2022 12:03:52 -0500 Subject: [PATCH 20/81] Removed some prefixes from Makefile variables where not necessary Also renamed GCI -> CI, this holds .ci files, though there is a risk of confusion with continuous integration. Also added unused but generated .ci files to clean rule. --- Makefile | 46 ++++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 11c6b111..e6ccd854 100644 --- a/Makefile +++ b/Makefile @@ -30,19 +30,20 @@ SRC ?= $(filter-out $(wildcard *.*.c),$(wildcard *.c)) OBJ := $(SRC:%.c=$(BUILDDIR)%.o) DEP := $(SRC:%.c=$(BUILDDIR)%.d) ASM := $(SRC:%.c=$(BUILDDIR)%.s) -CGI := $(SRC:%.c=$(BUILDDIR)%.ci) -TAGCDA := $(SRC:%.c=$(BUILDDIR)%.t.a.gcda) +CI := $(SRC:%.c=$(BUILDDIR)%.ci) +GCDA := $(SRC:%.c=$(BUILDDIR)%.t.a.gcda) TESTS ?= $(wildcard tests/*.toml) TEST_SRC ?= $(SRC) \ $(filter-out $(wildcard bd/*.*.c),$(wildcard bd/*.c)) \ runners/test_runner.c -TEST_TSRC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) $(TEST_SRC:%.c=$(BUILDDIR)%.t.c) -TEST_TASRC := $(TEST_TSRC:%.t.c=%.t.a.c) -TEST_TAOBJ := $(TEST_TASRC:%.t.a.c=%.t.a.o) -TEST_TADEP := $(TEST_TASRC:%.t.a.c=%.t.a.d) -TEST_TAGCNO := $(TEST_TASRC:%.t.a.c=%.t.a.gcno) -TEST_TAGCDA := $(TEST_TASRC:%.t.a.c=%.t.a.gcda) +TEST_TC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) $(TEST_SRC:%.c=$(BUILDDIR)%.t.c) +TEST_TAC := $(TEST_TC:%.t.c=%.t.a.c) +TEST_OBJ := $(TEST_TAC:%.t.a.c=%.t.a.o) +TEST_DEP := $(TEST_TAC:%.t.a.c=%.t.a.d) +TEST_CI := $(TEST_TAC:%.t.a.c=%.t.a.ci) +TEST_GCNO := $(TEST_TAC:%.t.a.c=%.t.a.gcno) +TEST_GCDA := $(TEST_TAC:%.t.a.c=%.t.a.gcda) ifdef DEBUG override CFLAGS += -O0 @@ -112,7 +113,7 @@ test-runner: $(BUILDDIR)runners/test_runner .PHONY: test test: test-runner - rm -f $(TEST_TAGCDA) + rm -f $(TEST_GCDA) ./scripts/test.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS) .PHONY: test-list @@ -128,7 +129,7 @@ data: $(OBJ) ./scripts/data.py $^ -S $(DATAFLAGS) .PHONY: stack -stack: $(CGI) +stack: $(CI) ./scripts/stack.py $^ -S $(STACKFLAGS) .PHONY: structs @@ -136,7 +137,7 @@ structs: $(OBJ) ./scripts/structs.py $^ -S $(STRUCTSFLAGS) .PHONY: coverage -coverage: $(TAGCDA) +coverage: $(GCDA) ./scripts/coverage.py $^ -s $(COVERAGEFLAGS) .PHONY: summary @@ -146,7 +147,7 @@ summary: $(BUILDDIR)lfs.csv # rules -include $(DEP) --include $(TEST_TADEP) +-include $(TEST_DEP) .SUFFIXES: .SECONDARY: @@ -156,13 +157,13 @@ $(BUILDDIR)lfs: $(OBJ) $(BUILDDIR)lfs.a: $(OBJ) $(AR) rcs $@ $^ -$(BUILDDIR)lfs.csv: $(OBJ) $(CGI) +$(BUILDDIR)lfs.csv: $(OBJ) $(CI) ./scripts/code.py $(OBJ) -q $(CODEFLAGS) -o $@ ./scripts/data.py $(OBJ) -q -m $@ $(DATAFLAGS) -o $@ - ./scripts/stack.py $(CGI) -q -m $@ $(STACKFLAGS) -o $@ + ./scripts/stack.py $(CI) -q -m $@ $(STACKFLAGS) -o $@ ./scripts/structs.py $(OBJ) -q -m $@ $(STRUCTSFLAGS) -o $@ -$(BUILDDIR)runners/test_runner: $(TEST_TAOBJ) +$(BUILDDIR)runners/test_runner: $(TEST_OBJ) $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ # our main build rule generates .o, .d, and .ci files, the latter @@ -195,10 +196,11 @@ clean: rm -f $(OBJ) rm -f $(DEP) rm -f $(ASM) - rm -f $(CGI) - rm -f $(TEST_TSRC) - rm -f $(TEST_TASRC) - rm -f $(TEST_TAOBJ) - rm -f $(TEST_TADEP) - rm -f $(TEST_TAGCNO) - rm -f $(TEST_TAGCDA) + rm -f $(CI) + rm -f $(TEST_TC) + rm -f $(TEST_TAC) + rm -f $(TEST_OBJ) + rm -f $(TEST_DEP) + rm -f $(TEST_CI) + rm -f $(TEST_GCNO) + rm -f $(TEST_GCDA) From b08463f8de7ec7d698a046ede60b293b3cb4d6f7 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 16 Aug 2022 11:41:46 -0500 Subject: [PATCH 21/81] Reworked scripts/pretty_asserts.py a bit - Renamed explode_asserts.py -> pretty_asserts.py, this name is hopefully a bit more descriptive - Small cleanup of the parser rules - Added recognization of memcmp/strcmp => 0 statements and generate the relevant memory inspecting assert messages I attempted to fix the incorrect column numbers for the generated asserts, but unfortunately this didn't go anywhere and I don't think it's actually possible. There is no column control analogous to the #line directive. I thought you might be able to intermix #line directives to put arguments at the right column like so: assert(a == b); __PRETTY_ASSERT_INT_EQ( #line 1 a, #line 1 b); But this doesn't work as preprocessor directives are not allowed in macros arguments in standard C. Unfortunately this is probably not possible to fix without better support in the language. --- Makefile | 4 +- scripts/explode_asserts.py | 391 ---------------------------------- scripts/pretty_asserts.py | 426 +++++++++++++++++++++++++++++++++++++ 3 files changed, 428 insertions(+), 393 deletions(-) delete mode 100755 scripts/explode_asserts.py create mode 100755 scripts/pretty_asserts.py diff --git a/Makefile b/Makefile index e6ccd854..bd829dd7 100644 --- a/Makefile +++ b/Makefile @@ -175,10 +175,10 @@ $(BUILDDIR)%.s: %.c $(CC) -S $(CFLAGS) $< -o $@ $(BUILDDIR)%.a.c: %.c - ./scripts/explode_asserts.py $< -o $@ + ./scripts/pretty_asserts.py -p LFS_ASSERT $< -o $@ $(BUILDDIR)%.a.c: $(BUILDDIR)%.c - ./scripts/explode_asserts.py $< -o $@ + ./scripts/pretty_asserts.py -p LFS_ASSERT $< -o $@ $(BUILDDIR)%.t.c: %.toml ./scripts/test.py -c $< $(TESTCFLAGS) -o $@ diff --git a/scripts/explode_asserts.py b/scripts/explode_asserts.py deleted file mode 100755 index e49dbc0c..00000000 --- a/scripts/explode_asserts.py +++ /dev/null @@ -1,391 +0,0 @@ -#!/usr/bin/env python3 - -import re -import sys - -PATTERN = ['LFS_ASSERT', 'assert'] -PREFIX = 'LFS' -MAXWIDTH = 16 - -ASSERT = "__{PREFIX}_ASSERT_{TYPE}_{COMP}" -FAIL = """ -__attribute__((unused)) -static void __{prefix}_assert_fail_{type}( - const char *file, int line, const char *comp, - {ctype} lh, size_t lsize, - {ctype} rh, size_t rsize) {{ - printf("%s:%d:assert: assert failed with ", file, line); - __{prefix}_assert_print_{type}(lh, lsize); - printf(", expected %s ", comp); - __{prefix}_assert_print_{type}(rh, rsize); - printf("\\n"); - fflush(NULL); - raise(SIGABRT); -}} -""" - -COMP = { - '==': 'eq', - '!=': 'ne', - '<=': 'le', - '>=': 'ge', - '<': 'lt', - '>': 'gt', -} - -TYPE = { - 'int': { - 'ctype': 'intmax_t', - 'fail': FAIL, - 'print': """ - __attribute__((unused)) - static void __{prefix}_assert_print_{type}({ctype} v, size_t size) {{ - (void)size; - printf("%"PRIiMAX, v); - }} - """, - 'assert': """ - #define __{PREFIX}_ASSERT_{TYPE}_{COMP}(file, line, lh, rh) - do {{ - __typeof__(lh) _lh = lh; - __typeof__(lh) _rh = (__typeof__(lh))rh; - if (!(_lh {op} _rh)) {{ - __{prefix}_assert_fail_{type}(file, line, "{comp}", - (intmax_t)_lh, 0, (intmax_t)_rh, 0); - }} - }} while (0) - """ - }, - 'bool': { - 'ctype': 'bool', - 'fail': FAIL, - 'print': """ - __attribute__((unused)) - static void __{prefix}_assert_print_{type}({ctype} v, size_t size) {{ - (void)size; - printf("%s", v ? "true" : "false"); - }} - """, - 'assert': """ - #define __{PREFIX}_ASSERT_{TYPE}_{COMP}(file, line, lh, rh) - do {{ - bool _lh = !!(lh); - bool _rh = !!(rh); - if (!(_lh {op} _rh)) {{ - __{prefix}_assert_fail_{type}(file, line, "{comp}", - _lh, 0, _rh, 0); - }} - }} while (0) - """ - }, - 'mem': { - 'ctype': 'const void *', - 'fail': FAIL, - 'print': """ - __attribute__((unused)) - static void __{prefix}_assert_print_{type}({ctype} v, size_t size) {{ - const uint8_t *s = v; - printf("\\\""); - for (size_t i = 0; i < size && i < {maxwidth}; i++) {{ - if (s[i] >= ' ' && s[i] <= '~') {{ - printf("%c", s[i]); - }} else {{ - printf("\\\\x%02x", s[i]); - }} - }} - if (size > {maxwidth}) {{ - printf("..."); - }} - printf("\\\""); - }} - """, - 'assert': """ - #define __{PREFIX}_ASSERT_{TYPE}_{COMP}(file, line, lh, rh, size) - do {{ - const void *_lh = lh; - const void *_rh = rh; - if (!(memcmp(_lh, _rh, size) {op} 0)) {{ - __{prefix}_assert_fail_{type}(file, line, "{comp}", - _lh, size, _rh, size); - }} - }} while (0) - """ - }, - 'str': { - 'ctype': 'const char *', - 'fail': FAIL, - 'print': """ - __attribute__((unused)) - static void __{prefix}_assert_print_{type}({ctype} v, size_t size) {{ - __{prefix}_assert_print_mem(v, size); - }} - """, - 'assert': """ - #define __{PREFIX}_ASSERT_{TYPE}_{COMP}(file, line, lh, rh) - do {{ - const char *_lh = lh; - const char *_rh = rh; - if (!(strcmp(_lh, _rh) {op} 0)) {{ - __{prefix}_assert_fail_{type}(file, line, "{comp}", - _lh, strlen(_lh), _rh, strlen(_rh)); - }} - }} while (0) - """ - } -} - -def openio(path, mode='r'): - if path == '-': - if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), 'r') - else: - return os.fdopen(os.dup(sys.stdout.fileno()), 'w') - else: - return open(path, mode) - -def mkdecls(outf, maxwidth=16): - outf.write("#include \n") - outf.write("#include \n") - outf.write("#include \n") - outf.write("#include \n") - outf.write("#include \n") - - for type, desc in sorted(TYPE.items()): - format = { - 'type': type.lower(), 'TYPE': type.upper(), - 'ctype': desc['ctype'], - 'prefix': PREFIX.lower(), 'PREFIX': PREFIX.upper(), - 'maxwidth': maxwidth, - } - outf.write(re.sub('\s+', ' ', - desc['print'].strip().format(**format))+'\n') - outf.write(re.sub('\s+', ' ', - desc['fail'].strip().format(**format))+'\n') - - for op, comp in sorted(COMP.items()): - format.update({ - 'comp': comp.lower(), 'COMP': comp.upper(), - 'op': op, - }) - outf.write(re.sub('\s+', ' ', - desc['assert'].strip().format(**format))+'\n') - -def mkassert(type, comp, lh, rh, size=None): - format = { - 'type': type.lower(), 'TYPE': type.upper(), - 'comp': comp.lower(), 'COMP': comp.upper(), - 'prefix': PREFIX.lower(), 'PREFIX': PREFIX.upper(), - 'lh': lh.strip(' '), - 'rh': rh.strip(' '), - 'size': size, - } - if size: - return ((ASSERT + '(__FILE__, __LINE__, {lh}, {rh}, {size})') - .format(**format)) - else: - return ((ASSERT + '(__FILE__, __LINE__, {lh}, {rh})') - .format(**format)) - - -# simple recursive descent parser -LEX = { - 'ws': [r'(?:\s|\n|#.*?\n|//.*?\n|/\*.*?\*/)+'], - 'assert': PATTERN, - 'string': [r'"(?:\\.|[^"])*"', r"'(?:\\.|[^'])\'"], - 'arrow': ['=>'], - 'paren': ['\(', '\)'], - 'op': ['strcmp', 'memcmp', '->'], - 'comp': ['==', '!=', '<=', '>=', '<', '>'], - 'logic': ['\&\&', '\|\|'], - 'sep': [':', ';', '\{', '\}', ','], -} - -class ParseFailure(Exception): - def __init__(self, expected, found): - self.expected = expected - self.found = found - - def __str__(self): - return "expected %r, found %s..." % ( - self.expected, repr(self.found)[:70]) - -class Parse: - def __init__(self, inf, lexemes): - p = '|'.join('(?P<%s>%s)' % (n, '|'.join(l)) - for n, l in lexemes.items()) - p = re.compile(p, re.DOTALL) - data = inf.read() - tokens = [] - while True: - m = p.search(data) - if m: - if m.start() > 0: - tokens.append((None, data[:m.start()])) - tokens.append((m.lastgroup, m.group())) - data = data[m.end():] - else: - tokens.append((None, data)) - break - self.tokens = tokens - self.off = 0 - - def lookahead(self, *pattern): - if self.off < len(self.tokens): - token = self.tokens[self.off] - if token[0] in pattern or token[1] in pattern: - self.m = token[1] - return self.m - self.m = None - return self.m - - def accept(self, *patterns): - m = self.lookahead(*patterns) - if m is not None: - self.off += 1 - return m - - def expect(self, *patterns): - m = self.accept(*patterns) - if not m: - raise ParseFailure(patterns, self.tokens[self.off:]) - return m - - def push(self): - return self.off - - def pop(self, state): - self.off = state - -def passert(p): - def pastr(p): - p.expect('assert') ; p.accept('ws') ; p.expect('(') ; p.accept('ws') - p.expect('strcmp') ; p.accept('ws') ; p.expect('(') ; p.accept('ws') - lh = pexpr(p) ; p.accept('ws') - p.expect(',') ; p.accept('ws') - rh = pexpr(p) ; p.accept('ws') - p.expect(')') ; p.accept('ws') - comp = p.expect('comp') ; p.accept('ws') - p.expect('0') ; p.accept('ws') - p.expect(')') - return mkassert('str', COMP[comp], lh, rh) - - def pamem(p): - p.expect('assert') ; p.accept('ws') ; p.expect('(') ; p.accept('ws') - p.expect('memcmp') ; p.accept('ws') ; p.expect('(') ; p.accept('ws') - lh = pexpr(p) ; p.accept('ws') - p.expect(',') ; p.accept('ws') - rh = pexpr(p) ; p.accept('ws') - p.expect(',') ; p.accept('ws') - size = pexpr(p) ; p.accept('ws') - p.expect(')') ; p.accept('ws') - comp = p.expect('comp') ; p.accept('ws') - p.expect('0') ; p.accept('ws') - p.expect(')') - return mkassert('mem', COMP[comp], lh, rh, size) - - def paint(p): - p.expect('assert') ; p.accept('ws') ; p.expect('(') ; p.accept('ws') - lh = pexpr(p) ; p.accept('ws') - comp = p.expect('comp') ; p.accept('ws') - rh = pexpr(p) ; p.accept('ws') - p.expect(')') - return mkassert('int', COMP[comp], lh, rh) - - def pabool(p): - p.expect('assert') ; p.accept('ws') ; p.expect('(') ; p.accept('ws') - lh = pexprs(p) ; p.accept('ws') - p.expect(')') - return mkassert('bool', 'eq', lh, 'true') - - def pa(p): - return p.expect('assert') - - state = p.push() - lastf = None - for pa in [pastr, pamem, paint, pabool, pa]: - try: - return pa(p) - except ParseFailure as f: - p.pop(state) - lastf = f - else: - raise lastf - -def pexpr(p): - res = [] - while True: - if p.accept('('): - res.append(p.m) - while True: - res.append(pexprs(p)) - if p.accept('sep'): - res.append(p.m) - else: - break - res.append(p.expect(')')) - elif p.lookahead('assert'): - res.append(passert(p)) - elif p.accept('assert', 'ws', 'string', 'op', None): - res.append(p.m) - else: - return ''.join(res) - -def pexprs(p): - res = [] - while True: - res.append(pexpr(p)) - if p.accept('comp', 'logic', ','): - res.append(p.m) - else: - return ''.join(res) - -def pstmt(p): - ws = p.accept('ws') or '' - lh = pexprs(p) - if p.accept('=>'): - rh = pexprs(p) - return ws + mkassert('int', 'eq', lh, rh) - else: - return ws + lh - - -def main(args): - with openio(args.input or '-', 'r') as inf: - with openio(args.output or '-', 'w') as outf: - lexemes = LEX.copy() - if args.pattern: - lexemes['assert'] = args.pattern - p = Parse(inf, lexemes) - - # write extra verbose asserts - mkdecls(outf, maxwidth=args.maxwidth) - if args.input: - outf.write("#line %d \"%s\"\n" % (1, args.input)) - - # parse and write out stmt at a time - try: - while True: - outf.write(pstmt(p)) - if p.accept('sep'): - outf.write(p.m) - else: - break - except ParseFailure as f: - pass - - for i in range(p.off, len(p.tokens)): - outf.write(p.tokens[i][1]) - -if __name__ == "__main__": - import argparse - parser = argparse.ArgumentParser( - description="Cpp step that increases assert verbosity") - parser.add_argument('input', nargs='?', - help="Input C file after cpp.") - parser.add_argument('-o', '--output', required=True, - help="Output C file.") - parser.add_argument('-p', '--pattern', action='append', - help="Patterns to search for starting an assert statement.") - parser.add_argument('--maxwidth', default=MAXWIDTH, type=int, - help="Maximum number of characters to display for strcmp and memcmp.") - main(parser.parse_args()) diff --git a/scripts/pretty_asserts.py b/scripts/pretty_asserts.py new file mode 100755 index 00000000..ceb206ca --- /dev/null +++ b/scripts/pretty_asserts.py @@ -0,0 +1,426 @@ +#!/usr/bin/env python3 + +import re +import sys + +# NOTE the use of macros here helps keep a consistent stack depth which +# tools may rely on. +# +# If compilation errors are noisy consider using -ftrack-macro-expansion=0. +# + +LIMIT = 16 + +CMP = { + '==': 'eq', + '!=': 'ne', + '<=': 'le', + '>=': 'ge', + '<': 'lt', + '>': 'gt', +} + +LEXEMES = { + 'ws': [r'(?:\s|\n|#.*?\n|//.*?\n|/\*.*?\*/)+'], + 'assert': ['assert'], + 'arrow': ['=>'], + 'string': [r'"(?:\\.|[^"])*"', r"'(?:\\.|[^'])\'"], + 'paren': ['\(', '\)'], + 'cmp': CMP.keys(), + 'logic': ['\&\&', '\|\|'], + 'sep': [':', ';', '\{', '\}', ','], +} + + +def openio(path, mode='r'): + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) + +def write_header(f, limit=LIMIT): + f.writeln("// Generated by %s:" % sys.argv[0]) + f.writeln("//") + f.writeln("// %s" % ' '.join(sys.argv)) + f.writeln("//") + f.writeln() + + f.writeln("#include ") + f.writeln("#include ") + f.writeln("#include ") + f.writeln("#include ") + f.writeln("#include ") + f.writeln() + + # write print macros + f.writeln("__attribute__((unused))") + f.writeln("static void __pretty_assert_print_bool(") + f.writeln(" const void *v, size_t size) {") + f.writeln(" (void)size;") + f.writeln(" printf(\"%s\", *(const bool*)v ? \"true\" : \"false\");") + f.writeln("}") + f.writeln() + f.writeln("__attribute__((unused))") + f.writeln("static void __pretty_assert_print_int(") + f.writeln(" const void *v, size_t size) {") + f.writeln(" (void)size;") + f.writeln(" printf(\"%\"PRIiMAX, *(const intmax_t*)v);") + f.writeln("}") + f.writeln() + f.writeln("__attribute__((unused))") + f.writeln("static void __pretty_assert_print_mem(") + f.writeln(" const void *v, size_t size) {") + f.writeln(" const uint8_t *v_ = v;") + f.writeln(" printf(\"\\\"\");") + f.writeln(" for (size_t i = 0; i < size && i < %d; i++) {" % limit) + f.writeln(" if (v_[i] >= ' ' && v_[i] <= '~') {") + f.writeln(" printf(\"%c\", v_[i]);") + f.writeln(" } else {") + f.writeln(" printf(\"\\\\x%02x\", v_[i]);") + f.writeln(" }") + f.writeln(" }") + f.writeln(" if (size > %d) {" % limit) + f.writeln(" printf(\"...\");") + f.writeln(" }") + f.writeln(" printf(\"\\\"\");") + f.writeln("}") + f.writeln() + f.writeln("__attribute__((unused))") + f.writeln("static void __pretty_assert_print_str(") + f.writeln(" const void *v, size_t size) {") + f.writeln(" __pretty_assert_print_mem(v, size);") + f.writeln("}") + f.writeln() + f.writeln("__attribute__((unused, noinline))") + f.writeln("static void __pretty_assert_fail(") + f.writeln(" const char *file, int line,") + f.writeln(" void (*type_print_cb)(const void*, size_t),") + f.writeln(" const char *cmp,") + f.writeln(" const void *lh, size_t lsize,") + f.writeln(" const void *rh, size_t rsize) {") + f.writeln(" printf(\"%s:%d:assert: assert failed with \", file, line);") + f.writeln(" type_print_cb(lh, lsize);") + f.writeln(" printf(\", expected %s \", cmp);") + f.writeln(" type_print_cb(rh, rsize);") + f.writeln(" printf(\"\\n\");") + f.writeln(" fflush(NULL);") + f.writeln(" raise(SIGABRT);") + f.writeln("}") + f.writeln() + + # write assert macros + for op, cmp in sorted(CMP.items()): + f.writeln("#define __PRETTY_ASSERT_BOOL_%s(lh, rh) do { \\" + % cmp.upper()) + f.writeln(" bool _lh = !!(lh); \\") + f.writeln(" bool _rh = !!(rh); \\") + f.writeln(" if (!(_lh %s _rh)) { \\" % op) + f.writeln(" __pretty_assert_fail( \\") + f.writeln(" __FILE__, __LINE__, \\") + f.writeln(" __pretty_assert_print_bool, \"%s\", \\" + % cmp) + f.writeln(" &_lh, 0, \\") + f.writeln(" &_rh, 0); \\") + f.writeln(" } \\") + f.writeln("} while (0)") + for op, cmp in sorted(CMP.items()): + f.writeln("#define __PRETTY_ASSERT_INT_%s(lh, rh) do { \\" + % cmp.upper()) + f.writeln(" __typeof__(lh) _lh = lh; \\") + f.writeln(" __typeof__(lh) _rh = rh; \\") + f.writeln(" if (!(_lh %s _rh)) { \\" % op) + f.writeln(" __pretty_assert_fail( \\") + f.writeln(" __FILE__, __LINE__, \\") + f.writeln(" __pretty_assert_print_int, \"%s\", \\" + % cmp) + f.writeln(" &(intmax_t){_lh}, 0, \\") + f.writeln(" &(intmax_t){_rh}, 0); \\") + f.writeln(" } \\") + f.writeln("} while (0)") + for op, cmp in sorted(CMP.items()): + f.writeln("#define __PRETTY_ASSERT_MEM_%s(lh, rh, size) do { \\" + % cmp.upper()) + f.writeln(" const void *_lh = lh; \\") + f.writeln(" const void *_rh = rh; \\") + f.writeln(" if (!(memcmp(_lh, _rh, size) %s 0)) { \\" % op) + f.writeln(" __pretty_assert_fail( \\") + f.writeln(" __FILE__, __LINE__, \\") + f.writeln(" __pretty_assert_print_mem, \"%s\", \\" + % cmp) + f.writeln(" _lh, size, \\") + f.writeln(" _rh, size); \\") + f.writeln(" } \\") + f.writeln("} while (0)") + for op, cmp in sorted(CMP.items()): + f.writeln("#define __PRETTY_ASSERT_STR_%s(lh, rh) do { \\" + % cmp.upper()) + f.writeln(" const char *_lh = lh; \\") + f.writeln(" const char *_rh = rh; \\") + f.writeln(" if (!(strcmp(_lh, _rh) %s 0)) { \\" % op) + f.writeln(" __pretty_assert_fail( \\") + f.writeln(" __FILE__, __LINE__, \\") + f.writeln(" __pretty_assert_print_str, \"%s\", \\" + % cmp) + f.writeln(" _lh, strlen(_lh), \\") + f.writeln(" _rh, strlen(_rh)); \\") + f.writeln(" } \\") + f.writeln("} while (0)") + f.writeln() + f.writeln() + +def mkassert(type, cmp, lh, rh, size=None): + if size is not None: + return ("__PRETTY_ASSERT_%s_%s(%s, %s, %s)" + % (type.upper(), cmp.upper(), lh, rh, size)) + else: + return ("__PRETTY_ASSERT_%s_%s(%s, %s)" + % (type.upper(), cmp.upper(), lh, rh)) + + +# simple recursive descent parser +class ParseFailure(Exception): + def __init__(self, expected, found): + self.expected = expected + self.found = found + + def __str__(self): + return "expected %r, found %s..." % ( + self.expected, repr(self.found)[:70]) + +class Parser: + def __init__(self, in_f, lexemes=LEXEMES): + p = '|'.join('(?P<%s>%s)' % (n, '|'.join(l)) + for n, l in lexemes.items()) + p = re.compile(p, re.DOTALL) + data = in_f.read() + tokens = [] + line = 1 + col = 0 + while True: + m = p.search(data) + if m: + if m.start() > 0: + tokens.append((None, data[:m.start()], line, col)) + tokens.append((m.lastgroup, m.group(), line, col)) + data = data[m.end():] + else: + tokens.append((None, data, line, col)) + break + self.tokens = tokens + self.off = 0 + + def lookahead(self, *pattern): + if self.off < len(self.tokens): + token = self.tokens[self.off] + if token[0] in pattern or token[1] in pattern: + self.m = token[1] + return self.m + self.m = None + return self.m + + def accept(self, *patterns): + m = self.lookahead(*patterns) + if m is not None: + self.off += 1 + return m + + def expect(self, *patterns): + m = self.accept(*patterns) + if not m: + raise ParseFailure(patterns, self.tokens[self.off:]) + return m + + def push(self): + return self.off + + def pop(self, state): + self.off = state + +def p_assert(p): + state = p.push() + + # assert(memcmp(a,b,size) cmp 0)? + try: + p.expect('assert') ; p.accept('ws') + p.expect('(') ; p.accept('ws') + p.expect('memcmp') ; p.accept('ws') + p.expect('(') ; p.accept('ws') + lh = p_expr(p) ; p.accept('ws') + p.expect(',') ; p.accept('ws') + rh = p_expr(p) ; p.accept('ws') + p.expect(',') ; p.accept('ws') + size = p_expr(p) ; p.accept('ws') + p.expect(')') ; p.accept('ws') + cmp = p.expect('cmp') ; p.accept('ws') + p.expect('0') ; p.accept('ws') + p.expect(')') + return mkassert('mem', CMP[cmp], lh, rh, size) + except ParseFailure: + p.pop(state) + + # assert(strcmp(a,b) cmp 0)? + try: + p.expect('assert') ; p.accept('ws') + p.expect('(') ; p.accept('ws') + p.expect('strcmp') ; p.accept('ws') + p.expect('(') ; p.accept('ws') + lh = p_expr(p) ; p.accept('ws') + p.expect(',') ; p.accept('ws') + rh = p_expr(p) ; p.accept('ws') + p.expect(')') ; p.accept('ws') + cmp = p.expect('cmp') ; p.accept('ws') + p.expect('0') ; p.accept('ws') + p.expect(')') + return mkassert('str', CMP[cmp], lh, rh) + except ParseFailure: + p.pop(state) + + # assert(a cmp b)? + try: + p.expect('assert') ; p.accept('ws') + p.expect('(') ; p.accept('ws') + lh = p_expr(p) ; p.accept('ws') + cmp = p.expect('cmp') ; p.accept('ws') + rh = p_expr(p) ; p.accept('ws') + p.expect(')') + return mkassert('int', CMP[cmp], lh, rh) + except ParseFailure: + p.pop(state) + + # assert(a)? + p.expect('assert') ; p.accept('ws') + p.expect('(') ; p.accept('ws') + lh = p_exprs(p) ; p.accept('ws') + p.expect(')') + return mkassert('bool', 'eq', lh, 'true') + +def p_expr(p): + res = [] + while True: + if p.accept('('): + res.append(p.m) + while True: + res.append(p_exprs(p)) + if p.accept('sep'): + res.append(p.m) + else: + break + res.append(p.expect(')')) + elif p.lookahead('assert'): + state = p.push() + try: + res.append(p_assert(p)) + except ParseFailure: + p.pop(state) + res.append(p.expect('assert')) + elif p.accept('string', None, 'ws'): + res.append(p.m) + else: + return ''.join(res) + +def p_exprs(p): + res = [] + while True: + res.append(p_expr(p)) + if p.accept('cmp', 'logic', ','): + res.append(p.m) + else: + return ''.join(res) + +def p_stmt(p): + ws = p.accept('ws') or '' + + # memcmp(lh,rh,size) => 0? + if p.lookahead('memcmp'): + state = p.push() + try: + p.expect('memcmp') ; p.accept('ws') + p.expect('(') ; p.accept('ws') + lh = p_expr(p) ; p.accept('ws') + p.expect(',') ; p.accept('ws') + rh = p_expr(p) ; p.accept('ws') + p.expect(',') ; p.accept('ws') + size = p_expr(p) ; p.accept('ws') + p.expect(')') ; p.accept('ws') + p.expect('=>') ; p.accept('ws') + p.expect('0') ; p.accept('ws') + return ws + mkassert('mem', 'eq', lh, rh, size) + except ParseFailure: + p.pop(state) + + # strcmp(lh,rh) => 0? + if p.lookahead('strcmp'): + state = p.push() + try: + p.expect('strcmp') ; p.accept('ws') ; p.expect('(') ; p.accept('ws') + lh = p_expr(p) ; p.accept('ws') + p.expect(',') ; p.accept('ws') + rh = p_expr(p) ; p.accept('ws') + p.expect(')') ; p.accept('ws') + p.expect('=>') ; p.accept('ws') + p.expect('0') ; p.accept('ws') + return ws + mkassert('str', 'eq', lh, rh) + except ParseFailure: + p.pop(state) + + # lh => rh? + lh = p_exprs(p) + if p.accept('=>'): + rh = p_exprs(p) + return ws + mkassert('int', 'eq', lh, rh) + else: + return ws + lh + +def main(input=None, output=None, pattern=[], limit=LIMIT): + with openio(input or '-', 'r') as in_f: + # create parser + lexemes = LEXEMES.copy() + lexemes['assert'] += pattern + p = Parser(in_f, lexemes) + + with openio(output or '-', 'w') as f: + def writeln(s=''): + f.write(s) + f.write('\n') + f.writeln = writeln + + # write extra verbose asserts + write_header(f, limit=limit) + if input is not None: + f.writeln("#line %d \"%s\"" % (1, input)) + + # parse and write out stmt at a time + try: + while True: + f.write(p_stmt(p)) + if p.accept('sep'): + f.write(p.m) + else: + break + except ParseFailure as f: + pass + + for i in range(p.off, len(p.tokens)): + f.write(p.tokens[i][1]) + + +if __name__ == "__main__": + import argparse + import sys + parser = argparse.ArgumentParser( + description="Preprocessor that makes asserts easy to debug.") + parser.add_argument('input', + help="Input C file.") + parser.add_argument('-o', '--output', required=True, + help="Output C file.") + parser.add_argument('-p', '--pattern', action='append', + help="Regex patterns to search for starting an assert statement. This" + " implicitly includes \"assert\" and \"=>\".") + parser.add_argument('-l', '--limit', default=LIMIT, type=int, + help="Maximum number of characters to display in strcmp and memcmp.") + sys.exit(main(**{k: v + for k, v in vars(parser.parse_args()).items() + if v is not None})) From a368d3a07c53a7e5ca7b41d1c90780a7fba7aee5 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 17 Aug 2022 11:50:45 -0500 Subject: [PATCH 22/81] Moved emulation of erase values up into lfs_testbd Yes this is more expensive, since small programs need to rewrite the whole block in order to conform to the block device API. However, it reduces code duplication and keeps all of the test-related block device emulation in lfs_testbd. Some people have used lfs_filebd/lfs_rambd as a starting point for new block devices and I think it should be clear that erase does not need to have side effects. Though to be fair this also just means we should have more examples of block devices... --- bd/lfs_filebd.c | 111 ++++++++++++------------------------------------ bd/lfs_filebd.h | 11 ----- bd/lfs_rambd.c | 41 ++++++------------ bd/lfs_rambd.h | 4 -- bd/lfs_testbd.c | 98 ++++++++++++++++++++++++++++++++++-------- bd/lfs_testbd.h | 9 ++-- 6 files changed, 127 insertions(+), 147 deletions(-) diff --git a/bd/lfs_filebd.c b/bd/lfs_filebd.c index 98e5abc8..3040735b 100644 --- a/bd/lfs_filebd.c +++ b/bd/lfs_filebd.c @@ -15,39 +15,6 @@ #include #endif -int lfs_filebd_createcfg(const struct lfs_config *cfg, const char *path, - const struct lfs_filebd_config *bdcfg) { - LFS_FILEBD_TRACE("lfs_filebd_createcfg(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " - "\"%s\", " - "%p {.erase_value=%"PRId32"})", - (void*)cfg, cfg->context, - (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, - (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - path, (void*)bdcfg, bdcfg->erase_value); - lfs_filebd_t *bd = cfg->context; - bd->cfg = bdcfg; - - // open file - #ifdef _WIN32 - bd->fd = open(path, O_RDWR | O_CREAT | O_BINARY, 0666); - #else - bd->fd = open(path, O_RDWR | O_CREAT, 0666); - #endif - - if (bd->fd < 0) { - int err = -errno; - LFS_FILEBD_TRACE("lfs_filebd_createcfg -> %d", err); - return err; - } - - LFS_FILEBD_TRACE("lfs_filebd_createcfg -> %d", 0); - return 0; -} - int lfs_filebd_create(const struct lfs_config *cfg, const char *path) { LFS_FILEBD_TRACE("lfs_filebd_create(%p {.context=%p, " ".read=%p, .prog=%p, .erase=%p, .sync=%p, " @@ -58,11 +25,24 @@ int lfs_filebd_create(const struct lfs_config *cfg, const char *path) { (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - path); - static const struct lfs_filebd_config defaults = {.erase_value=-1}; - int err = lfs_filebd_createcfg(cfg, path, &defaults); - LFS_FILEBD_TRACE("lfs_filebd_create -> %d", err); - return err; + path, (void*)bdcfg, bdcfg->erase_value); + lfs_filebd_t *bd = cfg->context; + + // open file + #ifdef _WIN32 + bd->fd = open(path, O_RDWR | O_CREAT | O_BINARY, 0666); + #else + bd->fd = open(path, O_RDWR | O_CREAT, 0666); + #endif + + if (bd->fd < 0) { + int err = -errno; + LFS_FILEBD_TRACE("lfs_filebd_create -> %d", err); + return err; + } + + LFS_FILEBD_TRACE("lfs_filebd_create -> %d", 0); + return 0; } int lfs_filebd_destroy(const struct lfs_config *cfg) { @@ -86,14 +66,13 @@ int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_filebd_t *bd = cfg->context; // check if read is valid + LFS_ASSERT(block < cfg->block_count); LFS_ASSERT(off % cfg->read_size == 0); LFS_ASSERT(size % cfg->read_size == 0); - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(off+size <= cfg->block_size); // zero for reproducibility (in case file is truncated) - if (bd->cfg->erase_value != -1) { - memset(buffer, bd->cfg->erase_value, size); - } + memset(buffer, 0, size); // read off_t res1 = lseek(bd->fd, @@ -122,32 +101,10 @@ int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_filebd_t *bd = cfg->context; // check if write is valid + LFS_ASSERT(block < cfg->block_count); LFS_ASSERT(off % cfg->prog_size == 0); LFS_ASSERT(size % cfg->prog_size == 0); - LFS_ASSERT(block < cfg->block_count); - - // check that data was erased? only needed for testing - if (bd->cfg->erase_value != -1) { - off_t res1 = lseek(bd->fd, - (off_t)block*cfg->block_size + (off_t)off, SEEK_SET); - if (res1 < 0) { - int err = -errno; - LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", err); - return err; - } - - for (lfs_off_t i = 0; i < size; i++) { - uint8_t c; - ssize_t res2 = read(bd->fd, &c, 1); - if (res2 < 0) { - int err = -errno; - LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", err); - return err; - } - - LFS_ASSERT(c == bd->cfg->erase_value); - } - } + LFS_ASSERT(off+size <= cfg->block_size); // program data off_t res1 = lseek(bd->fd, @@ -171,29 +128,12 @@ int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block, int lfs_filebd_erase(const struct lfs_config *cfg, lfs_block_t block) { LFS_FILEBD_TRACE("lfs_filebd_erase(%p, 0x%"PRIx32")", (void*)cfg, block); - lfs_filebd_t *bd = cfg->context; // check if erase is valid LFS_ASSERT(block < cfg->block_count); - // erase, only needed for testing - if (bd->cfg->erase_value != -1) { - off_t res1 = lseek(bd->fd, (off_t)block*cfg->block_size, SEEK_SET); - if (res1 < 0) { - int err = -errno; - LFS_FILEBD_TRACE("lfs_filebd_erase -> %d", err); - return err; - } - - for (lfs_off_t i = 0; i < cfg->block_size; i++) { - ssize_t res2 = write(bd->fd, &(uint8_t){bd->cfg->erase_value}, 1); - if (res2 < 0) { - int err = -errno; - LFS_FILEBD_TRACE("lfs_filebd_erase -> %d", err); - return err; - } - } - } + // erase is a noop + (void)block; LFS_FILEBD_TRACE("lfs_filebd_erase -> %d", 0); return 0; @@ -201,6 +141,7 @@ int lfs_filebd_erase(const struct lfs_config *cfg, lfs_block_t block) { int lfs_filebd_sync(const struct lfs_config *cfg) { LFS_FILEBD_TRACE("lfs_filebd_sync(%p)", (void*)cfg); + // file sync lfs_filebd_t *bd = cfg->context; #ifdef _WIN32 diff --git a/bd/lfs_filebd.h b/bd/lfs_filebd.h index 0ed1909a..0f24996a 100644 --- a/bd/lfs_filebd.h +++ b/bd/lfs_filebd.h @@ -26,25 +26,14 @@ extern "C" #endif #endif -// filebd config (optional) -struct lfs_filebd_config { - // 8-bit erase value to use for simulating erases. -1 does not simulate - // erases, which can speed up testing by avoiding all the extra block-device - // operations to store the erase value. - int32_t erase_value; -}; - // filebd state typedef struct lfs_filebd { int fd; - const struct lfs_filebd_config *cfg; } lfs_filebd_t; // Create a file block device using the geometry in lfs_config int lfs_filebd_create(const struct lfs_config *cfg, const char *path); -int lfs_filebd_createcfg(const struct lfs_config *cfg, const char *path, - const struct lfs_filebd_config *bdcfg); // Clean up memory associated with block device int lfs_filebd_destroy(const struct lfs_config *cfg); diff --git a/bd/lfs_rambd.c b/bd/lfs_rambd.c index 39bb8150..cbdd7d0f 100644 --- a/bd/lfs_rambd.c +++ b/bd/lfs_rambd.c @@ -13,12 +13,12 @@ int lfs_rambd_createcfg(const struct lfs_config *cfg, ".read=%p, .prog=%p, .erase=%p, .sync=%p, " ".read_size=%"PRIu32", .prog_size=%"PRIu32", " ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " - "%p {.erase_value=%"PRId32", .buffer=%p})", + "%p {.buffer=%p})", (void*)cfg, cfg->context, (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - (void*)bdcfg, bdcfg->erase_value, bdcfg->buffer); + (void*)bdcfg, bdcfg->buffer); lfs_rambd_t *bd = cfg->context; bd->cfg = bdcfg; @@ -33,13 +33,8 @@ int lfs_rambd_createcfg(const struct lfs_config *cfg, } } - // zero for reproducibility? - if (bd->cfg->erase_value != -1) { - memset(bd->buffer, bd->cfg->erase_value, - cfg->block_size * cfg->block_count); - } else { - memset(bd->buffer, 0, cfg->block_size * cfg->block_count); - } + // zero for reproducibility + memset(bd->buffer, 0, cfg->block_size * cfg->block_count); LFS_RAMBD_TRACE("lfs_rambd_createcfg -> %d", 0); return 0; @@ -54,7 +49,7 @@ int lfs_rambd_create(const struct lfs_config *cfg) { (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count); - static const struct lfs_rambd_config defaults = {.erase_value=-1}; + static const struct lfs_rambd_config defaults = {0}; int err = lfs_rambd_createcfg(cfg, &defaults); LFS_RAMBD_TRACE("lfs_rambd_create -> %d", err); return err; @@ -79,9 +74,10 @@ int lfs_rambd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_rambd_t *bd = cfg->context; // check if read is valid + LFS_ASSERT(block < cfg->block_count); LFS_ASSERT(off % cfg->read_size == 0); LFS_ASSERT(size % cfg->read_size == 0); - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(off+size <= cfg->block_size); // read data memcpy(buffer, &bd->buffer[block*cfg->block_size + off], size); @@ -98,17 +94,10 @@ int lfs_rambd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_rambd_t *bd = cfg->context; // check if write is valid + LFS_ASSERT(block < cfg->block_count); LFS_ASSERT(off % cfg->prog_size == 0); LFS_ASSERT(size % cfg->prog_size == 0); - LFS_ASSERT(block < cfg->block_count); - - // check that data was erased? only needed for testing - if (bd->cfg->erase_value != -1) { - for (lfs_off_t i = 0; i < size; i++) { - LFS_ASSERT(bd->buffer[block*cfg->block_size + off + i] == - bd->cfg->erase_value); - } - } + LFS_ASSERT(off+size <= cfg->block_size); // program data memcpy(&bd->buffer[block*cfg->block_size + off], buffer, size); @@ -119,16 +108,12 @@ int lfs_rambd_prog(const struct lfs_config *cfg, lfs_block_t block, int lfs_rambd_erase(const struct lfs_config *cfg, lfs_block_t block) { LFS_RAMBD_TRACE("lfs_rambd_erase(%p, 0x%"PRIx32")", (void*)cfg, block); - lfs_rambd_t *bd = cfg->context; // check if erase is valid LFS_ASSERT(block < cfg->block_count); - // erase, only needed for testing - if (bd->cfg->erase_value != -1) { - memset(&bd->buffer[block*cfg->block_size], - bd->cfg->erase_value, cfg->block_size); - } + // erase is a noop + (void)block; LFS_RAMBD_TRACE("lfs_rambd_erase -> %d", 0); return 0; @@ -136,8 +121,10 @@ int lfs_rambd_erase(const struct lfs_config *cfg, lfs_block_t block) { int lfs_rambd_sync(const struct lfs_config *cfg) { LFS_RAMBD_TRACE("lfs_rambd_sync(%p)", (void*)cfg); - // sync does nothing because we aren't backed by anything real + + // sync is a noop (void)cfg; + LFS_RAMBD_TRACE("lfs_rambd_sync -> %d", 0); return 0; } diff --git a/bd/lfs_rambd.h b/bd/lfs_rambd.h index b7629a9b..34246802 100644 --- a/bd/lfs_rambd.h +++ b/bd/lfs_rambd.h @@ -28,10 +28,6 @@ extern "C" // rambd config (optional) struct lfs_rambd_config { - // 8-bit erase value to simulate erasing with. -1 indicates no erase - // occurs, which is still a valid block device - int32_t erase_value; - // Optional statically allocated buffer for the block device. void *buffer; }; diff --git a/bd/lfs_testbd.c b/bd/lfs_testbd.c index 1f0877d4..e9a96a96 100644 --- a/bd/lfs_testbd.c +++ b/bd/lfs_testbd.c @@ -35,6 +35,20 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, bd->persist = path; bd->power_cycles = bd->cfg->power_cycles; + // create scratch block if we need it (for emulating erase values) + if (bd->cfg->erase_value != -1) { + if (bd->cfg->scratch_buffer) { + bd->scratch = bd->cfg->scratch_buffer; + } else { + bd->scratch = lfs_malloc(cfg->block_size); + if (!bd->scratch) { + LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM); + return LFS_ERR_NOMEM; + } + } + } + + // create map of wear if (bd->cfg->erase_cycles) { if (bd->cfg->wear_buffer) { bd->wear = bd->cfg->wear_buffer; @@ -51,15 +65,11 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, // create underlying block device if (bd->persist) { - bd->u.file.cfg = (struct lfs_filebd_config){ - .erase_value = bd->cfg->erase_value, - }; - int err = lfs_filebd_createcfg(cfg, path, &bd->u.file.cfg); + int err = lfs_filebd_create(cfg, path); LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", err); return err; } else { bd->u.ram.cfg = (struct lfs_rambd_config){ - .erase_value = bd->cfg->erase_value, .buffer = bd->cfg->buffer, }; int err = lfs_rambd_createcfg(cfg, &bd->u.ram.cfg); @@ -88,6 +98,10 @@ int lfs_testbd_create(const struct lfs_config *cfg, const char *path) { int lfs_testbd_destroy(const struct lfs_config *cfg) { LFS_TESTBD_TRACE("lfs_testbd_destroy(%p)", (void*)cfg); lfs_testbd_t *bd = cfg->context; + + if (bd->cfg->erase_value != -1 && !bd->cfg->scratch_buffer) { + lfs_free(bd->scratch); + } if (bd->cfg->erase_cycles && !bd->cfg->wear_buffer) { lfs_free(bd->wear); } @@ -152,9 +166,10 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_testbd_t *bd = cfg->context; // check if read is valid + LFS_ASSERT(block < cfg->block_count); LFS_ASSERT(off % cfg->read_size == 0); LFS_ASSERT(size % cfg->read_size == 0); - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(off+size <= cfg->block_size); // block bad? if (bd->cfg->erase_cycles && bd->wear[block] >= bd->cfg->erase_cycles && @@ -177,9 +192,10 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_testbd_t *bd = cfg->context; // check if write is valid + LFS_ASSERT(block < cfg->block_count); LFS_ASSERT(off % cfg->prog_size == 0); LFS_ASSERT(size % cfg->prog_size == 0); - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(off+size <= cfg->block_size); // block bad? if (bd->cfg->erase_cycles && bd->wear[block] >= bd->cfg->erase_cycles) { @@ -196,11 +212,41 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, } } - // prog - int err = lfs_testbd_rawprog(cfg, block, off, buffer, size); - if (err) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); - return err; + // emulate an erase value? + if (bd->cfg->erase_value != -1) { + int err = lfs_testbd_rawread(cfg, block, 0, + bd->scratch, cfg->block_size); + if (err) { + LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); + return err; + } + + // assert that program block was erased + for (lfs_off_t i = 0; i < size; i++) { + LFS_ASSERT(bd->scratch[off+i] == bd->cfg->erase_value); + } + + memcpy(&bd->scratch[off], buffer, size); + + err = lfs_testbd_rawerase(cfg, block); + if (err) { + LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); + return err; + } + + err = lfs_testbd_rawprog(cfg, block, 0, + bd->scratch, cfg->block_size); + if (err) { + LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); + return err; + } + } else { + // prog + int err = lfs_testbd_rawprog(cfg, block, off, buffer, size); + if (err) { + LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); + return err; + } } // lose power? @@ -243,11 +289,29 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { } } - // erase - int err = lfs_testbd_rawerase(cfg, block); - if (err) { - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); - return err; + // emulate an erase value? + if (bd->cfg->erase_value != -1) { + memset(bd->scratch, bd->cfg->erase_value, cfg->block_size); + + int err = lfs_testbd_rawerase(cfg, block); + if (err) { + LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); + return err; + } + + err = lfs_testbd_rawprog(cfg, block, 0, + bd->scratch, cfg->block_size); + if (err) { + LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); + return err; + } + } else { + // erase + int err = lfs_testbd_rawerase(cfg, block); + if (err) { + LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); + return err; + } } // lose power? diff --git a/bd/lfs_testbd.h b/bd/lfs_testbd.h index 06794e72..3eafb8f4 100644 --- a/bd/lfs_testbd.h +++ b/bd/lfs_testbd.h @@ -50,7 +50,7 @@ typedef int32_t lfs_testbd_swear_t; // testbd config, this is required for testing struct lfs_testbd_config { // 8-bit erase value to use for simulating erases. -1 does not simulate - // erases, which can speed up testing by avoiding all the extra block-device + // erases, which can speed up testing by avoiding the extra block-device // operations to store the erase value. int32_t erase_value; @@ -68,8 +68,11 @@ struct lfs_testbd_config { // Optional buffer for RAM block device. void *buffer; - // Optional buffer for wear + // Optional buffer for wear. void *wear_buffer; + + // Optional buffer for scratch memory, needed when erase_value != -1. + void *scratch_buffer; }; // testbd state @@ -77,7 +80,6 @@ typedef struct lfs_testbd { union { struct { lfs_filebd_t bd; - struct lfs_filebd_config cfg; } file; struct { lfs_rambd_t bd; @@ -88,6 +90,7 @@ typedef struct lfs_testbd { bool persist; uint32_t power_cycles; lfs_testbd_wear_t *wear; + uint8_t *scratch; const struct lfs_testbd_config *cfg; } lfs_testbd_t; From 01b11da31b72e3c755bdce7b3c750d7e91d6fd1a Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 17 Aug 2022 12:29:11 -0500 Subject: [PATCH 23/81] Added a simple test that the block device works On one hand this seems like the wrong place for these tests, on the other hand, it's good to know that the block device is behaving as expected when debugging the filesystem. Maybe this should be moved to an external program for users to test their block devices in the future? --- tests/test_bd.toml | 248 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 tests/test_bd.toml diff --git a/tests/test_bd.toml b/tests/test_bd.toml new file mode 100644 index 00000000..3cbc178f --- /dev/null +++ b/tests/test_bd.toml @@ -0,0 +1,248 @@ +# These tests don't really test littlefs at all, they are here only to make +# sure the underlying block device is working. +# +# Note we use 251, a prime, in places to avoid aliasing powers of 2. +# + +[cases.one_block] +defines.READ = ['READ_SIZE', 'BLOCK_SIZE'] +defines.PROG = ['PROG_SIZE', 'BLOCK_SIZE'] +code = ''' + uint8_t buffer[lfs_max(READ, PROG)]; + + // write data + cfg->erase(cfg, 0) => 0; + for (lfs_off_t i = 0; i < cfg->block_size; i += PROG) { + for (lfs_off_t j = 0; j < PROG; j++) { + buffer[j] = (i+j) % 251; + } + cfg->prog(cfg, 0, i, buffer, PROG) => 0; + } + + // read data + for (lfs_off_t i = 0; i < cfg->block_size; i += READ) { + cfg->read(cfg, 0, i, buffer, READ) => 0; + + for (lfs_off_t j = 0; j < READ; j++) { + LFS_ASSERT(buffer[j] == (i+j) % 251); + } + } +''' + +[cases.two_block] +defines.READ = ['READ_SIZE', 'BLOCK_SIZE'] +defines.PROG = ['PROG_SIZE', 'BLOCK_SIZE'] +code = ''' + uint8_t buffer[lfs_max(READ, PROG)]; + lfs_block_t block; + + // write block 0 + block = 0; + cfg->erase(cfg, block) => 0; + for (lfs_off_t i = 0; i < cfg->block_size; i += PROG) { + for (lfs_off_t j = 0; j < PROG; j++) { + buffer[j] = (block+i+j) % 251; + } + cfg->prog(cfg, block, i, buffer, PROG) => 0; + } + + // read block 0 + block = 0; + for (lfs_off_t i = 0; i < cfg->block_size; i += READ) { + cfg->read(cfg, block, i, buffer, READ) => 0; + + for (lfs_off_t j = 0; j < READ; j++) { + LFS_ASSERT(buffer[j] == (block+i+j) % 251); + } + } + + // write block 1 + block = 1; + cfg->erase(cfg, block) => 0; + for (lfs_off_t i = 0; i < cfg->block_size; i += PROG) { + for (lfs_off_t j = 0; j < PROG; j++) { + buffer[j] = (block+i+j) % 251; + } + cfg->prog(cfg, block, i, buffer, PROG) => 0; + } + + // read block 1 + block = 1; + for (lfs_off_t i = 0; i < cfg->block_size; i += READ) { + cfg->read(cfg, block, i, buffer, READ) => 0; + + for (lfs_off_t j = 0; j < READ; j++) { + LFS_ASSERT(buffer[j] == (block+i+j) % 251); + } + } + + // read block 0 again + block = 0; + for (lfs_off_t i = 0; i < cfg->block_size; i += READ) { + cfg->read(cfg, block, i, buffer, READ) => 0; + + for (lfs_off_t j = 0; j < READ; j++) { + LFS_ASSERT(buffer[j] == (block+i+j) % 251); + } + } +''' + +[cases.last_block] +defines.READ = ['READ_SIZE', 'BLOCK_SIZE'] +defines.PROG = ['PROG_SIZE', 'BLOCK_SIZE'] +code = ''' + uint8_t buffer[lfs_max(READ, PROG)]; + lfs_block_t block; + + // write block 0 + block = 0; + cfg->erase(cfg, block) => 0; + for (lfs_off_t i = 0; i < cfg->block_size; i += PROG) { + for (lfs_off_t j = 0; j < PROG; j++) { + buffer[j] = (block+i+j) % 251; + } + cfg->prog(cfg, block, i, buffer, PROG) => 0; + } + + // read block 0 + block = 0; + for (lfs_off_t i = 0; i < cfg->block_size; i += READ) { + cfg->read(cfg, block, i, buffer, READ) => 0; + + for (lfs_off_t j = 0; j < READ; j++) { + LFS_ASSERT(buffer[j] == (block+i+j) % 251); + } + } + + // write block n-1 + block = cfg->block_count-1; + cfg->erase(cfg, block) => 0; + for (lfs_off_t i = 0; i < cfg->block_size; i += PROG) { + for (lfs_off_t j = 0; j < PROG; j++) { + buffer[j] = (block+i+j) % 251; + } + cfg->prog(cfg, block, i, buffer, PROG) => 0; + } + + // read block n-1 + block = cfg->block_count-1; + for (lfs_off_t i = 0; i < cfg->block_size; i += READ) { + cfg->read(cfg, block, i, buffer, READ) => 0; + + for (lfs_off_t j = 0; j < READ; j++) { + LFS_ASSERT(buffer[j] == (block+i+j) % 251); + } + } + + // read block 0 again + block = 0; + for (lfs_off_t i = 0; i < cfg->block_size; i += READ) { + cfg->read(cfg, block, i, buffer, READ) => 0; + + for (lfs_off_t j = 0; j < READ; j++) { + LFS_ASSERT(buffer[j] == (block+i+j) % 251); + } + } +''' + +[cases.powers_of_two] +defines.READ = ['READ_SIZE', 'BLOCK_SIZE'] +defines.PROG = ['PROG_SIZE', 'BLOCK_SIZE'] +code = ''' + uint8_t buffer[lfs_max(READ, PROG)]; + + // write/read every power of 2 + lfs_block_t block = 1; + while (block < cfg->block_count) { + // write + cfg->erase(cfg, block) => 0; + for (lfs_off_t i = 0; i < cfg->block_size; i += PROG) { + for (lfs_off_t j = 0; j < PROG; j++) { + buffer[j] = (block+i+j) % 251; + } + cfg->prog(cfg, block, i, buffer, PROG) => 0; + } + + // read + for (lfs_off_t i = 0; i < cfg->block_size; i += READ) { + cfg->read(cfg, block, i, buffer, READ) => 0; + + for (lfs_off_t j = 0; j < READ; j++) { + LFS_ASSERT(buffer[j] == (block+i+j) % 251); + } + } + + block *= 2; + } + + // read every power of 2 again + block = 1; + while (block < cfg->block_count) { + // read + for (lfs_off_t i = 0; i < cfg->block_size; i += READ) { + cfg->read(cfg, block, i, buffer, READ) => 0; + + for (lfs_off_t j = 0; j < READ; j++) { + LFS_ASSERT(buffer[j] == (block+i+j) % 251); + } + } + + block *= 2; + } +''' + +[cases.fibonacci] +defines.READ = ['READ_SIZE', 'BLOCK_SIZE'] +defines.PROG = ['PROG_SIZE', 'BLOCK_SIZE'] +code = ''' + uint8_t buffer[lfs_max(READ, PROG)]; + + // write/read every fibonacci number on our device + lfs_block_t block = 1; + lfs_block_t block_ = 1; + while (block < cfg->block_count) { + // write + cfg->erase(cfg, block) => 0; + for (lfs_off_t i = 0; i < cfg->block_size; i += PROG) { + for (lfs_off_t j = 0; j < PROG; j++) { + buffer[j] = (block+i+j) % 251; + } + cfg->prog(cfg, block, i, buffer, PROG) => 0; + } + + // read + for (lfs_off_t i = 0; i < cfg->block_size; i += READ) { + cfg->read(cfg, block, i, buffer, READ) => 0; + + for (lfs_off_t j = 0; j < READ; j++) { + LFS_ASSERT(buffer[j] == (block+i+j) % 251); + } + } + + lfs_block_t nblock = block + block_; + block_ = block; + block = nblock; + } + + // read every fibonacci number again + block = 1; + block_ = 1; + while (block < cfg->block_count) { + // read + for (lfs_off_t i = 0; i < cfg->block_size; i += READ) { + cfg->read(cfg, block, i, buffer, READ) => 0; + + for (lfs_off_t j = 0; j < READ; j++) { + LFS_ASSERT(buffer[j] == (block+i+j) % 251); + } + } + + lfs_block_t nblock = block + block_; + block_ = block; + block = nblock; + } +''' + + + + From 61455b6191bb5da9d8b799f4b7056570d3df8820 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 19 Aug 2022 18:57:55 -0500 Subject: [PATCH 24/81] Added back heuristic-based power-loss testing The main change here from the previous test framework design is: 1. Powerloss testing remains in-process, speeding up testing. 2. The state of a test, included all powerlosses, is encoded in the test id + leb16 encoded powerloss string. This means exhaustive testing can be run in CI, but then easily reproduced locally with full debugger support. For example: ./scripts/test.py test_dirs#reentrant_many_dir#10#1248g1g2 --gdb Will run the test test_dir, case reentrant_many_dir, permutation #10, with powerlosses at 1, 2, 4, 8, 16, and 32 cycles. Dropping into gdb if an assert fails. The changes to the block-device are a work-in-progress for a lazily-allocated/copy-on-write block device that I'm hoping will keep exhaustive testing relatively low-cost. --- Makefile | 2 +- bd/lfs_testbd.c | 448 +++++++++------ bd/lfs_testbd.h | 119 +++- lfs.h | 2 - lfs_util.h | 1 + runners/test_runner.c | 1257 +++++++++++++++++++++++++++++++---------- runners/test_runner.h | 27 +- scripts/test.py | 106 ++-- 8 files changed, 1391 insertions(+), 571 deletions(-) diff --git a/Makefile b/Makefile index bd829dd7..416e7606 100644 --- a/Makefile +++ b/Makefile @@ -110,10 +110,10 @@ tags: .PHONY: test-runner test-runner: override CFLAGS+=--coverage test-runner: $(BUILDDIR)runners/test_runner + rm -f $(TEST_GCDA) .PHONY: test test: test-runner - rm -f $(TEST_GCDA) ./scripts/test.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS) .PHONY: test-list diff --git a/bd/lfs_testbd.c b/bd/lfs_testbd.c index e9a96a96..4100ea8f 100644 --- a/bd/lfs_testbd.c +++ b/bd/lfs_testbd.c @@ -11,6 +11,79 @@ #include +// access to lazily-allocated/copy-on-write blocks +// +// Note we can only modify a block if we have exclusive access to it (rc == 1) +// + +// TODO +__attribute__((unused)) +static void lfs_testbd_incblock(lfs_testbd_t *bd, lfs_block_t block) { + if (bd->blocks[block]) { + bd->blocks[block]->rc += 1; + } +} + +static void lfs_testbd_decblock(lfs_testbd_t *bd, lfs_block_t block) { + if (bd->blocks[block]) { + bd->blocks[block]->rc -= 1; + if (bd->blocks[block]->rc == 0) { + free(bd->blocks[block]); + bd->blocks[block] = NULL; + } + } +} + +static const lfs_testbd_block_t *lfs_testbd_getblock(lfs_testbd_t *bd, + lfs_block_t block) { + return bd->blocks[block]; +} + +static lfs_testbd_block_t *lfs_testbd_mutblock(lfs_testbd_t *bd, + lfs_block_t block, lfs_size_t block_size) { + if (bd->blocks[block] && bd->blocks[block]->rc == 1) { + // rc == 1? can modify + return bd->blocks[block]; + + } else if (bd->blocks[block]) { + // rc > 1? need to create a copy + lfs_testbd_block_t *b = malloc( + sizeof(lfs_testbd_block_t) + block_size); + if (!b) { + return NULL; + } + + memcpy(b, bd->blocks[block], sizeof(lfs_testbd_block_t) + block_size); + b->rc = 1; + + lfs_testbd_decblock(bd, block); + bd->blocks[block] = b; + return b; + + } else { + // no block? need to allocate + lfs_testbd_block_t *b = malloc( + sizeof(lfs_testbd_block_t) + block_size); + if (!b) { + return NULL; + } + + b->rc = 1; + b->wear = 0; + + // zero for consistency + memset(b->data, + (bd->cfg->erase_value != -1) ? bd->cfg->erase_value : 0, + block_size); + + bd->blocks[block] = b; + return b; + } +} + + +// testbd create/destroy + int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, const struct lfs_testbd_config *bdcfg) { LFS_TESTBD_TRACE("lfs_testbd_createcfg(%p {.context=%p, " @@ -20,62 +93,35 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, "\"%s\", " "%p {.erase_value=%"PRId32", .erase_cycles=%"PRIu32", " ".badblock_behavior=%"PRIu8", .power_cycles=%"PRIu32", " - ".buffer=%p, .wear_buffer=%p})", + ".powerloss_behavior=%"PRIu8", .powerloss_cb=%p, " + ".powerloss_data=%p, .track_branches=%d})", (void*)cfg, cfg->context, (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, path, (void*)bdcfg, bdcfg->erase_value, bdcfg->erase_cycles, bdcfg->badblock_behavior, bdcfg->power_cycles, - bdcfg->buffer, bdcfg->wear_buffer); + bdcfg->powerloss_behavior, (void*)(uintptr_t)bdcfg->powerloss_cb, + bdcfg->powerloss_data, bdcfg->track_branches); lfs_testbd_t *bd = cfg->context; bd->cfg = bdcfg; + // allocate our block array, all blocks start as uninitialized + bd->blocks = malloc(cfg->block_count * sizeof(lfs_testbd_block_t*)); + if (!bd->blocks) { + LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM); + return LFS_ERR_NOMEM; + } + memset(bd->blocks, 0, cfg->block_count * sizeof(lfs_testbd_block_t*)); + // setup testing things - bd->persist = path; bd->power_cycles = bd->cfg->power_cycles; + bd->branches = NULL; + bd->branch_capacity = 0; + bd->branch_count = 0; - // create scratch block if we need it (for emulating erase values) - if (bd->cfg->erase_value != -1) { - if (bd->cfg->scratch_buffer) { - bd->scratch = bd->cfg->scratch_buffer; - } else { - bd->scratch = lfs_malloc(cfg->block_size); - if (!bd->scratch) { - LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM); - return LFS_ERR_NOMEM; - } - } - } - - // create map of wear - if (bd->cfg->erase_cycles) { - if (bd->cfg->wear_buffer) { - bd->wear = bd->cfg->wear_buffer; - } else { - bd->wear = lfs_malloc(sizeof(lfs_testbd_wear_t)*cfg->block_count); - if (!bd->wear) { - LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM); - return LFS_ERR_NOMEM; - } - } - - memset(bd->wear, 0, sizeof(lfs_testbd_wear_t) * cfg->block_count); - } - - // create underlying block device - if (bd->persist) { - int err = lfs_filebd_create(cfg, path); - LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", err); - return err; - } else { - bd->u.ram.cfg = (struct lfs_rambd_config){ - .buffer = bd->cfg->buffer, - }; - int err = lfs_rambd_createcfg(cfg, &bd->u.ram.cfg); - LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", err); - return err; - } + LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", 0); + return 0; } int lfs_testbd_create(const struct lfs_config *cfg, const char *path) { @@ -99,65 +145,65 @@ int lfs_testbd_destroy(const struct lfs_config *cfg) { LFS_TESTBD_TRACE("lfs_testbd_destroy(%p)", (void*)cfg); lfs_testbd_t *bd = cfg->context; - if (bd->cfg->erase_value != -1 && !bd->cfg->scratch_buffer) { - lfs_free(bd->scratch); - } - if (bd->cfg->erase_cycles && !bd->cfg->wear_buffer) { - lfs_free(bd->wear); + // decrement reference counts + for (lfs_block_t i = 0; i < cfg->block_count; i++) { + lfs_testbd_decblock(bd, i); } - if (bd->persist) { - int err = lfs_filebd_destroy(cfg); - LFS_TESTBD_TRACE("lfs_testbd_destroy -> %d", err); - return err; - } else { - int err = lfs_rambd_destroy(cfg); - LFS_TESTBD_TRACE("lfs_testbd_destroy -> %d", err); - return err; - } + // free memory + free(bd->blocks); + free(bd->branches); + + LFS_TESTBD_TRACE("lfs_testbd_destroy -> %d", 0); + return 0; } -/// Internal mapping to block devices /// -static int lfs_testbd_rawread(const struct lfs_config *cfg, lfs_block_t block, - lfs_off_t off, void *buffer, lfs_size_t size) { - lfs_testbd_t *bd = cfg->context; - if (bd->persist) { - return lfs_filebd_read(cfg, block, off, buffer, size); - } else { - return lfs_rambd_read(cfg, block, off, buffer, size); - } -} -static int lfs_testbd_rawprog(const struct lfs_config *cfg, lfs_block_t block, - lfs_off_t off, const void *buffer, lfs_size_t size) { - lfs_testbd_t *bd = cfg->context; - if (bd->persist) { - return lfs_filebd_prog(cfg, block, off, buffer, size); - } else { - return lfs_rambd_prog(cfg, block, off, buffer, size); - } -} -static int lfs_testbd_rawerase(const struct lfs_config *cfg, - lfs_block_t block) { - lfs_testbd_t *bd = cfg->context; - if (bd->persist) { - return lfs_filebd_erase(cfg, block); - } else { - return lfs_rambd_erase(cfg, block); - } -} +///// Internal mapping to block devices /// +//static int lfs_testbd_rawread(const struct lfs_config *cfg, lfs_block_t block, +// lfs_off_t off, void *buffer, lfs_size_t size) { +// lfs_testbd_t *bd = cfg->context; +// if (bd->persist) { +// return lfs_filebd_read(cfg, block, off, buffer, size); +// } else { +// return lfs_rambd_read(cfg, block, off, buffer, size); +// } +//} +// +//static int lfs_testbd_rawprog(const struct lfs_config *cfg, lfs_block_t block, +// lfs_off_t off, const void *buffer, lfs_size_t size) { +// lfs_testbd_t *bd = cfg->context; +// if (bd->persist) { +// return lfs_filebd_prog(cfg, block, off, buffer, size); +// } else { +// return lfs_rambd_prog(cfg, block, off, buffer, size); +// } +//} +// +//static int lfs_testbd_rawerase(const struct lfs_config *cfg, +// lfs_block_t block) { +// lfs_testbd_t *bd = cfg->context; +// if (bd->persist) { +// return lfs_filebd_erase(cfg, block); +// } else { +// return lfs_rambd_erase(cfg, block); +// } +//} +// +//static int lfs_testbd_rawsync(const struct lfs_config *cfg) { +// lfs_testbd_t *bd = cfg->context; +// if (bd->persist) { +// return lfs_filebd_sync(cfg); +// } else { +// return lfs_rambd_sync(cfg); +// } +//} -static int lfs_testbd_rawsync(const struct lfs_config *cfg) { - lfs_testbd_t *bd = cfg->context; - if (bd->persist) { - return lfs_filebd_sync(cfg); - } else { - return lfs_rambd_sync(cfg); - } -} -/// block device API /// + +// block device API + int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { LFS_TESTBD_TRACE("lfs_testbd_read(%p, " @@ -171,17 +217,27 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, LFS_ASSERT(size % cfg->read_size == 0); LFS_ASSERT(off+size <= cfg->block_size); - // block bad? - if (bd->cfg->erase_cycles && bd->wear[block] >= bd->cfg->erase_cycles && - bd->cfg->badblock_behavior == LFS_TESTBD_BADBLOCK_READERROR) { - LFS_TESTBD_TRACE("lfs_testbd_read -> %d", LFS_ERR_CORRUPT); - return LFS_ERR_CORRUPT; + // get the block + const lfs_testbd_block_t *b = lfs_testbd_getblock(bd, block); + if (b) { + // block bad? + if (bd->cfg->erase_cycles && b->wear >= bd->cfg->erase_cycles && + bd->cfg->badblock_behavior == LFS_TESTBD_BADBLOCK_READERROR) { + LFS_TESTBD_TRACE("lfs_testbd_read -> %d", LFS_ERR_CORRUPT); + return LFS_ERR_CORRUPT; + } + + // read data + memcpy(buffer, &b->data[off], size); + } else { + // zero for consistency + memset(buffer, + (bd->cfg->erase_value != -1) ? bd->cfg->erase_value : 0, + size); } - // read - int err = lfs_testbd_rawread(cfg, block, off, buffer, size); - LFS_TESTBD_TRACE("lfs_testbd_read -> %d", err); - return err; + LFS_TESTBD_TRACE("lfs_testbd_read -> %d", 0); + return 0; } int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, @@ -197,8 +253,15 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, LFS_ASSERT(size % cfg->prog_size == 0); LFS_ASSERT(off+size <= cfg->block_size); + // get the block + lfs_testbd_block_t *b = lfs_testbd_mutblock(bd, block, cfg->block_size); + if (!b) { + LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_NOMEM); + return LFS_ERR_NOMEM; + } + // block bad? - if (bd->cfg->erase_cycles && bd->wear[block] >= bd->cfg->erase_cycles) { + if (bd->cfg->erase_cycles && b->wear >= bd->cfg->erase_cycles) { if (bd->cfg->badblock_behavior == LFS_TESTBD_BADBLOCK_PROGERROR) { LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_CORRUPT); @@ -212,54 +275,34 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, } } - // emulate an erase value? + // were we erased properly? if (bd->cfg->erase_value != -1) { - int err = lfs_testbd_rawread(cfg, block, 0, - bd->scratch, cfg->block_size); - if (err) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); - return err; - } - - // assert that program block was erased for (lfs_off_t i = 0; i < size; i++) { - LFS_ASSERT(bd->scratch[off+i] == bd->cfg->erase_value); - } - - memcpy(&bd->scratch[off], buffer, size); - - err = lfs_testbd_rawerase(cfg, block); - if (err) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); - return err; - } - - err = lfs_testbd_rawprog(cfg, block, 0, - bd->scratch, cfg->block_size); - if (err) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); - return err; - } - } else { - // prog - int err = lfs_testbd_rawprog(cfg, block, off, buffer, size); - if (err) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); - return err; + LFS_ASSERT(b->data[off+i] == bd->cfg->erase_value); } } + // prog data + memcpy(&b->data[off], buffer, size); + // lose power? if (bd->power_cycles > 0) { bd->power_cycles -= 1; if (bd->power_cycles == 0) { - // sync to make sure we persist the last changes - LFS_ASSERT(lfs_testbd_rawsync(cfg) == 0); // simulate power loss - exit(33); + bd->cfg->powerloss_cb(bd->cfg->powerloss_data); } } +// // track power-loss branch? +// if (bd->cfg->track_branches) { +// int err = lfs_testbd_trackbranch(bd); +// if (err) { +// LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); +// return err; +// } +// } + LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0); return 0; } @@ -271,9 +314,16 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { // check if erase is valid LFS_ASSERT(block < cfg->block_count); + // get the block + lfs_testbd_block_t *b = lfs_testbd_mutblock(bd, block, cfg->block_size); + if (!b) { + LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_NOMEM); + return LFS_ERR_NOMEM; + } + // block bad? if (bd->cfg->erase_cycles) { - if (bd->wear[block] >= bd->cfg->erase_cycles) { + if (b->wear >= bd->cfg->erase_cycles) { if (bd->cfg->badblock_behavior == LFS_TESTBD_BADBLOCK_ERASEERROR) { LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", LFS_ERR_CORRUPT); @@ -285,70 +335,69 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { } } else { // mark wear - bd->wear[block] += 1; + b->wear += 1; } } // emulate an erase value? if (bd->cfg->erase_value != -1) { - memset(bd->scratch, bd->cfg->erase_value, cfg->block_size); - - int err = lfs_testbd_rawerase(cfg, block); - if (err) { - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); - return err; - } - - err = lfs_testbd_rawprog(cfg, block, 0, - bd->scratch, cfg->block_size); - if (err) { - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); - return err; - } - } else { - // erase - int err = lfs_testbd_rawerase(cfg, block); - if (err) { - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); - return err; - } + memset(b->data, bd->cfg->erase_value, cfg->block_size); } // lose power? if (bd->power_cycles > 0) { bd->power_cycles -= 1; if (bd->power_cycles == 0) { - // sync to make sure we persist the last changes - LFS_ASSERT(lfs_testbd_rawsync(cfg) == 0); // simulate power loss - exit(33); + bd->cfg->powerloss_cb(bd->cfg->powerloss_data); } } +// // track power-loss branch? +// if (bd->cfg->track_branches) { +// int err = lfs_testbd_trackbranch(bd); +// if (err) { +// LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); +// return err; +// } +// } + LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0); return 0; } int lfs_testbd_sync(const struct lfs_config *cfg) { LFS_TESTBD_TRACE("lfs_testbd_sync(%p)", (void*)cfg); - int err = lfs_testbd_rawsync(cfg); - LFS_TESTBD_TRACE("lfs_testbd_sync -> %d", err); - return err; + + // do nothing + (void)cfg; + + LFS_TESTBD_TRACE("lfs_testbd_sync -> %d", 0); + return 0; } -/// simulated wear operations /// +// simulated wear operations + lfs_testbd_swear_t lfs_testbd_getwear(const struct lfs_config *cfg, lfs_block_t block) { LFS_TESTBD_TRACE("lfs_testbd_getwear(%p, %"PRIu32")", (void*)cfg, block); lfs_testbd_t *bd = cfg->context; // check if block is valid - LFS_ASSERT(bd->cfg->erase_cycles); LFS_ASSERT(block < cfg->block_count); - LFS_TESTBD_TRACE("lfs_testbd_getwear -> %"PRIu32, bd->wear[block]); - return bd->wear[block]; + // get the wear + lfs_testbd_wear_t wear; + const lfs_testbd_block_t *b = lfs_testbd_getblock(bd, block); + if (b) { + wear = b->wear; + } else { + wear = 0; + } + + LFS_TESTBD_TRACE("lfs_testbd_getwear -> %"PRIu32, wear); + return wear; } int lfs_testbd_setwear(const struct lfs_config *cfg, @@ -357,11 +406,58 @@ int lfs_testbd_setwear(const struct lfs_config *cfg, lfs_testbd_t *bd = cfg->context; // check if block is valid - LFS_ASSERT(bd->cfg->erase_cycles); LFS_ASSERT(block < cfg->block_count); - bd->wear[block] = wear; + // set the wear + lfs_testbd_block_t *b = lfs_testbd_mutblock(bd, block, cfg->block_size); + if (!b) { + LFS_TESTBD_TRACE("lfs_testbd_setwear -> %"PRIu32, LFS_ERR_NOMEM); + return LFS_ERR_NOMEM; + } + b->wear = wear; - LFS_TESTBD_TRACE("lfs_testbd_setwear -> %d", 0); + LFS_TESTBD_TRACE("lfs_testbd_setwear -> %"PRIu32, 0); return 0; } + +lfs_testbd_spowercycles_t lfs_testbd_getpowercycles( + const struct lfs_config *cfg) { + LFS_TESTBD_TRACE("lfs_testbd_getpowercycles(%p)", (void*)cfg); + lfs_testbd_t *bd = cfg->context; + + LFS_TESTBD_TRACE("lfs_testbd_getpowercycles -> %"PRIi32, bd->power_cycles); + return bd->power_cycles; +} + +int lfs_testbd_setpowercycles(const struct lfs_config *cfg, + lfs_testbd_powercycles_t power_cycles) { + LFS_TESTBD_TRACE("lfs_testbd_setpowercycles(%p, %"PRIi32")", + (void*)cfg, power_cycles); + lfs_testbd_t *bd = cfg->context; + + bd->power_cycles = power_cycles; + + LFS_TESTBD_TRACE("lfs_testbd_getpowercycles -> %d", 0); + return 0; +} + +//int lfs_testbd_getbranch(const struct lfs_config *cfg, +// lfs_testbd_powercycles_t branch, lfs_testbd_t *bd) { +// LFS_TESTBD_TRACE("lfs_testbd_getbranch(%p, %zu, %p)", +// (void*)cfg, branch, bd); +// lfs_testbd_t *bd = cfg->context; +// +// // TODO +// +// LFS_TESTBD_TRACE("lfs_testbd_getbranch -> %d", 0); +// return 0; +//} + +lfs_testbd_spowercycles_t lfs_testbd_getbranchcount( + const struct lfs_config *cfg) { + LFS_TESTBD_TRACE("lfs_testbd_getbranchcount(%p)", (void*)cfg); + lfs_testbd_t *bd = cfg->context; + + LFS_TESTBD_TRACE("lfs_testbd_getbranchcount -> %"PRIu32, bd->branch_count); + return bd->branch_count; +} diff --git a/bd/lfs_testbd.h b/bd/lfs_testbd.h index 3eafb8f4..9b51cd85 100644 --- a/bd/lfs_testbd.h +++ b/bd/lfs_testbd.h @@ -29,23 +29,33 @@ extern "C" #endif #endif -// Mode determining how "bad blocks" behave during testing. This simulates +// Mode determining how "bad-blocks" behave during testing. This simulates // some real-world circumstances such as progs not sticking (prog-noop), // a readonly disk (erase-noop), and ECC failures (read-error). // // Not that read-noop is not allowed. Read _must_ return a consistent (but // may be arbitrary) value on every read. -enum lfs_testbd_badblock_behavior { +typedef enum lfs_testbd_badblock_behavior { LFS_TESTBD_BADBLOCK_PROGERROR, LFS_TESTBD_BADBLOCK_ERASEERROR, LFS_TESTBD_BADBLOCK_READERROR, LFS_TESTBD_BADBLOCK_PROGNOOP, LFS_TESTBD_BADBLOCK_ERASENOOP, -}; +} lfs_testbd_badblock_behavior_t; + +// Mode determining how power-loss behaves during testing. For now this +// only supports a noop behavior, leaving the data on-disk untouched. +typedef enum lfs_testbd_powerloss_behavior { + LFS_TESTBD_POWERLOSS_NOOP, +} lfs_testbd_powerloss_behavior_t; // Type for measuring wear typedef uint32_t lfs_testbd_wear_t; -typedef int32_t lfs_testbd_swear_t; +typedef int32_t lfs_testbd_swear_t; + +// Type for tracking power-cycles +typedef uint32_t lfs_testbd_powercycles_t; +typedef int32_t lfs_testbd_spowercycles_t; // testbd config, this is required for testing struct lfs_testbd_config { @@ -55,42 +65,77 @@ struct lfs_testbd_config { int32_t erase_value; // Number of erase cycles before a block becomes "bad". The exact behavior - // of bad blocks is controlled by the badblock_mode. + // of bad blocks is controlled by badblock_behavior. uint32_t erase_cycles; - // The mode determining how bad blocks fail - uint8_t badblock_behavior; + // The mode determining how bad-blocks fail + lfs_testbd_badblock_behavior_t badblock_behavior; - // Number of write operations (erase/prog) before forcefully killing - // the program with exit. Simulates power-loss. 0 disables. - uint32_t power_cycles; + // Number of write operations (erase/prog) before triggering a power-loss. + // power_cycles=0 disables this. The exact behavior of power-loss is + // controlled by a combination of powerloss_behavior and powerloss_cb. + lfs_testbd_powercycles_t power_cycles; - // Optional buffer for RAM block device. - void *buffer; + // The mode determining how power-loss affects disk + lfs_testbd_powerloss_behavior_t powerloss_behavior; - // Optional buffer for wear. - void *wear_buffer; + // Function to call to emulate power-loss. The exact behavior of power-loss + // is up to the runner to provide. + void (*powerloss_cb)(void*); - // Optional buffer for scratch memory, needed when erase_value != -1. - void *scratch_buffer; + // Data for power-loss callback + void *powerloss_data; + + // True to track when power-loss could have occured. Note this involves + // heavy memory usage! + bool track_branches; + +// // Optional buffer for RAM block device. +// void *buffer; +// +// // Optional buffer for wear. +// void *wear_buffer; +// +// // Optional buffer for scratch memory, needed when erase_value != -1. +// void *scratch_buffer; }; +// A reference counted block +typedef struct lfs_testbd_block { + uint32_t rc; + lfs_testbd_wear_t wear; + + uint8_t data[]; +} lfs_testbd_block_t; + // testbd state typedef struct lfs_testbd { - union { - struct { - lfs_filebd_t bd; - } file; - struct { - lfs_rambd_t bd; - struct lfs_rambd_config cfg; - } ram; - } u; - - bool persist; + // array of copy-on-write blocks + lfs_testbd_block_t **blocks; uint32_t power_cycles; - lfs_testbd_wear_t *wear; - uint8_t *scratch; + + // array of tracked branches + struct lfs_testbd *branches; + lfs_testbd_powercycles_t branch_count; + lfs_testbd_powercycles_t branch_capacity; + + // TODO file? + + +// union { +// struct { +// lfs_filebd_t bd; +// } file; +// struct { +// lfs_rambd_t bd; +// struct lfs_rambd_config cfg; +// } ram; +// } u; +// +// bool persist; +// uint32_t power_cycles; +// lfs_testbd_wear_t *wear; +// uint8_t *scratch; const struct lfs_testbd_config *cfg; } lfs_testbd_t; @@ -139,6 +184,22 @@ lfs_testbd_swear_t lfs_testbd_getwear(const struct lfs_config *cfg, int lfs_testbd_setwear(const struct lfs_config *cfg, lfs_block_t block, lfs_testbd_wear_t wear); +// Get the remaining power-cycles +lfs_testbd_spowercycles_t lfs_testbd_getpowercycles( + const struct lfs_config *cfg); + +// Manually set the remaining power-cycles +int lfs_testbd_setpowercycles(const struct lfs_config *cfg, + lfs_testbd_powercycles_t power_cycles); + +// Get a power-loss branch, requires track_branches=true +int lfs_testbd_getbranch(const struct lfs_config *cfg, + lfs_testbd_powercycles_t branch, lfs_testbd_t *bd); + +// Get the current number of power-loss branches +lfs_testbd_spowercycles_t lfs_testbd_getbranchcount( + const struct lfs_config *cfg); + #ifdef __cplusplus } /* extern "C" */ diff --git a/lfs.h b/lfs.h index 3fc1e982..9bdfaa0c 100644 --- a/lfs.h +++ b/lfs.h @@ -8,8 +8,6 @@ #ifndef LFS_H #define LFS_H -#include -#include #include "lfs_util.h" #ifdef __cplusplus diff --git a/lfs_util.h b/lfs_util.h index 0cbc2a31..3971882b 100644 --- a/lfs_util.h +++ b/lfs_util.h @@ -23,6 +23,7 @@ // System includes #include #include +#include #include #include diff --git a/runners/test_runner.c b/runners/test_runner.c index cc932b73..392915af 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -5,6 +5,7 @@ #include #include #include +#include // test suites in a custom ld section @@ -75,7 +76,7 @@ intmax_t test_define(size_t define) { exit(-1); } -static void test_define_geometry(const struct test_geometry *geometry) { +static void define_geometry(const struct test_geometry *geometry) { test_geometry_defines = geometry->defines; } @@ -98,7 +99,7 @@ static void test_define_overrides( } } -static void test_define_suite(const struct test_suite *suite) { +static void define_suite(const struct test_suite *suite) { test_define_names = suite->define_names; test_define_count = suite->define_count; @@ -123,7 +124,7 @@ static void test_define_suite(const struct test_suite *suite) { } } -static void test_define_perm( +static void define_perm( const struct test_suite *suite, const struct test_case *case_, size_t perm) { @@ -136,12 +137,110 @@ static void test_define_perm( } -// other miscellany -static const char *test_suite = NULL; -static const char *test_case = NULL; -static size_t test_perm = -1; +// a quick encoding scheme for sequences of power-loss +static void leb16_print( + const lfs_testbd_powercycles_t *cycles, + size_t cycle_count) { + for (size_t i = 0; i < cycle_count; i++) { + lfs_testbd_powercycles_t x = cycles[i]; + while (true) { + lfs_testbd_powercycles_t nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0); + printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10); + if (x <= 0xf) { + break; + } + x >>= 4; + } + } +} + +static size_t leb16_parse(const char *s, char **tail, + lfs_testbd_powercycles_t **cycles) { + // first lets count how many number we're dealing with + size_t count = 0; + size_t len = 0; + for (size_t i = 0;; i++) { + if ((s[i] >= '0' && s[i] <= '9') + || (s[i] >= 'a' && s[i] <= 'f')) { + len = i+1; + count += 1; + } else if ((s[i] >= 'g' && s[i] <= 'v')) { + // do nothing + } else { + break; + } + } + + // then parse + lfs_testbd_powercycles_t *cycles_ = malloc( + count * sizeof(lfs_testbd_powercycles_t)); + size_t i = 0; + lfs_testbd_powercycles_t x = 0; + size_t k = 0; + for (size_t j = 0; j < len; j++) { + lfs_testbd_powercycles_t nibble = s[j]; + nibble = (nibble < 'a') ? nibble-'0' : nibble-'a'+10; + x |= (nibble & 0xf) << (4*k); + k += 1; + if (!(nibble & 0x10)) { + cycles_[i] = x; + i += 1; + x = 0; + k = 0; + } + } + + if (tail) { + *tail = (char*)s + len; + } + *cycles = cycles_; + return count; +} + + +// test state +typedef struct test_powerloss { + char short_name; + const char *long_name; + + void (*run)( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm, + const lfs_testbd_powercycles_t *cycles, + size_t cycle_count); + const lfs_testbd_powercycles_t *cycles; + size_t cycle_count; +} test_powerloss_t; + +static void run_powerloss_none( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm, + const lfs_testbd_powercycles_t *cycles, + size_t cycle_count); +static const test_powerloss_t *test_powerlosses = (const test_powerloss_t[]){ + {'0', "none", run_powerloss_none, NULL, 0}, +}; +static size_t test_powerloss_count = 1; + + +typedef struct test_id { + const char *suite; + const char *case_; + size_t perm; + const lfs_testbd_powercycles_t *cycles; + size_t cycle_count; +} test_id_t; + +static const test_id_t *test_ids = (const test_id_t[]) { + {NULL, NULL, -1, NULL, 0}, +}; +static size_t test_id_count = 1; + + static const char *test_geometry = NULL; -static test_types_t test_types = 0; + static size_t test_start = 0; static size_t test_stop = -1; static size_t test_step = 1; @@ -149,243 +248,294 @@ static size_t test_step = 1; static const char *test_disk = NULL; FILE *test_trace = NULL; -// note, these skips are different than filtered tests -static bool test_suite_skip(const struct test_suite *suite) { - return (test_suite && strcmp(suite->name, test_suite) != 0) - || (test_types && (suite->types & test_types) == 0); -} -static bool test_case_skip(const struct test_case *case_) { - return (test_case && strcmp(case_->name, test_case) != 0) - || (test_types && (case_->types & test_types) == 0); -} - -static bool test_perm_skip(size_t perm) { - size_t geom_perm = perm % TEST_GEOMETRY_COUNT; - return (test_perm != (size_t)-1 && perm != test_perm) - || (test_geometry && (strcmp( - test_geometries[geom_perm].name, - test_geometry) != 0)); -} - -static bool test_step_skip(size_t step) { - return !(step >= test_start - && step < test_stop - && (step-test_start) % test_step == 0); -} - -static void test_case_permcount( +// how many permutations are there actually in a test case +static void count_perms( const struct test_suite *suite, const struct test_case *case_, + size_t perm, + const lfs_testbd_powercycles_t *cycles, + size_t cycle_count, size_t *perms, size_t *filtered) { + (void)cycle_count; size_t perms_ = 0; size_t filtered_ = 0; - for (size_t perm = 0; - perm < TEST_GEOMETRY_COUNT - * case_->permutations; - perm++) { - if (test_perm_skip(perm)) { + for (size_t p = 0; p < (cycles ? 1 : test_powerloss_count); p++) { + if (!cycles + && test_powerlosses[p].short_name != '0' + && !(case_->flags & TEST_REENTRANT)) { continue; } - perms_ += 1; - - // setup defines - size_t case_perm = perm / TEST_GEOMETRY_COUNT; - size_t geom_perm = perm % TEST_GEOMETRY_COUNT; - test_define_perm(suite, case_, case_perm); - test_define_geometry(&test_geometries[geom_perm]); - - if (case_->filter) { - if (!case_->filter()) { + size_t perm_ = 0; + for (size_t g = 0; g < TEST_GEOMETRY_COUNT; g++) { + if (test_geometry && strcmp( + test_geometries[g].name, test_geometry) != 0) { continue; } - } - filtered_ += 1; + for (size_t k = 0; k < case_->permutations; k++) { + perm_ += 1; + + if (perm != (size_t)-1 && perm_ != perm) { + continue; + } + + perms_ += 1; + + // setup defines + define_perm(suite, case_, k); + define_geometry(&test_geometries[g]); + + if (case_->filter && !case_->filter()) { + continue; + } + + filtered_ += 1; + } + } } *perms += perms_; *filtered += filtered_; -} +} // operations we can do static void summary(void) { printf("%-36s %7s %7s %7s %11s\n", - "", "types", "suites", "cases", "perms"); + "", "flags", "suites", "cases", "perms"); size_t cases = 0; - test_types_t types = 0; + test_flags_t flags = 0; size_t perms = 0; size_t filtered = 0; - for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { - if (test_suite_skip(&test_suites[i])) { - continue; - } - test_define_suite(&test_suites[i]); - - for (size_t j = 0; j < test_suites[i].case_count; j++) { - if (test_case_skip(&test_suites[i].cases[j])) { + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + if (test_ids[t].suite && strcmp( + test_suites[i].name, test_ids[t].suite) != 0) { continue; } - test_case_permcount(&test_suites[i], &test_suites[i].cases[j], - &perms, &filtered); - } + define_suite(&test_suites[i]); - cases += test_suites[i].case_count; - types |= test_suites[i].types; + for (size_t j = 0; j < test_suites[i].case_count; j++) { + if (test_ids[t].case_ && strcmp( + test_suites[i].cases[j].name, test_ids[t].case_) != 0) { + continue; + } + + count_perms(&test_suites[i], &test_suites[i].cases[j], + test_ids[t].perm, + test_ids[t].cycles, + test_ids[t].cycle_count, + &perms, &filtered); + } + + cases += test_suites[i].case_count; + flags |= test_suites[i].flags; + } } char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", filtered, perms); - char type_buf[64]; - sprintf(type_buf, "%s%s", - (types & TEST_NORMAL) ? "n" : "", - (types & TEST_REENTRANT) ? "r" : ""); + char flag_buf[64]; + sprintf(flag_buf, "%s%s", + (flags & TEST_REENTRANT) ? "r" : "", + (!flags) ? "-" : ""); printf("%-36s %7s %7zu %7zu %11s\n", "TOTAL", - type_buf, + flag_buf, TEST_SUITE_COUNT, cases, perm_buf); } static void list_suites(void) { - printf("%-36s %7s %7s %11s\n", "suite", "types", "cases", "perms"); - for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { - if (test_suite_skip(&test_suites[i])) { - continue; - } + printf("%-36s %7s %7s %11s\n", "suite", "flags", "cases", "perms"); - test_define_suite(&test_suites[i]); - - size_t perms = 0; - size_t filtered = 0; - for (size_t j = 0; j < test_suites[i].case_count; j++) { - if (test_case_skip(&test_suites[i].cases[j])) { + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + if (test_ids[t].suite && strcmp( + test_suites[i].name, test_ids[t].suite) != 0) { continue; } - test_case_permcount(&test_suites[i], &test_suites[i].cases[j], - &perms, &filtered); - } - - char perm_buf[64]; - sprintf(perm_buf, "%zu/%zu", filtered, perms); - char type_buf[64]; - sprintf(type_buf, "%s%s", - (test_suites[i].types & TEST_NORMAL) ? "n" : "", - (test_suites[i].types & TEST_REENTRANT) ? "r" : ""); - printf("%-36s %7s %7zu %11s\n", - test_suites[i].id, - type_buf, - test_suites[i].case_count, - perm_buf); - } -} - -static void list_cases(void) { - printf("%-36s %7s %11s\n", "case", "types", "perms"); - for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { - if (test_suite_skip(&test_suites[i])) { - continue; - } - - test_define_suite(&test_suites[i]); - - for (size_t j = 0; j < test_suites[i].case_count; j++) { - if (test_case_skip(&test_suites[i].cases[j])) { - continue; - } + define_suite(&test_suites[i]); size_t perms = 0; size_t filtered = 0; - test_case_permcount(&test_suites[i], &test_suites[i].cases[j], - &perms, &filtered); - test_types_t types = test_suites[i].cases[j].types; + + for (size_t j = 0; j < test_suites[i].case_count; j++) { + if (test_ids[t].case_ && strcmp( + test_suites[i].cases[j].name, test_ids[t].case_) != 0) { + continue; + } + + count_perms(&test_suites[i], &test_suites[i].cases[j], + test_ids[t].perm, + test_ids[t].cycles, + test_ids[t].cycle_count, + &perms, &filtered); + } char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", filtered, perms); - char type_buf[64]; - sprintf(type_buf, "%s%s", - (types & TEST_NORMAL) ? "n" : "", - (types & TEST_REENTRANT) ? "r" : ""); - printf("%-36s %7s %11s\n", - test_suites[i].cases[j].id, - type_buf, + char flag_buf[64]; + sprintf(flag_buf, "%s%s", + (test_suites[i].flags & TEST_REENTRANT) ? "r" : "", + (!test_suites[i].flags) ? "-" : ""); + printf("%-36s %7s %7zu %11s\n", + test_suites[i].id, + flag_buf, + test_suites[i].case_count, perm_buf); } } } -static void list_paths(void) { - for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { - if (test_suite_skip(&test_suites[i])) { - continue; - } +static void list_cases(void) { + printf("%-36s %7s %11s\n", "case", "flags", "perms"); - for (size_t j = 0; j < test_suites[i].case_count; j++) { - if (test_case_skip(&test_suites[i].cases[j])) { + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + if (test_ids[t].suite && strcmp( + test_suites[i].name, test_ids[t].suite) != 0) { continue; } - printf("%-36s %-36s\n", - test_suites[i].cases[j].id, - test_suites[i].cases[j].path); + define_suite(&test_suites[i]); + + for (size_t j = 0; j < test_suites[i].case_count; j++) { + if (test_ids[t].case_ && strcmp( + test_suites[i].cases[j].name, test_ids[t].case_) != 0) { + continue; + } + + size_t perms = 0; + size_t filtered = 0; + + count_perms(&test_suites[i], &test_suites[i].cases[j], + test_ids[t].perm, + test_ids[t].cycles, + test_ids[t].cycle_count, + &perms, &filtered); + + char perm_buf[64]; + sprintf(perm_buf, "%zu/%zu", filtered, perms); + char flag_buf[64]; + sprintf(flag_buf, "%s%s", + (test_suites[i].cases[j].flags & TEST_REENTRANT) + ? "r" : "", + (!test_suites[i].cases[j].flags) + ? "-" : ""); + printf("%-36s %7s %11s\n", + test_suites[i].cases[j].id, + flag_buf, + perm_buf); + } + } + } +} + +static void list_paths(void) { + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + if (test_ids[t].suite && strcmp( + test_suites[i].name, test_ids[t].suite) != 0) { + continue; + } + + for (size_t j = 0; j < test_suites[i].case_count; j++) { + if (test_ids[t].case_ && strcmp( + test_suites[i].cases[j].name, test_ids[t].case_) != 0) { + continue; + } + + printf("%-36s %-36s\n", + test_suites[i].cases[j].id, + test_suites[i].cases[j].path); + } } } } static void list_defines(void) { - for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { - if (test_suite_skip(&test_suites[i])) { - continue; - } - - test_define_suite(&test_suites[i]); - - for (size_t j = 0; j < test_suites[i].case_count; j++) { - if (test_case_skip(&test_suites[i].cases[j])) { + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + if (test_ids[t].suite && strcmp( + test_suites[i].name, test_ids[t].suite) != 0) { continue; } - for (size_t perm = 0; - perm < TEST_GEOMETRY_COUNT - * test_suites[i].cases[j].permutations; - perm++) { - if (test_perm_skip(perm)) { + define_suite(&test_suites[i]); + + for (size_t j = 0; j < test_suites[i].case_count; j++) { + if (test_ids[t].case_ && strcmp( + test_suites[i].cases[j].name, test_ids[t].case_) != 0) { continue; } - // setup defines - size_t case_perm = perm / TEST_GEOMETRY_COUNT; - size_t geom_perm = perm % TEST_GEOMETRY_COUNT; - test_define_perm(&test_suites[i], - &test_suites[i].cases[j], case_perm); - test_define_geometry(&test_geometries[geom_perm]); + for (size_t p = 0; + p < (test_ids[t].cycles ? 1 : test_powerloss_count); + p++) { + if (!test_ids[t].cycles + && test_powerlosses[p].short_name != '0' + && !(test_suites[i].cases[j].flags + & TEST_REENTRANT)) { + continue; + } - // print the case - char id_buf[256]; - sprintf(id_buf, "%s#%zu", test_suites[i].cases[j].id, perm); - printf("%-36s ", id_buf); + size_t perm_ = 0; + for (size_t g = 0; g < TEST_GEOMETRY_COUNT; g++) { + if (test_geometry && strcmp( + test_geometries[g].name, test_geometry) != 0) { + continue; + } - // special case for the current geometry - printf("GEOMETRY=%s ", test_geometries[geom_perm].name); + for (size_t k = 0; + k < test_suites[i].cases[j].permutations; + k++) { + perm_ += 1; - // print each define - for (size_t k = 0; k < test_suites[i].define_count; k++) { - if (test_suites[i].cases[j].defines - && test_suites[i].cases[j].defines[case_perm][k]) { - printf("%s=%jd ", - test_suites[i].define_names[k], - test_define(k)); + if (test_ids[t].perm != (size_t)-1 + && perm_ != test_ids[t].perm) { + continue; + } + + // setup defines + define_perm(&test_suites[i], + &test_suites[i].cases[j], + k); + define_geometry(&test_geometries[g]); + + // print the case + char id_buf[256]; + sprintf(id_buf, "%s#%zu", + test_suites[i].cases[j].id, perm_); + printf("%-36s ", id_buf); + + // special case for the current geometry + printf("GEOMETRY=%s ", test_geometries[g].name); + + // print each define + for (size_t l = 0; + l < test_suites[i].define_count; + l++) { + if (test_suites[i].cases[j].defines + && test_suites[i].cases[j] + .defines[k][l]) { + printf("%s=%jd ", + test_suites[i].define_names[l], + test_define(l)); + } + } + printf("\n"); + } } } - printf("\n"); } } } @@ -399,7 +549,7 @@ static void list_geometries(void) { continue; } - test_define_geometry(&test_geometries[i]); + define_geometry(&test_geometries[i]); printf("%-36s ", test_geometries[i].name); // print each define @@ -424,96 +574,420 @@ static void list_defaults(void) { printf("\n"); } -static void run(void) { - size_t step = 0; - for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { - if (test_suite_skip(&test_suites[i])) { + + +// scenarios to run tests under power-loss + +static void run_powerloss_none( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm, + const lfs_testbd_powercycles_t *cycles, + size_t cycle_count) { + (void)cycles; + (void)cycle_count; + (void)suite; + + // create block device and configuration + lfs_testbd_t bd; + + struct lfs_config cfg = { + .context = &bd, + .read = lfs_testbd_read, + .prog = lfs_testbd_prog, + .erase = lfs_testbd_erase, + .sync = lfs_testbd_sync, + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .block_size = BLOCK_SIZE, + .block_count = BLOCK_COUNT, + .block_cycles = BLOCK_CYCLES, + .cache_size = CACHE_SIZE, + .lookahead_size = LOOKAHEAD_SIZE, + }; + + struct lfs_testbd_config bdcfg = { + .erase_value = ERASE_VALUE, + .erase_cycles = ERASE_CYCLES, + .badblock_behavior = BADBLOCK_BEHAVIOR, + }; + + int err = lfs_testbd_createcfg(&cfg, test_disk, &bdcfg); + if (err) { + fprintf(stderr, "error: could not create block device: %d\n", err); + exit(-1); + } + + // run the test + printf("running %s#%zu\n", case_->id, perm); + + case_->run(&cfg); + + printf("finished %s#%zu\n", case_->id, perm); + + // cleanup + err = lfs_testbd_destroy(&cfg); + if (err) { + fprintf(stderr, "error: could not destroy block device: %d\n", err); + exit(-1); + } +} + +static void powerloss_longjmp(void *c) { + jmp_buf *powerloss_jmp = c; + longjmp(*powerloss_jmp, 1); +} + +static void run_powerloss_linear( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm, + const lfs_testbd_powercycles_t *cycles, + size_t cycle_count) { + (void)cycles; + (void)cycle_count; + (void)suite; + + // create block device and configuration + lfs_testbd_t bd; + jmp_buf powerloss_jmp; + volatile lfs_testbd_powercycles_t i = 1; + + struct lfs_config cfg = { + .context = &bd, + .read = lfs_testbd_read, + .prog = lfs_testbd_prog, + .erase = lfs_testbd_erase, + .sync = lfs_testbd_sync, + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .block_size = BLOCK_SIZE, + .block_count = BLOCK_COUNT, + .block_cycles = BLOCK_CYCLES, + .cache_size = CACHE_SIZE, + .lookahead_size = LOOKAHEAD_SIZE, + }; + + struct lfs_testbd_config bdcfg = { + .erase_value = ERASE_VALUE, + .erase_cycles = ERASE_CYCLES, + .badblock_behavior = BADBLOCK_BEHAVIOR, + .power_cycles = i, + .powerloss_behavior = POWERLOSS_BEHAVIOR, + .powerloss_cb = powerloss_longjmp, + .powerloss_data = &powerloss_jmp, + }; + + int err = lfs_testbd_createcfg(&cfg, test_disk, &bdcfg); + if (err) { + fprintf(stderr, "error: could not create block device: %d\n", err); + exit(-1); + } + + // run the test, increasing power-cycles as power-loss events occur + printf("running %s#%zu\n", case_->id, perm); + + while (true) { + if (!setjmp(powerloss_jmp)) { + case_->run(&cfg); + break; + } + + // power-loss! + printf("powerloss %s#%zu#", case_->id, perm); + for (lfs_testbd_powercycles_t j = 1; j <= i; j++) { + leb16_print(&j, 1); + } + printf("\n"); + + i += 1; + lfs_testbd_setpowercycles(&cfg, i); + } + + printf("finished %s#%zu\n", case_->id, perm); + + // cleanup + err = lfs_testbd_destroy(&cfg); + if (err) { + fprintf(stderr, "error: could not destroy block device: %d\n", err); + exit(-1); + } +} + +static void run_powerloss_exponential( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm, + const lfs_testbd_powercycles_t *cycles, + size_t cycle_count) { + (void)cycles; + (void)cycle_count; + (void)suite; + + // create block device and configuration + lfs_testbd_t bd; + jmp_buf powerloss_jmp; + volatile lfs_testbd_powercycles_t i = 1; + + struct lfs_config cfg = { + .context = &bd, + .read = lfs_testbd_read, + .prog = lfs_testbd_prog, + .erase = lfs_testbd_erase, + .sync = lfs_testbd_sync, + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .block_size = BLOCK_SIZE, + .block_count = BLOCK_COUNT, + .block_cycles = BLOCK_CYCLES, + .cache_size = CACHE_SIZE, + .lookahead_size = LOOKAHEAD_SIZE, + }; + + struct lfs_testbd_config bdcfg = { + .erase_value = ERASE_VALUE, + .erase_cycles = ERASE_CYCLES, + .badblock_behavior = BADBLOCK_BEHAVIOR, + .power_cycles = i, + .powerloss_behavior = POWERLOSS_BEHAVIOR, + .powerloss_cb = powerloss_longjmp, + .powerloss_data = &powerloss_jmp, + }; + + int err = lfs_testbd_createcfg(&cfg, test_disk, &bdcfg); + if (err) { + fprintf(stderr, "error: could not create block device: %d\n", err); + exit(-1); + } + + // run the test, increasing power-cycles as power-loss events occur + printf("running %s#%zu\n", case_->id, perm); + + while (true) { + if (!setjmp(powerloss_jmp)) { + case_->run(&cfg); + break; + } + + // power-loss! + printf("powerloss %s#%zu#", case_->id, perm); + for (lfs_testbd_powercycles_t j = 1; j <= i; j *= 2) { + leb16_print(&j, 1); + } + printf("\n"); + + i *= 2; + lfs_testbd_setpowercycles(&cfg, i); + } + + printf("finished %s#%zu\n", case_->id, perm); + + // cleanup + err = lfs_testbd_destroy(&cfg); + if (err) { + fprintf(stderr, "error: could not destroy block device: %d\n", err); + exit(-1); + } +} + +static void run_powerloss_cycles( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm, + const lfs_testbd_powercycles_t *cycles, + size_t cycle_count) { + (void)suite; + + // create block device and configuration + lfs_testbd_t bd; + jmp_buf powerloss_jmp; + volatile size_t i = 0; + + struct lfs_config cfg = { + .context = &bd, + .read = lfs_testbd_read, + .prog = lfs_testbd_prog, + .erase = lfs_testbd_erase, + .sync = lfs_testbd_sync, + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .block_size = BLOCK_SIZE, + .block_count = BLOCK_COUNT, + .block_cycles = BLOCK_CYCLES, + .cache_size = CACHE_SIZE, + .lookahead_size = LOOKAHEAD_SIZE, + }; + + struct lfs_testbd_config bdcfg = { + .erase_value = ERASE_VALUE, + .erase_cycles = ERASE_CYCLES, + .badblock_behavior = BADBLOCK_BEHAVIOR, + .power_cycles = (i < cycle_count) ? cycles[i] : 0, + .powerloss_behavior = POWERLOSS_BEHAVIOR, + .powerloss_cb = powerloss_longjmp, + .powerloss_data = &powerloss_jmp, + }; + + int err = lfs_testbd_createcfg(&cfg, test_disk, &bdcfg); + if (err) { + fprintf(stderr, "error: could not create block device: %d\n", err); + exit(-1); + } + + // run the test, increasing power-cycles as power-loss events occur + printf("running %s#%zu\n", case_->id, perm); + + while (true) { + if (!setjmp(powerloss_jmp)) { + case_->run(&cfg); + break; + } + + // power-loss! + assert(i <= cycle_count); + printf("powerloss %s#%zu#", case_->id, perm); + leb16_print(cycles, i+1); + printf("\n"); + + i += 1; + lfs_testbd_setpowercycles(&cfg, + (i < cycle_count) ? cycles[i] : 0); + } + + printf("finished %s#%zu\n", case_->id, perm); + + // cleanup + err = lfs_testbd_destroy(&cfg); + if (err) { + fprintf(stderr, "error: could not destroy block device: %d\n", err); + exit(-1); + } +} + +//static void run_powerloss_n(void *data, +// +//static void run_powerloss_incremental(void *data, + +const test_powerloss_t builtin_powerlosses[] = { + {'0', "none", run_powerloss_none, NULL, 0}, + {'e', "exponential", run_powerloss_exponential, NULL, 0}, + {'l', "linear", run_powerloss_linear, NULL, 0}, + //{'x', "exhaustive", run_powerloss_exhaustive} + {0, NULL, NULL, NULL, 0}, +}; + +const char *const builtin_powerlosses_help[] = { + "Run with no power-losses.", + "Run with linearly-decreasing power-losses.", + "Run with exponentially-decreasing power-losses.", + //"Run a all permutations of power-losses, this may take a while.", + "Run a all permutations of n power-losses.", + "Run a custom comma-separated set of power-losses.", + "Run a custom leb16-encoded set of power-losses.", +}; + +static void list_powerlosses(void) { + printf("%-24s %s\n", "scenario", "description"); + size_t i = 0; + for (; builtin_powerlosses[i].long_name; i++) { + printf("%c,%-22s %s\n", + builtin_powerlosses[i].short_name, + builtin_powerlosses[i].long_name, + builtin_powerlosses_help[i]); + } + + // a couple more options with special parsing + printf("%-24s %s\n", "1,2,3", builtin_powerlosses_help[i+0]); + printf("%-24s %s\n", "{1,2,3}", builtin_powerlosses_help[i+1]); + printf("%-24s %s\n", "#1248g1", builtin_powerlosses_help[i+2]); +} + + +// global test step count +static size_t step = 0; + +// run the tests +static void run_perms( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm, + const lfs_testbd_powercycles_t *cycles, + size_t cycle_count) { + for (size_t p = 0; p < (cycles ? 1 : test_powerloss_count); p++) { + if (!cycles + && test_powerlosses[p].short_name != '0' + && !(case_->flags & TEST_REENTRANT)) { continue; } - test_define_suite(&test_suites[i]); - - for (size_t j = 0; j < test_suites[i].case_count; j++) { - if (test_case_skip(&test_suites[i].cases[j])) { + size_t perm_ = 0; + for (size_t g = 0; g < TEST_GEOMETRY_COUNT; g++) { + if (test_geometry && strcmp( + test_geometries[g].name, test_geometry) != 0) { continue; } - for (size_t perm = 0; - perm < TEST_GEOMETRY_COUNT - * test_suites[i].cases[j].permutations; - perm++) { - if (test_perm_skip(perm)) { + for (size_t k = 0; k < case_->permutations; k++) { + perm_ += 1; + + if (perm != (size_t)-1 && perm_ != perm) { continue; } - if (test_step_skip(step)) { + + if (!(step >= test_start + && step < test_stop + && (step-test_start) % test_step == 0)) { step += 1; continue; } step += 1; // setup defines - size_t case_perm = perm / TEST_GEOMETRY_COUNT; - size_t geom_perm = perm % TEST_GEOMETRY_COUNT; - test_define_perm(&test_suites[i], - &test_suites[i].cases[j], case_perm); - test_define_geometry(&test_geometries[geom_perm]); + define_perm(suite, case_, k); + define_geometry(&test_geometries[g]); // filter? - if (test_suites[i].cases[j].filter) { - if (!test_suites[i].cases[j].filter()) { - printf("skipped %s#%zu\n", - test_suites[i].cases[j].id, - perm); - continue; - } + if (case_->filter && !case_->filter()) { + printf("skipped %s#%zu\n", case_->id, perm_); + continue; } - // create block device and configuration - lfs_testbd_t bd; + if (cycles) { + run_powerloss_cycles( + suite, case_, perm_, + cycles, + cycle_count); + } else { + test_powerlosses[p].run( + suite, case_, perm_, + test_powerlosses[p].cycles, + test_powerlosses[p].cycle_count); + } + } + } + } +} - struct lfs_config cfg = { - .context = &bd, - .read = lfs_testbd_read, - .prog = lfs_testbd_prog, - .erase = lfs_testbd_erase, - .sync = lfs_testbd_sync, - .read_size = READ_SIZE, - .prog_size = PROG_SIZE, - .block_size = BLOCK_SIZE, - .block_count = BLOCK_COUNT, - .block_cycles = BLOCK_CYCLES, - .cache_size = CACHE_SIZE, - .lookahead_size = LOOKAHEAD_SIZE, - }; +static void run(void) { + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + if (test_ids[t].suite && strcmp( + test_suites[i].name, test_ids[t].suite) != 0) { + continue; + } - struct lfs_testbd_config bdcfg = { - .erase_value = ERASE_VALUE, - .erase_cycles = ERASE_CYCLES, - .badblock_behavior = BADBLOCK_BEHAVIOR, - .power_cycles = 0, - }; + define_suite(&test_suites[i]); - int err = lfs_testbd_createcfg(&cfg, test_disk, &bdcfg); - if (err) { - fprintf(stderr, "error: " - "could not create block device: %d\n", err); - exit(-1); + for (size_t j = 0; j < test_suites[i].case_count; j++) { + if (test_ids[t].case_ && strcmp( + test_suites[i].cases[j].name, test_ids[t].case_) != 0) { + continue; } - // run the test - printf("running %s#%zu\n", test_suites[i].cases[j].id, perm); - - test_suites[i].cases[j].run(&cfg); - - printf("finished %s#%zu\n", test_suites[i].cases[j].id, perm); - - // cleanup - err = lfs_testbd_destroy(&cfg); - if (err) { - fprintf(stderr, "error: " - "could not destroy block device: %d\n", err); - exit(-1); - } + run_perms(&test_suites[i], &test_suites[i].cases[j], + test_ids[t].perm, + test_ids[t].cycles, + test_ids[t].cycle_count); } } } @@ -521,48 +995,47 @@ static void run(void) { - // option handling enum opt_flags { - OPT_HELP = 'h', - OPT_SUMMARY = 'Y', - OPT_LIST_SUITES = 'l', - OPT_LIST_CASES = 'L', - OPT_LIST_PATHS = 1, - OPT_LIST_DEFINES = 2, - OPT_LIST_GEOMETRIES = 3, - OPT_LIST_DEFAULTS = 4, - OPT_DEFINE = 'D', - OPT_GEOMETRY = 'G', - OPT_NORMAL = 'n', - OPT_REENTRANT = 'r', - OPT_START = 5, - OPT_STEP = 6, - OPT_STOP = 7, - OPT_DISK = 'd', - OPT_TRACE = 't', + OPT_HELP = 'h', + OPT_SUMMARY = 'Y', + OPT_LIST_SUITES = 'l', + OPT_LIST_CASES = 'L', + OPT_LIST_PATHS = 1, + OPT_LIST_DEFINES = 2, + OPT_LIST_GEOMETRIES = 3, + OPT_LIST_DEFAULTS = 4, + OPT_LIST_POWERLOSSES = 5, + OPT_DEFINE = 'D', + OPT_GEOMETRY = 'G', + OPT_POWERLOSS = 'p', + OPT_START = 6, + OPT_STEP = 7, + OPT_STOP = 8, + OPT_DISK = 'd', + OPT_TRACE = 't', }; -const char *short_opts = "hYlLD:G:nrVp:t:"; +const char *short_opts = "hYlLD:G:p:nrVd:t:"; const struct option long_opts[] = { - {"help", no_argument, NULL, OPT_HELP}, - {"summary", no_argument, NULL, OPT_SUMMARY}, - {"list-suites", no_argument, NULL, OPT_LIST_SUITES}, - {"list-cases", no_argument, NULL, OPT_LIST_CASES}, - {"list-paths", no_argument, NULL, OPT_LIST_PATHS}, - {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, - {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, - {"list-defaults", no_argument, NULL, OPT_LIST_DEFAULTS}, - {"define", required_argument, NULL, OPT_DEFINE}, - {"geometry", required_argument, NULL, OPT_GEOMETRY}, - {"normal", no_argument, NULL, OPT_NORMAL}, - {"reentrant", no_argument, NULL, OPT_REENTRANT}, - {"start", required_argument, NULL, OPT_START}, - {"stop", required_argument, NULL, OPT_STOP}, - {"step", required_argument, NULL, OPT_STEP}, - {"disk", required_argument, NULL, OPT_DISK}, - {"trace", required_argument, NULL, OPT_TRACE}, + {"help", no_argument, NULL, OPT_HELP}, + {"summary", no_argument, NULL, OPT_SUMMARY}, + {"list-suites", no_argument, NULL, OPT_LIST_SUITES}, + {"list-cases", no_argument, NULL, OPT_LIST_CASES}, + {"list-paths", no_argument, NULL, OPT_LIST_PATHS}, + {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, + {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, + {"list-defaults", no_argument, NULL, OPT_LIST_DEFAULTS}, + {"list-powerlosses", no_argument, NULL, OPT_LIST_POWERLOSSES}, + {"define", required_argument, NULL, OPT_DEFINE}, + {"geometry", required_argument, NULL, OPT_GEOMETRY}, + {"powerloss", required_argument, NULL, OPT_POWERLOSS}, + {"start", required_argument, NULL, OPT_START}, + {"stop", required_argument, NULL, OPT_STOP}, + {"step", required_argument, NULL, OPT_STEP}, + {"disk", required_argument, NULL, OPT_DISK}, + {"trace", required_argument, NULL, OPT_TRACE}, {NULL, 0, NULL, 0}, }; @@ -575,24 +1048,27 @@ const char *const help_text[] = { "List the defines for each test permutation.", "List the disk geometries used for testing.", "List the default defines in this test-runner.", + "List the available power-loss scenarios.", "Override a test define.", "Filter by geometry.", - "Filter for normal tests. Can be combined.", - "Filter for reentrant tests. Can be combined.", + "Comma-separated list of power-loss scenarios to test. Defaults to 0,l.", "Start at the nth test.", "Stop before the nth test.", "Only run every n tests, calculated after --start and --stop.", - "Use this file as the disk.", + "Redirect block device operations to this file.", "Redirect trace output to this file.", }; int main(int argc, char **argv) { void (*op)(void) = run; - static const char **override_names = NULL; - static intmax_t *override_defines = NULL; - static size_t override_count = 0; - static size_t override_cap = 0; + const char **override_names = NULL; + intmax_t *override_defines = NULL; + size_t override_count = 0; + size_t override_capacity = 0; + + size_t test_powerloss_capacity = 0; + size_t test_id_capacity = 0; // parse options while (true) { @@ -676,6 +1152,9 @@ int main(int argc, char **argv) { case OPT_LIST_DEFAULTS: op = list_defaults; break; + case OPT_LIST_POWERLOSSES: + op = list_powerlosses; + break; // configuration case OPT_DEFINE: { // special case for -DGEOMETRY=, we treat this the same @@ -687,12 +1166,14 @@ int main(int argc, char **argv) { // realloc if necessary override_count += 1; - if (override_count > override_cap) { - override_cap = (2*override_cap > 4) ? 2*override_cap : 4; - override_names = realloc(override_names, override_cap - * sizeof(const char *)); - override_defines = realloc(override_defines, override_cap - * sizeof(intmax_t)); + if (override_count > override_capacity) { + override_capacity = (2*override_capacity > 4) + ? 2*override_capacity + : 4; + override_names = realloc(override_names, + override_capacity * sizeof(const char *)); + override_defines = realloc(override_defines, + override_capacity * sizeof(intmax_t)); } // parse into string key/intmax_t value, cannibalizing the @@ -719,12 +1200,134 @@ invalid_define: case OPT_GEOMETRY: test_geometry = optarg; break; - case OPT_NORMAL: - test_types |= TEST_NORMAL; - break; - case OPT_REENTRANT: - test_types |= TEST_REENTRANT; + case OPT_POWERLOSS: { + // reset our powerloss scenarios + if (test_powerloss_capacity > 0) { + free((test_powerloss_t*)test_powerlosses); + } + test_powerlosses = NULL; + test_powerloss_count = 0; + test_powerloss_capacity = 0; + + // parse the comma separated list of power-loss scenarios + while (*optarg) { + // allocate space + test_powerloss_count += 1; + if (test_powerloss_count > test_powerloss_capacity) { + test_powerloss_capacity + = (2*test_powerloss_capacity > 4) + ? 2*test_powerloss_capacity + : 4; + test_powerlosses = realloc( + (test_powerloss_t*)test_powerlosses, + test_powerloss_capacity + * sizeof(test_powerloss_t)); + } + + // parse the power-loss scenario + optarg += strspn(optarg, " "); + + // named power-loss scenario + size_t len = strcspn(optarg, " ,"); + for (size_t i = 0; builtin_powerlosses[i].long_name; i++) { + if ((len == 1 + && *optarg == builtin_powerlosses[i].short_name) + || (len == strlen( + builtin_powerlosses[i].long_name) + && memcmp(optarg, + builtin_powerlosses[i].long_name, + len) == 0)) { + ((test_powerloss_t*)test_powerlosses)[ + test_powerloss_count-1] + = builtin_powerlosses[i]; + optarg += len; + goto powerloss_next; + } + } + + // exhaustive permutations + // TODO + + // comma-separated permutation + if (*optarg == '{') { + // how many cycles? + size_t count = 1; + for (size_t i = 0; optarg[i]; i++) { + if (optarg[i] == ',') { + count += 1; + } + } + + // parse cycles + lfs_testbd_powercycles_t *cycles = malloc( + count * sizeof(lfs_testbd_powercycles_t)); + size_t i = 0; + char *s = optarg + 1; + while (true) { + char *parsed = NULL; + cycles[i] = strtoumax(s, &parsed, 0); + if (parsed == s) { + count -= 1; + i -= 1; + } + i += 1; + + s = parsed + strspn(parsed, " "); + if (*s == ',') { + s += 1; + continue; + } else if (*s == '}') { + s += 1; + break; + } else { + goto powerloss_unknown; + } + } + + ((test_powerloss_t*)test_powerlosses)[ + test_powerloss_count-1] = (test_powerloss_t){ + .run = run_powerloss_cycles, + .cycles = cycles, + .cycle_count = count, + }; + optarg = s; + goto powerloss_next; + } + + // leb16-encoded permutation + if (*optarg == '#') { + lfs_testbd_powercycles_t *cycles; + char *parsed = NULL; + size_t count = leb16_parse(optarg+1, &parsed, &cycles); + if (parsed == optarg+1) { + goto powerloss_unknown; + } + + ((test_powerloss_t*)test_powerlosses)[ + test_powerloss_count-1] = (test_powerloss_t){ + .run = run_powerloss_cycles, + .cycles = cycles, + .cycle_count = count, + }; + optarg = (char*)parsed; + goto powerloss_next; + } + +powerloss_unknown: + // unknown scenario? + fprintf(stderr, "error: " + "unknown power-loss scenario: %s\n", + optarg); + exit(-1); + +powerloss_next: + optarg += strcspn(optarg, ","); + if (*optarg == ',') { + optarg += 1; + } + } break; + } case OPT_START: { char *parsed = NULL; test_start = strtoumax(optarg, &parsed, 0); @@ -777,36 +1380,55 @@ invalid_define: } getopt_done: ; - // parse test identifier, if any, cannibalizing the arg in the process if (argc > optind) { - if (argc - optind > 1) { - fprintf(stderr, "error: more than one test identifier\n"); - exit(-1); - } + // reset our test identifier list + test_ids = NULL; + test_id_count = 0; + test_id_capacity = 0; + } + // parse test identifier, if any, cannibalizing the arg in the process + for (; argc > optind; optind++) { // parse suite char *suite = argv[optind]; char *case_ = strchr(suite, '#'); + size_t perm = -1; + lfs_testbd_powercycles_t *cycles = NULL; + size_t cycle_count = 0; if (case_) { *case_ = '\0'; case_ += 1; // parse case - char *perm = strchr(case_, '#'); - if (perm) { - *perm = '\0'; - perm += 1; + char *perm_ = strchr(case_, '#'); + if (perm_) { + *perm_ = '\0'; + perm_ += 1; + + // parse power cycles + char *cycles_ = strchr(perm_, '#'); + if (cycles_) { + *cycles_ = '\0'; + cycles_ += 1; + + char *parsed = NULL; + cycle_count = leb16_parse(cycles_, &parsed, &cycles); + if (parsed == cycles_) { + fprintf(stderr, "error: " + "could not parse test cycles: %s\n", cycles_); + exit(-1); + } + } char *parsed = NULL; - test_perm = strtoumax(perm, &parsed, 10); - if (parsed == perm) { - fprintf(stderr, "error: could not parse test identifier\n"); + perm = strtoumax(perm_, &parsed, 10); + if (parsed == perm_) { + fprintf(stderr, "error: " + "could not parse test permutation: %s\n", perm_); exit(-1); } } - - test_case = case_; } // remove optional path and .toml suffix @@ -820,7 +1442,22 @@ getopt_done: ; suite[suite_len-5] = '\0'; } - test_suite = suite; + // append to identifier list + test_id_count += 1; + if (test_id_count > test_id_capacity) { + test_id_capacity = (2*test_id_capacity > 4) + ? 2*test_id_capacity + : 4; + test_ids = realloc((test_id_t*)test_ids, + test_id_capacity * sizeof(test_id_t)); + } + ((test_id_t*)test_ids)[test_id_count-1] = (test_id_t){ + .suite = suite, + .case_ = case_, + .perm = perm, + .cycles = cycles, + .cycle_count = cycle_count, + }; } // register overrides @@ -832,4 +1469,16 @@ getopt_done: ; // cleanup (need to be done for valgrind testing) free(override_names); free(override_defines); + if (test_powerloss_capacity) { + for (size_t i = 0; i < test_powerloss_count; i++) { + free((lfs_testbd_powercycles_t*)test_powerlosses[i].cycles); + } + free((test_powerloss_t*)test_powerlosses); + } + if (test_id_capacity) { + for (size_t i = 0; i < test_id_count; i++) { + free((lfs_testbd_powercycles_t*)test_ids[i].cycles); + } + free((test_id_t*)test_ids); + } } diff --git a/runners/test_runner.h b/runners/test_runner.h index 64ad15d7..27229ad3 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -5,18 +5,16 @@ // generated test configurations -enum test_types { - TEST_NORMAL = 0x1, - TEST_REENTRANT = 0x2, +enum test_flags { + TEST_REENTRANT = 0x1, }; - -typedef uint8_t test_types_t; +typedef uint8_t test_flags_t; struct test_case { const char *id; const char *name; const char *path; - test_types_t types; + test_flags_t flags; size_t permutations; intmax_t (*const *const *defines)(void); @@ -29,7 +27,7 @@ struct test_suite { const char *id; const char *name; const char *path; - test_types_t types; + test_flags_t flags; const char *const *define_names; size_t define_count; @@ -54,6 +52,7 @@ intmax_t test_define(size_t define); #define ERASE_VALUE test_predefine(7) #define ERASE_CYCLES test_predefine(8) #define BADBLOCK_BEHAVIOR test_predefine(9) +#define POWERLOSS_BEHAVIOR test_predefine(10) #define TEST_PREDEFINE_NAMES { \ "READ_SIZE", \ @@ -66,17 +65,19 @@ intmax_t test_define(size_t define); "ERASE_VALUE", \ "ERASE_CYCLES", \ "BADBLOCK_BEHAVIOR", \ + "POWERLOSS_BEHAVIOR", \ } -#define TEST_PREDEFINE_COUNT 10 +#define TEST_PREDEFINE_COUNT 11 // default predefines #define TEST_DEFAULTS { \ - /* LOOKAHEAD_SIZE */ 16, \ - /* BLOCK_CYCLES */ -1, \ - /* ERASE_VALUE */ 0xff, \ - /* ERASE_CYCLES */ 0, \ - /* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \ + /* LOOKAHEAD_SIZE */ 16, \ + /* BLOCK_CYCLES */ -1, \ + /* ERASE_VALUE */ 0xff, \ + /* ERASE_CYCLES */ 0, \ + /* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \ + /* POWERLOSS_BEHAVIOR */ LFS_TESTBD_POWERLOSS_NOOP, \ } #define TEST_DEFAULT_DEFINE_COUNT 5 diff --git a/scripts/test.py b/scripts/test.py index 281265eb..cbc7ab93 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -73,8 +73,6 @@ class TestCase: self.in_ = config.pop('in', config.pop('suite_in', None)) - self.normal = config.pop('normal', - config.pop('suite_normal', True)) self.reentrant = config.pop('reentrant', config.pop('suite_reentrant', False)) @@ -159,7 +157,6 @@ class TestSuite: # a couple of these we just forward to all cases defines = config.pop('defines', {}) in_ = config.pop('in', None) - normal = config.pop('normal', True) reentrant = config.pop('reentrant', False) self.cases = [] @@ -172,7 +169,6 @@ class TestSuite: 'suite': self.name, 'suite_defines': defines, 'suite_in': in_, - 'suite_normal': normal, 'suite_reentrant': reentrant, **case})) @@ -181,7 +177,6 @@ class TestSuite: set(case.defines) for case in self.cases)) # combine other per-case things - self.normal = any(case.normal for case in self.cases) self.reentrant = any(case.reentrant for case in self.cases) for k in config.keys(): @@ -236,6 +231,12 @@ def compile(**args): f.write = write f.writeln = writeln + f.writeln("// Generated by %s:" % sys.argv[0]) + f.writeln("//") + f.writeln("// %s" % ' '.join(sys.argv)) + f.writeln("//") + f.writeln() + # redirect littlefs tracing f.writeln('#define LFS_TRACE_(fmt, ...) do { \\') f.writeln(8*' '+'extern FILE *test_trace; \\') @@ -366,10 +367,10 @@ def compile(**args): f.writeln(4*' '+'.id = "%s",' % suite.id()) f.writeln(4*' '+'.name = "%s",' % suite.name) f.writeln(4*' '+'.path = "%s",' % suite.path) - f.writeln(4*' '+'.types = %s,' - % ' | '.join(filter(None, [ - 'TEST_NORMAL' if suite.normal else None, - 'TEST_REENTRANT' if suite.reentrant else None]))) + f.writeln(4*' '+'.flags = %s,' + % (' | '.join(filter(None, [ + 'TEST_REENTRANT' if suite.reentrant else None])) + or 0)) if suite.defines: # create suite define names f.writeln(4*' '+'.define_names = (const char *const[]){') @@ -384,10 +385,10 @@ def compile(**args): f.writeln(12*' '+'.id = "%s",' % case.id()) f.writeln(12*' '+'.name = "%s",' % case.name) f.writeln(12*' '+'.path = "%s",' % case.path) - f.writeln(12*' '+'.types = %s,' - % ' | '.join(filter(None, [ - 'TEST_NORMAL' if case.normal else None, - 'TEST_REENTRANT' if case.reentrant else None]))) + f.writeln(12*' '+'.flags = %s,' + % (' | '.join(filter(None, [ + 'TEST_REENTRANT' if case.reentrant else None])) + or 0)) f.writeln(12*' '+'.permutations = %d,' % len(case.permutations)) if case.defines: @@ -461,12 +462,13 @@ def runner(**args): '--error-exitcode=4', '-q']) - # filter tests? - if args.get('normal'): cmd.append('-n') - if args.get('reentrant'): cmd.append('-r') + # other context if args.get('geometry'): cmd.append('-G%s' % args.get('geometry')) + if args.get('powerloss'): + cmd.append('-p%s' % args.get('powerloss')) + # defines? if args.get('define'): for define in args.get('define'): @@ -476,12 +478,13 @@ def runner(**args): def list_(**args): cmd = runner(**args) - if args.get('summary'): cmd.append('--summary') - if args.get('list_suites'): cmd.append('--list-suites') - if args.get('list_cases'): cmd.append('--list-cases') - if args.get('list_paths'): cmd.append('--list-paths') - if args.get('list_defines'): cmd.append('--list-defines') - if args.get('list_geometries'): cmd.append('--list-geometries') + if args.get('summary'): cmd.append('--summary') + if args.get('list_suites'): cmd.append('--list-suites') + if args.get('list_cases'): cmd.append('--list-cases') + if args.get('list_paths'): cmd.append('--list-paths') + if args.get('list_defines'): cmd.append('--list-defines') + if args.get('list_geometries'): cmd.append('--list-geometries') + if args.get('list_powerlosses'): cmd.append('--list-powerlosses') if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) @@ -598,11 +601,12 @@ def run_stage(name, runner_, **args): passed_suite_perms = co.defaultdict(lambda: 0) passed_case_perms = co.defaultdict(lambda: 0) passed_perms = 0 + powerlosses = 0 failures = [] killed = False pattern = re.compile('^(?:' - '(?Prunning|finished|skipped) ' + '(?Prunning|finished|skipped|powerloss) ' '(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)' '|' '(?P[^:]+):(?P\d+):(?Passert):' ' *(?P.*)' ')$') @@ -613,6 +617,7 @@ def run_stage(name, runner_, **args): nonlocal passed_suite_perms nonlocal passed_case_perms nonlocal passed_perms + nonlocal powerlosses nonlocal locals # run the tests! @@ -659,6 +664,9 @@ def run_stage(name, runner_, **args): last_id = m.group('id') last_output = [] last_assert = None + elif op == 'powerloss': + last_id = m.group('id') + powerlosses += 1 elif op == 'finished': passed_suite_perms[m.group('suite')] += 1 passed_case_perms[m.group('case')] += 1 @@ -766,6 +774,8 @@ def run_stage(name, runner_, **args): len(expected_case_perms)) if not args.get('by_cases') else None, '%d/%d perms' % (passed_perms, expected_perms), + '%dpls!' % powerlosses + if powerlosses else None, '\x1b[31m%d/%d failures\x1b[m' % (len(failures), expected_perms) if failures else None])))) @@ -785,6 +795,7 @@ def run_stage(name, runner_, **args): return ( expected_perms, passed_perms, + powerlosses, failures, killed) @@ -806,33 +817,34 @@ def run(**args): expected = 0 passed = 0 + powerlosses = 0 failures = [] - for type, by in it.product( - ['normal', 'reentrant'], - expected_case_perms.keys() if args.get('by_cases') - else expected_suite_perms.keys() if args.get('by_suites') - else [None]): + for by in (expected_case_perms.keys() if args.get('by_cases') + else expected_suite_perms.keys() if args.get('by_suites') + else [None]): # rebuild runner for each stage to override test identifier if needed stage_runner = runner(**args | { - 'test_ids': [by] if by is not None else args.get('test_ids', []), - 'normal': type == 'normal', - 'reentrant': type == 'reentrant'}) + 'test_ids': [by] if by is not None else args.get('test_ids', [])}) # spawn jobs for stage - expected_, passed_, failures_, killed = run_stage( - '%s %s' % (type, by or 'tests'), stage_runner, **args) + expected_, passed_, powerlosses_, failures_, killed = run_stage( + by or 'tests', stage_runner, **args) expected += expected_ passed += passed_ + powerlosses += powerlosses_ failures.extend(failures_) if (failures and not args.get('keep_going')) or killed: break # show summary print() - print('\x1b[%dmdone:\x1b[m %d/%d passed, %d/%d failed, in %.2fs' + print('\x1b[%dmdone:\x1b[m %s' # %d/%d passed, %d/%d failed%s, in %.2fs' % (32 if not failures else 31, - passed, expected, len(failures), expected, - time.time()-start)) + ', '.join(filter(None, [ + '%d/%d passed' % (passed, expected), + '%d/%d failed' % (len(failures), expected), + '%dpls!' % powerlosses if powerlosses else None, + 'in %.2fs' % (time.time()-start)])))) print() # print each failure @@ -844,7 +856,7 @@ def run(**args): for failure in failures: # show summary of failure path, lineno = runner_paths[testcase(failure.id)] - defines = runner_defines[failure.id] + defines = runner_defines.get(failure.id, {}) print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s%s failed' % (path, lineno, failure.id, @@ -913,8 +925,9 @@ def main(**args): or args.get('list_cases') or args.get('list_paths') or args.get('list_defines') + or args.get('list_defaults') or args.get('list_geometries') - or args.get('list_defaults')): + or args.get('list_powerlosses')): list_(**args) else: run(**args) @@ -930,7 +943,7 @@ if __name__ == "__main__": help="Description of testis to run. May be a directory, path, or \ test identifier. Test identifiers are of the form \ ##, but suffixes can be \ - dropped to run any matching tests. Defaults to %r." % TEST_PATHS) + dropped to run any matching tests. Defaults to %s." % TEST_PATHS) parser.add_argument('-v', '--verbose', action='store_true', help="Output commands that run behind the scenes.") # test flags @@ -945,20 +958,21 @@ if __name__ == "__main__": help="List the path for each test case.") test_parser.add_argument('--list-defines', action='store_true', help="List the defines for each test permutation.") - test_parser.add_argument('--list-geometries', action='store_true', - help="List the disk geometries used for testing.") test_parser.add_argument('--list-defaults', action='store_true', help="List the default defines in this test-runner.") + test_parser.add_argument('--list-geometries', action='store_true', + help="List the disk geometries used for testing.") + test_parser.add_argument('--list-powerlosses', action='store_true', + help="List the available power-loss scenarios.") test_parser.add_argument('-D', '--define', action='append', help="Override a test define.") test_parser.add_argument('-G', '--geometry', help="Filter by geometry.") - test_parser.add_argument('-n', '--normal', action='store_true', - help="Filter for normal tests. Can be combined.") - test_parser.add_argument('-r', '--reentrant', action='store_true', - help="Filter for reentrant tests. Can be combined.") + test_parser.add_argument('-p', '--powerloss', + help="Comma-separated list of power-loss scenarios to test. \ + Defaults to 0,l.") test_parser.add_argument('-d', '--disk', - help="Use this file as the disk.") + help="Redirect block device operations to this file.") test_parser.add_argument('-t', '--trace', help="Redirect trace output to this file.") test_parser.add_argument('-o', '--output', From 4689678208e7339a448cfbc5619a09c73afc7ae3 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 22 Aug 2022 12:46:59 -0500 Subject: [PATCH 25/81] Added --color to test.py, fixed some terminal-clobbering issues With more features being added to test.py, the one-line status is starting to get quite long and pass the ~80 column readability heuristic. To make this worse this clobbers the terminal output when the terminal is not wide enough. Simple solution is to disable line-wrapping, potentially printing some garbage if line-wrapping-disable is not supported, but also printing a final status update to fix any garbage and avoid a race condition where the script would show a non-final status. Also added --color which disables any of this attempting-to-be-clever stuff. --- scripts/coverage.py | 1 - scripts/test.py | 131 +++++++++++++++++++++++++++----------------- 2 files changed, 82 insertions(+), 50 deletions(-) diff --git a/scripts/coverage.py b/scripts/coverage.py index 8c81e6fc..91445c2b 100755 --- a/scripts/coverage.py +++ b/scripts/coverage.py @@ -503,7 +503,6 @@ if __name__ == "__main__": help="Show a additional lines of context. Defaults to 3.") parser.add_argument('-w', '--width', type=lambda x: int(x, 0), default=80, help="Assume source is styled with this many columns. Defaults to 80.") - # TODO add this to test.py? parser.add_argument('--color', choices=['never', 'always', 'auto'], default='auto', help="When to use terminal colors.") diff --git a/scripts/test.py b/scripts/test.py index cbc7ab93..d58dfd65 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -58,6 +58,14 @@ def openio(path, mode='r'): else: return open(path, mode) +def color(**args): + if args.get('color') == 'auto': + return sys.stdout.isatty() + elif args.get('color') == 'always': + return True + else: + return False + class TestCase: # create a TestCase object from a config def __init__(self, config, args={}): @@ -98,8 +106,11 @@ class TestCase: (suite_defines_ | defines_).items()))))) for k in config.keys(): - print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r' - % (self.id(), k), + print('%swarning:%s in %s, found unused key %r' % ( + '\x1b[01;33m' if color(**args) else '', + '\x1b[m' if color(**args) else '', + self.id(), + k), file=sys.stderr) def id(self): @@ -170,7 +181,8 @@ class TestSuite: 'suite_defines': defines, 'suite_in': in_, 'suite_reentrant': reentrant, - **case})) + **case}, + args=args)) # combine per-case defines self.defines = set.union(*( @@ -180,8 +192,11 @@ class TestSuite: self.reentrant = any(case.reentrant for case in self.cases) for k in config.keys(): - print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r' - % (self.id(), k), + print('%swarning:%s in %s, found unused key %r' % ( + '\x1b[01;33m' if color(**args) else '', + '\x1b[m' if color(**args) else '', + self.id(), + k), file=sys.stderr) def id(self): @@ -210,10 +225,10 @@ def compile(**args): sys.exit(-1) # load our suite - suite = TestSuite(paths[0]) + suite = TestSuite(paths[0], args) else: # load all suites - suites = [TestSuite(path) for path in paths] + suites = [TestSuite(path, args) for path in paths] suites.sort(key=lambda s: s.name) # write generated test source @@ -748,46 +763,52 @@ def run_stage(name, runner_, **args): runners.append(th.Thread( target=run_job, args=(runner_, None, None))) + def print_update(done): + if not args.get('verbose') and (color(**args) or done): + sys.stdout.write('%s%srunning %s%s:%s %s%s' % ( + '\r\x1b[K' if color(**args) else '', + '\x1b[?7l' if not done else '', + ('\x1b[32m' if not failures else '\x1b[31m') + if color(**args) else '', + name, + '\x1b[m' if color(**args) else '', + ', '.join(filter(None, [ + '%d/%d suites' % ( + sum(passed_suite_perms[k] == v + for k, v in expected_suite_perms.items()), + len(expected_suite_perms)) + if (not args.get('by_suites') + and not args.get('by_cases')) else None, + '%d/%d cases' % ( + sum(passed_case_perms[k] == v + for k, v in expected_case_perms.items()), + len(expected_case_perms)) + if not args.get('by_cases') else None, + '%d/%d perms' % (passed_perms, expected_perms), + '%dpls!' % powerlosses + if powerlosses else None, + '%s%d/%d failures%s' % ( + '\x1b[31m' if color(**args) else '', + len(failures), + expected_perms, + '\x1b[m' if color(**args) else '') + if failures else None])), + '\x1b[?7h' if not done else '\n')) + sys.stdout.flush() + for r in runners: r.start() - needs_newline = False try: while any(r.is_alive() for r in runners): time.sleep(0.01) - - if not args.get('verbose'): - sys.stdout.write('\r\x1b[K' - 'running \x1b[%dm%s:\x1b[m %s ' - % (32 if not failures else 31, - name, - ', '.join(filter(None, [ - '%d/%d suites' % ( - sum(passed_suite_perms[k] == v - for k, v in expected_suite_perms.items()), - len(expected_suite_perms)) - if (not args.get('by_suites') - and not args.get('by_cases')) else None, - '%d/%d cases' % ( - sum(passed_case_perms[k] == v - for k, v in expected_case_perms.items()), - len(expected_case_perms)) - if not args.get('by_cases') else None, - '%d/%d perms' % (passed_perms, expected_perms), - '%dpls!' % powerlosses - if powerlosses else None, - '\x1b[31m%d/%d failures\x1b[m' - % (len(failures), expected_perms) - if failures else None])))) - sys.stdout.flush() - needs_newline = True + print_update(False) except KeyboardInterrupt: # this is handled by the runner threads, we just # need to not abort here killed = True finally: - if needs_newline: - print() + print_update(True) for r in runners: r.join() @@ -838,13 +859,15 @@ def run(**args): # show summary print() - print('\x1b[%dmdone:\x1b[m %s' # %d/%d passed, %d/%d failed%s, in %.2fs' - % (32 if not failures else 31, - ', '.join(filter(None, [ - '%d/%d passed' % (passed, expected), - '%d/%d failed' % (len(failures), expected), - '%dpls!' % powerlosses if powerlosses else None, - 'in %.2fs' % (time.time()-start)])))) + print('%sdone:%s %s' % ( + ('\x1b[32m' if not failures else '\x1b[31m') + if color(**args) else '', + '\x1b[m' if color(**args) else '', + ', '.join(filter(None, [ + '%d/%d passed' % (passed, expected), + '%d/%d failed' % (len(failures), expected), + '%dpls!' % powerlosses if powerlosses else None, + 'in %.2fs' % (time.time()-start)])))) print() # print each failure @@ -858,10 +881,13 @@ def run(**args): path, lineno = runner_paths[testcase(failure.id)] defines = runner_defines.get(failure.id, {}) - print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s%s failed' - % (path, lineno, failure.id, - ' (%s)' % ', '.join( - '%s=%s' % (k, v) for k, v in defines.items()) + print('%s%s:%d:%sfailure:%s %s%s failed' % ( + '\x1b[01m' if color(**args) else '', + path, lineno, + '\x1b[01;31m' if color(**args) else '', + '\x1b[m' if color(**args) else '', + failure.id, + ' (%s)' % ', '.join('%s=%s' % (k,v) for k,v in defines.items()) if defines else '')) if failure.output: @@ -873,8 +899,12 @@ def run(**args): if failure.assert_ is not None: path, lineno, message = failure.assert_ - print('\x1b[01m%s:%d:\x1b[01;31massert:\x1b[m %s' - % (path, lineno, message)) + print('%s%s:%d:%sassert:%s %s' % ( + '\x1b[01m' if color(**args) else '', + path, lineno, + '\x1b[01;31m' if color(**args) else '', + '\x1b[m' if color(**args) else '', + message)) with open(path) as f: line = next(it.islice(f, lineno-1, None)).strip('\n') print(line) @@ -946,6 +976,9 @@ if __name__ == "__main__": dropped to run any matching tests. Defaults to %s." % TEST_PATHS) parser.add_argument('-v', '--verbose', action='store_true', help="Output commands that run behind the scenes.") + parser.add_argument('--color', + choices=['never', 'always', 'auto'], default='auto', + help="When to use terminal colors.") # test flags test_parser = parser.add_argument_group('test options') test_parser.add_argument('-Y', '--summary', action='store_true', From 3f4f85986e0a1ecc7d2b4717c0cc24b8070b1e75 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 23 Aug 2022 14:48:29 -0500 Subject: [PATCH 26/81] Readded support for mirror writes to a file in testbd Before this was available implicitly by supporting both rambd and filebd as backends, but now that testbd is a bit more complicated and no longer maps directly to a block-device, this needs to be explicitly supported. --- bd/lfs_testbd.c | 126 ++++++++++++++++++++++++++++-------------- bd/lfs_testbd.h | 15 +++-- runners/test_runner.c | 4 ++ 3 files changed, 95 insertions(+), 50 deletions(-) diff --git a/bd/lfs_testbd.c b/bd/lfs_testbd.c index 4100ea8f..6e53a67e 100644 --- a/bd/lfs_testbd.c +++ b/bd/lfs_testbd.c @@ -6,9 +6,17 @@ * Copyright (c) 2017, Arm Limited. All rights reserved. * SPDX-License-Identifier: BSD-3-Clause */ + #include "bd/lfs_testbd.h" #include +#include +#include +#include + +#ifdef _WIN32 +#include +#endif // access to lazily-allocated/copy-on-write blocks @@ -116,10 +124,41 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, // setup testing things bd->power_cycles = bd->cfg->power_cycles; + bd->disk_fd = -1; + bd->disk_scratch_block = NULL; + bd->branches = NULL; bd->branch_capacity = 0; bd->branch_count = 0; + if (bd->cfg->disk_path) { + #ifdef _WIN32 + bd->disk_fd = open(bd->cfg->disk_path, + O_RDWR | O_CREAT | O_BINARY, 0666); + #else + bd->disk_fd = open(bd->cfg->disk_path, + O_RDWR | O_CREAT, 0666); + #endif + if (bd->disk_fd < 0) { + int err = -errno; + LFS_TESTBD_TRACE("lfs_testbd_create -> %d", err); + return err; + } + + // if we're emulating erase values, we can keep a block around in + // memory of just the erase state to speed up emulated erases + if (bd->cfg->erase_value != -1) { + bd->disk_scratch_block = malloc(cfg->block_size); + if (!bd->disk_scratch_block) { + LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM); + return LFS_ERR_NOMEM; + } + memset(bd->disk_scratch_block, + bd->cfg->erase_value, + cfg->block_size); + } + } + LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", 0); return 0; } @@ -154,54 +193,17 @@ int lfs_testbd_destroy(const struct lfs_config *cfg) { free(bd->blocks); free(bd->branches); + if (bd->disk_fd >= 0) { + close(bd->disk_fd); + free(bd->disk_scratch_block); + } + LFS_TESTBD_TRACE("lfs_testbd_destroy -> %d", 0); return 0; } -///// Internal mapping to block devices /// -//static int lfs_testbd_rawread(const struct lfs_config *cfg, lfs_block_t block, -// lfs_off_t off, void *buffer, lfs_size_t size) { -// lfs_testbd_t *bd = cfg->context; -// if (bd->persist) { -// return lfs_filebd_read(cfg, block, off, buffer, size); -// } else { -// return lfs_rambd_read(cfg, block, off, buffer, size); -// } -//} -// -//static int lfs_testbd_rawprog(const struct lfs_config *cfg, lfs_block_t block, -// lfs_off_t off, const void *buffer, lfs_size_t size) { -// lfs_testbd_t *bd = cfg->context; -// if (bd->persist) { -// return lfs_filebd_prog(cfg, block, off, buffer, size); -// } else { -// return lfs_rambd_prog(cfg, block, off, buffer, size); -// } -//} -// -//static int lfs_testbd_rawerase(const struct lfs_config *cfg, -// lfs_block_t block) { -// lfs_testbd_t *bd = cfg->context; -// if (bd->persist) { -// return lfs_filebd_erase(cfg, block); -// } else { -// return lfs_rambd_erase(cfg, block); -// } -//} -// -//static int lfs_testbd_rawsync(const struct lfs_config *cfg) { -// lfs_testbd_t *bd = cfg->context; -// if (bd->persist) { -// return lfs_filebd_sync(cfg); -// } else { -// return lfs_rambd_sync(cfg); -// } -//} - - - // block device API int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, @@ -285,6 +287,25 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, // prog data memcpy(&b->data[off], buffer, size); + // mirror to disk file? + if (bd->disk_fd >= 0) { + off_t res1 = lseek(bd->disk_fd, + (off_t)block*cfg->block_size + (off_t)off, + SEEK_SET); + if (res1 < 0) { + int err = -errno; + LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); + return err; + } + + ssize_t res2 = write(bd->disk_fd, buffer, size); + if (res2 < 0) { + int err = -errno; + LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); + return err; + } + } + // lose power? if (bd->power_cycles > 0) { bd->power_cycles -= 1; @@ -342,6 +363,27 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { // emulate an erase value? if (bd->cfg->erase_value != -1) { memset(b->data, bd->cfg->erase_value, cfg->block_size); + + // mirror to disk file? + if (bd->disk_fd >= 0) { + off_t res1 = lseek(bd->disk_fd, + (off_t)block*cfg->block_size, + SEEK_SET); + if (res1 < 0) { + int err = -errno; + LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); + return err; + } + + ssize_t res2 = write(bd->disk_fd, + bd->disk_scratch_block, + cfg->block_size); + if (res2 < 0) { + int err = -errno; + LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); + return err; + } + } } // lose power? diff --git a/bd/lfs_testbd.h b/bd/lfs_testbd.h index 9b51cd85..64c13840 100644 --- a/bd/lfs_testbd.h +++ b/bd/lfs_testbd.h @@ -90,14 +90,9 @@ struct lfs_testbd_config { // heavy memory usage! bool track_branches; -// // Optional buffer for RAM block device. -// void *buffer; -// -// // Optional buffer for wear. -// void *wear_buffer; -// -// // Optional buffer for scratch memory, needed when erase_value != -1. -// void *scratch_buffer; + // Path to file to use as a mirror of the disk. This provides a way to view + // the current state of the block device. + const char *disk_path; }; // A reference counted block @@ -112,7 +107,11 @@ typedef struct lfs_testbd_block { typedef struct lfs_testbd { // array of copy-on-write blocks lfs_testbd_block_t **blocks; + + // some other test state uint32_t power_cycles; + int disk_fd; + uint8_t *disk_scratch_block; // array of tracked branches struct lfs_testbd *branches; diff --git a/runners/test_runner.c b/runners/test_runner.c index 392915af..b94bad5c 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -610,6 +610,7 @@ static void run_powerloss_none( .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, + .disk_path = test_disk, }; int err = lfs_testbd_createcfg(&cfg, test_disk, &bdcfg); @@ -672,6 +673,7 @@ static void run_powerloss_linear( .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, + .disk_path = test_disk, .power_cycles = i, .powerloss_behavior = POWERLOSS_BEHAVIOR, .powerloss_cb = powerloss_longjmp, @@ -748,6 +750,7 @@ static void run_powerloss_exponential( .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, + .disk_path = test_disk, .power_cycles = i, .powerloss_behavior = POWERLOSS_BEHAVIOR, .powerloss_cb = powerloss_longjmp, @@ -822,6 +825,7 @@ static void run_powerloss_cycles( .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, + .disk_path = test_disk, .power_cycles = (i < cycle_count) ? cycles[i] : 0, .powerloss_behavior = POWERLOSS_BEHAVIOR, .powerloss_cb = powerloss_longjmp, From 552336eba9a1f32ed967232da3e160bc13ee561d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 23 Aug 2022 17:01:04 -0500 Subject: [PATCH 27/81] Added optional read/prog/erase delays to testbd These have no real purpose other than slowing down the simulation for inspection/fun. Note this did reveal an issue in pretty_asserts.py which was clobbering feature macros. Added explicit, and maybe a bit hacky, #undef _FEATURE_H to avoid this. --- bd/lfs_testbd.c | 41 +++++++++++++++++++++++++++++ bd/lfs_testbd.h | 16 ++++++++++++ runners/test_runner.c | 54 +++++++++++++++++++++++++++++++++++++++ scripts/pretty_asserts.py | 5 +++- scripts/test.py | 14 ++++++++++ 5 files changed, 129 insertions(+), 1 deletion(-) diff --git a/bd/lfs_testbd.c b/bd/lfs_testbd.c index 6e53a67e..762512af 100644 --- a/bd/lfs_testbd.c +++ b/bd/lfs_testbd.c @@ -7,12 +7,17 @@ * SPDX-License-Identifier: BSD-3-Clause */ +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 199309L +#endif + #include "bd/lfs_testbd.h" #include #include #include #include +#include #ifdef _WIN32 #include @@ -238,6 +243,18 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, size); } + if (bd->cfg->read_delay) { + int err = nanosleep(&(struct timespec){ + .tv_sec=bd->cfg->read_delay/1000000000, + .tv_nsec=bd->cfg->read_delay%1000000000}, + NULL); + if (err) { + err = -errno; + LFS_TESTBD_TRACE("lfs_testbd_read -> %d", err); + return err; + } + } + LFS_TESTBD_TRACE("lfs_testbd_read -> %d", 0); return 0; } @@ -306,6 +323,18 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, } } + if (bd->cfg->prog_delay) { + int err = nanosleep(&(struct timespec){ + .tv_sec=bd->cfg->prog_delay/1000000000, + .tv_nsec=bd->cfg->prog_delay%1000000000}, + NULL); + if (err) { + err = -errno; + LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); + return err; + } + } + // lose power? if (bd->power_cycles > 0) { bd->power_cycles -= 1; @@ -386,6 +415,18 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { } } + if (bd->cfg->erase_delay) { + int err = nanosleep(&(struct timespec){ + .tv_sec=bd->cfg->erase_delay/1000000000, + .tv_nsec=bd->cfg->erase_delay%1000000000}, + NULL); + if (err) { + err = -errno; + LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); + return err; + } + } + // lose power? if (bd->power_cycles > 0) { bd->power_cycles -= 1; diff --git a/bd/lfs_testbd.h b/bd/lfs_testbd.h index 64c13840..c89d717a 100644 --- a/bd/lfs_testbd.h +++ b/bd/lfs_testbd.h @@ -57,6 +57,10 @@ typedef int32_t lfs_testbd_swear_t; typedef uint32_t lfs_testbd_powercycles_t; typedef int32_t lfs_testbd_spowercycles_t; +// Type for delays in nanoseconds +typedef uint64_t lfs_testbd_delay_t; +typedef int64_t lfs_testbd_sdelay_t; + // testbd config, this is required for testing struct lfs_testbd_config { // 8-bit erase value to use for simulating erases. -1 does not simulate @@ -93,6 +97,18 @@ struct lfs_testbd_config { // Path to file to use as a mirror of the disk. This provides a way to view // the current state of the block device. const char *disk_path; + + // Artificial delay in nanoseconds, there is no purpose for this other + // than slowing down the simulation. + lfs_testbd_delay_t read_delay; + + // Artificial delay in nanoseconds, there is no purpose for this other + // than slowing down the simulation. + lfs_testbd_delay_t prog_delay; + + // Artificial delay in nanoseconds, there is no purpose for this other + // than slowing down the simulation. + lfs_testbd_delay_t erase_delay; }; // A reference counted block diff --git a/runners/test_runner.c b/runners/test_runner.c index b94bad5c..f89d7f40 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -247,6 +247,9 @@ static size_t test_step = 1; static const char *test_disk = NULL; FILE *test_trace = NULL; +static lfs_testbd_delay_t test_read_delay = 0.0; +static lfs_testbd_delay_t test_prog_delay = 0.0; +static lfs_testbd_delay_t test_erase_delay = 0.0; // how many permutations are there actually in a test case @@ -611,6 +614,9 @@ static void run_powerloss_none( .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, .disk_path = test_disk, + .read_delay = test_read_delay, + .prog_delay = test_prog_delay, + .erase_delay = test_erase_delay, }; int err = lfs_testbd_createcfg(&cfg, test_disk, &bdcfg); @@ -674,6 +680,9 @@ static void run_powerloss_linear( .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, .disk_path = test_disk, + .read_delay = test_read_delay, + .prog_delay = test_prog_delay, + .erase_delay = test_erase_delay, .power_cycles = i, .powerloss_behavior = POWERLOSS_BEHAVIOR, .powerloss_cb = powerloss_longjmp, @@ -751,6 +760,9 @@ static void run_powerloss_exponential( .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, .disk_path = test_disk, + .read_delay = test_read_delay, + .prog_delay = test_prog_delay, + .erase_delay = test_erase_delay, .power_cycles = i, .powerloss_behavior = POWERLOSS_BEHAVIOR, .powerloss_cb = powerloss_longjmp, @@ -826,6 +838,9 @@ static void run_powerloss_cycles( .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, .disk_path = test_disk, + .read_delay = test_read_delay, + .prog_delay = test_prog_delay, + .erase_delay = test_erase_delay, .power_cycles = (i < cycle_count) ? cycles[i] : 0, .powerloss_behavior = POWERLOSS_BEHAVIOR, .powerloss_cb = powerloss_longjmp, @@ -1018,6 +1033,9 @@ enum opt_flags { OPT_STOP = 8, OPT_DISK = 'd', OPT_TRACE = 't', + OPT_READ_DELAY = 9, + OPT_PROG_DELAY = 10, + OPT_ERASE_DELAY = 11, }; const char *short_opts = "hYlLD:G:p:nrVd:t:"; @@ -1040,6 +1058,9 @@ const struct option long_opts[] = { {"step", required_argument, NULL, OPT_STEP}, {"disk", required_argument, NULL, OPT_DISK}, {"trace", required_argument, NULL, OPT_TRACE}, + {"read-delay", required_argument, NULL, OPT_READ_DELAY}, + {"prog-delay", required_argument, NULL, OPT_PROG_DELAY}, + {"erase-delay", required_argument, NULL, OPT_ERASE_DELAY}, {NULL, 0, NULL, 0}, }; @@ -1061,6 +1082,9 @@ const char *const help_text[] = { "Only run every n tests, calculated after --start and --stop.", "Redirect block device operations to this file.", "Redirect trace output to this file.", + "Artificial read delay in seconds.", + "Artificial prog delay in seconds.", + "Artificial erase delay in seconds.", }; int main(int argc, char **argv) { @@ -1374,6 +1398,36 @@ powerloss_next: } } break; + case OPT_READ_DELAY: { + char *parsed = NULL; + double read_delay = strtod(optarg, &parsed); + if (parsed == optarg) { + fprintf(stderr, "error: invalid read-delay: %s\n", optarg); + exit(-1); + } + test_read_delay = read_delay*1.0e9; + break; + } + case OPT_PROG_DELAY: { + char *parsed = NULL; + double prog_delay = strtod(optarg, &parsed); + if (parsed == optarg) { + fprintf(stderr, "error: invalid prog-delay: %s\n", optarg); + exit(-1); + } + test_prog_delay = prog_delay*1.0e9; + break; + } + case OPT_ERASE_DELAY: { + char *parsed = NULL; + double erase_delay = strtod(optarg, &parsed); + if (parsed == optarg) { + fprintf(stderr, "error: invalid erase-delay: %s\n", optarg); + exit(-1); + } + test_erase_delay = erase_delay*1.0e9; + break; + } // done parsing case -1: goto getopt_done; diff --git a/scripts/pretty_asserts.py b/scripts/pretty_asserts.py index ceb206ca..33c64c9b 100755 --- a/scripts/pretty_asserts.py +++ b/scripts/pretty_asserts.py @@ -48,11 +48,14 @@ def write_header(f, limit=LIMIT): f.writeln("//") f.writeln() - f.writeln("#include ") f.writeln("#include ") f.writeln("#include ") f.writeln("#include ") + f.writeln("#include ") + f.writeln("#include ") f.writeln("#include ") + # give source a chance to define feature macros + f.writeln("#undef _FEATURES_H") f.writeln() # write print macros diff --git a/scripts/test.py b/scripts/test.py index d58dfd65..4aa0f116 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -498,6 +498,7 @@ def list_(**args): if args.get('list_cases'): cmd.append('--list-cases') if args.get('list_paths'): cmd.append('--list-paths') if args.get('list_defines'): cmd.append('--list-defines') + if args.get('list_defaults'): cmd.append('--list-defaults') if args.get('list_geometries'): cmd.append('--list-geometries') if args.get('list_powerlosses'): cmd.append('--list-powerlosses') @@ -641,8 +642,15 @@ def run_stage(name, runner_, **args): cmd.append('--disk=%s' % args['disk']) if args.get('trace'): cmd.append('--trace=%s' % args['trace']) + if args.get('read_delay'): + cmd.append('--read-delay=%s' % args['read_delay']) + if args.get('prog_delay'): + cmd.append('--prog-delay=%s' % args['prog_delay']) + if args.get('erase_delay'): + cmd.append('--erase-delay=%s' % args['erase_delay']) if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) + mpty, spty = pty.openpty() proc = sp.Popen(cmd, stdout=spty, stderr=spty) os.close(spty) @@ -1010,6 +1018,12 @@ if __name__ == "__main__": help="Redirect trace output to this file.") test_parser.add_argument('-o', '--output', help="Redirect stdout and stderr to this file.") + test_parser.add_argument('--read-delay', + help="Artificial read delay in seconds.") + test_parser.add_argument('--prog-delay', + help="Artificial prog delay in seconds.") + test_parser.add_argument('--erase-delay', + help="Artificial erase delay in seconds.") test_parser.add_argument('--runner', default=[RUNNER_PATH], type=lambda x: x.split(), help="Path to runner, defaults to %r" % RUNNER_PATH) From 5279fc6022eb56d532c685b855b7637b74662ce3 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 25 Aug 2022 11:06:19 -0500 Subject: [PATCH 28/81] Implemented exhaustive testing of n nested powerlosses As expected this takes a significant amount of time (~10 minutes for all 1 powerlosses, >10 hours for all 2 powerlosses) but this may be reducible in the future by optimizing tests for powerloss testing. Currently test_files does a lot of work that doesn't really have testing value. --- bd/lfs_testbd.c | 199 ++++++++++++++++++++---------------------- bd/lfs_testbd.h | 42 +++------ runners/test_runner.c | 195 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 292 insertions(+), 144 deletions(-) diff --git a/bd/lfs_testbd.c b/bd/lfs_testbd.c index 762512af..5a7e6b6b 100644 --- a/bd/lfs_testbd.c +++ b/bd/lfs_testbd.c @@ -29,68 +29,65 @@ // Note we can only modify a block if we have exclusive access to it (rc == 1) // -// TODO -__attribute__((unused)) -static void lfs_testbd_incblock(lfs_testbd_t *bd, lfs_block_t block) { - if (bd->blocks[block]) { - bd->blocks[block]->rc += 1; +static lfs_testbd_block_t *lfs_testbd_incblock(lfs_testbd_block_t *block) { + if (block) { + block->rc += 1; } + return block; } -static void lfs_testbd_decblock(lfs_testbd_t *bd, lfs_block_t block) { - if (bd->blocks[block]) { - bd->blocks[block]->rc -= 1; - if (bd->blocks[block]->rc == 0) { - free(bd->blocks[block]); - bd->blocks[block] = NULL; +static void lfs_testbd_decblock(lfs_testbd_block_t *block) { + if (block) { + block->rc -= 1; + if (block->rc == 0) { + free(block); } } } -static const lfs_testbd_block_t *lfs_testbd_getblock(lfs_testbd_t *bd, - lfs_block_t block) { - return bd->blocks[block]; -} - -static lfs_testbd_block_t *lfs_testbd_mutblock(lfs_testbd_t *bd, - lfs_block_t block, lfs_size_t block_size) { - if (bd->blocks[block] && bd->blocks[block]->rc == 1) { +static lfs_testbd_block_t *lfs_testbd_mutblock( + const struct lfs_config *cfg, + lfs_testbd_block_t **block) { + lfs_testbd_block_t *block_ = *block; + if (block_ && block_->rc == 1) { // rc == 1? can modify - return bd->blocks[block]; + return block_; - } else if (bd->blocks[block]) { + } else if (block_) { // rc > 1? need to create a copy - lfs_testbd_block_t *b = malloc( - sizeof(lfs_testbd_block_t) + block_size); - if (!b) { + lfs_testbd_block_t *nblock = malloc( + sizeof(lfs_testbd_block_t) + cfg->block_size); + if (!nblock) { return NULL; } - memcpy(b, bd->blocks[block], sizeof(lfs_testbd_block_t) + block_size); - b->rc = 1; + memcpy(nblock, block_, + sizeof(lfs_testbd_block_t) + cfg->block_size); + nblock->rc = 1; - lfs_testbd_decblock(bd, block); - bd->blocks[block] = b; - return b; + lfs_testbd_decblock(block_); + *block = nblock; + return nblock; } else { // no block? need to allocate - lfs_testbd_block_t *b = malloc( - sizeof(lfs_testbd_block_t) + block_size); - if (!b) { + lfs_testbd_block_t *nblock = malloc( + sizeof(lfs_testbd_block_t) + cfg->block_size); + if (!nblock) { return NULL; } - b->rc = 1; - b->wear = 0; + nblock->rc = 1; + nblock->wear = 0; // zero for consistency - memset(b->data, + lfs_testbd_t *bd = cfg->context; + memset(nblock->data, (bd->cfg->erase_value != -1) ? bd->cfg->erase_value : 0, - block_size); + cfg->block_size); - bd->blocks[block] = b; - return b; + *block = nblock; + return nblock; } } @@ -129,22 +126,25 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, // setup testing things bd->power_cycles = bd->cfg->power_cycles; - bd->disk_fd = -1; - bd->disk_scratch_block = NULL; - - bd->branches = NULL; - bd->branch_capacity = 0; - bd->branch_count = 0; + bd->disk = NULL; if (bd->cfg->disk_path) { + bd->disk = malloc(sizeof(lfs_testbd_disk_t)); + if (!bd->disk) { + LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM); + return LFS_ERR_NOMEM; + } + bd->disk->rc = 1; + bd->disk->scratch = NULL; + #ifdef _WIN32 - bd->disk_fd = open(bd->cfg->disk_path, + bd->disk->fd = open(bd->cfg->disk_path, O_RDWR | O_CREAT | O_BINARY, 0666); #else - bd->disk_fd = open(bd->cfg->disk_path, + bd->disk->fd = open(bd->cfg->disk_path, O_RDWR | O_CREAT, 0666); #endif - if (bd->disk_fd < 0) { + if (bd->disk->fd < 0) { int err = -errno; LFS_TESTBD_TRACE("lfs_testbd_create -> %d", err); return err; @@ -153,12 +153,12 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, // if we're emulating erase values, we can keep a block around in // memory of just the erase state to speed up emulated erases if (bd->cfg->erase_value != -1) { - bd->disk_scratch_block = malloc(cfg->block_size); - if (!bd->disk_scratch_block) { + bd->disk->scratch = malloc(cfg->block_size); + if (!bd->disk->scratch) { LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } - memset(bd->disk_scratch_block, + memset(bd->disk->scratch, bd->cfg->erase_value, cfg->block_size); } @@ -191,16 +191,18 @@ int lfs_testbd_destroy(const struct lfs_config *cfg) { // decrement reference counts for (lfs_block_t i = 0; i < cfg->block_count; i++) { - lfs_testbd_decblock(bd, i); + lfs_testbd_decblock(bd->blocks[i]); } - - // free memory free(bd->blocks); - free(bd->branches); - if (bd->disk_fd >= 0) { - close(bd->disk_fd); - free(bd->disk_scratch_block); + // clean up other resources + if (bd->disk) { + bd->disk->rc -= 1; + if (bd->disk->rc == 0) { + close(bd->disk->fd); + free(bd->disk->scratch); + free(bd->disk); + } } LFS_TESTBD_TRACE("lfs_testbd_destroy -> %d", 0); @@ -225,7 +227,7 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, LFS_ASSERT(off+size <= cfg->block_size); // get the block - const lfs_testbd_block_t *b = lfs_testbd_getblock(bd, block); + const lfs_testbd_block_t *b = bd->blocks[block]; if (b) { // block bad? if (bd->cfg->erase_cycles && b->wear >= bd->cfg->erase_cycles && @@ -273,7 +275,7 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, LFS_ASSERT(off+size <= cfg->block_size); // get the block - lfs_testbd_block_t *b = lfs_testbd_mutblock(bd, block, cfg->block_size); + lfs_testbd_block_t *b = lfs_testbd_mutblock(cfg, &bd->blocks[block]); if (!b) { LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; @@ -305,8 +307,8 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, memcpy(&b->data[off], buffer, size); // mirror to disk file? - if (bd->disk_fd >= 0) { - off_t res1 = lseek(bd->disk_fd, + if (bd->disk) { + off_t res1 = lseek(bd->disk->fd, (off_t)block*cfg->block_size + (off_t)off, SEEK_SET); if (res1 < 0) { @@ -315,7 +317,7 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, return err; } - ssize_t res2 = write(bd->disk_fd, buffer, size); + ssize_t res2 = write(bd->disk->fd, buffer, size); if (res2 < 0) { int err = -errno; LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); @@ -344,15 +346,6 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, } } -// // track power-loss branch? -// if (bd->cfg->track_branches) { -// int err = lfs_testbd_trackbranch(bd); -// if (err) { -// LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); -// return err; -// } -// } - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0); return 0; } @@ -365,7 +358,7 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { LFS_ASSERT(block < cfg->block_count); // get the block - lfs_testbd_block_t *b = lfs_testbd_mutblock(bd, block, cfg->block_size); + lfs_testbd_block_t *b = lfs_testbd_mutblock(cfg, &bd->blocks[block]); if (!b) { LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; @@ -394,8 +387,8 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { memset(b->data, bd->cfg->erase_value, cfg->block_size); // mirror to disk file? - if (bd->disk_fd >= 0) { - off_t res1 = lseek(bd->disk_fd, + if (bd->disk) { + off_t res1 = lseek(bd->disk->fd, (off_t)block*cfg->block_size, SEEK_SET); if (res1 < 0) { @@ -404,8 +397,8 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { return err; } - ssize_t res2 = write(bd->disk_fd, - bd->disk_scratch_block, + ssize_t res2 = write(bd->disk->fd, + bd->disk->scratch, cfg->block_size); if (res2 < 0) { int err = -errno; @@ -436,15 +429,6 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { } } -// // track power-loss branch? -// if (bd->cfg->track_branches) { -// int err = lfs_testbd_trackbranch(bd); -// if (err) { -// LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); -// return err; -// } -// } - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0); return 0; } @@ -472,7 +456,7 @@ lfs_testbd_swear_t lfs_testbd_getwear(const struct lfs_config *cfg, // get the wear lfs_testbd_wear_t wear; - const lfs_testbd_block_t *b = lfs_testbd_getblock(bd, block); + const lfs_testbd_block_t *b = bd->blocks[block]; if (b) { wear = b->wear; } else { @@ -492,7 +476,7 @@ int lfs_testbd_setwear(const struct lfs_config *cfg, LFS_ASSERT(block < cfg->block_count); // set the wear - lfs_testbd_block_t *b = lfs_testbd_mutblock(bd, block, cfg->block_size); + lfs_testbd_block_t *b = lfs_testbd_mutblock(cfg, &bd->blocks[block]); if (!b) { LFS_TESTBD_TRACE("lfs_testbd_setwear -> %"PRIu32, LFS_ERR_NOMEM); return LFS_ERR_NOMEM; @@ -524,23 +508,30 @@ int lfs_testbd_setpowercycles(const struct lfs_config *cfg, return 0; } -//int lfs_testbd_getbranch(const struct lfs_config *cfg, -// lfs_testbd_powercycles_t branch, lfs_testbd_t *bd) { -// LFS_TESTBD_TRACE("lfs_testbd_getbranch(%p, %zu, %p)", -// (void*)cfg, branch, bd); -// lfs_testbd_t *bd = cfg->context; -// -// // TODO -// -// LFS_TESTBD_TRACE("lfs_testbd_getbranch -> %d", 0); -// return 0; -//} - -lfs_testbd_spowercycles_t lfs_testbd_getbranchcount( - const struct lfs_config *cfg) { - LFS_TESTBD_TRACE("lfs_testbd_getbranchcount(%p)", (void*)cfg); +int lfs_testbd_copy(const struct lfs_config *cfg, lfs_testbd_t *copy) { + LFS_TESTBD_TRACE("lfs_testbd_copy(%p, %p)", (void*)cfg, (void*)copy); lfs_testbd_t *bd = cfg->context; - LFS_TESTBD_TRACE("lfs_testbd_getbranchcount -> %"PRIu32, bd->branch_count); - return bd->branch_count; + // lazily copy over our block array + copy->blocks = malloc(cfg->block_count * sizeof(lfs_testbd_block_t*)); + if (!copy->blocks) { + LFS_TESTBD_TRACE("lfs_testbd_copy -> %d", LFS_ERR_NOMEM); + return LFS_ERR_NOMEM; + } + + for (size_t i = 0; i < cfg->block_count; i++) { + copy->blocks[i] = lfs_testbd_incblock(bd->blocks[i]); + } + + // other state + copy->power_cycles = bd->power_cycles; + copy->disk = bd->disk; + if (copy->disk) { + copy->disk->rc += 1; + } + copy->cfg = bd->cfg; + + LFS_TESTBD_TRACE("lfs_testbd_copy -> %d", 0); + return 0; } + diff --git a/bd/lfs_testbd.h b/bd/lfs_testbd.h index c89d717a..4d26a4be 100644 --- a/bd/lfs_testbd.h +++ b/bd/lfs_testbd.h @@ -119,6 +119,13 @@ typedef struct lfs_testbd_block { uint8_t data[]; } lfs_testbd_block_t; +// Disk mirror +typedef struct lfs_testbd_disk { + uint32_t rc; + int fd; + uint8_t *scratch; +} lfs_testbd_disk_t; + // testbd state typedef struct lfs_testbd { // array of copy-on-write blocks @@ -126,31 +133,7 @@ typedef struct lfs_testbd { // some other test state uint32_t power_cycles; - int disk_fd; - uint8_t *disk_scratch_block; - - // array of tracked branches - struct lfs_testbd *branches; - lfs_testbd_powercycles_t branch_count; - lfs_testbd_powercycles_t branch_capacity; - - // TODO file? - - -// union { -// struct { -// lfs_filebd_t bd; -// } file; -// struct { -// lfs_rambd_t bd; -// struct lfs_rambd_config cfg; -// } ram; -// } u; -// -// bool persist; -// uint32_t power_cycles; -// lfs_testbd_wear_t *wear; -// uint8_t *scratch; + lfs_testbd_disk_t *disk; const struct lfs_testbd_config *cfg; } lfs_testbd_t; @@ -207,13 +190,8 @@ lfs_testbd_spowercycles_t lfs_testbd_getpowercycles( int lfs_testbd_setpowercycles(const struct lfs_config *cfg, lfs_testbd_powercycles_t power_cycles); -// Get a power-loss branch, requires track_branches=true -int lfs_testbd_getbranch(const struct lfs_config *cfg, - lfs_testbd_powercycles_t branch, lfs_testbd_t *bd); - -// Get the current number of power-loss branches -lfs_testbd_spowercycles_t lfs_testbd_getbranchcount( - const struct lfs_config *cfg); +// Create a copy-on-write copy of the state of this block device +int lfs_testbd_copy(const struct lfs_config *cfg, lfs_testbd_t *copy); #ifdef __cplusplus diff --git a/runners/test_runner.c b/runners/test_runner.c index f89d7f40..b187fcb3 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -700,6 +700,7 @@ static void run_powerloss_linear( while (true) { if (!setjmp(powerloss_jmp)) { + // run the test case_->run(&cfg); break; } @@ -780,6 +781,7 @@ static void run_powerloss_exponential( while (true) { if (!setjmp(powerloss_jmp)) { + // run the test case_->run(&cfg); break; } @@ -858,6 +860,7 @@ static void run_powerloss_cycles( while (true) { if (!setjmp(powerloss_jmp)) { + // run the test case_->run(&cfg); break; } @@ -883,15 +886,177 @@ static void run_powerloss_cycles( } } -//static void run_powerloss_n(void *data, -// -//static void run_powerloss_incremental(void *data, +struct powerloss_exhaustive_state { + struct lfs_config *cfg; + + lfs_testbd_t *branches; + size_t branch_count; + size_t branch_capacity; +}; + +struct powerloss_exhaustive_cycles { + lfs_testbd_powercycles_t *cycles; + size_t cycle_count; + size_t cycle_capacity; +}; + +static void powerloss_exhaustive_branch(void *c) { + // append to branches + struct powerloss_exhaustive_state *state = c; + state->branch_count += 1; + if (state->branch_count > state->branch_capacity) { + state->branch_capacity = (2*state->branch_capacity > 4) + ? 2*state->branch_capacity + : 4; + state->branches = realloc(state->branches, + state->branch_capacity * sizeof(lfs_testbd_t)); + if (!state->branches) { + fprintf(stderr, "error: exhaustive: out of memory\n"); + exit(-1); + } + } + + // create copy-on-write copy + int err = lfs_testbd_copy(state->cfg, + &state->branches[state->branch_count-1]); + if (err) { + fprintf(stderr, "error: exhaustive: could not create bd copy\n"); + exit(-1); + } + + // also trigger on next power cycle + lfs_testbd_setpowercycles(state->cfg, 1); +} + +static void run_powerloss_exhaustive_layer( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm, + struct lfs_config *cfg, + struct lfs_testbd_config *bdcfg, + size_t depth, + struct powerloss_exhaustive_cycles *cycles) { + (void)suite; + + struct powerloss_exhaustive_state state = { + .cfg = cfg, + .branches = NULL, + .branch_count = 0, + .branch_capacity = 0, + }; + + // run through the test without additional powerlosses, collecting possible + // branches as we do so + lfs_testbd_setpowercycles(state.cfg, depth > 0 ? 1 : 0); + bdcfg->powerloss_data = &state; + + // run the tests + case_->run(cfg); + + // aggressively clean up memory here to try to keep our memory usage low + int err = lfs_testbd_destroy(cfg); + if (err) { + fprintf(stderr, "error: could not destroy block device: %d\n", err); + exit(-1); + } + + // recurse into each branch + for (size_t i = 0; i < state.branch_count; i++) { + // first push and print the branch + cycles->cycle_count += 1; + if (cycles->cycle_count > cycles->cycle_capacity) { + cycles->cycle_capacity = (2*cycles->cycle_capacity > 4) + ? 2*cycles->cycle_capacity + : 4; + cycles->cycles = realloc(cycles->cycles, + cycles->cycle_capacity * sizeof(lfs_testbd_powercycles_t)); + if (!cycles->cycles) { + fprintf(stderr, "error: exhaustive: out of memory\n"); + exit(-1); + } + } + cycles->cycles[cycles->cycle_count-1] = i; + + printf("powerloss %s#%zu#", case_->id, perm); + leb16_print(cycles->cycles, cycles->cycle_count); + printf("\n"); + + // now recurse + cfg->context = &state.branches[i]; + run_powerloss_exhaustive_layer(suite, case_, perm, + cfg, bdcfg, depth-1, cycles); + + // pop the cycle + cycles->cycle_count -= 1; + } + + // clean up memory + free(state.branches); +} + +static void run_powerloss_exhaustive( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm, + const lfs_testbd_powercycles_t *cycles, + size_t cycle_count) { + (void)cycles; + (void)suite; + + // create block device and configuration + lfs_testbd_t bd; + + struct lfs_config cfg = { + .context = &bd, + .read = lfs_testbd_read, + .prog = lfs_testbd_prog, + .erase = lfs_testbd_erase, + .sync = lfs_testbd_sync, + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .block_size = BLOCK_SIZE, + .block_count = BLOCK_COUNT, + .block_cycles = BLOCK_CYCLES, + .cache_size = CACHE_SIZE, + .lookahead_size = LOOKAHEAD_SIZE, + }; + + struct lfs_testbd_config bdcfg = { + .erase_value = ERASE_VALUE, + .erase_cycles = ERASE_CYCLES, + .badblock_behavior = BADBLOCK_BEHAVIOR, + .disk_path = test_disk, + .read_delay = test_read_delay, + .prog_delay = test_prog_delay, + .erase_delay = test_erase_delay, + .powerloss_behavior = POWERLOSS_BEHAVIOR, + .powerloss_cb = powerloss_exhaustive_branch, + .powerloss_data = NULL, + }; + + int err = lfs_testbd_createcfg(&cfg, test_disk, &bdcfg); + if (err) { + fprintf(stderr, "error: could not create block device: %d\n", err); + exit(-1); + } + + // run the test, increasing power-cycles as power-loss events occur + printf("running %s#%zu\n", case_->id, perm); + + // recursively exhaust each layer of powerlosses + run_powerloss_exhaustive_layer(suite, case_, perm, + &cfg, &bdcfg, cycle_count, + &(struct powerloss_exhaustive_cycles){NULL, 0, 0}); + + printf("finished %s#%zu\n", case_->id, perm); +} + const test_powerloss_t builtin_powerlosses[] = { {'0', "none", run_powerloss_none, NULL, 0}, {'e', "exponential", run_powerloss_exponential, NULL, 0}, {'l', "linear", run_powerloss_linear, NULL, 0}, - //{'x', "exhaustive", run_powerloss_exhaustive} + {'x', "exhaustive", run_powerloss_exhaustive, NULL, SIZE_MAX}, {0, NULL, NULL, NULL, 0}, }; @@ -899,7 +1064,7 @@ const char *const builtin_powerlosses_help[] = { "Run with no power-losses.", "Run with linearly-decreasing power-losses.", "Run with exponentially-decreasing power-losses.", - //"Run a all permutations of power-losses, this may take a while.", + "Run a all permutations of power-losses, this may take a while.", "Run a all permutations of n power-losses.", "Run a custom comma-separated set of power-losses.", "Run a custom leb16-encoded set of power-losses.", @@ -1273,9 +1438,6 @@ invalid_define: } } - // exhaustive permutations - // TODO - // comma-separated permutation if (*optarg == '{') { // how many cycles? @@ -1341,6 +1503,23 @@ invalid_define: goto powerloss_next; } + // exhaustive permutations + { + char *parsed = NULL; + size_t count = strtoumax(optarg, &parsed, 0); + if (parsed == optarg) { + goto powerloss_unknown; + } + ((test_powerloss_t*)test_powerlosses)[ + test_powerloss_count-1] = (test_powerloss_t){ + .run = run_powerloss_exhaustive, + .cycles = NULL, + .cycle_count = count, + }; + optarg = (char*)parsed; + goto powerloss_next; + } + powerloss_unknown: // unknown scenario? fprintf(stderr, "error: " From c9a6e3a95b3b432be17a3cad97e1deff223a8527 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 27 Aug 2022 16:50:03 -0500 Subject: [PATCH 29/81] Added tailpipe.py and improved redirecting test trace/log output over fifos This mostly involved futzing around with some of the less intuitive parts of Unix's named-pipes behavior. This is a bit important since the tests can quickly generate several gigabytes of trace output. --- runners/test_runner.c | 117 +++++++++++++++++++++++++++----------- runners/test_runner.h | 23 +++++++- scripts/pretty_asserts.py | 3 +- scripts/tailpipe.py | 102 +++++++++++++++++++++++++++++++++ scripts/test.py | 113 ++++++++++++++++++------------------ 5 files changed, 268 insertions(+), 90 deletions(-) create mode 100755 scripts/tailpipe.py diff --git a/runners/test_runner.c b/runners/test_runner.c index b187fcb3..4ce22ab6 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -1,4 +1,8 @@ +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 199309L +#endif + #include "runners/test_runner.h" #include "bd/lfs_testbd.h" @@ -6,6 +10,10 @@ #include #include #include +#include +#include +#include +#include // test suites in a custom ld section @@ -219,10 +227,10 @@ static void run_powerloss_none( size_t perm, const lfs_testbd_powercycles_t *cycles, size_t cycle_count); -static const test_powerloss_t *test_powerlosses = (const test_powerloss_t[]){ +const test_powerloss_t *test_powerlosses = (const test_powerloss_t[]){ {'0', "none", run_powerloss_none, NULL, 0}, }; -static size_t test_powerloss_count = 1; +size_t test_powerloss_count = 1; typedef struct test_id { @@ -233,23 +241,70 @@ typedef struct test_id { size_t cycle_count; } test_id_t; -static const test_id_t *test_ids = (const test_id_t[]) { +const test_id_t *test_ids = (const test_id_t[]) { {NULL, NULL, -1, NULL, 0}, }; -static size_t test_id_count = 1; +size_t test_id_count = 1; -static const char *test_geometry = NULL; +const char *test_geometry = NULL; -static size_t test_start = 0; -static size_t test_stop = -1; -static size_t test_step = 1; +size_t test_start = 0; +size_t test_stop = -1; +size_t test_step = 1; -static const char *test_disk = NULL; -FILE *test_trace = NULL; -static lfs_testbd_delay_t test_read_delay = 0.0; -static lfs_testbd_delay_t test_prog_delay = 0.0; -static lfs_testbd_delay_t test_erase_delay = 0.0; +const char *test_disk_path = NULL; +const char *test_trace_path = NULL; +FILE *test_trace_file = NULL; +uint32_t test_trace_cycles = 0; +lfs_testbd_delay_t test_read_delay = 0.0; +lfs_testbd_delay_t test_prog_delay = 0.0; +lfs_testbd_delay_t test_erase_delay = 0.0; + + +// trace printing +void test_trace(const char *fmt, ...) { + if (test_trace_path) { + if (!test_trace_file) { + // Tracing output is heavy and trying to open every trace + // call is slow, so we only try to open the trace file every + // so often. Note this doesn't affect successfully opened files + if (test_trace_cycles % 128 != 0) { + test_trace_cycles += 1; + return; + } + test_trace_cycles += 1; + + int fd; + if (strcmp(test_trace_path, "-") == 0) { + fd = dup(1); + } else { + fd = open( + test_trace_path, + O_WRONLY | O_CREAT | O_APPEND | O_NONBLOCK, + 0666); + } + if (fd < 0) { + return; + } + + FILE *f = fdopen(fd, "a"); + assert(f); + int err = setvbuf(f, NULL, _IOLBF, BUFSIZ); + assert(!err); + test_trace_file = f; + } + + va_list va; + va_start(va, fmt); + int res = vfprintf(test_trace_file, fmt, va); + if (res < 0) { + fclose(test_trace_file); + test_trace_file = NULL; + } + va_end(va); + } +} // how many permutations are there actually in a test case @@ -613,13 +668,13 @@ static void run_powerloss_none( .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, - .disk_path = test_disk, + .disk_path = test_disk_path, .read_delay = test_read_delay, .prog_delay = test_prog_delay, .erase_delay = test_erase_delay, }; - int err = lfs_testbd_createcfg(&cfg, test_disk, &bdcfg); + int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -679,7 +734,7 @@ static void run_powerloss_linear( .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, - .disk_path = test_disk, + .disk_path = test_disk_path, .read_delay = test_read_delay, .prog_delay = test_prog_delay, .erase_delay = test_erase_delay, @@ -689,7 +744,7 @@ static void run_powerloss_linear( .powerloss_data = &powerloss_jmp, }; - int err = lfs_testbd_createcfg(&cfg, test_disk, &bdcfg); + int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -760,7 +815,7 @@ static void run_powerloss_exponential( .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, - .disk_path = test_disk, + .disk_path = test_disk_path, .read_delay = test_read_delay, .prog_delay = test_prog_delay, .erase_delay = test_erase_delay, @@ -770,7 +825,7 @@ static void run_powerloss_exponential( .powerloss_data = &powerloss_jmp, }; - int err = lfs_testbd_createcfg(&cfg, test_disk, &bdcfg); + int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -839,7 +894,7 @@ static void run_powerloss_cycles( .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, - .disk_path = test_disk, + .disk_path = test_disk_path, .read_delay = test_read_delay, .prog_delay = test_prog_delay, .erase_delay = test_erase_delay, @@ -849,7 +904,7 @@ static void run_powerloss_cycles( .powerloss_data = &powerloss_jmp, }; - int err = lfs_testbd_createcfg(&cfg, test_disk, &bdcfg); + int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1025,7 +1080,7 @@ static void run_powerloss_exhaustive( .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, - .disk_path = test_disk, + .disk_path = test_disk_path, .read_delay = test_read_delay, .prog_delay = test_prog_delay, .erase_delay = test_erase_delay, @@ -1034,7 +1089,7 @@ static void run_powerloss_exhaustive( .powerloss_data = NULL, }; - int err = lfs_testbd_createcfg(&cfg, test_disk, &bdcfg); + int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1153,6 +1208,9 @@ static void run_perms( } static void run(void) { + // ignore disconnected pipes + signal(SIGPIPE, SIG_IGN); + for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { if (test_ids[t].suite && strcmp( @@ -1563,19 +1621,10 @@ powerloss_next: break; } case OPT_DISK: - test_disk = optarg; + test_disk_path = optarg; break; case OPT_TRACE: - if (strcmp(optarg, "-") == 0) { - test_trace = stdout; - } else { - test_trace = fopen(optarg, "w"); - if (!test_trace) { - fprintf(stderr, "error: could not open for trace: %d\n", - -errno); - exit(-1); - } - } + test_trace_path = optarg; break; case OPT_READ_DELAY: { char *parsed = NULL; diff --git a/runners/test_runner.h b/runners/test_runner.h index 27229ad3..4459a5c9 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -1,7 +1,26 @@ #ifndef TEST_RUNNER_H #define TEST_RUNNER_H -#include "lfs.h" + +// override LFS_TRACE +void test_trace(const char *fmt, ...); + +#define LFS_TRACE_(fmt, ...) \ + test_trace("%s:%d:trace: " fmt "%s\n", \ + __FILE__, \ + __LINE__, \ + __VA_ARGS__) +#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") +#define LFS_TESTBD_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") + + +// note these are indirectly included in any generated files +#include "bd/lfs_testbd.h" +#include + +// give source a chance to define feature macros +#undef _FEATURES_H +#undef _STDIO_H // generated test configurations @@ -10,6 +29,8 @@ enum test_flags { }; typedef uint8_t test_flags_t; +struct lfs_config; + struct test_case { const char *id; const char *name; diff --git a/scripts/pretty_asserts.py b/scripts/pretty_asserts.py index 33c64c9b..8afa6545 100755 --- a/scripts/pretty_asserts.py +++ b/scripts/pretty_asserts.py @@ -422,7 +422,8 @@ if __name__ == "__main__": parser.add_argument('-p', '--pattern', action='append', help="Regex patterns to search for starting an assert statement. This" " implicitly includes \"assert\" and \"=>\".") - parser.add_argument('-l', '--limit', default=LIMIT, type=int, + parser.add_argument('-l', '--limit', + default=LIMIT, type=lambda x: int(x, 0), help="Maximum number of characters to display in strcmp and memcmp.") sys.exit(main(**{k: v for k, v in vars(parser.parse_args()).items() diff --git a/scripts/tailpipe.py b/scripts/tailpipe.py new file mode 100755 index 00000000..985f571c --- /dev/null +++ b/scripts/tailpipe.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +import os +import sys +import threading as th +import time + + +def openio(path, mode='r'): + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) + +def main(path, lines=1, keep_open=False): + ring = [None] * lines + i = 0 + count = 0 + lock = th.Lock() + event = th.Event() + done = False + + # do the actual reading in a background thread + def read(): + nonlocal i + nonlocal count + nonlocal done + while True: + with openio(path, 'r') as f: + for line in f: + with lock: + ring[i] = line + i = (i + 1) % lines + count = min(lines, count + 1) + event.set() + if not keep_open: + break + done = True + + th.Thread(target=read, daemon=True).start() + + try: + last_count = 1 + while not done: + time.sleep(0.01) + event.wait() + event.clear() + + # create a copy to avoid corrupt output + with lock: + ring_ = ring.copy() + i_ = i + count_ = count + + # first thing first, give ourself a canvas + while last_count < count_: + sys.stdout.write('\n') + last_count += 1 + + for j in range(count_): + # move cursor, clear line, disable/reenable line wrapping + sys.stdout.write('\r%s\x1b[K\x1b[?7l%s\x1b[?7h%s' % ( + '\x1b[%dA' % (count_-1-j) if count_-1-j > 0 else '', + ring_[(i_-count+j) % lines][:-1], + '\x1b[%dB' % (count_-1-j) if count_-1-j > 0 else '')) + + sys.stdout.flush() + + except KeyboardInterrupt: + pass + + sys.stdout.write('\n') + + +if __name__ == "__main__": + import sys + import argparse + parser = argparse.ArgumentParser( + description="Efficiently displays the last n lines of a file/pipe.") + parser.add_argument( + 'path', + nargs='?', + default='-', + help="Path to read from.") + parser.add_argument( + '-n', + '--lines', + type=lambda x: int(x, 0), + default=1, + help="Number of lines to show, defaults to 1.") + parser.add_argument( + '-k', + '--keep-open', + action='store_true', + help="Reopen the pipe on EOF, useful when multiple " + "processes are writing.") + sys.exit(main(**{k: v + for k, v in vars(parser.parse_args()).items() + if v is not None})) diff --git a/scripts/test.py b/scripts/test.py index 4aa0f116..a29423bb 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -22,16 +22,7 @@ import toml TEST_PATHS = ['tests'] RUNNER_PATH = './runners/test_runner' - -SUITE_PROLOGUE = """ -#include "runners/test_runner.h" -#include "bd/lfs_testbd.h" -#include -""" -CASE_PROLOGUE = """ -""" -CASE_EPILOGUE = """ -""" +HEADER_PATH = 'runners/test_runner.h' def testpath(path): @@ -49,14 +40,21 @@ def testcase(path): _, case, *_ = path.split('#', 2) return '%s#%s' % (testsuite(path), case) -def openio(path, mode='r'): +def openio(path, mode='r', buffering=-1, nb=False): if path == '-': if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + return os.fdopen(os.dup(sys.stdin.fileno()), 'r', buffering) else: - return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + return os.fdopen(os.dup(sys.stdout.fileno()), 'w', buffering) + elif nb and 'a' in mode: + return os.fdopen(os.open( + path, + os.O_WRONLY | os.O_CREAT | os.O_APPEND | os.O_NONBLOCK, + 0o666), + mode, + buffering) else: - return open(path, mode) + return open(path, mode, buffering) def color(**args): if args.get('color') == 'auto': @@ -252,19 +250,8 @@ def compile(**args): f.writeln("//") f.writeln() - # redirect littlefs tracing - f.writeln('#define LFS_TRACE_(fmt, ...) do { \\') - f.writeln(8*' '+'extern FILE *test_trace; \\') - f.writeln(8*' '+'if (test_trace) { \\') - f.writeln(12*' '+'fprintf(test_trace, ' - '"%s:%d:trace: " fmt "%s\\n", \\') - f.writeln(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\') - f.writeln(8*' '+'} \\') - f.writeln(4*' '+'} while (0)') - f.writeln('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")') - f.writeln('#define LFS_TESTBD_TRACE(...) ' - 'LFS_TRACE_(__VA_ARGS__, "")') - f.writeln() + # include test_runner.h in every generated file + f.writeln("#include \"%s\"" % HEADER_PATH) # write out generated functions, this can end up in different # files depending on the "in" attribute @@ -316,10 +303,6 @@ def compile(**args): f.writeln('void __test__%s__%s__run(' '__attribute__((unused)) struct lfs_config *cfg) {' % (suite.name, case.name)) - if CASE_PROLOGUE.strip(): - f.writeln(4*' '+'%s' - % CASE_PROLOGUE.strip().replace('\n', '\n'+4*' ')) - f.writeln() f.writeln(4*' '+'// test case %s' % case.id()) if case.code_lineno is not None: f.writeln(4*' '+'#line %d "%s"' @@ -328,17 +311,10 @@ def compile(**args): if case.code_lineno is not None: f.writeln(4*' '+'#line %d "%s"' % (f.lineno+1, args['output'])) - if CASE_EPILOGUE.strip(): - f.writeln() - f.writeln(4*' '+'%s' - % CASE_EPILOGUE.strip().replace('\n', '\n'+4*' ')) f.writeln('}') f.writeln() if not args.get('source'): - # write test suite prologue - f.writeln('%s' % SUITE_PROLOGUE.strip()) - f.writeln() if suite.code is not None: if suite.code_lineno is not None: f.writeln('#line %d "%s"' @@ -427,9 +403,6 @@ def compile(**args): shutil.copyfileobj(sf, f) f.writeln() - f.write(SUITE_PROLOGUE) - f.writeln() - # write any internal tests for suite in suites: for case in suite.cases: @@ -638,6 +611,7 @@ def run_stage(name, runner_, **args): # run the tests! cmd = runner_.copy() + # TODO move all these to runner? if args.get('disk'): cmd.append('--disk=%s' % args['disk']) if args.get('trace'): @@ -656,8 +630,7 @@ def run_stage(name, runner_, **args): os.close(spty) children.add(proc) mpty = os.fdopen(mpty, 'r', 1) - if args.get('output'): - output = openio(args['output'], 'w') + output = None last_id = None last_output = [] @@ -675,8 +648,18 @@ def run_stage(name, runner_, **args): break last_output.append(line) if args.get('output'): - output.write(line) - elif args.get('verbose'): + try: + if not output: + output = openio(args['output'], 'a', 1, nb=True) + output.write(line) + except OSError as e: + if e.errno not in [ + errno.ENXIO, + errno.EPIPE, + errno.EAGAIN]: + raise + output = None + if args.get('verbose'): sys.stdout.write(line) m = pattern.match(line) @@ -709,8 +692,6 @@ def run_stage(name, runner_, **args): finally: children.remove(proc) mpty.close() - if args.get('output'): - output.close() proc.wait() if proc.returncode != 0: @@ -722,6 +703,7 @@ def run_stage(name, runner_, **args): def run_job(runner, start=None, step=None): nonlocal failures + nonlocal killed nonlocal locals start = start or 0 @@ -756,6 +738,7 @@ def run_stage(name, runner_, **args): continue else: # stop other tests + killed = True for child in children.copy(): child.kill() break @@ -766,10 +749,12 @@ def run_stage(name, runner_, **args): if 'jobs' in args: for job in range(args['jobs']): runners.append(th.Thread( - target=run_job, args=(runner_, job, args['jobs']))) + target=run_job, args=(runner_, job, args['jobs']), + daemon=True)) else: runners.append(th.Thread( - target=run_job, args=(runner_, None, None))) + target=run_job, args=(runner_, None, None), + daemon=True)) def print_update(done): if not args.get('verbose') and (color(**args) or done): @@ -830,8 +815,10 @@ def run_stage(name, runner_, **args): def run(**args): + # measure runtime start = time.time() + # query runner for tests runner_ = runner(**args) print('using runner: %s' % ' '.join(shlex.quote(c) for c in runner_)) @@ -844,6 +831,15 @@ def run(**args): total_perms)) print() + # truncate and open logs here so they aren't disconnected between tests + output = None + if args.get('output'): + output = openio(args['output'], 'w', 1) + trace = None + if args.get('trace'): + trace = openio(args['trace'], 'w', 1) + + # spawn runners expected = 0 passed = 0 powerlosses = 0 @@ -857,7 +853,9 @@ def run(**args): # spawn jobs for stage expected_, passed_, powerlosses_, failures_, killed = run_stage( - by or 'tests', stage_runner, **args) + by or 'tests', + stage_runner, + **args) expected += expected_ passed += passed_ powerlosses += powerlosses_ @@ -865,6 +863,11 @@ def run(**args): if (failures and not args.get('keep_going')) or killed: break + if output: + output.close() + if trace: + trace.close() + # show summary print() print('%sdone:%s %s' % ( @@ -974,9 +977,11 @@ def main(**args): if __name__ == "__main__": import argparse import sys + argparse.ArgumentParser._handle_conflict_ignore = lambda *_: None + argparse._ArgumentGroup._handle_conflict_ignore = lambda *_: None parser = argparse.ArgumentParser( description="Build and run tests.", - conflict_handler='resolve') + conflict_handler='ignore') parser.add_argument('test_ids', nargs='*', help="Description of testis to run. May be a directory, path, or \ test identifier. Test identifiers are of the form \ @@ -1013,11 +1018,11 @@ if __name__ == "__main__": help="Comma-separated list of power-loss scenarios to test. \ Defaults to 0,l.") test_parser.add_argument('-d', '--disk', - help="Redirect block device operations to this file.") + help="Direct block device operations to this file.") test_parser.add_argument('-t', '--trace', - help="Redirect trace output to this file.") + help="Direct trace output to this file.") test_parser.add_argument('-o', '--output', - help="Redirect stdout and stderr to this file.") + help="Direct stdout and stderr to this file.") test_parser.add_argument('--read-delay', help="Artificial read delay in seconds.") test_parser.add_argument('--prog-delay', From 91200e6678d0fbd2196b092afceb038e9e696812 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 2 Sep 2022 03:08:28 -0500 Subject: [PATCH 30/81] Added tracebd.py, a script for rendering block device operations Based on a handful of local hacky variations, this sort of trace rendering is surprisingly useful for getting an understanding of how different filesystem operations interact with the underlying block-device. At some point it would probably be good to reimplement this in a compiled language. Parsing and tracking the trace output quickly becomes a bottleneck with the amount of trace output the tests generate. Note also that since tracebd.py run on trace output, it can also be used to debug logged block-device operations post-run. --- bd/lfs_testbd.c | 18 +- bd/lfs_testbd.h | 10 +- runners/test_runner.c | 83 ++--- scripts/coverage.py | 2 +- scripts/tailpipe.py | 29 +- scripts/test.py | 39 ++- scripts/tracebd.py | 777 ++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 878 insertions(+), 80 deletions(-) create mode 100755 scripts/tracebd.py diff --git a/bd/lfs_testbd.c b/bd/lfs_testbd.c index 5a7e6b6b..61063af8 100644 --- a/bd/lfs_testbd.c +++ b/bd/lfs_testbd.c @@ -245,10 +245,10 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, size); } - if (bd->cfg->read_delay) { + if (bd->cfg->read_sleep) { int err = nanosleep(&(struct timespec){ - .tv_sec=bd->cfg->read_delay/1000000000, - .tv_nsec=bd->cfg->read_delay%1000000000}, + .tv_sec=bd->cfg->read_sleep/1000000000, + .tv_nsec=bd->cfg->read_sleep%1000000000}, NULL); if (err) { err = -errno; @@ -325,10 +325,10 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, } } - if (bd->cfg->prog_delay) { + if (bd->cfg->prog_sleep) { int err = nanosleep(&(struct timespec){ - .tv_sec=bd->cfg->prog_delay/1000000000, - .tv_nsec=bd->cfg->prog_delay%1000000000}, + .tv_sec=bd->cfg->prog_sleep/1000000000, + .tv_nsec=bd->cfg->prog_sleep%1000000000}, NULL); if (err) { err = -errno; @@ -408,10 +408,10 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { } } - if (bd->cfg->erase_delay) { + if (bd->cfg->erase_sleep) { int err = nanosleep(&(struct timespec){ - .tv_sec=bd->cfg->erase_delay/1000000000, - .tv_nsec=bd->cfg->erase_delay%1000000000}, + .tv_sec=bd->cfg->erase_sleep/1000000000, + .tv_nsec=bd->cfg->erase_sleep%1000000000}, NULL); if (err) { err = -errno; diff --git a/bd/lfs_testbd.h b/bd/lfs_testbd.h index 4d26a4be..b0c9d005 100644 --- a/bd/lfs_testbd.h +++ b/bd/lfs_testbd.h @@ -58,8 +58,8 @@ typedef uint32_t lfs_testbd_powercycles_t; typedef int32_t lfs_testbd_spowercycles_t; // Type for delays in nanoseconds -typedef uint64_t lfs_testbd_delay_t; -typedef int64_t lfs_testbd_sdelay_t; +typedef uint64_t lfs_testbd_sleep_t; +typedef int64_t lfs_testbd_ssleep_t; // testbd config, this is required for testing struct lfs_testbd_config { @@ -100,15 +100,15 @@ struct lfs_testbd_config { // Artificial delay in nanoseconds, there is no purpose for this other // than slowing down the simulation. - lfs_testbd_delay_t read_delay; + lfs_testbd_sleep_t read_sleep; // Artificial delay in nanoseconds, there is no purpose for this other // than slowing down the simulation. - lfs_testbd_delay_t prog_delay; + lfs_testbd_sleep_t prog_sleep; // Artificial delay in nanoseconds, there is no purpose for this other // than slowing down the simulation. - lfs_testbd_delay_t erase_delay; + lfs_testbd_sleep_t erase_sleep; }; // A reference counted block diff --git a/runners/test_runner.c b/runners/test_runner.c index 4ce22ab6..eea4b5f3 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -257,9 +257,9 @@ const char *test_disk_path = NULL; const char *test_trace_path = NULL; FILE *test_trace_file = NULL; uint32_t test_trace_cycles = 0; -lfs_testbd_delay_t test_read_delay = 0.0; -lfs_testbd_delay_t test_prog_delay = 0.0; -lfs_testbd_delay_t test_erase_delay = 0.0; +lfs_testbd_sleep_t test_read_sleep = 0.0; +lfs_testbd_sleep_t test_prog_sleep = 0.0; +lfs_testbd_sleep_t test_erase_sleep = 0.0; // trace printing @@ -278,14 +278,19 @@ void test_trace(const char *fmt, ...) { int fd; if (strcmp(test_trace_path, "-") == 0) { fd = dup(1); + if (fd < 0) { + return; + } } else { fd = open( test_trace_path, O_WRONLY | O_CREAT | O_APPEND | O_NONBLOCK, 0666); - } - if (fd < 0) { - return; + if (fd < 0) { + return; + } + int err = fcntl(fd, F_SETFL, O_WRONLY | O_CREAT | O_APPEND); + assert(!err); } FILE *f = fdopen(fd, "a"); @@ -669,9 +674,9 @@ static void run_powerloss_none( .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, .disk_path = test_disk_path, - .read_delay = test_read_delay, - .prog_delay = test_prog_delay, - .erase_delay = test_erase_delay, + .read_sleep = test_read_sleep, + .prog_sleep = test_prog_sleep, + .erase_sleep = test_erase_sleep, }; int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); @@ -735,9 +740,9 @@ static void run_powerloss_linear( .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, .disk_path = test_disk_path, - .read_delay = test_read_delay, - .prog_delay = test_prog_delay, - .erase_delay = test_erase_delay, + .read_sleep = test_read_sleep, + .prog_sleep = test_prog_sleep, + .erase_sleep = test_erase_sleep, .power_cycles = i, .powerloss_behavior = POWERLOSS_BEHAVIOR, .powerloss_cb = powerloss_longjmp, @@ -816,9 +821,9 @@ static void run_powerloss_exponential( .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, .disk_path = test_disk_path, - .read_delay = test_read_delay, - .prog_delay = test_prog_delay, - .erase_delay = test_erase_delay, + .read_sleep = test_read_sleep, + .prog_sleep = test_prog_sleep, + .erase_sleep = test_erase_sleep, .power_cycles = i, .powerloss_behavior = POWERLOSS_BEHAVIOR, .powerloss_cb = powerloss_longjmp, @@ -895,9 +900,9 @@ static void run_powerloss_cycles( .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, .disk_path = test_disk_path, - .read_delay = test_read_delay, - .prog_delay = test_prog_delay, - .erase_delay = test_erase_delay, + .read_sleep = test_read_sleep, + .prog_sleep = test_prog_sleep, + .erase_sleep = test_erase_sleep, .power_cycles = (i < cycle_count) ? cycles[i] : 0, .powerloss_behavior = POWERLOSS_BEHAVIOR, .powerloss_cb = powerloss_longjmp, @@ -1081,9 +1086,9 @@ static void run_powerloss_exhaustive( .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, .disk_path = test_disk_path, - .read_delay = test_read_delay, - .prog_delay = test_prog_delay, - .erase_delay = test_erase_delay, + .read_sleep = test_read_sleep, + .prog_sleep = test_prog_sleep, + .erase_sleep = test_erase_sleep, .powerloss_behavior = POWERLOSS_BEHAVIOR, .powerloss_cb = powerloss_exhaustive_branch, .powerloss_data = NULL, @@ -1256,9 +1261,9 @@ enum opt_flags { OPT_STOP = 8, OPT_DISK = 'd', OPT_TRACE = 't', - OPT_READ_DELAY = 9, - OPT_PROG_DELAY = 10, - OPT_ERASE_DELAY = 11, + OPT_READ_SLEEP = 9, + OPT_PROG_SLEEP = 10, + OPT_ERASE_SLEEP = 11, }; const char *short_opts = "hYlLD:G:p:nrVd:t:"; @@ -1281,9 +1286,9 @@ const struct option long_opts[] = { {"step", required_argument, NULL, OPT_STEP}, {"disk", required_argument, NULL, OPT_DISK}, {"trace", required_argument, NULL, OPT_TRACE}, - {"read-delay", required_argument, NULL, OPT_READ_DELAY}, - {"prog-delay", required_argument, NULL, OPT_PROG_DELAY}, - {"erase-delay", required_argument, NULL, OPT_ERASE_DELAY}, + {"read-sleep", required_argument, NULL, OPT_READ_SLEEP}, + {"prog-sleep", required_argument, NULL, OPT_PROG_SLEEP}, + {"erase-sleep", required_argument, NULL, OPT_ERASE_SLEEP}, {NULL, 0, NULL, 0}, }; @@ -1626,34 +1631,34 @@ powerloss_next: case OPT_TRACE: test_trace_path = optarg; break; - case OPT_READ_DELAY: { + case OPT_READ_SLEEP: { char *parsed = NULL; - double read_delay = strtod(optarg, &parsed); + double read_sleep = strtod(optarg, &parsed); if (parsed == optarg) { - fprintf(stderr, "error: invalid read-delay: %s\n", optarg); + fprintf(stderr, "error: invalid read-sleep: %s\n", optarg); exit(-1); } - test_read_delay = read_delay*1.0e9; + test_read_sleep = read_sleep*1.0e9; break; } - case OPT_PROG_DELAY: { + case OPT_PROG_SLEEP: { char *parsed = NULL; - double prog_delay = strtod(optarg, &parsed); + double prog_sleep = strtod(optarg, &parsed); if (parsed == optarg) { - fprintf(stderr, "error: invalid prog-delay: %s\n", optarg); + fprintf(stderr, "error: invalid prog-sleep: %s\n", optarg); exit(-1); } - test_prog_delay = prog_delay*1.0e9; + test_prog_sleep = prog_sleep*1.0e9; break; } - case OPT_ERASE_DELAY: { + case OPT_ERASE_SLEEP: { char *parsed = NULL; - double erase_delay = strtod(optarg, &parsed); + double erase_sleep = strtod(optarg, &parsed); if (parsed == optarg) { - fprintf(stderr, "error: invalid erase-delay: %s\n", optarg); + fprintf(stderr, "error: invalid erase-sleep: %s\n", optarg); exit(-1); } - test_erase_delay = erase_delay*1.0e9; + test_erase_sleep = erase_sleep*1.0e9; break; } // done parsing diff --git a/scripts/coverage.py b/scripts/coverage.py index 91445c2b..d30b3ffd 100755 --- a/scripts/coverage.py +++ b/scripts/coverage.py @@ -501,7 +501,7 @@ if __name__ == "__main__": help="Show uncovered branches.") parser.add_argument('-c', '--context', type=lambda x: int(x, 0), default=3, help="Show a additional lines of context. Defaults to 3.") - parser.add_argument('-w', '--width', type=lambda x: int(x, 0), default=80, + parser.add_argument('-W', '--width', type=lambda x: int(x, 0), default=80, help="Assume source is styled with this many columns. Defaults to 80.") parser.add_argument('--color', choices=['never', 'always', 'auto'], default='auto', diff --git a/scripts/tailpipe.py b/scripts/tailpipe.py index 985f571c..101fd98c 100755 --- a/scripts/tailpipe.py +++ b/scripts/tailpipe.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +# +# Efficiently displays the last n lines of a file/pipe. +# import os import sys @@ -15,7 +18,7 @@ def openio(path, mode='r'): else: return open(path, mode) -def main(path, lines=1, keep_open=False): +def main(path='-', *, lines=1, sleep=0.01, keep_open=False): ring = [None] * lines i = 0 count = 0 @@ -29,7 +32,7 @@ def main(path, lines=1, keep_open=False): nonlocal count nonlocal done while True: - with openio(path, 'r') as f: + with openio(path) as f: for line in f: with lock: ring[i] = line @@ -45,7 +48,7 @@ def main(path, lines=1, keep_open=False): try: last_count = 1 while not done: - time.sleep(0.01) + time.sleep(sleep) event.wait() event.clear() @@ -62,10 +65,15 @@ def main(path, lines=1, keep_open=False): for j in range(count_): # move cursor, clear line, disable/reenable line wrapping - sys.stdout.write('\r%s\x1b[K\x1b[?7l%s\x1b[?7h%s' % ( - '\x1b[%dA' % (count_-1-j) if count_-1-j > 0 else '', - ring_[(i_-count+j) % lines][:-1], - '\x1b[%dB' % (count_-1-j) if count_-1-j > 0 else '')) + sys.stdout.write('\r') + if count_-1-j > 0: + sys.stdout.write('\x1b[%dA' % (count_-1-j)) + sys.stdout.write('\x1b[K') + sys.stdout.write('\x1b[?7l') + sys.stdout.write(ring_[(i_-count_+j) % lines][:-1]) + sys.stdout.write('\x1b[?7h') + if count_-1-j > 0: + sys.stdout.write('\x1b[%dB' % (count_-1-j)) sys.stdout.flush() @@ -83,14 +91,17 @@ if __name__ == "__main__": parser.add_argument( 'path', nargs='?', - default='-', help="Path to read from.") parser.add_argument( '-n', '--lines', type=lambda x: int(x, 0), - default=1, help="Number of lines to show, defaults to 1.") + parser.add_argument( + '-s', + '--sleep', + type=float, + help="Seconds to sleep between reads, defaults to 0.01.") parser.add_argument( '-k', '--keep-open', diff --git a/scripts/test.py b/scripts/test.py index a29423bb..c7c6b0f7 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -489,7 +489,8 @@ def find_cases(runner_, **args): stdout=sp.PIPE, stderr=sp.PIPE if not args.get('verbose') else None, universal_newlines=True, - errors='replace') + errors='replace', + close_fds=False) expected_suite_perms = co.defaultdict(lambda: 0) expected_case_perms = co.defaultdict(lambda: 0) expected_perms = 0 @@ -528,7 +529,8 @@ def find_paths(runner_, **args): stdout=sp.PIPE, stderr=sp.PIPE if not args.get('verbose') else None, universal_newlines=True, - errors='replace') + errors='replace', + close_fds=False) paths = co.OrderedDict() pattern = re.compile( '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' @@ -555,7 +557,8 @@ def find_defines(runner_, **args): stdout=sp.PIPE, stderr=sp.PIPE if not args.get('verbose') else None, universal_newlines=True, - errors='replace') + errors='replace', + close_fds=False) defines = co.OrderedDict() pattern = re.compile( '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' @@ -616,17 +619,17 @@ def run_stage(name, runner_, **args): cmd.append('--disk=%s' % args['disk']) if args.get('trace'): cmd.append('--trace=%s' % args['trace']) - if args.get('read_delay'): - cmd.append('--read-delay=%s' % args['read_delay']) - if args.get('prog_delay'): - cmd.append('--prog-delay=%s' % args['prog_delay']) - if args.get('erase_delay'): - cmd.append('--erase-delay=%s' % args['erase_delay']) + if args.get('read_sleep'): + cmd.append('--read-sleep=%s' % args['read_sleep']) + if args.get('prog_sleep'): + cmd.append('--prog-sleep=%s' % args['prog_sleep']) + if args.get('erase_sleep'): + cmd.append('--erase-sleep=%s' % args['erase_sleep']) if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) mpty, spty = pty.openpty() - proc = sp.Popen(cmd, stdout=spty, stderr=spty) + proc = sp.Popen(cmd, stdout=spty, stderr=spty, close_fds=False) os.close(spty) children.add(proc) mpty = os.fdopen(mpty, 'r', 1) @@ -815,9 +818,6 @@ def run_stage(name, runner_, **args): def run(**args): - # measure runtime - start = time.time() - # query runner for tests runner_ = runner(**args) print('using runner: %s' @@ -839,6 +839,9 @@ def run(**args): if args.get('trace'): trace = openio(args['trace'], 'w', 1) + # measure runtime + start = time.time() + # spawn runners expected = 0 passed = 0 @@ -863,6 +866,8 @@ def run(**args): if (failures and not args.get('keep_going')) or killed: break + stop = time.time() + if output: output.close() if trace: @@ -878,7 +883,7 @@ def run(**args): '%d/%d passed' % (passed, expected), '%d/%d failed' % (len(failures), expected), '%dpls!' % powerlosses if powerlosses else None, - 'in %.2fs' % (time.time()-start)])))) + 'in %.2fs' % (stop-start)])))) print() # print each failure @@ -1023,11 +1028,11 @@ if __name__ == "__main__": help="Direct trace output to this file.") test_parser.add_argument('-o', '--output', help="Direct stdout and stderr to this file.") - test_parser.add_argument('--read-delay', + test_parser.add_argument('--read-sleep', help="Artificial read delay in seconds.") - test_parser.add_argument('--prog-delay', + test_parser.add_argument('--prog-sleep', help="Artificial prog delay in seconds.") - test_parser.add_argument('--erase-delay', + test_parser.add_argument('--erase-sleep', help="Artificial erase delay in seconds.") test_parser.add_argument('--runner', default=[RUNNER_PATH], type=lambda x: x.split(), diff --git a/scripts/tracebd.py b/scripts/tracebd.py new file mode 100755 index 00000000..3638fef9 --- /dev/null +++ b/scripts/tracebd.py @@ -0,0 +1,777 @@ +#!/usr/bin/env python3 +# +# Display operations on block devices based on trace output +# + +import collections as co +import itertools as it +import math as m +import re +import shutil +import threading as th +import time + + +def openio(path, mode='r'): + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) + +# space filling Hilbert-curve +def hilbert_curve(width, height): + # memoize the last curve + if getattr(hilbert_curve, 'last', (None,))[0] == (width, height): + return hilbert_curve.last[1] + + # based on generalized Hilbert curves: + # https://github.com/jakubcerveny/gilbert + # + def hilbert_(x, y, a_x, a_y, b_x, b_y): + w = abs(a_x+a_y) + h = abs(b_x+b_y) + a_dx = -1 if a_x < 0 else +1 if a_x > 0 else 0 + a_dy = -1 if a_y < 0 else +1 if a_y > 0 else 0 + b_dx = -1 if b_x < 0 else +1 if b_x > 0 else 0 + b_dy = -1 if b_y < 0 else +1 if b_y > 0 else 0 + + # trivial row + if h == 1: + for _ in range(w): + yield (x,y) + x, y = x+a_dx, y+a_dy + return + + # trivial column + if w == 1: + for _ in range(h): + yield (x,y) + x, y = x+b_dx, y+b_dy + return + + a_x_, a_y_ = a_x//2, a_y//2 + b_x_, b_y_ = b_x//2, b_y//2 + w_ = abs(a_x_+a_y_) + h_ = abs(b_x_+b_y_) + + if 2*w > 3*h: + # prefer even steps + if w_ % 2 != 0 and w > 2: + a_x_, a_y_ = a_x_+a_dx, a_y_+a_dy + + # split in two + yield from hilbert_(x, y, a_x_, a_y_, b_x, b_y) + yield from hilbert_(x+a_x_, y+a_y_, a_x-a_x_, a_y-a_y_, b_x, b_y) + else: + # prefer even steps + if h_ % 2 != 0 and h > 2: + b_x_, b_y_ = b_x_+b_dx, b_y_+b_dy + + # split in three + yield from hilbert_(x, y, b_x_, b_y_, a_x_, a_y_) + yield from hilbert_(x+b_x_, y+b_y_, a_x, a_y, b_x-b_x_, b_y-b_y_) + yield from hilbert_( + x+(a_x-a_dx)+(b_x_-b_dx), y+(a_y-a_dy)+(b_y_-b_dy), + -b_x_, -b_y_, -(a_x-a_x_), -(a_y-a_y_)) + + if width >= height: + curve = hilbert_(0, 0, +width, 0, 0, +height) + else: + curve = hilbert_(0, 0, 0, +height, +width, 0) + + curve = list(curve) + hilbert_curve.last = ((width, height), curve) + return curve + +# space filling Z-curve/Lebesgue-curve +def lebesgue_curve(width, height): + # memoize the last curve + if getattr(lebesgue_curve, 'last', (None,))[0] == (width, height): + return lebesgue_curve.last[1] + + # we create a truncated Z-curve by simply filtering out the points + # that are outside our region + curve = [] + for i in range(2**(2*m.ceil(m.log2(max(width, height))))): + # we just operate on binary strings here because it's easier + b = '{:0{}b}'.format(i, 2*m.ceil(m.log2(i+1)/2)) + x = int(b[1::2], 2) if b[1::2] else 0 + y = int(b[0::2], 2) if b[0::2] else 0 + if x < width and y < height: + curve.append((x, y)) + + lebesgue_curve.last = ((width, height), curve) + return curve + + +class Block: + def __init__(self, wear=0, readed=False, proged=False, erased=False): + self._ = ((wear << 3) + | (1 if readed else 0) + | (2 if proged else 0) + | (4 if erased else False)) + + @property + def wear(self): + return self._ >> 3 + + @property + def readed(self): + return (self._ & 1) != 0 + + @property + def proged(self): + return (self._ & 2) != 0 + + @property + def erased(self): + return (self._ & 4) != 0 + + def read(self): + self._ |= 1 + + def prog(self): + self._ |= 2 + + def erase(self): + self._ = (self._ | 4) + 8 + + def clear(self): + self._ &= ~7 + + def reset(self): + self._ = 0 + + def copy(self): + return Block(self.wear, self.readed, self.proged, self.erased) + + def __add__(self, other): + return Block( + max(self.wear, other.wear), + self.readed | other.readed, + self.proged | other.proged, + self.erased | other.erased) + + def draw(self, + ascii=False, + chars=None, + wear_chars=None, + color='always', + read=True, + prog=True, + erase=True, + wear=False, + max_wear=None, + block_cycles=None): + if not chars: chars = '.rpe' + c = chars[0] + f = [] + + if wear: + if not wear_chars and ascii: wear_chars = '0123456789' + elif not wear_chars: wear_chars = '.₁₂₃₄₅₆789' + + if block_cycles: + w = self.wear / block_cycles + else: + w = self.wear / max(max_wear, len(wear_chars)-1) + + c = wear_chars[min( + int(w*(len(wear_chars)-1)), + len(wear_chars)-1)] + if color == 'wear' or ( + color == 'always' and not read and not prog and not erase): + if w*9 >= 9: f.append('\x1b[1;31m') + elif w*9 >= 7: f.append('\x1b[35m') + + if erase and self.erased: c = chars[3] + elif prog and self.proged: c = chars[2] + elif read and self.readed: c = chars[1] + + if color == 'ops' or color == 'always': + if erase and self.erased: f.append('\x1b[44m') + elif prog and self.proged: f.append('\x1b[45m') + elif read and self.readed: f.append('\x1b[42m') + + if color in ['always', 'wear', 'ops'] and f: + return '%s%c\x1b[m' % (''.join(f), c) + else: + return c + +class Bd: + def __init__(self, *, blocks=None, size=1, count=1, width=80): + if blocks is not None: + self.blocks = blocks + self.size = size + self.count = count + self.width = width + else: + self.blocks = [] + self.size = None + self.count = None + self.width = None + self.smoosh(size=size, count=count, width=width) + + def get(self, block=slice(None), off=slice(None)): + if not isinstance(block, slice): + block = slice(block, block+1) + if not isinstance(off, slice): + off = slice(off, off+1) + + if (not self.blocks + or not self.width + or not self.size + or not self.count): + return + + if self.count >= self.width: + scale = (self.count+self.width-1) // self.width + for i in range( + (block.start if block.start is not None else 0)//scale, + (min(block.stop if block.stop is not None else self.count, + self.count)+scale-1)//scale): + yield self.blocks[i] + else: + scale = self.width // self.count + for i in range( + block.start if block.start is not None else 0, + min(block.stop if block.stop is not None else self.count, + self.count)): + for j in range( + ((off.start if off.start is not None else 0) + *scale)//self.size, + (min(off.stop if off.stop is not None else self.size, + self.size)*scale+self.size-1)//self.size): + yield self.blocks[i*scale+j] + + def __getitem__(self, block=slice(None), off=slice(None)): + if isinstance(block, tuple): + block, off = block + if not isinstance(block, slice): + block = slice(block, block+1) + if not isinstance(off, slice): + off = slice(off, off+1) + + # needs resize? + if ((block.stop is not None and block.stop > self.count) + or (off.stop is not None and off.stop > self.size)): + self.smoosh( + count=max(block.stop or self.count, self.count), + size=max(off.stop or self.size, self.size)) + + return self.get(block, off) + + def smoosh(self, *, size=None, count=None, width=None): + size = size or self.size + count = count or self.count + width = width or self.width + + if count >= width: + scale = (count+width-1) // width + self.blocks = [ + sum(self.get(slice(i,i+scale)), start=Block()) + for i in range(0, count, scale)] + else: + scale = width // count + self.blocks = [ + sum(self.get(i, slice(j*(size//width),(j+1)*(size//width))), + start=Block()) + for i in range(0, count) + for j in range(scale)] + + self.size = size + self.count = count + self.width = width + + def read(self, block=slice(None), off=slice(None)): + for c in self[block, off]: + c.read() + + def prog(self, block=slice(None), off=slice(None)): + for c in self[block, off]: + c.prog() + + def erase(self, block=slice(None), off=slice(None)): + for c in self[block, off]: + c.erase() + + def clear(self, block=slice(None), off=slice(None)): + for c in self[block, off]: + c.clear() + + def reset(self, block=slice(None), off=slice(None)): + for c in self[block, off]: + c.reset() + + def copy(self): + return Bd( + blocks=[b.copy() for b in self.blocks], + size=self.size, count=self.count, width=self.width) + + +def main(path='-', *, + read=False, + prog=False, + erase=False, + wear=False, + reset=False, + ascii=False, + chars=None, + wear_chars=None, + color='auto', + block=None, + start=None, + stop=None, + start_off=None, + stop_off=None, + block_size=None, + block_count=None, + block_cycles=None, + width=None, + height=1, + scale=None, + lines=None, + coalesce=None, + sleep=None, + hilbert=False, + lebesgue=False, + keep_open=False): + if not read and not prog and not erase and not wear: + read = True + prog = True + erase = True + if color == 'auto': + color = 'always' if sys.stdout.isatty() else 'never' + + start = (start if start is not None + else block if block is not None + else 0) + stop = (stop if stop is not None + else block+1 if block is not None + else block_count if block_count is not None + else None) + start_off = (start_off if start_off is not None + else 0) + stop_off = (stop_off if stop_off is not None + else block_size if block_size is not None + else None) + + bd = Bd( + size=(block_size if block_size is not None + else stop_off-start_off if stop_off is not None + else 1), + count=(block_count if block_count is not None + else stop-start if stop is not None + else 1), + width=(width or 80)*height) + lock = th.Lock() + event = th.Event() + done = False + + # adjust width? + def resmoosh(): + if width is None: + w = shutil.get_terminal_size((80, 0))[0] * height + elif width == 0: + w = max(int(bd.count*(scale or 1)), 1) + else: + w = width * height + + if scale and int(bd.count*scale) > w: + c = int(w/scale) + elif scale and int(bd.count*scale) < w: + w = max(int(bd.count*(scale or 1)), 1) + c = bd.count + else: + c = bd.count + + if w != bd.width or c != bd.count: + bd.smoosh(width=w, count=c) + resmoosh() + + # parse a line of trace output + pattern = re.compile( + 'trace.*?bd_(?:' + '(?Pcreate\w*)\(' + '(?:' + 'block_size=(?P\w+)' + '|' 'block_count=(?P\w+)' + '|' '.*?' ')*' '\)' + '|' '(?Pread)\(' + '\s*(?P\w+)\s*' ',' + '\s*(?P\w+)\s*' ',' + '\s*(?P\w+)\s*' ',' + '\s*(?P\w+)\s*' ',' + '\s*(?P\w+)\s*' '\)' + '|' '(?Pprog)\(' + '\s*(?P\w+)\s*' ',' + '\s*(?P\w+)\s*' ',' + '\s*(?P\w+)\s*' ',' + '\s*(?P\w+)\s*' ',' + '\s*(?P\w+)\s*' '\)' + '|' '(?Perase)\(' + '\s*(?P\w+)\s*' ',' + '\s*(?P\w+)\s*' '\)' + '|' '(?Psync)\(' + '\s*(?P\w+)\s*' '\)' ')') + def parse_line(line): + # string searching is actually much faster than + # the regex here + if 'trace' not in line or 'bd' not in line: + return False + m = pattern.search(line) + if not m: + return False + + if m.group('create'): + # update our block size/count + size = int(m.group('block_size'), 0) + count = int(m.group('block_count'), 0) + + if stop_off is not None: + size = stop_off-start_off + if stop is not None: + count = stop-start + + with lock: + if reset: + bd.reset() + + # ignore the new values is stop/stop_off is explicit + bd.smoosh( + size=(size if stop_off is None + else stop_off-start_off), + count=(count if stop is None + else stop-start)) + return True + + elif m.group('read') and read: + block = int(m.group('read_block'), 0) + off = int(m.group('read_off'), 0) + size = int(m.group('read_size'), 0) + + if stop is not None and block >= stop: + return False + block -= start + if stop_off is not None: + if off >= stop_off: + return False + size = min(size, stop_off-off) + off -= start_off + + with lock: + bd.read(block, slice(off,off+size)) + return True + + elif m.group('prog') and prog: + block = int(m.group('prog_block'), 0) + off = int(m.group('prog_off'), 0) + size = int(m.group('prog_size'), 0) + + if stop is not None and block >= stop: + return False + block -= start + if stop_off is not None: + if off >= stop_off: + return False + size = min(size, stop_off-off) + off -= start_off + + with lock: + bd.prog(block, slice(off,off+size)) + return True + + elif m.group('erase') and (erase or wear): + block = int(m.group('erase_block'), 0) + + if stop is not None and block >= stop: + return False + block -= start + + with lock: + bd.erase(block) + return True + + else: + return False + + + # print a pretty line of trace output + history = [] + def push_line(): + # create copy to avoid corrupt output + with lock: + resmoosh() + bd_ = bd.copy() + bd.clear() + + max_wear = None + if wear: + max_wear = max(b.wear for b in bd_.blocks) + + def draw(b): + return b.draw( + ascii=ascii, + chars=chars, + wear_chars=wear_chars, + color=color, + read=read, + prog=prog, + erase=erase, + wear=wear, + max_wear=max_wear, + block_cycles=block_cycles) + + # fold via a curve? + if height > 1: + w = (len(bd.blocks)+height-1) // height + if hilbert: + grid = {} + for (x,y),b in zip(hilbert_curve(w, height), bd_.blocks): + grid[(x,y)] = draw(b) + line = [ + ''.join(grid.get((x,y), ' ') for x in range(w)) + for y in range(height)] + elif lebesgue: + grid = {} + for (x,y),b in zip(lebesgue_curve(w, height), bd_.blocks): + grid[(x,y)] = draw(b) + line = [ + ''.join(grid.get((x,y), ' ') for x in range(w)) + for y in range(height)] + else: + line = [ + ''.join(draw(b) for b in bd_.blocks[y*w:y*w+w]) + for y in range(height)] + else: + line = [''.join(draw(b) for b in bd_.blocks)] + + if not lines: + # just go ahead and print here + for row in line: + sys.stdout.write(row) + sys.stdout.write('\n') + sys.stdout.flush() + else: + history.append(line) + del history[:-lines] + + last_rows = 1 + def print_line(): + nonlocal last_rows + if not lines: + return + + # give ourself a canvas + while last_rows < len(history)*height: + sys.stdout.write('\n') + last_rows += 1 + + for i, row in enumerate(it.chain.from_iterable(history)): + jump = len(history)*height-1-i + # move cursor, clear line, disable/reenable line wrapping + sys.stdout.write('\r') + if jump > 0: + sys.stdout.write('\x1b[%dA' % jump) + sys.stdout.write('\x1b[K') + sys.stdout.write('\x1b[?7l') + sys.stdout.write(row) + sys.stdout.write('\x1b[?7h') + if jump > 0: + sys.stdout.write('\x1b[%dB' % jump) + + + if sleep is None or (coalesce and not lines): + # read/parse coalesce number of operations + try: + while True: + with openio(path) as f: + changes = 0 + for line in f: + change = parse_line(line) + changes += change + if change and changes % (coalesce or 1) == 0: + push_line() + print_line() + # sleep between coalesced lines? + if sleep is not None: + time.sleep(sleep) + if not keep_open: + break + except KeyboardInterrupt: + pass + else: + # read/parse in a background thread + def parse(): + nonlocal done + while True: + with openio(path) as f: + changes = 0 + for line in f: + change = parse_line(line) + changes += change + if change and changes % (coalesce or 1) == 0: + if coalesce: + push_line() + event.set() + if not keep_open: + break + done = True + + th.Thread(target=parse, daemon=True).start() + + try: + while not done: + time.sleep(sleep) + event.wait() + event.clear() + if not coalesce: + push_line() + print_line() + except KeyboardInterrupt: + pass + + if lines: + sys.stdout.write('\n') + + +if __name__ == "__main__": + import sys + import argparse + parser = argparse.ArgumentParser( + description="Display operations on block devices based on " + "trace output.") + parser.add_argument( + 'path', + nargs='?', + help="Path to read from.") + parser.add_argument( + '-r', + '--read', + action='store_true', + help="Render reads.") + parser.add_argument( + '-p', + '--prog', + action='store_true', + help="Render progs.") + parser.add_argument( + '-e', + '--erase', + action='store_true', + help="Render erases.") + parser.add_argument( + '-w', + '--wear', + action='store_true', + help="Render wear.") + parser.add_argument( + '-R', + '--reset', + action='store_true', + help="Reset wear on block device initialization.") + parser.add_argument( + '-A', + '--ascii', + action='store_true', + help="Don't use unicode characters.") + parser.add_argument( + '--chars', + help="Characters to use for noop, read, prog, erase operations.") + parser.add_argument( + '--wear-chars', + help="Characters to use to show wear.") + parser.add_argument( + '--color', + choices=['never', 'always', 'auto', 'ops', 'wear'], + help="When to use terminal colors, defaults to auto.") + parser.add_argument( + '-b', + '--block', + type=lambda x: int(x, 0), + help="Show a specific block.") + parser.add_argument( + '--start', + type=lambda x: int(x, 0), + help="Start at this block.") + parser.add_argument( + '--stop', + type=lambda x: int(x, 0), + help="Stop before this block.") + parser.add_argument( + '--start-off', + type=lambda x: int(x, 0), + help="Start at this offset.") + parser.add_argument( + '--stop-off', + type=lambda x: int(x, 0), + help="Stop before this offset.") + parser.add_argument( + '-B', + '--block-size', + type=lambda x: int(x, 0), + help="Assume a specific block size.") + parser.add_argument( + '--block-count', + type=lambda x: int(x, 0), + help="Assume a specific block count.") + parser.add_argument( + '-C', + '--block-cycles', + type=lambda x: int(x, 0), + help="Assumed maximum number of erase cycles when measuring wear.") + parser.add_argument( + '-W', + '--width', + type=lambda x: int(x, 0), + help="Width in columns. A width of 0 indicates no limit. Defaults " + "to terminal width or 80.") + parser.add_argument( + '-H', + '--height', + type=lambda x: int(x, 0), + help="Height in rows. Defaults to 1.") + parser.add_argument( + '-x', + '--scale', + type=float, + help="Number of characters per block, ignores --width if set.") + parser.add_argument( + '-n', + '--lines', + type=lambda x: int(x, 0), + help="Number of lines to show, with 0 indicating no limit. " + "Defaults to 0.") + parser.add_argument( + '-c', + '--coalesce', + type=lambda x: int(x, 0), + help="Number of operations to coalesce together. Defaults to 1.") + parser.add_argument( + '-s', + '--sleep', + type=float, + help="Time in seconds to sleep between reads, while coalescing " + "operations.") + parser.add_argument( + '-I', + '--hilbert', + action='store_true', + help="Render as a space-filling Hilbert curve.") + parser.add_argument( + '-Z', + '--lebesgue', + action='store_true', + help="Render as a space-filling Z-curve.") + parser.add_argument( + '-k', + '--keep-open', + action='store_true', + help="Reopen the pipe on EOF, useful when multiple " + "processes are writing.") + sys.exit(main(**{k: v + for k, v in vars(parser.parse_args()).items() + if v is not None})) From a208d848e5b93d8e8f917ff5d6d2b9cf91303597 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 5 Sep 2022 22:53:12 -0500 Subject: [PATCH 31/81] Reworked test defines a bit to use one common array layout Previously didn't think this would work without making test.py aware of the number of implicit defines, which risks being incredibly fragile. Fortunately it turns out we can defer the actual array size calculation until the C preprocessor. This simplifies a few things. Also a bitmap-based caching layer for the defines. Since the test defines have been upgraded to callbacks recursive defines risk spending a decent amount of time evaluating on every lookup. Some quick testing shows 408015154 hits to 46160 misses so that's a good sign. Also changed the geometries to be their own leb16-encoded part of the test identifier. This means any geometry can be captured and reproduced with just the test identifier. Here are the current test geometries: ./runners/test_runner --list-geometries geometry read prog erase count size leb16 d,default 16 16 512 2048 1048576 g1gg2 e,eeprom 1 1 512 2048 1048576 1gg2 E,emmc 512 512 512 2048 1048576 gg2 n,nor 1 1 4096 256 1048576 1ggg1 N,nand 4096 4096 32768 32 1048576 ggg1ggg8 --- runners/test_runner.c | 1416 +++++++++++++++++++++++++++-------------- runners/test_runner.h | 88 ++- scripts/test.py | 157 +++-- 3 files changed, 1075 insertions(+), 586 deletions(-) diff --git a/runners/test_runner.c b/runners/test_runner.c index eea4b5f3..e505e04e 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -16,197 +16,89 @@ #include -// test suites in a custom ld section -extern struct test_suite __start__test_suites; -extern struct test_suite __stop__test_suites; +// some helpers -const struct test_suite *test_suites = &__start__test_suites; -#define TEST_SUITE_COUNT \ - ((size_t)(&__stop__test_suites - &__start__test_suites)) +// append to an array with amortized doubling +void *mappend(void **p, + size_t size, + size_t *count, + size_t *capacity) { + uint8_t *p_ = *p; + size_t count_ = *count; + size_t capacity_ = *capacity; -// test geometries -struct test_geometry { - const char *name; - intmax_t defines[TEST_GEOMETRY_DEFINE_COUNT]; -}; + count_ += 1; + if (count_ > capacity_) { + capacity_ = (2*capacity_ < 4) ? 4 : 2*capacity_; -const struct test_geometry test_geometries[TEST_GEOMETRY_COUNT] - = TEST_GEOMETRIES; - -// test define lookup and management -const intmax_t *test_override_defines; -intmax_t (*const *test_case_defines)(void); -const intmax_t *test_geometry_defines; -const intmax_t test_default_defines[TEST_PREDEFINE_COUNT] - = TEST_DEFAULTS; - -uint8_t test_override_predefine_map[TEST_PREDEFINE_COUNT]; -uint8_t test_override_define_map[256]; -uint8_t test_case_predefine_map[TEST_PREDEFINE_COUNT]; - -const char *const *test_override_names; -size_t test_override_count; - -const char *const test_predefine_names[TEST_PREDEFINE_COUNT] - = TEST_PREDEFINE_NAMES; - -const char *const *test_define_names; -size_t test_define_count; - - -intmax_t test_predefine(size_t define) { - if (test_override_defines - && test_override_predefine_map[define] != 0xff) { - return test_override_defines[test_override_predefine_map[define]]; - } else if (test_case_defines - && test_case_predefine_map[define] != 0xff - && test_case_defines[test_case_predefine_map[define]]) { - return test_case_defines[test_case_predefine_map[define]](); - } else if (define < TEST_GEOMETRY_DEFINE_COUNT) { - return test_geometry_defines[define]; - } else { - return test_default_defines[define-TEST_GEOMETRY_DEFINE_COUNT]; - } -} - -intmax_t test_define(size_t define) { - if (test_override_defines - && test_override_define_map[define] != 0xff) { - return test_override_defines[test_override_define_map[define]]; - } else if (test_case_defines - && test_case_defines[define]) { - return test_case_defines[define](); - } - - fprintf(stderr, "error: undefined define %s\n", - test_define_names[define]); - assert(false); - exit(-1); -} - -static void define_geometry(const struct test_geometry *geometry) { - test_geometry_defines = geometry->defines; -} - -static void test_define_overrides( - const char *const *override_names, - const intmax_t *override_defines, - size_t override_count) { - test_override_defines = override_defines; - test_override_names = override_names; - test_override_count = override_count; - - // map any override predefines - memset(test_override_predefine_map, 0xff, TEST_PREDEFINE_COUNT); - for (size_t i = 0; i < test_override_count; i++) { - for (size_t j = 0; j < TEST_PREDEFINE_COUNT; j++) { - if (strcmp(test_override_names[i], test_predefine_names[j]) == 0) { - test_override_predefine_map[j] = i; - } - } - } -} - -static void define_suite(const struct test_suite *suite) { - test_define_names = suite->define_names; - test_define_count = suite->define_count; - - // map any override defines - memset(test_override_define_map, 0xff, suite->define_count); - for (size_t i = 0; i < test_override_count; i++) { - for (size_t j = 0; j < suite->define_count; j++) { - if (strcmp(test_override_names[i], suite->define_names[j]) == 0) { - test_override_define_map[j] = i; - } + p_ = realloc(p_, capacity_*size); + if (!p_) { + return NULL; } } - // map any suite/case predefines - memset(test_case_predefine_map, 0xff, TEST_PREDEFINE_COUNT); - for (size_t i = 0; i < suite->define_count; i++) { - for (size_t j = 0; j < TEST_PREDEFINE_COUNT; j++) { - if (strcmp(suite->define_names[i], test_predefine_names[j]) == 0) { - test_case_predefine_map[j] = i; - } + *p = p_; + *count = count_; + *capacity = capacity_; + return &p_[(count_-1)*size]; +} + +// a quick self-terminating text-safe varint scheme +static void leb16_print(uintmax_t x) { + while (true) { + lfs_testbd_powercycles_t nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0); + printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10); + if (x <= 0xf) { + break; } + x >>= 4; } } -static void define_perm( - const struct test_suite *suite, - const struct test_case *case_, - size_t perm) { - (void)suite; - if (case_->defines) { - test_case_defines = case_->defines[perm]; - } else { - test_case_defines = NULL; - } -} - - -// a quick encoding scheme for sequences of power-loss -static void leb16_print( - const lfs_testbd_powercycles_t *cycles, - size_t cycle_count) { - for (size_t i = 0; i < cycle_count; i++) { - lfs_testbd_powercycles_t x = cycles[i]; - while (true) { - lfs_testbd_powercycles_t nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0); - printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10); - if (x <= 0xf) { - break; - } - x >>= 4; - } - } -} - -static size_t leb16_parse(const char *s, char **tail, - lfs_testbd_powercycles_t **cycles) { - // first lets count how many number we're dealing with - size_t count = 0; - size_t len = 0; - for (size_t i = 0;; i++) { - if ((s[i] >= '0' && s[i] <= '9') - || (s[i] >= 'a' && s[i] <= 'f')) { - len = i+1; - count += 1; - } else if ((s[i] >= 'g' && s[i] <= 'v')) { - // do nothing +static uintmax_t leb16_parse(const char *s, char **tail) { + uintmax_t x = 0; + size_t i = 0; + while (true) { + uintmax_t nibble = s[i]; + if (nibble >= '0' && nibble <= '9') { + nibble = nibble - '0'; + } else if (nibble >= 'a' && nibble <= 'v') { + nibble = nibble - 'a' + 10; } else { + // invalid? + if (tail) { + *tail = (char*)s; + } + return 0; + } + + x |= (nibble & 0xf) << (4*i); + i += 1; + if (!(nibble & 0x10)) { break; } } - // then parse - lfs_testbd_powercycles_t *cycles_ = malloc( - count * sizeof(lfs_testbd_powercycles_t)); - size_t i = 0; - lfs_testbd_powercycles_t x = 0; - size_t k = 0; - for (size_t j = 0; j < len; j++) { - lfs_testbd_powercycles_t nibble = s[j]; - nibble = (nibble < 'a') ? nibble-'0' : nibble-'a'+10; - x |= (nibble & 0xf) << (4*k); - k += 1; - if (!(nibble & 0x10)) { - cycles_[i] = x; - i += 1; - x = 0; - k = 0; - } - } - if (tail) { - *tail = (char*)s + len; + *tail = (char*)s + i; } - *cycles = cycles_; - return count; + return x; } -// test state + +// test_runner types + +typedef struct test_geometry { + char short_name; + const char *long_name; + + lfs_size_t read_size; + lfs_size_t prog_size; + lfs_size_t block_size; + lfs_size_t block_count; +} test_geometry_t; + typedef struct test_powerloss { char short_name; const char *long_name; @@ -221,34 +113,270 @@ typedef struct test_powerloss { size_t cycle_count; } test_powerloss_t; -static void run_powerloss_none( - const struct test_suite *suite, - const struct test_case *case_, - size_t perm, - const lfs_testbd_powercycles_t *cycles, - size_t cycle_count); -const test_powerloss_t *test_powerlosses = (const test_powerloss_t[]){ - {'0', "none", run_powerloss_none, NULL, 0}, -}; -size_t test_powerloss_count = 1; - - typedef struct test_id { const char *suite; const char *case_; size_t perm; + const test_geometry_t *geometry; const lfs_testbd_powercycles_t *cycles; size_t cycle_count; } test_id_t; +static void print_id( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm, + const lfs_testbd_powercycles_t *cycles, + size_t cycle_count) { + (void)suite; + // suite[#case[#perm[#geometry[#powercycles]]]] + printf("%s#%zu#", case_->id, perm); + + // reduce duplication in geometry, this is appended to every test + if (READ_SIZE != BLOCK_SIZE || PROG_SIZE != BLOCK_SIZE) { + if (READ_SIZE != PROG_SIZE) { + leb16_print(READ_SIZE); + } + leb16_print(PROG_SIZE); + } + leb16_print(BLOCK_SIZE); + if (BLOCK_COUNT*BLOCK_SIZE != 1024*1024) { + leb16_print(BLOCK_COUNT); + } + + // only print power-cycles if any occured + if (cycles) { + printf("#"); + for (size_t i = 0; i < cycle_count; i++) { + leb16_print(cycles[i]); + } + } +} + + +// test suites are linked into a custom ld section +extern struct test_suite __start__test_suites; +extern struct test_suite __stop__test_suites; + +const struct test_suite *test_suites = &__start__test_suites; +#define TEST_SUITE_COUNT \ + ((size_t)(&__stop__test_suites - &__start__test_suites)) + + +// test define management +typedef struct test_define_map { + intmax_t (*const *defines)(size_t); + const char *const *names; + size_t count; +} test_define_map_t; + +extern const test_geometry_t *test_geometry; + +#define TEST_DEFINE(k, v) \ + intmax_t test_define_##k(__attribute__((unused)) size_t define) { \ + return v; \ + } + + TEST_IMPLICIT_DEFINES +#undef TEST_DEFINE + +#define TEST_DEFINE_MAP_COUNT 3 +test_define_map_t test_define_maps[TEST_DEFINE_MAP_COUNT] = { + {NULL, NULL, 0}, + {NULL, NULL, 0}, + { + (intmax_t (*const[TEST_IMPLICIT_DEFINE_COUNT])(size_t)){ + #define TEST_DEFINE(k, v) \ + [k##_i] = test_define_##k, + + TEST_IMPLICIT_DEFINES + #undef TEST_DEFINE + }, + (const char *const[TEST_IMPLICIT_DEFINE_COUNT]){ + #define TEST_DEFINE(k, v) \ + [k##_i] = #k, + + TEST_IMPLICIT_DEFINES + #undef TEST_DEFINE + }, + TEST_IMPLICIT_DEFINE_COUNT, + }, +}; + +intmax_t *test_define_cache; +size_t test_define_cache_count; +unsigned *test_define_cache_mask; + +const char *test_define_name(size_t define) { + // lookup in our test defines + for (size_t i = 0; i < TEST_DEFINE_MAP_COUNT; i++) { + if (define < test_define_maps[i].count + && test_define_maps[i].names + && test_define_maps[i].names[define]) { + return test_define_maps[i].names[define]; + } + } + + return NULL; +} + +intmax_t test_define(size_t define) { + // is the define in our cache? + if (define < test_define_cache_count + && (test_define_cache_mask[define/(8*sizeof(unsigned))] + & (1 << (define%(8*sizeof(unsigned)))))) { + return test_define_cache[define]; + } + + // lookup in our test defines + for (size_t i = 0; i < TEST_DEFINE_MAP_COUNT; i++) { + if (define < test_define_maps[i].count + && test_define_maps[i].defines[define]) { + intmax_t v = test_define_maps[i].defines[define](define); + + // insert into cache! + test_define_cache[define] = v; + test_define_cache_mask[define / (8*sizeof(unsigned))] + |= 1 << (define%(8*sizeof(unsigned))); + + return v; + } + } + + // not found? + const char *name = test_define_name(define); + fprintf(stderr, "error: undefined define %s (%zd)\n", + name ? name : "(unknown)", + define); + assert(false); + exit(-1); +} + +void test_define_flush(void) { + // clear cache between permutations + memset(test_define_cache_mask, 0, + sizeof(unsigned)*( + (test_define_cache_count+(8*sizeof(unsigned))-1) + / (8*sizeof(unsigned)))); +} + +// geometry updates +const test_geometry_t *test_geometry = NULL; + +void test_define_geometry(const test_geometry_t *geometry) { + test_geometry = geometry; +} + +// override updates +typedef struct test_override { + const char *name; + intmax_t define; +} test_override_t; + +const test_override_t *test_overrides = NULL; +size_t test_override_count = 0; +intmax_t *test_override_map = NULL; + +intmax_t test_define_override(size_t define) { + return test_override_map[define]; +} + +void test_define_overrides( + const test_override_t *overrides, + size_t override_count) { + test_overrides = overrides; + test_override_count = override_count; +} + +// suite/perm updates +void test_define_suite(const struct test_suite *suite) { + test_define_maps[1].names = suite->define_names; + test_define_maps[1].count = suite->define_count; + + // make sure our cache is large enough + if (lfs_max(suite->define_count, TEST_IMPLICIT_DEFINE_COUNT) + > test_define_cache_count) { + // align to power of two to avoid any superlinear growth + size_t ncount = 1 << lfs_npw2( + lfs_max(suite->define_count, TEST_IMPLICIT_DEFINE_COUNT)); + test_define_cache = realloc(test_define_cache, ncount*sizeof(intmax_t)); + test_define_cache_mask = realloc(test_define_cache_mask, + sizeof(unsigned)*( + (ncount+(8*sizeof(unsigned))-1) + / (8*sizeof(unsigned)))); + test_define_cache_count = ncount; + } + + // map any overrides + if (test_override_count > 0) { + // make sure our override arrays are big enough + if (suite->define_count > test_define_maps[0].count) { + // align to power of two to avoid any superlinear growth + size_t ncount = 1 << lfs_npw2(suite->define_count); + test_define_maps[0].defines = realloc( + (intmax_t (**)(size_t))test_define_maps[0].defines, + ncount*sizeof(intmax_t (*)(size_t))); + test_override_map = realloc( + test_override_map, + ncount*sizeof(intmax_t)); + test_define_maps[0].count = ncount; + } + + for (size_t i = 0; i < test_define_maps[0].count; i++) { + ((intmax_t (**)(size_t))test_define_maps[0].defines)[i] = NULL; + + const char *name = test_define_name(i); + if (!name) { + continue; + } + + for (size_t j = 0; j < test_override_count; j++) { + if (strcmp(name, test_overrides[j].name) == 0) { + test_override_map[i] = test_overrides[j].define; + ((intmax_t (**)(size_t))test_define_maps[0].defines)[i] + = test_define_override; + break; + } + } + } + } +} + +void test_define_perm( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm) { + if (case_->defines) { + test_define_maps[1].defines = case_->defines[perm]; + test_define_maps[1].count = suite->define_count; + } else { + test_define_maps[1].defines = NULL; + test_define_maps[1].count = 0; + } +} + +void test_define_cleanup(void) { + // test define management can allocate a few things + free(test_define_cache); + free(test_define_cache_mask); + free(test_override_map); + free((intmax_t (**)(size_t))test_define_maps[0].defines); +} + + + +// test state +extern const test_geometry_t *test_geometries; +extern size_t test_geometry_count; + +extern const test_powerloss_t *test_powerlosses; +extern size_t test_powerloss_count; + const test_id_t *test_ids = (const test_id_t[]) { - {NULL, NULL, -1, NULL, 0}, + {NULL, NULL, -1, NULL, NULL, 0}, }; size_t test_id_count = 1; - -const char *test_geometry = NULL; - size_t test_start = 0; size_t test_stop = -1; size_t test_step = 1; @@ -317,6 +445,7 @@ static void count_perms( const struct test_suite *suite, const struct test_case *case_, size_t perm, + const test_geometry_t *geometry, const lfs_testbd_powercycles_t *cycles, size_t cycle_count, size_t *perms, @@ -325,33 +454,29 @@ static void count_perms( size_t perms_ = 0; size_t filtered_ = 0; - for (size_t p = 0; p < (cycles ? 1 : test_powerloss_count); p++) { - if (!cycles - && test_powerlosses[p].short_name != '0' - && !(case_->flags & TEST_REENTRANT)) { + for (size_t k = 0; k < case_->permutations; k++) { + if (perm != (size_t)-1 && k != perm) { continue; } - size_t perm_ = 0; - for (size_t g = 0; g < TEST_GEOMETRY_COUNT; g++) { - if (test_geometry && strcmp( - test_geometries[g].name, test_geometry) != 0) { - continue; - } + // define permutation + test_define_perm(suite, case_, k); - for (size_t k = 0; k < case_->permutations; k++) { - perm_ += 1; + for (size_t g = 0; g < (geometry ? 1 : test_geometry_count); g++) { + // define geometry + test_define_geometry(geometry ? geometry : &test_geometries[g]); + test_define_flush(); - if (perm != (size_t)-1 && perm_ != perm) { + for (size_t p = 0; p < (cycles ? 1 : test_powerloss_count); p++) { + // skip non-reentrant tests when powerloss testing + if (!cycles + && test_powerlosses[p].short_name != '0' + && !(case_->flags & TEST_REENTRANT)) { continue; } perms_ += 1; - // setup defines - define_perm(suite, case_, k); - define_geometry(&test_geometries[g]); - if (case_->filter && !case_->filter()) { continue; } @@ -370,6 +495,7 @@ static void count_perms( static void summary(void) { printf("%-36s %7s %7s %7s %11s\n", "", "flags", "suites", "cases", "perms"); + size_t suites = 0; size_t cases = 0; test_flags_t flags = 0; size_t perms = 0; @@ -382,7 +508,7 @@ static void summary(void) { continue; } - define_suite(&test_suites[i]); + test_define_suite(&test_suites[i]); for (size_t j = 0; j < test_suites[i].case_count; j++) { if (test_ids[t].case_ && strcmp( @@ -390,14 +516,16 @@ static void summary(void) { continue; } + cases += 1; count_perms(&test_suites[i], &test_suites[i].cases[j], test_ids[t].perm, + test_ids[t].geometry, test_ids[t].cycles, test_ids[t].cycle_count, &perms, &filtered); } - cases += test_suites[i].case_count; + suites += 1; flags |= test_suites[i].flags; } } @@ -411,7 +539,7 @@ static void summary(void) { printf("%-36s %7s %7zu %7zu %11s\n", "TOTAL", flag_buf, - TEST_SUITE_COUNT, + suites, cases, perm_buf); } @@ -426,8 +554,9 @@ static void list_suites(void) { continue; } - define_suite(&test_suites[i]); + test_define_suite(&test_suites[i]); + size_t cases = 0; size_t perms = 0; size_t filtered = 0; @@ -437,8 +566,10 @@ static void list_suites(void) { continue; } + cases += 1; count_perms(&test_suites[i], &test_suites[i].cases[j], test_ids[t].perm, + test_ids[t].geometry, test_ids[t].cycles, test_ids[t].cycle_count, &perms, &filtered); @@ -453,7 +584,7 @@ static void list_suites(void) { printf("%-36s %7s %7zu %11s\n", test_suites[i].id, flag_buf, - test_suites[i].case_count, + cases, perm_buf); } } @@ -469,7 +600,7 @@ static void list_cases(void) { continue; } - define_suite(&test_suites[i]); + test_define_suite(&test_suites[i]); for (size_t j = 0; j < test_suites[i].case_count; j++) { if (test_ids[t].case_ && strcmp( @@ -482,6 +613,7 @@ static void list_cases(void) { count_perms(&test_suites[i], &test_suites[i].cases[j], test_ids[t].perm, + test_ids[t].geometry, test_ids[t].cycles, test_ids[t].cycle_count, &perms, &filtered); @@ -503,7 +635,26 @@ static void list_cases(void) { } } -static void list_paths(void) { +static void list_suite_paths(void) { + printf("%-36s %s\n", "suite", "path"); + + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + if (test_ids[t].suite && strcmp( + test_suites[i].name, test_ids[t].suite) != 0) { + continue; + } + + printf("%-36s %s\n", + test_suites[i].id, + test_suites[i].path); + } + } +} + +static void list_case_paths(void) { + printf("%-36s %s\n", "case", "path"); + for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { if (test_ids[t].suite && strcmp( @@ -517,7 +668,7 @@ static void list_paths(void) { continue; } - printf("%-36s %-36s\n", + printf("%-36s %s\n", test_suites[i].cases[j].id, test_suites[i].cases[j].path); } @@ -525,7 +676,101 @@ static void list_paths(void) { } } +struct list_define { + const char *name; + intmax_t *values; + size_t value_count; + size_t value_capacity; +}; + +static void list_defines_perms( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm, + const test_geometry_t *geometry, + struct list_define **defines, + size_t *define_count, + size_t *define_capacity) { + struct list_define *defines_ = *defines; + size_t define_count_ = *define_count; + size_t define_capacity_ = *define_capacity; + + for (size_t k = 0; k < case_->permutations; k++) { + if (perm != (size_t)-1 && k != perm) { + continue; + } + + // define permutation + test_define_perm(suite, case_, k); + + for (size_t g = 0; g < (geometry ? 1 : test_geometry_count); g++) { + // define geometry + test_define_geometry(geometry ? geometry : &test_geometries[g]); + test_define_flush(); + + // collect defines + for (size_t d = 0; + d < lfs_max(suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + if (!(d < TEST_IMPLICIT_DEFINE_COUNT || ( + case_->defines + && case_->defines[k] + && case_->defines[k][d]))) { + continue; + } + const char *name = test_define_name(d); + intmax_t value = test_define(d); + + // define already in defines? + for (size_t i = 0; i < define_count_; i++) { + if (strcmp(defines_[i].name, name) == 0) { + // value already in values? + for (size_t j = 0; j < defines_[i].value_count; j++) { + if (defines_[i].values[j] == value) { + goto next_define; + } + } + + *(intmax_t*)mappend( + (void**)&defines_[i].values, + sizeof(intmax_t), + &defines_[i].value_count, + &defines_[i].value_capacity) = value; + + goto next_define; + } + } + + { + // new define? + struct list_define *define = mappend( + (void**)&defines_, + sizeof(struct list_define), + &define_count_, + &define_capacity_); + define->name = name; + define->values = malloc(sizeof(intmax_t)); + define->values[0] = value; + define->value_count = 1; + define->value_capacity = 1; + } + + next_define:; + } + } + } + + *defines = defines_; + *define_count = define_count_; + *define_capacity = define_capacity_; +} + static void list_defines(void) { + struct list_define *defines = NULL; + size_t define_count = 0; + size_t define_capacity = 0; + for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { if (test_ids[t].suite && strcmp( @@ -533,7 +778,7 @@ static void list_defines(void) { continue; } - define_suite(&test_suites[i]); + test_define_suite(&test_suites[i]); for (size_t j = 0; j < test_suites[i].case_count; j++) { if (test_ids[t].case_ && strcmp( @@ -541,104 +786,123 @@ static void list_defines(void) { continue; } - for (size_t p = 0; - p < (test_ids[t].cycles ? 1 : test_powerloss_count); - p++) { - if (!test_ids[t].cycles - && test_powerlosses[p].short_name != '0' - && !(test_suites[i].cases[j].flags - & TEST_REENTRANT)) { - continue; - } - - size_t perm_ = 0; - for (size_t g = 0; g < TEST_GEOMETRY_COUNT; g++) { - if (test_geometry && strcmp( - test_geometries[g].name, test_geometry) != 0) { - continue; - } - - for (size_t k = 0; - k < test_suites[i].cases[j].permutations; - k++) { - perm_ += 1; - - if (test_ids[t].perm != (size_t)-1 - && perm_ != test_ids[t].perm) { - continue; - } - - // setup defines - define_perm(&test_suites[i], - &test_suites[i].cases[j], - k); - define_geometry(&test_geometries[g]); - - // print the case - char id_buf[256]; - sprintf(id_buf, "%s#%zu", - test_suites[i].cases[j].id, perm_); - printf("%-36s ", id_buf); - - // special case for the current geometry - printf("GEOMETRY=%s ", test_geometries[g].name); - - // print each define - for (size_t l = 0; - l < test_suites[i].define_count; - l++) { - if (test_suites[i].cases[j].defines - && test_suites[i].cases[j] - .defines[k][l]) { - printf("%s=%jd ", - test_suites[i].define_names[l], - test_define(l)); - } - } - printf("\n"); - } - } - } + list_defines_perms(&test_suites[i], &test_suites[i].cases[j], + test_ids[t].perm, + test_ids[t].geometry, + &defines, + &define_count, + &define_capacity); } } } -} -static void list_geometries(void) { - for (size_t i = 0; i < TEST_GEOMETRY_COUNT; i++) { - if (test_geometry && strcmp( - test_geometries[i].name, - test_geometry) != 0) { - continue; - } - - define_geometry(&test_geometries[i]); - - printf("%-36s ", test_geometries[i].name); - // print each define - for (size_t k = 0; k < TEST_GEOMETRY_DEFINE_COUNT; k++) { - printf("%s=%jd ", - test_predefine_names[k], - test_predefine(k)); + for (size_t i = 0; i < define_count; i++) { + printf("%s=", defines[i].name); + for (size_t j = 0; j < defines[i].value_count; j++) { + printf("%jd", defines[i].values[j]); + if (j != defines[i].value_count-1) { + printf(","); + } } printf("\n"); - } + + for (size_t i = 0; i < define_count; i++) { + free(defines[i].values); + } + free(defines); } -static void list_defaults(void) { - printf("%-36s ", "defaults"); - // print each define - for (size_t k = 0; k < TEST_DEFAULT_DEFINE_COUNT; k++) { - printf("%s=%jd ", - test_predefine_names[k+TEST_GEOMETRY_DEFINE_COUNT], - test_predefine(k+TEST_GEOMETRY_DEFINE_COUNT)); +static void list_implicit(void) { + struct list_define *defines = NULL; + size_t define_count = 0; + size_t define_capacity = 0; + + for (size_t t = 0; t < test_id_count; t++) { + // yes we do need to define a suite, this does a bit of bookeeping + // such as setting up the define cache + test_define_suite(&(const struct test_suite){0}); + list_defines_perms( + &(const struct test_suite){0}, + &(const struct test_case){.permutations=1}, + -1, + test_ids[t].geometry, + &defines, + &define_count, + &define_capacity); } - printf("\n"); + + for (size_t i = 0; i < define_count; i++) { + printf("%s=", defines[i].name); + for (size_t j = 0; j < defines[i].value_count; j++) { + printf("%jd", defines[i].values[j]); + if (j != defines[i].value_count-1) { + printf(","); + } + } + printf("\n"); + } + + for (size_t i = 0; i < define_count; i++) { + free(defines[i].values); + } + free(defines); } +// geometries to test + +const test_geometry_t builtin_geometries[] = { + {'d', "default", 16, 16, 512, (1024*1024)/512}, + {'e', "eeprom", 1, 1, 512, (1024*1024)/512}, + {'E', "emmc", 512, 512, 512, (1024*1024)/512}, + {'n', "nor", 1, 1, 4096, (1024*1024)/4096}, + {'N', "nand", 4096, 4096, 32768, (1024*1024)/(32*1024)}, + {0, NULL, 0, 0, 0, 0}, +}; + +const test_geometry_t *test_geometries = (const test_geometry_t[]){ + {'d', "default", 16, 16, 512, (1024*1024)/512}, + {'e', "eeprom", 1, 1, 512, (1024*1024)/512}, + {'E', "emmc", 512, 512, 512, (1024*1024)/512}, + {'n', "nor", 1, 1, 4096, (1024*1024)/4096}, + {'N', "nand", 4096, 4096, 32768, (1024*1024)/(32*1024)}, +}; +size_t test_geometry_count = 5; + +static void list_geometries(void) { + printf("%-24s %7s %7s %7s %7s %11s %s\n", + "geometry", "read", "prog", "erase", "count", "size", "leb16"); + size_t i = 0; + for (; builtin_geometries[i].long_name; i++) { + uintmax_t read_size = builtin_geometries[i].read_size; + uintmax_t prog_size = builtin_geometries[i].prog_size; + uintmax_t block_size = builtin_geometries[i].block_size; + uintmax_t block_count = builtin_geometries[i].block_count; + printf("%c,%-22s %7ju %7ju %7ju %7ju %11ju ", + builtin_geometries[i].short_name, + builtin_geometries[i].long_name, + read_size, + prog_size, + block_size, + block_count, + block_size*block_count); + if (read_size != block_size || prog_size != block_size) { + if (read_size != prog_size) { + leb16_print(read_size); + } + leb16_print(prog_size); + } + leb16_print(block_size); + if (block_count*block_size != 1024*1024) { + leb16_print(block_count); + } + printf("\n"); + } +} + + // scenarios to run tests under power-loss static void run_powerloss_none( @@ -686,11 +950,15 @@ static void run_powerloss_none( } // run the test - printf("running %s#%zu\n", case_->id, perm); + printf("running "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); case_->run(&cfg); - printf("finished %s#%zu\n", case_->id, perm); + printf("finished "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); // cleanup err = lfs_testbd_destroy(&cfg); @@ -756,7 +1024,9 @@ static void run_powerloss_linear( } // run the test, increasing power-cycles as power-loss events occur - printf("running %s#%zu\n", case_->id, perm); + printf("running "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); while (true) { if (!setjmp(powerloss_jmp)) { @@ -766,9 +1036,11 @@ static void run_powerloss_linear( } // power-loss! - printf("powerloss %s#%zu#", case_->id, perm); + printf("powerloss "); + print_id(suite, case_, perm, NULL, 0); + printf("#"); for (lfs_testbd_powercycles_t j = 1; j <= i; j++) { - leb16_print(&j, 1); + leb16_print(j); } printf("\n"); @@ -776,7 +1048,9 @@ static void run_powerloss_linear( lfs_testbd_setpowercycles(&cfg, i); } - printf("finished %s#%zu\n", case_->id, perm); + printf("finished "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); // cleanup err = lfs_testbd_destroy(&cfg); @@ -837,7 +1111,9 @@ static void run_powerloss_exponential( } // run the test, increasing power-cycles as power-loss events occur - printf("running %s#%zu\n", case_->id, perm); + printf("running "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); while (true) { if (!setjmp(powerloss_jmp)) { @@ -847,9 +1123,11 @@ static void run_powerloss_exponential( } // power-loss! - printf("powerloss %s#%zu#", case_->id, perm); + printf("powerloss "); + print_id(suite, case_, perm, NULL, 0); + printf("#"); for (lfs_testbd_powercycles_t j = 1; j <= i; j *= 2) { - leb16_print(&j, 1); + leb16_print(j); } printf("\n"); @@ -857,7 +1135,9 @@ static void run_powerloss_exponential( lfs_testbd_setpowercycles(&cfg, i); } - printf("finished %s#%zu\n", case_->id, perm); + printf("finished "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); // cleanup err = lfs_testbd_destroy(&cfg); @@ -916,7 +1196,9 @@ static void run_powerloss_cycles( } // run the test, increasing power-cycles as power-loss events occur - printf("running %s#%zu\n", case_->id, perm); + printf("running "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); while (true) { if (!setjmp(powerloss_jmp)) { @@ -927,8 +1209,8 @@ static void run_powerloss_cycles( // power-loss! assert(i <= cycle_count); - printf("powerloss %s#%zu#", case_->id, perm); - leb16_print(cycles, i+1); + printf("powerloss "); + print_id(suite, case_, perm, cycles, i+1); printf("\n"); i += 1; @@ -936,7 +1218,9 @@ static void run_powerloss_cycles( (i < cycle_count) ? cycles[i] : 0); } - printf("finished %s#%zu\n", case_->id, perm); + printf("finished "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); // cleanup err = lfs_testbd_destroy(&cfg); @@ -961,24 +1245,20 @@ struct powerloss_exhaustive_cycles { }; static void powerloss_exhaustive_branch(void *c) { - // append to branches struct powerloss_exhaustive_state *state = c; - state->branch_count += 1; - if (state->branch_count > state->branch_capacity) { - state->branch_capacity = (2*state->branch_capacity > 4) - ? 2*state->branch_capacity - : 4; - state->branches = realloc(state->branches, - state->branch_capacity * sizeof(lfs_testbd_t)); - if (!state->branches) { - fprintf(stderr, "error: exhaustive: out of memory\n"); - exit(-1); - } + // append to branches + lfs_testbd_t *branch = mappend( + (void**)&state->branches, + sizeof(lfs_testbd_t), + &state->branch_count, + &state->branch_capacity); + if (!branch) { + fprintf(stderr, "error: exhaustive: out of memory\n"); + exit(-1); } // create copy-on-write copy - int err = lfs_testbd_copy(state->cfg, - &state->branches[state->branch_count-1]); + int err = lfs_testbd_copy(state->cfg, branch); if (err) { fprintf(stderr, "error: exhaustive: could not create bd copy\n"); exit(-1); @@ -1023,22 +1303,19 @@ static void run_powerloss_exhaustive_layer( // recurse into each branch for (size_t i = 0; i < state.branch_count; i++) { // first push and print the branch - cycles->cycle_count += 1; - if (cycles->cycle_count > cycles->cycle_capacity) { - cycles->cycle_capacity = (2*cycles->cycle_capacity > 4) - ? 2*cycles->cycle_capacity - : 4; - cycles->cycles = realloc(cycles->cycles, - cycles->cycle_capacity * sizeof(lfs_testbd_powercycles_t)); - if (!cycles->cycles) { - fprintf(stderr, "error: exhaustive: out of memory\n"); - exit(-1); - } + lfs_testbd_powercycles_t *cycle = mappend( + (void**)&cycles->cycles, + sizeof(lfs_testbd_powercycles_t), + &cycles->cycle_count, + &cycles->cycle_capacity); + if (!cycle) { + fprintf(stderr, "error: exhaustive: out of memory\n"); + exit(-1); } - cycles->cycles[cycles->cycle_count-1] = i; + *cycle = i; - printf("powerloss %s#%zu#", case_->id, perm); - leb16_print(cycles->cycles, cycles->cycle_count); + printf("powerloss "); + print_id(suite, case_, perm, cycles->cycles, cycles->cycle_count); printf("\n"); // now recurse @@ -1122,14 +1399,19 @@ const test_powerloss_t builtin_powerlosses[] = { const char *const builtin_powerlosses_help[] = { "Run with no power-losses.", - "Run with linearly-decreasing power-losses.", "Run with exponentially-decreasing power-losses.", + "Run with linearly-decreasing power-losses.", "Run a all permutations of power-losses, this may take a while.", "Run a all permutations of n power-losses.", "Run a custom comma-separated set of power-losses.", "Run a custom leb16-encoded set of power-losses.", }; +const test_powerloss_t *test_powerlosses = (const test_powerloss_t[]){ + {'0', "none", run_powerloss_none, NULL, 0}, +}; +size_t test_powerloss_count = 1; + static void list_powerlosses(void) { printf("%-24s %s\n", "scenario", "description"); size_t i = 0; @@ -1148,33 +1430,34 @@ static void list_powerlosses(void) { // global test step count -static size_t step = 0; +size_t step = 0; // run the tests static void run_perms( const struct test_suite *suite, const struct test_case *case_, size_t perm, + const test_geometry_t *geometry, const lfs_testbd_powercycles_t *cycles, size_t cycle_count) { - for (size_t p = 0; p < (cycles ? 1 : test_powerloss_count); p++) { - if (!cycles - && test_powerlosses[p].short_name != '0' - && !(case_->flags & TEST_REENTRANT)) { + for (size_t k = 0; k < case_->permutations; k++) { + if (perm != (size_t)-1 && k != perm) { continue; } - size_t perm_ = 0; - for (size_t g = 0; g < TEST_GEOMETRY_COUNT; g++) { - if (test_geometry && strcmp( - test_geometries[g].name, test_geometry) != 0) { - continue; - } + // define permutation + test_define_perm(suite, case_, k); - for (size_t k = 0; k < case_->permutations; k++) { - perm_ += 1; + for (size_t g = 0; g < (geometry ? 1 : test_geometry_count); g++) { + // define geometry + test_define_geometry(geometry ? geometry : &test_geometries[g]); + test_define_flush(); - if (perm != (size_t)-1 && perm_ != perm) { + for (size_t p = 0; p < (cycles ? 1 : test_powerloss_count); p++) { + // skip non-reentrant tests when powerloss testing + if (!cycles + && test_powerlosses[p].short_name != '0' + && !(case_->flags & TEST_REENTRANT)) { continue; } @@ -1186,24 +1469,20 @@ static void run_perms( } step += 1; - // setup defines - define_perm(suite, case_, k); - define_geometry(&test_geometries[g]); - // filter? if (case_->filter && !case_->filter()) { - printf("skipped %s#%zu\n", case_->id, perm_); + printf("skipped %s#%zu\n", case_->id, k); continue; } if (cycles) { run_powerloss_cycles( - suite, case_, perm_, + suite, case_, k, cycles, cycle_count); } else { test_powerlosses[p].run( - suite, case_, perm_, + suite, case_, k, test_powerlosses[p].cycles, test_powerlosses[p].cycle_count); } @@ -1223,7 +1502,7 @@ static void run(void) { continue; } - define_suite(&test_suites[i]); + test_define_suite(&test_suites[i]); for (size_t j = 0; j < test_suites[i].case_count; j++) { if (test_ids[t].case_ && strcmp( @@ -1233,6 +1512,7 @@ static void run(void) { run_perms(&test_suites[i], &test_suites[i].cases[j], test_ids[t].perm, + test_ids[t].geometry, test_ids[t].cycles, test_ids[t].cycle_count); } @@ -1248,35 +1528,37 @@ enum opt_flags { OPT_SUMMARY = 'Y', OPT_LIST_SUITES = 'l', OPT_LIST_CASES = 'L', - OPT_LIST_PATHS = 1, - OPT_LIST_DEFINES = 2, - OPT_LIST_GEOMETRIES = 3, - OPT_LIST_DEFAULTS = 4, - OPT_LIST_POWERLOSSES = 5, + OPT_LIST_SUITE_PATHS = 1, + OPT_LIST_CASE_PATHS = 2, + OPT_LIST_DEFINES = 3, + OPT_LIST_IMPLICIT = 4, + OPT_LIST_GEOMETRIES = 5, + OPT_LIST_POWERLOSSES = 6, OPT_DEFINE = 'D', - OPT_GEOMETRY = 'G', + OPT_GEOMETRY = 'g', OPT_POWERLOSS = 'p', - OPT_START = 6, - OPT_STEP = 7, - OPT_STOP = 8, + OPT_START = 7, + OPT_STEP = 8, + OPT_STOP = 9, OPT_DISK = 'd', OPT_TRACE = 't', - OPT_READ_SLEEP = 9, - OPT_PROG_SLEEP = 10, - OPT_ERASE_SLEEP = 11, + OPT_READ_SLEEP = 10, + OPT_PROG_SLEEP = 11, + OPT_ERASE_SLEEP = 12, }; -const char *short_opts = "hYlLD:G:p:nrVd:t:"; +const char *short_opts = "hYlLD:g:p:d:t:"; const struct option long_opts[] = { {"help", no_argument, NULL, OPT_HELP}, {"summary", no_argument, NULL, OPT_SUMMARY}, {"list-suites", no_argument, NULL, OPT_LIST_SUITES}, {"list-cases", no_argument, NULL, OPT_LIST_CASES}, - {"list-paths", no_argument, NULL, OPT_LIST_PATHS}, + {"list-suite-paths", no_argument, NULL, OPT_LIST_SUITE_PATHS}, + {"list-case-paths", no_argument, NULL, OPT_LIST_CASE_PATHS}, {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, + {"list-implicit", no_argument, NULL, OPT_LIST_IMPLICIT}, {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, - {"list-defaults", no_argument, NULL, OPT_LIST_DEFAULTS}, {"list-powerlosses", no_argument, NULL, OPT_LIST_POWERLOSSES}, {"define", required_argument, NULL, OPT_DEFINE}, {"geometry", required_argument, NULL, OPT_GEOMETRY}, @@ -1297,13 +1579,14 @@ const char *const help_text[] = { "Show quick summary.", "List test suites.", "List test cases.", - "List the path for each test case.", - "List the defines for each test permutation.", - "List the disk geometries used for testing.", - "List the default defines in this test-runner.", + "List the path for each test suite.", + "List the path and line number for each test case.", + "List all defines in this test-runner.", + "List implicit defines in this test-runner.", + "List the available disk geometries.", "List the available power-loss scenarios.", "Override a test define.", - "Filter by geometry.", + "Comma-separated list of disk geometries to test. Defaults to d,e,E,n,N.", "Comma-separated list of power-loss scenarios to test. Defaults to 0,l.", "Start at the nth test.", "Stop before the nth test.", @@ -1318,11 +1601,11 @@ const char *const help_text[] = { int main(int argc, char **argv) { void (*op)(void) = run; - const char **override_names = NULL; - intmax_t *override_defines = NULL; + test_override_t *overrides = NULL; size_t override_count = 0; size_t override_capacity = 0; + size_t test_geometry_capacity = 0; size_t test_powerloss_capacity = 0; size_t test_id_capacity = 0; @@ -1396,41 +1679,32 @@ int main(int argc, char **argv) { case OPT_LIST_CASES: op = list_cases; break; - case OPT_LIST_PATHS: - op = list_paths; + case OPT_LIST_SUITE_PATHS: + op = list_suite_paths; + break; + case OPT_LIST_CASE_PATHS: + op = list_case_paths; break; case OPT_LIST_DEFINES: op = list_defines; break; + case OPT_LIST_IMPLICIT: + op = list_implicit; + break; case OPT_LIST_GEOMETRIES: op = list_geometries; break; - case OPT_LIST_DEFAULTS: - op = list_defaults; - break; case OPT_LIST_POWERLOSSES: op = list_powerlosses; break; // configuration case OPT_DEFINE: { - // special case for -DGEOMETRY=, we treat this the same - // as --geometry= - if (strncmp(optarg, "GEOMETRY=", strlen("GEOMETRY=")) == 0) { - test_geometry = &optarg[strlen("GEOMETRY=")]; - break; - } - - // realloc if necessary - override_count += 1; - if (override_count > override_capacity) { - override_capacity = (2*override_capacity > 4) - ? 2*override_capacity - : 4; - override_names = realloc(override_names, - override_capacity * sizeof(const char *)); - override_defines = realloc(override_defines, - override_capacity * sizeof(intmax_t)); - } + // allocate space + test_override_t *override = mappend( + (void**)&overrides, + sizeof(test_override_t), + &override_count, + &override_capacity); // parse into string key/intmax_t value, cannibalizing the // arg in the process @@ -1439,13 +1713,12 @@ int main(int argc, char **argv) { if (!sep) { goto invalid_define; } - override_defines[override_count-1] - = strtoumax(sep+1, &parsed, 0); + override->define = strtoumax(sep+1, &parsed, 0); if (parsed == sep+1) { goto invalid_define; } - override_names[override_count-1] = optarg; + override->name = optarg; *sep = '\0'; break; @@ -1453,9 +1726,136 @@ invalid_define: fprintf(stderr, "error: invalid define: %s\n", optarg); exit(-1); } - case OPT_GEOMETRY: - test_geometry = optarg; + case OPT_GEOMETRY: { + // reset our geometry scenarios + if (test_geometry_capacity > 0) { + free((test_geometry_t*)test_geometries); + } + test_geometries = NULL; + test_geometry_count = 0; + test_geometry_capacity = 0; + + // parse the comma separated list of disk geometries + while (*optarg) { + // allocate space + test_geometry_t *geometry = mappend( + (void**)&test_geometries, + sizeof(test_geometry_t), + &test_geometry_count, + &test_geometry_capacity); + + // parse the disk geometry + optarg += strspn(optarg, " "); + + // named disk geometry + size_t len = strcspn(optarg, " ,"); + for (size_t i = 0; builtin_geometries[i].long_name; i++) { + if ((len == 1 + && *optarg == builtin_geometries[i].short_name) + || (len == strlen( + builtin_geometries[i].long_name) + && memcmp(optarg, + builtin_geometries[i].long_name, + len) == 0)) { + *geometry = builtin_geometries[i]; + optarg += len; + goto geometry_next; + } + } + + // comma-separated read/prog/erase/count + if (*optarg == '{') { + lfs_size_t sizes[4]; + size_t count = 0; + + char *s = optarg + 1; + while (count < 4) { + char *parsed = NULL; + sizes[count] = strtoumax(s, &parsed, 0); + count += 1; + + s = parsed + strspn(parsed, " "); + if (*s == ',') { + s += 1; + continue; + } else if (*s == '}') { + s += 1; + break; + } else { + goto geometry_unknown; + } + } + + // allow implicit r=p and p=e for common geometries + geometry->read_size = sizes[0]; + geometry->prog_size + = count >= 3 ? sizes[1] + : sizes[0]; + geometry->block_size + = count >= 3 ? sizes[2] + : count >= 2 ? sizes[1] + : sizes[0]; + // if no block_count, figure out 1 MiB total size + geometry->block_count + = count >= 4 ? sizes[3] + : (1024*1024) / geometry->block_size; + optarg = s; + goto geometry_next; + } + + // leb16-encoded read/prog/erase/count + if (*optarg == '#') { + lfs_size_t sizes[4]; + size_t count = 0; + + char *s = optarg + 1; + while (true) { + char *parsed = NULL; + uintmax_t x = leb16_parse(s, &parsed); + if (parsed == s || count >= 4) { + break; + } + + sizes[count] = x; + count += 1; + s = parsed; + } + + // allow implicit r=p and p=e for common geometries + geometry->read_size = sizes[0]; + geometry->prog_size + = count >= 3 ? sizes[1] + : sizes[0]; + geometry->block_size + = count >= 3 ? sizes[2] + : count >= 2 ? sizes[1] + : sizes[0]; + // if no block_count, figure out 1 MiB total size + geometry->block_count + = count >= 4 ? sizes[3] + : (1024*1024) / geometry->block_size; + optarg = s; + goto geometry_next; + } + +geometry_unknown: + // unknown scenario? + fprintf(stderr, "error: unknown disk geometry: %s\n", + optarg); + exit(-1); + +geometry_next: + optarg += strspn(optarg, " "); + if (*optarg == ',') { + optarg += 1; + } else if (*optarg == '\0') { + break; + } else { + goto geometry_unknown; + } + } break; + } case OPT_POWERLOSS: { // reset our powerloss scenarios if (test_powerloss_capacity > 0) { @@ -1468,17 +1868,11 @@ invalid_define: // parse the comma separated list of power-loss scenarios while (*optarg) { // allocate space - test_powerloss_count += 1; - if (test_powerloss_count > test_powerloss_capacity) { - test_powerloss_capacity - = (2*test_powerloss_capacity > 4) - ? 2*test_powerloss_capacity - : 4; - test_powerlosses = realloc( - (test_powerloss_t*)test_powerlosses, - test_powerloss_capacity - * sizeof(test_powerloss_t)); - } + test_powerloss_t *powerloss = mappend( + (void**)&test_powerlosses, + sizeof(test_powerloss_t), + &test_powerloss_count, + &test_powerloss_capacity); // parse the power-loss scenario optarg += strspn(optarg, " "); @@ -1493,9 +1887,7 @@ invalid_define: && memcmp(optarg, builtin_powerlosses[i].long_name, len) == 0)) { - ((test_powerloss_t*)test_powerlosses)[ - test_powerloss_count-1] - = builtin_powerlosses[i]; + *powerloss = builtin_powerlosses[i]; optarg += len; goto powerloss_next; } @@ -1503,27 +1895,19 @@ invalid_define: // comma-separated permutation if (*optarg == '{') { - // how many cycles? - size_t count = 1; - for (size_t i = 0; optarg[i]; i++) { - if (optarg[i] == ',') { - count += 1; - } - } + lfs_testbd_powercycles_t *cycles = NULL; + size_t cycle_count = 0; + size_t cycle_capacity = 0; - // parse cycles - lfs_testbd_powercycles_t *cycles = malloc( - count * sizeof(lfs_testbd_powercycles_t)); - size_t i = 0; char *s = optarg + 1; while (true) { char *parsed = NULL; - cycles[i] = strtoumax(s, &parsed, 0); - if (parsed == s) { - count -= 1; - i -= 1; - } - i += 1; + *(lfs_testbd_powercycles_t*)mappend( + (void**)&cycles, + sizeof(lfs_testbd_powercycles_t), + &cycle_count, + &cycle_capacity) + = strtoumax(s, &parsed, 0); s = parsed + strspn(parsed, " "); if (*s == ',') { @@ -1537,11 +1921,10 @@ invalid_define: } } - ((test_powerloss_t*)test_powerlosses)[ - test_powerloss_count-1] = (test_powerloss_t){ + *powerloss = (test_powerloss_t){ .run = run_powerloss_cycles, .cycles = cycles, - .cycle_count = count, + .cycle_count = cycle_count, }; optarg = s; goto powerloss_next; @@ -1549,20 +1932,32 @@ invalid_define: // leb16-encoded permutation if (*optarg == '#') { - lfs_testbd_powercycles_t *cycles; - char *parsed = NULL; - size_t count = leb16_parse(optarg+1, &parsed, &cycles); - if (parsed == optarg+1) { - goto powerloss_unknown; + lfs_testbd_powercycles_t *cycles = NULL; + size_t cycle_count = 0; + size_t cycle_capacity = 0; + + char *s = optarg + 1; + while (true) { + char *parsed = NULL; + uintmax_t x = leb16_parse(s, &parsed); + if (parsed == s) { + break; + } + + *(lfs_testbd_powercycles_t*)mappend( + (void**)&cycles, + sizeof(lfs_testbd_powercycles_t), + &cycle_count, + &cycle_capacity) = x; + s = parsed; } - ((test_powerloss_t*)test_powerlosses)[ - test_powerloss_count-1] = (test_powerloss_t){ + *powerloss = (test_powerloss_t){ .run = run_powerloss_cycles, .cycles = cycles, - .cycle_count = count, + .cycle_count = cycle_count, }; - optarg = (char*)parsed; + optarg = s; goto powerloss_next; } @@ -1573,8 +1968,7 @@ invalid_define: if (parsed == optarg) { goto powerloss_unknown; } - ((test_powerloss_t*)test_powerlosses)[ - test_powerloss_count-1] = (test_powerloss_t){ + *powerloss = (test_powerloss_t){ .run = run_powerloss_exhaustive, .cycles = NULL, .cycle_count = count, @@ -1585,15 +1979,18 @@ invalid_define: powerloss_unknown: // unknown scenario? - fprintf(stderr, "error: " - "unknown power-loss scenario: %s\n", + fprintf(stderr, "error: unknown power-loss scenario: %s\n", optarg); exit(-1); powerloss_next: - optarg += strcspn(optarg, ","); + optarg += strspn(optarg, " "); if (*optarg == ',') { optarg += 1; + } else if (*optarg == '\0') { + break; + } else { + goto powerloss_unknown; } } break; @@ -1684,6 +2081,7 @@ getopt_done: ; char *suite = argv[optind]; char *case_ = strchr(suite, '#'); size_t perm = -1; + test_geometry_t *geometry = NULL; lfs_testbd_powercycles_t *cycles = NULL; size_t cycle_count = 0; @@ -1697,19 +2095,69 @@ getopt_done: ; *perm_ = '\0'; perm_ += 1; - // parse power cycles - char *cycles_ = strchr(perm_, '#'); - if (cycles_) { - *cycles_ = '\0'; - cycles_ += 1; + // parse geometry + char *geometry_ = strchr(perm_, '#'); + if (geometry_) { + *geometry_ = '\0'; + geometry_ += 1; - char *parsed = NULL; - cycle_count = leb16_parse(cycles_, &parsed, &cycles); - if (parsed == cycles_) { - fprintf(stderr, "error: " - "could not parse test cycles: %s\n", cycles_); - exit(-1); + // parse power cycles + char *cycles_ = strchr(geometry_, '#'); + if (cycles_) { + *cycles_ = '\0'; + cycles_ += 1; + + size_t cycle_capacity = 0; + while (*cycles_ != '\0') { + char *parsed = NULL; + *(lfs_testbd_powercycles_t*)mappend( + (void**)&cycles, + sizeof(lfs_testbd_powercycles_t), + &cycle_count, + &cycle_capacity) + = leb16_parse(cycles_, &parsed); + if (parsed == cycles_) { + fprintf(stderr, "error: " + "could not parse test cycles: %s\n", + cycles_); + exit(-1); + } + cycles_ = parsed; + } } + + geometry = malloc(sizeof(test_geometry_t)); + lfs_size_t sizes[4]; + size_t count = 0; + + while (*geometry_ != '\0') { + char *parsed = NULL; + uintmax_t x = leb16_parse(geometry_, &parsed); + if (parsed == geometry_ || count >= 4) { + fprintf(stderr, "error: " + "count not parse test geometry: %s\n", + geometry_); + exit(-1); + } + + sizes[count] = x; + count += 1; + geometry_ = parsed; + } + + // allow implicit r=p and p=e for common geometries + geometry->read_size = sizes[0]; + geometry->prog_size + = count >= 3 ? sizes[1] + : sizes[0]; + geometry->block_size + = count >= 3 ? sizes[2] + : count >= 2 ? sizes[1] + : sizes[0]; + // if no block_count, figure out 1 MiB total size + geometry->block_count + = count >= 4 ? sizes[3] + : (1024*1024) / geometry->block_size; } char *parsed = NULL; @@ -1734,32 +2182,33 @@ getopt_done: ; } // append to identifier list - test_id_count += 1; - if (test_id_count > test_id_capacity) { - test_id_capacity = (2*test_id_capacity > 4) - ? 2*test_id_capacity - : 4; - test_ids = realloc((test_id_t*)test_ids, - test_id_capacity * sizeof(test_id_t)); - } - ((test_id_t*)test_ids)[test_id_count-1] = (test_id_t){ + *(test_id_t*)mappend( + (void**)&test_ids, + sizeof(test_id_t), + &test_id_count, + &test_id_capacity) = (test_id_t){ .suite = suite, .case_ = case_, .perm = perm, + .geometry = geometry, .cycles = cycles, .cycle_count = cycle_count, }; } // register overrides - test_define_overrides(override_names, override_defines, override_count); + test_define_overrides(overrides, override_count); // do the thing op(); // cleanup (need to be done for valgrind testing) - free(override_names); - free(override_defines); + test_define_cleanup(); + free(overrides); + + if (test_geometry_capacity) { + free((test_geometry_t*)test_geometries); + } if (test_powerloss_capacity) { for (size_t i = 0; i < test_powerloss_count; i++) { free((lfs_testbd_powercycles_t*)test_powerlosses[i].cycles); @@ -1768,6 +2217,7 @@ getopt_done: ; } if (test_id_capacity) { for (size_t i = 0; i < test_id_count; i++) { + free((test_geometry_t*)test_ids[i].geometry); free((lfs_testbd_powercycles_t*)test_ids[i].cycles); } free((test_id_t*)test_ids); diff --git a/runners/test_runner.h b/runners/test_runner.h index 4459a5c9..810c7df2 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -38,7 +38,7 @@ struct test_case { test_flags_t flags; size_t permutations; - intmax_t (*const *const *defines)(void); + intmax_t (*const *const *defines)(size_t); bool (*filter)(void); void (*run)(struct lfs_config *cfg); @@ -59,60 +59,50 @@ struct test_suite { // access generated test defines -intmax_t test_predefine(size_t define); +//intmax_t test_predefine(size_t define); intmax_t test_define(size_t define); // a few preconfigured defines that control how tests run -#define READ_SIZE test_predefine(0) -#define PROG_SIZE test_predefine(1) -#define BLOCK_SIZE test_predefine(2) -#define BLOCK_COUNT test_predefine(3) -#define CACHE_SIZE test_predefine(4) -#define LOOKAHEAD_SIZE test_predefine(5) -#define BLOCK_CYCLES test_predefine(6) -#define ERASE_VALUE test_predefine(7) -#define ERASE_CYCLES test_predefine(8) -#define BADBLOCK_BEHAVIOR test_predefine(9) -#define POWERLOSS_BEHAVIOR test_predefine(10) + +#define READ_SIZE_i 0 +#define PROG_SIZE_i 1 +#define BLOCK_SIZE_i 2 +#define BLOCK_COUNT_i 3 +#define CACHE_SIZE_i 4 +#define LOOKAHEAD_SIZE_i 5 +#define BLOCK_CYCLES_i 6 +#define ERASE_VALUE_i 7 +#define ERASE_CYCLES_i 8 +#define BADBLOCK_BEHAVIOR_i 9 +#define POWERLOSS_BEHAVIOR_i 10 -#define TEST_PREDEFINE_NAMES { \ - "READ_SIZE", \ - "PROG_SIZE", \ - "BLOCK_SIZE", \ - "BLOCK_COUNT", \ - "CACHE_SIZE", \ - "LOOKAHEAD_SIZE", \ - "BLOCK_CYCLES", \ - "ERASE_VALUE", \ - "ERASE_CYCLES", \ - "BADBLOCK_BEHAVIOR", \ - "POWERLOSS_BEHAVIOR", \ -} -#define TEST_PREDEFINE_COUNT 11 +#define READ_SIZE test_define(READ_SIZE_i) +#define PROG_SIZE test_define(PROG_SIZE_i) +#define BLOCK_SIZE test_define(BLOCK_SIZE_i) +#define BLOCK_COUNT test_define(BLOCK_COUNT_i) +#define CACHE_SIZE test_define(CACHE_SIZE_i) +#define LOOKAHEAD_SIZE test_define(LOOKAHEAD_SIZE_i) +#define BLOCK_CYCLES test_define(BLOCK_CYCLES_i) +#define ERASE_VALUE test_define(ERASE_VALUE_i) +#define ERASE_CYCLES test_define(ERASE_CYCLES_i) +#define BADBLOCK_BEHAVIOR test_define(BADBLOCK_BEHAVIOR_i) +#define POWERLOSS_BEHAVIOR test_define(POWERLOSS_BEHAVIOR_i) +#define TEST_IMPLICIT_DEFINES \ + TEST_DEFINE(READ_SIZE, test_geometry->read_size) \ + TEST_DEFINE(PROG_SIZE, test_geometry->prog_size) \ + TEST_DEFINE(BLOCK_SIZE, test_geometry->block_size) \ + TEST_DEFINE(BLOCK_COUNT, test_geometry->block_count) \ + TEST_DEFINE(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ + TEST_DEFINE(LOOKAHEAD_SIZE, 16) \ + TEST_DEFINE(BLOCK_CYCLES, -1) \ + TEST_DEFINE(ERASE_VALUE, 0xff) \ + TEST_DEFINE(ERASE_CYCLES, 0) \ + TEST_DEFINE(BADBLOCK_BEHAVIOR, LFS_TESTBD_BADBLOCK_PROGERROR) \ + TEST_DEFINE(POWERLOSS_BEHAVIOR, LFS_TESTBD_POWERLOSS_NOOP) -// default predefines -#define TEST_DEFAULTS { \ - /* LOOKAHEAD_SIZE */ 16, \ - /* BLOCK_CYCLES */ -1, \ - /* ERASE_VALUE */ 0xff, \ - /* ERASE_CYCLES */ 0, \ - /* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \ - /* POWERLOSS_BEHAVIOR */ LFS_TESTBD_POWERLOSS_NOOP, \ -} -#define TEST_DEFAULT_DEFINE_COUNT 5 - -// test geometries -#define TEST_GEOMETRIES { \ - /*geometry, read, write, erase, count, cache */ \ - {"test", { 16, 16, 512, (1024*1024)/512, 64}}, \ - {"eeprom", { 1, 1, 512, (1024*1024)/512, 64}}, \ - {"emmc", { 512, 512, 512, (1024*1024)/512, 512}}, \ - {"nor", { 1, 1, 4096, (1024*1024)/4096, 64}}, \ - {"nand", {4096, 4096, 32*1024, (1024*1024)/(32*1024), 4096}}, \ -} -#define TEST_GEOMETRY_COUNT 5 -#define TEST_GEOMETRY_DEFINE_COUNT 5 +#define TEST_GEOMETRY_DEFINE_COUNT 4 +#define TEST_IMPLICIT_DEFINE_COUNT 11 #endif diff --git a/scripts/test.py b/scripts/test.py index c7c6b0f7..d224f9e9 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -252,6 +252,7 @@ def compile(**args): # include test_runner.h in every generated file f.writeln("#include \"%s\"" % HEADER_PATH) + f.writeln() # write out generated functions, this can end up in different # files depending on the "in" attribute @@ -270,20 +271,22 @@ def compile(**args): name = ('__test__%s__%s__%s__%d' % (suite.name, case.name, k, i)) define_cbs[v] = name - f.writeln('intmax_t %s(void) {' % name) + f.writeln('intmax_t %s(' + '__attribute__((unused)) ' + 'size_t define) {' % name) f.writeln(4*' '+'return %s;' % v) f.writeln('}') f.writeln() f.writeln('intmax_t (*const *const ' - '__test__%s__%s__defines[])(void) = {' + '__test__%s__%s__defines[])(size_t) = {' % (suite.name, case.name)) for defines in case.permutations: - f.writeln(4*' '+'(intmax_t (*const[])(void)){') - for define in sorted(suite.defines): - f.writeln(8*' '+'%s,' % ( - define_cbs[defines[define]] - if define in defines - else 'NULL')) + f.writeln(4*' '+'(intmax_t (*const[' + 'TEST_IMPLICIT_DEFINE_COUNT+%d])(size_t)){' % ( + len(suite.defines))) + for k, v in sorted(defines.items()): + f.writeln(8*' '+'[%-24s] = %s,' % ( + k+'_i', define_cbs[v])) f.writeln(4*' '+'},') f.writeln('};') f.writeln() @@ -328,8 +331,10 @@ def compile(**args): if suite.defines: for i, define in enumerate(sorted(suite.defines)): f.writeln('#ifndef %s' % define) - f.writeln('#define %-24s test_define(%d)' - % (define, i)) + f.writeln('#define %-24s ' + 'TEST_IMPLICIT_DEFINE_COUNT+%d' % (define+'_i', i)) + f.writeln('#define %-24s ' + 'test_define(%s)' % (define, define+'_i')) f.writeln('#endif') f.writeln() @@ -340,7 +345,7 @@ def compile(**args): else: if case.defines: f.writeln('extern intmax_t (*const *const ' - '__test__%s__%s__defines[])(void);' + '__test__%s__%s__defines[])(size_t);' % (suite.name, case.name)) if suite.if_ is not None or case.if_ is not None: f.writeln('extern bool __test__%s__%s__filter(' @@ -364,11 +369,14 @@ def compile(**args): or 0)) if suite.defines: # create suite define names - f.writeln(4*' '+'.define_names = (const char *const[]){') + f.writeln(4*' '+'.define_names = (const char *const[' + 'TEST_IMPLICIT_DEFINE_COUNT+%d]){' % ( + len(suite.defines))) for k in sorted(suite.defines): - f.writeln(8*' '+'"%s",' % k) + f.writeln(8*' '+'[%-24s] = "%s",' % (k+'_i', k)) f.writeln(4*' '+'},') - f.writeln(4*' '+'.define_count = %d,' % len(suite.defines)) + f.writeln(4*' '+'.define_count = ' + 'TEST_IMPLICIT_DEFINE_COUNT+%d,' % len(suite.defines)) f.writeln(4*' '+'.cases = (const struct test_case[]){') for case in suite.cases: # create case structs @@ -415,10 +423,15 @@ def compile(**args): for i, define in enumerate( sorted(suite.defines)): f.writeln('#ifndef %s' % define) - f.writeln('#define %-24s test_define(%d)' - % (define, i)) - f.writeln('#define __TEST__%s__NEEDS_UNDEF' - % define) + f.writeln('#define %-24s ' + 'TEST_IMPLICIT_DEFINE_COUNT+%d' % ( + define+'_i', i)) + f.writeln('#define %-24s ' + 'test_define(%s)' % ( + define, define+'_i')) + f.writeln('#define ' + '__TEST__%s__NEEDS_UNDEF' % ( + define)) f.writeln('#endif') f.writeln() @@ -431,6 +444,7 @@ def compile(**args): f.writeln('#undef __TEST__%s__NEEDS_UNDEF' % define) f.writeln('#undef %s' % define) + f.writeln('#undef %s' % (define+'_i')) f.writeln('#endif') f.writeln() @@ -469,9 +483,10 @@ def list_(**args): if args.get('summary'): cmd.append('--summary') if args.get('list_suites'): cmd.append('--list-suites') if args.get('list_cases'): cmd.append('--list-cases') - if args.get('list_paths'): cmd.append('--list-paths') + if args.get('list_suite_paths'): cmd.append('--list-suite-paths') + if args.get('list_case_paths'): cmd.append('--list-case-paths') if args.get('list_defines'): cmd.append('--list-defines') - if args.get('list_defaults'): cmd.append('--list-defaults') + if args.get('list_implicit'): cmd.append('--list-implicit') if args.get('list_geometries'): cmd.append('--list-geometries') if args.get('list_powerlosses'): cmd.append('--list-powerlosses') @@ -503,10 +518,11 @@ def find_cases(runner_, **args): m = pattern.match(line) if m: filtered = int(m.group('filtered')) + perms = int(m.group('perms')) expected_suite_perms[m.group('suite')] += filtered expected_case_perms[m.group('id')] += filtered expected_perms += filtered - total_perms += int(m.group('perms')) + total_perms += perms proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -520,9 +536,9 @@ def find_cases(runner_, **args): expected_perms, total_perms) -def find_paths(runner_, **args): +def find_path(runner_, id, **args): # query from runner - cmd = runner_ + ['--list-paths'] + cmd = runner_ + ['--list-case-paths', id] if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, @@ -531,14 +547,17 @@ def find_paths(runner_, **args): universal_newlines=True, errors='replace', close_fds=False) - paths = co.OrderedDict() + path = None pattern = re.compile( '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' '(?P[^:]+):(?P\d+)') - for line in proc.stdout: + # skip the first line + for line in it.islice(proc.stdout, 1, None): m = pattern.match(line) - if m: - paths[m.group('id')] = (m.group('path'), int(m.group('lineno'))) + if m and path is None: + path_ = m.group('path') + lineno = int(m.group('lineno')) + path = (path_, lineno) proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -546,11 +565,36 @@ def find_paths(runner_, **args): sys.stdout.write(line) sys.exit(-1) - return paths + return path -def find_defines(runner_, **args): - # query from runner - cmd = runner_ + ['--list-defines'] +def find_defines(runner_, id, **args): + # query implicit defines from runner + cmd = runner_ + ['--list-implicit'] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + implicit_defines = co.OrderedDict() + pattern = re.compile('^(?P\w+)=(?P.+)') + for line in proc.stdout: + m = pattern.match(line) + if m: + define = m.group('define') + values = m.group('values').split(',') + implicit_defines[define] = set(values) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + # query case defines from runner + cmd = runner_ + ['--list-defines', id] if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, @@ -560,14 +604,15 @@ def find_defines(runner_, **args): errors='replace', close_fds=False) defines = co.OrderedDict() - pattern = re.compile( - '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' - '(?P(?:\w+=\w+\s*)+)') + pattern = re.compile('^(?P\w+)=(?P.+)') for line in proc.stdout: m = pattern.match(line) if m: - defines[m.group('id')] = {k: v - for k, v in re.findall('(\w+)=(\w+)', m.group('defines'))} + define = m.group('define') + value = m.group('value') + if (define not in implicit_defines + or value not in implicit_defines[define]): + defines[define] = value proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -887,16 +932,16 @@ def run(**args): print() # print each failure - if failures: - # get some extra info from runner - runner_paths = find_paths(runner_, **args) - runner_defines = find_defines(runner_, **args) - for failure in failures: - # show summary of failure - path, lineno = runner_paths[testcase(failure.id)] - defines = runner_defines.get(failure.id, {}) + assert failure.id is not None, '%s broken? %r' % ( + ' '.join(shlex.quote(c) for c in runner_), + failure) + # get some extra info from runner + path, lineno = find_path(runner_, failure.id, **args) + defines = find_defines(runner_, failure.id, **args) + + # show summary of failure print('%s%s:%d:%sfailure:%s %s%s failed' % ( '\x1b[01m' if color(**args) else '', path, lineno, @@ -969,9 +1014,10 @@ def main(**args): elif (args.get('summary') or args.get('list_suites') or args.get('list_cases') - or args.get('list_paths') + or args.get('list_suite_paths') + or args.get('list_case_paths') or args.get('list_defines') - or args.get('list_defaults') + or args.get('list_implicit') or args.get('list_geometries') or args.get('list_powerlosses')): list_(**args) @@ -1005,20 +1051,23 @@ if __name__ == "__main__": help="List test suites.") test_parser.add_argument('-L', '--list-cases', action='store_true', help="List test cases.") - test_parser.add_argument('--list-paths', action='store_true', - help="List the path for each test case.") + test_parser.add_argument('--list-suite-paths', action='store_true', + help="List the path for each test suite.") + test_parser.add_argument('--list-case-paths', action='store_true', + help="List the path and line number for each test case.") test_parser.add_argument('--list-defines', action='store_true', - help="List the defines for each test permutation.") - test_parser.add_argument('--list-defaults', action='store_true', - help="List the default defines in this test-runner.") + help="List all defines in this test-runner.") + test_parser.add_argument('--list-implicit', action='store_true', + help="List implicit defines in this test-runner.") test_parser.add_argument('--list-geometries', action='store_true', - help="List the disk geometries used for testing.") + help="List the available disk geometries.") test_parser.add_argument('--list-powerlosses', action='store_true', help="List the available power-loss scenarios.") test_parser.add_argument('-D', '--define', action='append', help="Override a test define.") - test_parser.add_argument('-G', '--geometry', - help="Filter by geometry.") + test_parser.add_argument('-g', '--geometry', + help="Comma-separated list of disk geometries to test. \ + Defaults to d,e,E,n,N.") test_parser.add_argument('-p', '--powerloss', help="Comma-separated list of power-loss scenarios to test. \ Defaults to 0,l.") From c7f7094a064c56ddfab6958640bc37e3631686a9 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 8 Sep 2022 19:54:07 -0500 Subject: [PATCH 32/81] Several tweaks to test.py and test runner These are just some minor quality of life improvements - Added a "make build-test" alias - Made test runner a positional arg for test.py since it is almost always required. This shortens the command line invocation most of the time. - Added --context to test.py - Renamed --output in test.py to --stdout, note this still merges stderr. Maybe at some point these should be split, but it's not really worth it for now. - Reworked the test_id parsing code a bit. - Changed the test runner --step to take a range such as -s0,12,2 - Changed tracebd.py --block and --off to take ranges --- Makefile | 10 +- runners/test_runner.c | 192 +++++++++++++++++++---------------- scripts/test.py | 231 +++++++++++++++++++++--------------------- scripts/tracebd.py | 106 +++++++++---------- 4 files changed, 272 insertions(+), 267 deletions(-) diff --git a/Makefile b/Makefile index 416e7606..20b1979e 100644 --- a/Makefile +++ b/Makefile @@ -107,18 +107,18 @@ size: $(OBJ) tags: $(CTAGS) --totals --c-types=+p $(shell find -H -name '*.h') $(SRC) -.PHONY: test-runner -test-runner: override CFLAGS+=--coverage -test-runner: $(BUILDDIR)runners/test_runner +.PHONY: test-runner build-test +test-runner build-test: override CFLAGS+=--coverage +test-runner build-test: $(BUILDDIR)runners/test_runner rm -f $(TEST_GCDA) .PHONY: test test: test-runner - ./scripts/test.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS) + ./scripts/test.py $(BUILDDIR)runners/test_runner $(TESTFLAGS) .PHONY: test-list test-list: test-runner - ./scripts/test.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS) -l + ./scripts/test.py $(BUILDDIR)runners/test_runner $(TESTFLAGS) -l .PHONY: code code: $(OBJ) diff --git a/runners/test_runner.c b/runners/test_runner.c index e505e04e..51347f96 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -377,9 +377,9 @@ const test_id_t *test_ids = (const test_id_t[]) { }; size_t test_id_count = 1; -size_t test_start = 0; -size_t test_stop = -1; -size_t test_step = 1; +size_t test_step_start = 0; +size_t test_step_stop = -1; +size_t test_step_step = 1; const char *test_disk_path = NULL; const char *test_trace_path = NULL; @@ -1430,7 +1430,7 @@ static void list_powerlosses(void) { // global test step count -size_t step = 0; +size_t test_step = 0; // run the tests static void run_perms( @@ -1461,13 +1461,13 @@ static void run_perms( continue; } - if (!(step >= test_start - && step < test_stop - && (step-test_start) % test_step == 0)) { - step += 1; + if (!(test_step >= test_step_start + && test_step < test_step_stop + && (test_step-test_step_start) % test_step_step == 0)) { + test_step += 1; continue; } - step += 1; + test_step += 1; // filter? if (case_->filter && !case_->filter()) { @@ -1537,17 +1537,15 @@ enum opt_flags { OPT_DEFINE = 'D', OPT_GEOMETRY = 'g', OPT_POWERLOSS = 'p', - OPT_START = 7, - OPT_STEP = 8, - OPT_STOP = 9, + OPT_STEP = 's', OPT_DISK = 'd', OPT_TRACE = 't', - OPT_READ_SLEEP = 10, - OPT_PROG_SLEEP = 11, - OPT_ERASE_SLEEP = 12, + OPT_READ_SLEEP = 7, + OPT_PROG_SLEEP = 8, + OPT_ERASE_SLEEP = 9, }; -const char *short_opts = "hYlLD:g:p:d:t:"; +const char *short_opts = "hYlLD:g:p:s:d:t:"; const struct option long_opts[] = { {"help", no_argument, NULL, OPT_HELP}, @@ -1563,8 +1561,6 @@ const struct option long_opts[] = { {"define", required_argument, NULL, OPT_DEFINE}, {"geometry", required_argument, NULL, OPT_GEOMETRY}, {"powerloss", required_argument, NULL, OPT_POWERLOSS}, - {"start", required_argument, NULL, OPT_START}, - {"stop", required_argument, NULL, OPT_STOP}, {"step", required_argument, NULL, OPT_STEP}, {"disk", required_argument, NULL, OPT_DISK}, {"trace", required_argument, NULL, OPT_TRACE}, @@ -1588,9 +1584,7 @@ const char *const help_text[] = { "Override a test define.", "Comma-separated list of disk geometries to test. Defaults to d,e,E,n,N.", "Comma-separated list of power-loss scenarios to test. Defaults to 0,l.", - "Start at the nth test.", - "Stop before the nth test.", - "Only run every n tests, calculated after --start and --stop.", + "Comma-separated range of test permutations to run (start,stop,step).", "Redirect block device operations to this file.", "Redirect trace output to this file.", "Artificial read delay in seconds.", @@ -1995,32 +1989,51 @@ powerloss_next: } break; } - case OPT_START: { - char *parsed = NULL; - test_start = strtoumax(optarg, &parsed, 0); - if (parsed == optarg) { - fprintf(stderr, "error: invalid skip: %s\n", optarg); - exit(-1); - } - break; - } - case OPT_STOP: { - char *parsed = NULL; - test_stop = strtoumax(optarg, &parsed, 0); - if (parsed == optarg) { - fprintf(stderr, "error: invalid count: %s\n", optarg); - exit(-1); - } - break; - } case OPT_STEP: { char *parsed = NULL; - test_step = strtoumax(optarg, &parsed, 0); - if (parsed == optarg) { - fprintf(stderr, "error: invalid every: %s\n", optarg); - exit(-1); + size_t start = strtoumax(optarg, &parsed, 0); + // allow empty string for start=0 + if (parsed != optarg) { + test_step_start = start; } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != '\0') { + goto step_unknown; + } + + if (*optarg == ',') { + optarg += 1; + size_t stop = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=end + if (parsed != optarg) { + test_step_stop = stop; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != '\0') { + goto step_unknown; + } + + if (*optarg == ',') { + optarg += 1; + size_t step = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=1 + if (parsed != optarg) { + test_step_step = step; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != '\0') { + goto step_unknown; + } + } + } + break; +step_unknown: + fprintf(stderr, "error: invalid step: %s\n", optarg); + exit(-1); } case OPT_DISK: test_disk_path = optarg; @@ -2077,53 +2090,62 @@ getopt_done: ; // parse test identifier, if any, cannibalizing the arg in the process for (; argc > optind; optind++) { - // parse suite - char *suite = argv[optind]; - char *case_ = strchr(suite, '#'); size_t perm = -1; test_geometry_t *geometry = NULL; lfs_testbd_powercycles_t *cycles = NULL; size_t cycle_count = 0; + // parse suite + char *suite = argv[optind]; + char *case_ = strchr(suite, '#'); if (case_) { *case_ = '\0'; case_ += 1; + } + // remove optional path and .toml suffix + char *slash = strrchr(suite, '/'); + if (slash) { + suite = slash+1; + } + + size_t suite_len = strlen(suite); + if (suite_len > 5 && strcmp(&suite[suite_len-5], ".toml") == 0) { + suite[suite_len-5] = '\0'; + } + + if (case_) { // parse case char *perm_ = strchr(case_, '#'); if (perm_) { *perm_ = '\0'; perm_ += 1; + } - // parse geometry + // nothing really to do for case + + if (perm_) { + // parse permutation char *geometry_ = strchr(perm_, '#'); if (geometry_) { *geometry_ = '\0'; geometry_ += 1; + } - // parse power cycles + char *parsed = NULL; + perm = strtoumax(perm_, &parsed, 10); + if (parsed == perm_) { + fprintf(stderr, "error: " + "could not parse test permutation: %s\n", perm_); + exit(-1); + } + + if (geometry_) { + // parse geometry char *cycles_ = strchr(geometry_, '#'); if (cycles_) { *cycles_ = '\0'; cycles_ += 1; - - size_t cycle_capacity = 0; - while (*cycles_ != '\0') { - char *parsed = NULL; - *(lfs_testbd_powercycles_t*)mappend( - (void**)&cycles, - sizeof(lfs_testbd_powercycles_t), - &cycle_count, - &cycle_capacity) - = leb16_parse(cycles_, &parsed); - if (parsed == cycles_) { - fprintf(stderr, "error: " - "could not parse test cycles: %s\n", - cycles_); - exit(-1); - } - cycles_ = parsed; - } } geometry = malloc(sizeof(test_geometry_t)); @@ -2131,7 +2153,6 @@ getopt_done: ; size_t count = 0; while (*geometry_ != '\0') { - char *parsed = NULL; uintmax_t x = leb16_parse(geometry_, &parsed); if (parsed == geometry_ || count >= 4) { fprintf(stderr, "error: " @@ -2158,29 +2179,30 @@ getopt_done: ; geometry->block_count = count >= 4 ? sizes[3] : (1024*1024) / geometry->block_size; - } - char *parsed = NULL; - perm = strtoumax(perm_, &parsed, 10); - if (parsed == perm_) { - fprintf(stderr, "error: " - "could not parse test permutation: %s\n", perm_); - exit(-1); + if (cycles_) { + // parse power cycles + size_t cycle_capacity = 0; + while (*cycles_ != '\0') { + *(lfs_testbd_powercycles_t*)mappend( + (void**)&cycles, + sizeof(lfs_testbd_powercycles_t), + &cycle_count, + &cycle_capacity) + = leb16_parse(cycles_, &parsed); + if (parsed == cycles_) { + fprintf(stderr, "error: " + "could not parse test cycles: %s\n", + cycles_); + exit(-1); + } + cycles_ = parsed; + } + } } } } - // remove optional path and .toml suffix - char *slash = strrchr(suite, '/'); - if (slash) { - suite = slash+1; - } - - size_t suite_len = strlen(suite); - if (suite_len > 5 && strcmp(&suite[suite_len-5], ".toml") == 0) { - suite[suite_len-5] = '\0'; - } - // append to identifier list *(test_id_t*)mappend( (void**)&test_ids, diff --git a/scripts/test.py b/scripts/test.py index d224f9e9..df761be2 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -20,26 +20,10 @@ import time import toml -TEST_PATHS = ['tests'] -RUNNER_PATH = './runners/test_runner' +RUNNER_PATH = 'runners/test_runner' HEADER_PATH = 'runners/test_runner.h' -def testpath(path): - path, *_ = path.split('#', 1) - return path - -def testsuite(path): - suite = testpath(path) - suite = os.path.basename(suite) - if suite.endswith('.toml'): - suite = suite[:-len('.toml')] - return suite - -def testcase(path): - _, case, *_ = path.split('#', 2) - return '%s#%s' % (testsuite(path), case) - def openio(path, mode='r', buffering=-1, nb=False): if path == '-': if 'r' in mode: @@ -56,14 +40,6 @@ def openio(path, mode='r', buffering=-1, nb=False): else: return open(path, mode, buffering) -def color(**args): - if args.get('color') == 'auto': - return sys.stdout.isatty() - elif args.get('color') == 'always': - return True - else: - return False - class TestCase: # create a TestCase object from a config def __init__(self, config, args={}): @@ -105,8 +81,8 @@ class TestCase: for k in config.keys(): print('%swarning:%s in %s, found unused key %r' % ( - '\x1b[01;33m' if color(**args) else '', - '\x1b[m' if color(**args) else '', + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', self.id(), k), file=sys.stderr) @@ -118,8 +94,10 @@ class TestCase: class TestSuite: # create a TestSuite object from a toml file def __init__(self, path, args={}): - self.name = testsuite(path) - self.path = testpath(path) + self.path = path + self.name = os.path.basename(path) + if self.name.endswith('.toml'): + self.name = self.name[:-len('.toml')] # load toml file and parse test cases with open(self.path) as f: @@ -191,8 +169,8 @@ class TestSuite: for k in config.keys(): print('%swarning:%s in %s, found unused key %r' % ( - '\x1b[01;33m' if color(**args) else '', - '\x1b[m' if color(**args) else '', + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', self.id(), k), file=sys.stderr) @@ -202,10 +180,10 @@ class TestSuite: -def compile(**args): +def compile(test_paths, **args): # find .toml files paths = [] - for path in args.get('test_ids', TEST_PATHS): + for path in test_paths: if os.path.isdir(path): path = path + '/*.toml' @@ -213,13 +191,12 @@ def compile(**args): paths.append(path) if not paths: - print('no test suites found in %r?' % args['test_ids']) + print('no test suites found in %r?' % test_paths) sys.exit(-1) if not args.get('source'): if len(paths) > 1: - print('more than one test suite for compilation? (%r)' - % args['test_ids']) + print('more than one test suite for compilation? (%r)' % test_paths) sys.exit(-1) # load our suite @@ -251,7 +228,7 @@ def compile(**args): f.writeln() # include test_runner.h in every generated file - f.writeln("#include \"%s\"" % HEADER_PATH) + f.writeln("#include \"%s\"" % args['include']) f.writeln() # write out generated functions, this can end up in different @@ -448,9 +425,9 @@ def compile(**args): f.writeln('#endif') f.writeln() -def runner(**args): - cmd = args['runner'].copy() - cmd.extend(args.get('test_ids')) +def find_runner(runner, test_ids, **args): + cmd = runner.copy() + cmd.extend(test_ids) # run under some external command? cmd[:0] = args.get('exec', []) @@ -466,10 +443,19 @@ def runner(**args): # other context if args.get('geometry'): - cmd.append('-G%s' % args.get('geometry')) - + cmd.append('-g%s' % args['geometry']) if args.get('powerloss'): - cmd.append('-p%s' % args.get('powerloss')) + cmd.append('-p%s' % args['powerloss']) + if args.get('disk'): + cmd.append('-d%s' % args['disk']) + if args.get('trace'): + cmd.append('-t%s' % args['trace']) + if args.get('read_sleep'): + cmd.append('--read-sleep=%s' % args['read_sleep']) + if args.get('prog_sleep'): + cmd.append('--prog-sleep=%s' % args['prog_sleep']) + if args.get('erase_sleep'): + cmd.append('--erase-sleep=%s' % args['erase_sleep']) # defines? if args.get('define'): @@ -478,8 +464,8 @@ def runner(**args): return cmd -def list_(**args): - cmd = runner(**args) +def list_(runner, test_ids, **args): + cmd = find_runner(runner, test_ids, **args) if args.get('summary'): cmd.append('--summary') if args.get('list_suites'): cmd.append('--list-suites') if args.get('list_cases'): cmd.append('--list-cases') @@ -492,7 +478,7 @@ def list_(**args): if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) - sys.exit(sp.call(cmd)) + return sp.call(cmd) def find_cases(runner_, **args): @@ -624,10 +610,10 @@ def find_defines(runner_, id, **args): class TestFailure(Exception): - def __init__(self, id, returncode, output, assert_=None): + def __init__(self, id, returncode, stdout, assert_=None): self.id = id self.returncode = returncode - self.output = output + self.stdout = stdout self.assert_ = assert_ def run_stage(name, runner_, **args): @@ -659,17 +645,6 @@ def run_stage(name, runner_, **args): # run the tests! cmd = runner_.copy() - # TODO move all these to runner? - if args.get('disk'): - cmd.append('--disk=%s' % args['disk']) - if args.get('trace'): - cmd.append('--trace=%s' % args['trace']) - if args.get('read_sleep'): - cmd.append('--read-sleep=%s' % args['read_sleep']) - if args.get('prog_sleep'): - cmd.append('--prog-sleep=%s' % args['prog_sleep']) - if args.get('erase_sleep'): - cmd.append('--erase-sleep=%s' % args['erase_sleep']) if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) @@ -678,10 +653,10 @@ def run_stage(name, runner_, **args): os.close(spty) children.add(proc) mpty = os.fdopen(mpty, 'r', 1) - output = None + stdout = None last_id = None - last_output = [] + last_stdout = [] last_assert = None try: while True: @@ -694,19 +669,19 @@ def run_stage(name, runner_, **args): raise if not line: break - last_output.append(line) - if args.get('output'): + last_stdout.append(line) + if args.get('stdout'): try: - if not output: - output = openio(args['output'], 'a', 1, nb=True) - output.write(line) + if not stdout: + stdout = openio(args['stdout'], 'a', 1, nb=True) + stdout.write(line) except OSError as e: if e.errno not in [ errno.ENXIO, errno.EPIPE, errno.EAGAIN]: raise - output = None + stdout = None if args.get('verbose'): sys.stdout.write(line) @@ -716,7 +691,7 @@ def run_stage(name, runner_, **args): if op == 'running': locals.seen_perms += 1 last_id = m.group('id') - last_output = [] + last_stdout = [] last_assert = None elif op == 'powerloss': last_id = m.group('id') @@ -736,7 +711,7 @@ def run_stage(name, runner_, **args): if args.get('keep_going'): proc.kill() except KeyboardInterrupt: - raise TestFailure(last_id, 1, last_output) + raise TestFailure(last_id, 1, last_stdout) finally: children.remove(proc) mpty.close() @@ -746,7 +721,7 @@ def run_stage(name, runner_, **args): raise TestFailure( last_id, proc.returncode, - last_output, + last_stdout, last_assert) def run_job(runner, start=None, step=None): @@ -758,12 +733,10 @@ def run_stage(name, runner_, **args): step = step or 1 while start < total_perms: runner_ = runner.copy() - if start is not None: - runner_.append('--start=%d' % start) - if step is not None: - runner_.append('--step=%d' % step) if args.get('isolate') or args.get('valgrind'): - runner_.append('--stop=%d' % (start+step)) + runner_.append('-s%s,%s,%s' % (start, start+step, step)) + else: + runner_.append('-s%s,,%s' % (start, step)) try: # run the tests @@ -805,14 +778,14 @@ def run_stage(name, runner_, **args): daemon=True)) def print_update(done): - if not args.get('verbose') and (color(**args) or done): + if not args.get('verbose') and (args['color'] or done): sys.stdout.write('%s%srunning %s%s:%s %s%s' % ( - '\r\x1b[K' if color(**args) else '', + '\r\x1b[K' if args['color'] else '', '\x1b[?7l' if not done else '', ('\x1b[32m' if not failures else '\x1b[31m') - if color(**args) else '', + if args['color'] else '', name, - '\x1b[m' if color(**args) else '', + '\x1b[m' if args['color'] else '', ', '.join(filter(None, [ '%d/%d suites' % ( sum(passed_suite_perms[k] == v @@ -829,10 +802,10 @@ def run_stage(name, runner_, **args): '%dpls!' % powerlosses if powerlosses else None, '%s%d/%d failures%s' % ( - '\x1b[31m' if color(**args) else '', + '\x1b[31m' if args['color'] else '', len(failures), expected_perms, - '\x1b[m' if color(**args) else '') + '\x1b[m' if args['color'] else '') if failures else None])), '\x1b[?7h' if not done else '\n')) sys.stdout.flush() @@ -862,9 +835,9 @@ def run_stage(name, runner_, **args): killed) -def run(**args): +def run(runner, test_ids, **args): # query runner for tests - runner_ = runner(**args) + runner_ = find_runner(runner, test_ids, **args) print('using runner: %s' % ' '.join(shlex.quote(c) for c in runner_)) expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( @@ -877,9 +850,9 @@ def run(**args): print() # truncate and open logs here so they aren't disconnected between tests - output = None - if args.get('output'): - output = openio(args['output'], 'w', 1) + stdout = None + if args.get('stdout'): + stdout = openio(args['stdout'], 'w', 1) trace = None if args.get('trace'): trace = openio(args['trace'], 'w', 1) @@ -896,8 +869,8 @@ def run(**args): else expected_suite_perms.keys() if args.get('by_suites') else [None]): # rebuild runner for each stage to override test identifier if needed - stage_runner = runner(**args | { - 'test_ids': [by] if by is not None else args.get('test_ids', [])}) + stage_runner = find_runner(runner, + [by] if by is not None else test_ids, **args) # spawn jobs for stage expected_, passed_, powerlosses_, failures_, killed = run_stage( @@ -913,8 +886,8 @@ def run(**args): stop = time.time() - if output: - output.close() + if stdout: + stdout.close() if trace: trace.close() @@ -922,8 +895,8 @@ def run(**args): print() print('%sdone:%s %s' % ( ('\x1b[32m' if not failures else '\x1b[31m') - if color(**args) else '', - '\x1b[m' if color(**args) else '', + if args['color'] else '', + '\x1b[m' if args['color'] else '', ', '.join(filter(None, [ '%d/%d passed' % (passed, expected), '%d/%d failed' % (len(failures), expected), @@ -943,28 +916,28 @@ def run(**args): # show summary of failure print('%s%s:%d:%sfailure:%s %s%s failed' % ( - '\x1b[01m' if color(**args) else '', + '\x1b[01m' if args['color'] else '', path, lineno, - '\x1b[01;31m' if color(**args) else '', - '\x1b[m' if color(**args) else '', + '\x1b[01;31m' if args['color'] else '', + '\x1b[m' if args['color'] else '', failure.id, ' (%s)' % ', '.join('%s=%s' % (k,v) for k,v in defines.items()) if defines else '')) - if failure.output: - output = failure.output + if failure.stdout: + stdout = failure.stdout if failure.assert_ is not None: - output = output[:-1] - for line in output[-5:]: + stdout = stdout[:-1] + for line in stdout[-args.get('context', 5):]: sys.stdout.write(line) if failure.assert_ is not None: path, lineno, message = failure.assert_ print('%s%s:%d:%sassert:%s %s' % ( - '\x1b[01m' if color(**args) else '', + '\x1b[01m' if args['color'] else '', path, lineno, - '\x1b[01;31m' if color(**args) else '', - '\x1b[m' if color(**args) else '', + '\x1b[01;31m' if args['color'] else '', + '\x1b[m' if args['color'] else '', message)) with open(path) as f: line = next(it.islice(f, lineno-1, None)).strip('\n') @@ -976,7 +949,7 @@ def run(**args): or args.get('gdb_case') or args.get('gdb_main')): failure = failures[0] - runner_ = runner(**args | {'test_ids': [failure.id]}) + runner_ = find_runner(runner, [failure.id], **args) if args.get('gdb_main'): cmd = ['gdb', @@ -984,7 +957,7 @@ def run(**args): '-ex', 'run', '--args'] + runner_ elif args.get('gdb_case'): - path, lineno = runner_paths[testcase(failure.id)] + path, lineno = find_path(runner_, failure.id, **args) cmd = ['gdb', '-ex', 'break %s:%d' % (path, lineno), '-ex', 'run', @@ -1009,8 +982,16 @@ def run(**args): def main(**args): + # figure out what color should be + if args.get('color') == 'auto': + args['color'] = sys.stdout.isatty() + elif args.get('color') == 'always': + args['color'] = True + else: + args['color'] = False + if args.get('compile'): - compile(**args) + return compile(**args) elif (args.get('summary') or args.get('list_suites') or args.get('list_cases') @@ -1020,9 +1001,9 @@ def main(**args): or args.get('list_implicit') or args.get('list_geometries') or args.get('list_powerlosses')): - list_(**args) + return list_(**args) else: - run(**args) + return run(**args) if __name__ == "__main__": @@ -1033,18 +1014,19 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( description="Build and run tests.", conflict_handler='ignore') - parser.add_argument('test_ids', nargs='*', - help="Description of testis to run. May be a directory, path, or \ - test identifier. Test identifiers are of the form \ - ##, but suffixes can be \ - dropped to run any matching tests. Defaults to %s." % TEST_PATHS) parser.add_argument('-v', '--verbose', action='store_true', help="Output commands that run behind the scenes.") parser.add_argument('--color', choices=['never', 'always', 'auto'], default='auto', help="When to use terminal colors.") + # test flags test_parser = parser.add_argument_group('test options') + test_parser.add_argument('runner', nargs='?', + type=lambda x: x.split(), + help="Test runner to use for testing. Defaults to %r." % RUNNER_PATH) + test_parser.add_argument('test_ids', nargs='*', + help="Description of tests to run.") test_parser.add_argument('-Y', '--summary', action='store_true', help="Show quick summary.") test_parser.add_argument('-l', '--list-suites', action='store_true', @@ -1075,17 +1057,14 @@ if __name__ == "__main__": help="Direct block device operations to this file.") test_parser.add_argument('-t', '--trace', help="Direct trace output to this file.") - test_parser.add_argument('-o', '--output', - help="Direct stdout and stderr to this file.") + test_parser.add_argument('-O', '--stdout', + help="Direct stdout to this file. Note stderr is already merged here.") test_parser.add_argument('--read-sleep', help="Artificial read delay in seconds.") test_parser.add_argument('--prog-sleep', help="Artificial prog delay in seconds.") test_parser.add_argument('--erase-sleep', help="Artificial erase delay in seconds.") - test_parser.add_argument('--runner', default=[RUNNER_PATH], - type=lambda x: x.split(), - help="Path to runner, defaults to %r" % RUNNER_PATH) test_parser.add_argument('-j', '--jobs', nargs='?', type=int, const=len(os.sched_getaffinity(0)), help="Number of parallel runners to run.") @@ -1097,6 +1076,9 @@ if __name__ == "__main__": help="Step through tests by suite.") test_parser.add_argument('-B', '--by-cases', action='store_true', help="Step through tests by case.") + test_parser.add_argument('--context', type=lambda x: int(x, 0), + help="Show this many lines of stdout on test failure. \ + Defaults to 5.") test_parser.add_argument('--gdb', action='store_true', help="Drop into gdb on test failure.") test_parser.add_argument('--gdb-case', action='store_true', @@ -1110,14 +1092,27 @@ if __name__ == "__main__": test_parser.add_argument('--valgrind', action='store_true', help="Run under Valgrind to find memory errors. Implicitly sets \ --isolate.") + # compilation flags comp_parser = parser.add_argument_group('compilation options') + comp_parser.add_argument('test_paths', nargs='*', + help="Description of *.toml files to compile. May be a directory \ + or a list of paths.") comp_parser.add_argument('-c', '--compile', action='store_true', help="Compile a test suite or source file.") comp_parser.add_argument('-s', '--source', help="Source file to compile, possibly injecting internal tests.") + comp_parser.add_argument('--include', default=HEADER_PATH, + help="Inject this header file into every compiled test file. \ + Defaults to %r." % HEADER_PATH) comp_parser.add_argument('-o', '--output', help="Output file.") + + # runner + test_ids overlaps test_paths, so we need to do some munging here + args = parser.parse_args() + args.test_paths = [' '.join(args.runner or [])] + args.test_ids + args.runner = args.runner or [RUNNER_PATH] + sys.exit(main(**{k: v - for k, v in vars(parser.parse_args()).items() + for k, v in vars(args).items() if v is not None})) diff --git a/scripts/tracebd.py b/scripts/tracebd.py index 3638fef9..daae0c08 100755 --- a/scripts/tracebd.py +++ b/scripts/tracebd.py @@ -6,6 +6,7 @@ import collections as co import itertools as it import math as m +import os import re import shutil import threading as th @@ -322,11 +323,8 @@ def main(path='-', *, chars=None, wear_chars=None, color='auto', - block=None, - start=None, - stop=None, - start_off=None, - stop_off=None, + block=(None,None), + off=(None,None), block_size=None, block_count=None, block_cycles=None, @@ -346,25 +344,26 @@ def main(path='-', *, if color == 'auto': color = 'always' if sys.stdout.isatty() else 'never' - start = (start if start is not None - else block if block is not None - else 0) - stop = (stop if stop is not None - else block+1 if block is not None - else block_count if block_count is not None - else None) - start_off = (start_off if start_off is not None - else 0) - stop_off = (stop_off if stop_off is not None - else block_size if block_size is not None - else None) + block_start = block[0] + block_stop = block[1] if len(block) > 1 else block[0]+1 + off_start = off[0] + off_stop = off[1] if len(off) > 1 else off[0]+1 + + if block_start is None: + block_start = 0 + if block_stop is None and block_count is not None: + block_stop = block_count + if off_start is None: + off_start = 0 + if off_stop is None and block_size is not None: + off_stop = block_size bd = Bd( size=(block_size if block_size is not None - else stop_off-start_off if stop_off is not None + else off_stop-off_start if off_stop is not None else 1), count=(block_count if block_count is not None - else stop-start if stop is not None + else block_stop-block_start if block_stop is not None else 1), width=(width or 80)*height) lock = th.Lock() @@ -431,21 +430,21 @@ def main(path='-', *, size = int(m.group('block_size'), 0) count = int(m.group('block_count'), 0) - if stop_off is not None: - size = stop_off-start_off - if stop is not None: - count = stop-start + if off_stop is not None: + size = off_stop-off_start + if block_stop is not None: + count = block_stop-block_start with lock: if reset: bd.reset() - # ignore the new values is stop/stop_off is explicit + # ignore the new values if block_stop/off_stop is explicit bd.smoosh( - size=(size if stop_off is None - else stop_off-start_off), - count=(count if stop is None - else stop-start)) + size=(size if off_stop is None + else off_stop-off_start), + count=(count if block_stop is None + else block_stop-block_start)) return True elif m.group('read') and read: @@ -453,14 +452,14 @@ def main(path='-', *, off = int(m.group('read_off'), 0) size = int(m.group('read_size'), 0) - if stop is not None and block >= stop: + if block_stop is not None and block >= block_stop: return False - block -= start - if stop_off is not None: - if off >= stop_off: + block -= block_start + if off_stop is not None: + if off >= off_stop: return False - size = min(size, stop_off-off) - off -= start_off + size = min(size, off_stop-off) + off -= off_start with lock: bd.read(block, slice(off,off+size)) @@ -471,14 +470,14 @@ def main(path='-', *, off = int(m.group('prog_off'), 0) size = int(m.group('prog_size'), 0) - if stop is not None and block >= stop: + if block_stop is not None and block >= block_stop: return False - block -= start - if stop_off is not None: - if off >= stop_off: + block -= block_start + if off_stop is not None: + if off >= off_stop: return False - size = min(size, stop_off-off) - off -= start_off + size = min(size, off_stop-off) + off -= off_start with lock: bd.prog(block, slice(off,off+size)) @@ -487,9 +486,9 @@ def main(path='-', *, elif m.group('erase') and (erase or wear): block = int(m.group('erase_block'), 0) - if stop is not None and block >= stop: + if block_stop is not None and block >= block_stop: return False - block -= start + block -= block_start with lock: bd.erase(block) @@ -691,24 +690,13 @@ if __name__ == "__main__": parser.add_argument( '-b', '--block', - type=lambda x: int(x, 0), - help="Show a specific block.") + type=lambda x: tuple(int(x,0) if x else None for x in x.split(',',1)), + help="Show a specific block or range of blocks.") parser.add_argument( - '--start', - type=lambda x: int(x, 0), - help="Start at this block.") - parser.add_argument( - '--stop', - type=lambda x: int(x, 0), - help="Stop before this block.") - parser.add_argument( - '--start-off', - type=lambda x: int(x, 0), - help="Start at this offset.") - parser.add_argument( - '--stop-off', - type=lambda x: int(x, 0), - help="Stop before this offset.") + '-i', + '--off', + type=lambda x: tuple(int(x,0) if x else None for x in x.split(',',1)), + help="Show a specific offset or range of offsets.") parser.add_argument( '-B', '--block-size', From 5a2ff178e05444900d4b9cfd35e8199aa8f34360 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 9 Sep 2022 01:55:35 -0500 Subject: [PATCH 33/81] Changed test identifier separator # -> : Compare: - test_dirs#reentrant_many_dir#1#ggg1ggg8#123456789abcdef - test_dirs:reentrant_many_dir:1:ggg1ggg8:123456789abcdef --- runners/test_runner.c | 30 +++++++++++++++--------------- scripts/test.py | 8 ++++---- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/runners/test_runner.c b/runners/test_runner.c index 51347f96..c9413f50 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -129,8 +129,8 @@ static void print_id( const lfs_testbd_powercycles_t *cycles, size_t cycle_count) { (void)suite; - // suite[#case[#perm[#geometry[#powercycles]]]] - printf("%s#%zu#", case_->id, perm); + // suite[:case[:perm[:geometry[:powercycles]]]] + printf("%s:%zu:", case_->id, perm); // reduce duplication in geometry, this is appended to every test if (READ_SIZE != BLOCK_SIZE || PROG_SIZE != BLOCK_SIZE) { @@ -146,7 +146,7 @@ static void print_id( // only print power-cycles if any occured if (cycles) { - printf("#"); + printf(":"); for (size_t i = 0; i < cycle_count; i++) { leb16_print(cycles[i]); } @@ -1038,7 +1038,7 @@ static void run_powerloss_linear( // power-loss! printf("powerloss "); print_id(suite, case_, perm, NULL, 0); - printf("#"); + printf(":"); for (lfs_testbd_powercycles_t j = 1; j <= i; j++) { leb16_print(j); } @@ -1125,7 +1125,7 @@ static void run_powerloss_exponential( // power-loss! printf("powerloss "); print_id(suite, case_, perm, NULL, 0); - printf("#"); + printf(":"); for (lfs_testbd_powercycles_t j = 1; j <= i; j *= 2) { leb16_print(j); } @@ -1378,14 +1378,14 @@ static void run_powerloss_exhaustive( } // run the test, increasing power-cycles as power-loss events occur - printf("running %s#%zu\n", case_->id, perm); + printf("running %s:%zu\n", case_->id, perm); // recursively exhaust each layer of powerlosses run_powerloss_exhaustive_layer(suite, case_, perm, &cfg, &bdcfg, cycle_count, &(struct powerloss_exhaustive_cycles){NULL, 0, 0}); - printf("finished %s#%zu\n", case_->id, perm); + printf("finished %s:%zu\n", case_->id, perm); } @@ -1425,7 +1425,7 @@ static void list_powerlosses(void) { // a couple more options with special parsing printf("%-24s %s\n", "1,2,3", builtin_powerlosses_help[i+0]); printf("%-24s %s\n", "{1,2,3}", builtin_powerlosses_help[i+1]); - printf("%-24s %s\n", "#1248g1", builtin_powerlosses_help[i+2]); + printf("%-24s %s\n", ":1248g1", builtin_powerlosses_help[i+2]); } @@ -1471,7 +1471,7 @@ static void run_perms( // filter? if (case_->filter && !case_->filter()) { - printf("skipped %s#%zu\n", case_->id, k); + printf("skipped %s:%zu\n", case_->id, k); continue; } @@ -1798,7 +1798,7 @@ invalid_define: } // leb16-encoded read/prog/erase/count - if (*optarg == '#') { + if (*optarg == ':') { lfs_size_t sizes[4]; size_t count = 0; @@ -1925,7 +1925,7 @@ geometry_next: } // leb16-encoded permutation - if (*optarg == '#') { + if (*optarg == ':') { lfs_testbd_powercycles_t *cycles = NULL; size_t cycle_count = 0; size_t cycle_capacity = 0; @@ -2097,7 +2097,7 @@ getopt_done: ; // parse suite char *suite = argv[optind]; - char *case_ = strchr(suite, '#'); + char *case_ = strchr(suite, ':'); if (case_) { *case_ = '\0'; case_ += 1; @@ -2116,7 +2116,7 @@ getopt_done: ; if (case_) { // parse case - char *perm_ = strchr(case_, '#'); + char *perm_ = strchr(case_, ':'); if (perm_) { *perm_ = '\0'; perm_ += 1; @@ -2126,7 +2126,7 @@ getopt_done: ; if (perm_) { // parse permutation - char *geometry_ = strchr(perm_, '#'); + char *geometry_ = strchr(perm_, ':'); if (geometry_) { *geometry_ = '\0'; geometry_ += 1; @@ -2142,7 +2142,7 @@ getopt_done: ; if (geometry_) { // parse geometry - char *cycles_ = strchr(geometry_, '#'); + char *cycles_ = strchr(geometry_, ':'); if (cycles_) { *cycles_ = '\0'; cycles_ += 1; diff --git a/scripts/test.py b/scripts/test.py index df761be2..b12e4e02 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -88,7 +88,7 @@ class TestCase: file=sys.stderr) def id(self): - return '%s#%s' % (self.suite, self.name) + return '%s:%s' % (self.suite, self.name) class TestSuite: @@ -497,7 +497,7 @@ def find_cases(runner_, **args): expected_perms = 0 total_perms = 0 pattern = re.compile( - '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' + '^(?P(?P(?P[^:]+):[^\s:]+)[^\s]*)\s+' '[^\s]+\s+(?P\d+)/(?P\d+)') # skip the first line for line in it.islice(proc.stdout, 1, None): @@ -535,7 +535,7 @@ def find_path(runner_, id, **args): close_fds=False) path = None pattern = re.compile( - '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' + '^(?P(?P(?P[^:]+):[^\s:]+)[^\s]*)\s+' '(?P[^:]+):(?P\d+)') # skip the first line for line in it.islice(proc.stdout, 1, None): @@ -630,7 +630,7 @@ def run_stage(name, runner_, **args): pattern = re.compile('^(?:' '(?Prunning|finished|skipped|powerloss) ' - '(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)' + '(?P(?P(?P[^:]+):[^\s:]+)[^\s]*)' '|' '(?P[^:]+):(?P\d+):(?Passert):' ' *(?P.*)' ')$') locals = th.local() From bfbe44e70dbe3418c1ab4890058e212bba9468e0 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 10 Sep 2022 15:15:03 -0500 Subject: [PATCH 34/81] Dropped permutation number for full leb16-encoded defines This is probably how the test runner should have been implemented in the first place, but it took a few tries to get here. This makes it so the test identifier, which is a bit longer now, fully encodes the state of the defines in the test. This removes the need for the extra geometry field and allows reproduction of tests with custom defines at runtime. The test runner may have already seemed like a solved problem, but these changes are really to enable repurposing the test runner as a bench runner. --- runners/test_runner.c | 1131 ++++++++++++++++++++++------------------- runners/test_runner.h | 17 +- scripts/test.py | 61 +-- 3 files changed, 646 insertions(+), 563 deletions(-) diff --git a/runners/test_runner.c b/runners/test_runner.c index c9413f50..4a31ccfc 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -93,10 +93,7 @@ typedef struct test_geometry { char short_name; const char *long_name; - lfs_size_t read_size; - lfs_size_t prog_size; - lfs_size_t block_size; - lfs_size_t block_count; + test_define_t defines[TEST_GEOMETRY_DEFINE_COUNT]; } test_geometry_t; typedef struct test_powerloss { @@ -104,11 +101,10 @@ typedef struct test_powerloss { const char *long_name; void (*run)( - const struct test_suite *suite, - const struct test_case *case_, - size_t perm, const lfs_testbd_powercycles_t *cycles, - size_t cycle_count); + size_t cycle_count, + const struct test_suite *suite, + const struct test_case *case_); const lfs_testbd_powercycles_t *cycles; size_t cycle_count; } test_powerloss_t; @@ -116,43 +112,12 @@ typedef struct test_powerloss { typedef struct test_id { const char *suite; const char *case_; - size_t perm; - const test_geometry_t *geometry; + const test_define_t *defines; + size_t define_count; const lfs_testbd_powercycles_t *cycles; size_t cycle_count; } test_id_t; -static void print_id( - const struct test_suite *suite, - const struct test_case *case_, - size_t perm, - const lfs_testbd_powercycles_t *cycles, - size_t cycle_count) { - (void)suite; - // suite[:case[:perm[:geometry[:powercycles]]]] - printf("%s:%zu:", case_->id, perm); - - // reduce duplication in geometry, this is appended to every test - if (READ_SIZE != BLOCK_SIZE || PROG_SIZE != BLOCK_SIZE) { - if (READ_SIZE != PROG_SIZE) { - leb16_print(READ_SIZE); - } - leb16_print(PROG_SIZE); - } - leb16_print(BLOCK_SIZE); - if (BLOCK_COUNT*BLOCK_SIZE != 1024*1024) { - leb16_print(BLOCK_COUNT); - } - - // only print power-cycles if any occured - if (cycles) { - printf(":"); - for (size_t i = 0; i < cycle_count; i++) { - leb16_print(cycles[i]); - } - } -} - // test suites are linked into a custom ld section extern struct test_suite __start__test_suites; @@ -165,33 +130,56 @@ const struct test_suite *test_suites = &__start__test_suites; // test define management typedef struct test_define_map { - intmax_t (*const *defines)(size_t); - const char *const *names; + const test_define_t *defines; size_t count; } test_define_map_t; -extern const test_geometry_t *test_geometry; +typedef struct test_define_names { + const char *const *names; + size_t count; +} test_define_names_t; + +intmax_t test_define_lit(void *data) { + return (intmax_t)data; +} +#define TEST_LIT(x) {test_define_lit, (void*)(uintptr_t)(x)} + #define TEST_DEFINE(k, v) \ - intmax_t test_define_##k(__attribute__((unused)) size_t define) { \ + intmax_t test_define_##k(void *data) { \ + (void)data; \ return v; \ } TEST_IMPLICIT_DEFINES #undef TEST_DEFINE -#define TEST_DEFINE_MAP_COUNT 3 +#define TEST_DEFINE_MAP_EXPLICIT 0 +#define TEST_DEFINE_MAP_OVERRIDE 1 +#define TEST_DEFINE_MAP_PERMUTATION 2 +#define TEST_DEFINE_MAP_GEOMETRY 3 +#define TEST_DEFINE_MAP_IMPLICIT 4 +#define TEST_DEFINE_MAP_COUNT 5 + test_define_map_t test_define_maps[TEST_DEFINE_MAP_COUNT] = { - {NULL, NULL, 0}, - {NULL, NULL, 0}, - { - (intmax_t (*const[TEST_IMPLICIT_DEFINE_COUNT])(size_t)){ + [TEST_DEFINE_MAP_IMPLICIT] = { + (const test_define_t[TEST_IMPLICIT_DEFINE_COUNT]) { #define TEST_DEFINE(k, v) \ - [k##_i] = test_define_##k, + [k##_i] = {test_define_##k, NULL}, TEST_IMPLICIT_DEFINES #undef TEST_DEFINE }, + TEST_IMPLICIT_DEFINE_COUNT, + }, +}; + +#define TEST_DEFINE_NAMES_SUITE 0 +#define TEST_DEFINE_NAMES_IMPLICIT 1 +#define TEST_DEFINE_NAMES_COUNT 2 + +test_define_names_t test_define_names[TEST_DEFINE_NAMES_COUNT] = { + [TEST_DEFINE_NAMES_IMPLICIT] = { (const char *const[TEST_IMPLICIT_DEFINE_COUNT]){ #define TEST_DEFINE(k, v) \ [k##_i] = #k, @@ -208,18 +196,30 @@ size_t test_define_cache_count; unsigned *test_define_cache_mask; const char *test_define_name(size_t define) { - // lookup in our test defines - for (size_t i = 0; i < TEST_DEFINE_MAP_COUNT; i++) { - if (define < test_define_maps[i].count - && test_define_maps[i].names - && test_define_maps[i].names[define]) { - return test_define_maps[i].names[define]; + // lookup in our test names + for (size_t i = 0; i < TEST_DEFINE_NAMES_COUNT; i++) { + if (define < test_define_names[i].count + && test_define_names[i].names + && test_define_names[i].names[define]) { + return test_define_names[i].names[define]; } } return NULL; } +bool test_define_ispermutation(size_t define) { + // is this define specific to the permutation? + for (size_t i = 0; i < TEST_DEFINE_MAP_IMPLICIT; i++) { + if (define < test_define_maps[i].count + && test_define_maps[i].defines[define].cb) { + return true; + } + } + + return false; +} + intmax_t test_define(size_t define) { // is the define in our cache? if (define < test_define_cache_count @@ -231,8 +231,9 @@ intmax_t test_define(size_t define) { // lookup in our test defines for (size_t i = 0; i < TEST_DEFINE_MAP_COUNT; i++) { if (define < test_define_maps[i].count - && test_define_maps[i].defines[define]) { - intmax_t v = test_define_maps[i].defines[define](define); + && test_define_maps[i].defines[define].cb) { + intmax_t v = test_define_maps[i].defines[define].cb( + test_define_maps[i].defines[define].data); // insert into cache! test_define_cache[define] = v; @@ -243,6 +244,8 @@ intmax_t test_define(size_t define) { } } + return 0; + // not found? const char *name = test_define_name(define); fprintf(stderr, "error: undefined define %s (%zd)\n", @@ -264,7 +267,8 @@ void test_define_flush(void) { const test_geometry_t *test_geometry = NULL; void test_define_geometry(const test_geometry_t *geometry) { - test_geometry = geometry; + test_define_maps[TEST_DEFINE_MAP_GEOMETRY] = (test_define_map_t){ + geometry->defines, TEST_GEOMETRY_DEFINE_COUNT}; } // override updates @@ -275,11 +279,6 @@ typedef struct test_override { const test_override_t *test_overrides = NULL; size_t test_override_count = 0; -intmax_t *test_override_map = NULL; - -intmax_t test_define_override(size_t define) { - return test_override_map[define]; -} void test_define_overrides( const test_override_t *overrides, @@ -290,8 +289,8 @@ void test_define_overrides( // suite/perm updates void test_define_suite(const struct test_suite *suite) { - test_define_maps[1].names = suite->define_names; - test_define_maps[1].count = suite->define_count; + test_define_names[TEST_DEFINE_NAMES_SUITE] = (test_define_names_t){ + suite->define_names, suite->define_count}; // make sure our cache is large enough if (lfs_max(suite->define_count, TEST_IMPLICIT_DEFINE_COUNT) @@ -310,20 +309,23 @@ void test_define_suite(const struct test_suite *suite) { // map any overrides if (test_override_count > 0) { // make sure our override arrays are big enough - if (suite->define_count > test_define_maps[0].count) { + if (suite->define_count + > test_define_maps[TEST_DEFINE_MAP_OVERRIDE].count) { // align to power of two to avoid any superlinear growth size_t ncount = 1 << lfs_npw2(suite->define_count); - test_define_maps[0].defines = realloc( - (intmax_t (**)(size_t))test_define_maps[0].defines, - ncount*sizeof(intmax_t (*)(size_t))); - test_override_map = realloc( - test_override_map, - ncount*sizeof(intmax_t)); - test_define_maps[0].count = ncount; + test_define_maps[TEST_DEFINE_MAP_OVERRIDE].defines = realloc( + (test_define_t*)test_define_maps[ + TEST_DEFINE_MAP_OVERRIDE].defines, + ncount*sizeof(test_define_t)); + test_define_maps[TEST_DEFINE_MAP_OVERRIDE].count = ncount; } - for (size_t i = 0; i < test_define_maps[0].count; i++) { - ((intmax_t (**)(size_t))test_define_maps[0].defines)[i] = NULL; + for (size_t i = 0; + i < test_define_maps[TEST_DEFINE_MAP_OVERRIDE].count; + i++) { + ((test_define_t*)test_define_maps[ + TEST_DEFINE_MAP_OVERRIDE].defines)[i] + = (test_define_t){NULL}; const char *name = test_define_name(i); if (!name) { @@ -332,9 +334,9 @@ void test_define_suite(const struct test_suite *suite) { for (size_t j = 0; j < test_override_count; j++) { if (strcmp(name, test_overrides[j].name) == 0) { - test_override_map[i] = test_overrides[j].define; - ((intmax_t (**)(size_t))test_define_maps[0].defines)[i] - = test_define_override; + ((test_define_t*)test_define_maps[ + TEST_DEFINE_MAP_OVERRIDE].defines)[i] + = (test_define_t)TEST_LIT(test_overrides[j].define); break; } } @@ -347,20 +349,26 @@ void test_define_perm( const struct test_case *case_, size_t perm) { if (case_->defines) { - test_define_maps[1].defines = case_->defines[perm]; - test_define_maps[1].count = suite->define_count; + test_define_maps[TEST_DEFINE_MAP_PERMUTATION] = (test_define_map_t){ + case_->defines[perm], suite->define_count}; } else { - test_define_maps[1].defines = NULL; - test_define_maps[1].count = 0; + test_define_maps[TEST_DEFINE_MAP_PERMUTATION] = (test_define_map_t){ + NULL, 0}; } } +void test_define_explicit( + const test_define_t *defines, + size_t define_count) { + test_define_maps[TEST_DEFINE_MAP_EXPLICIT] = (test_define_map_t){ + defines, define_count}; +} + void test_define_cleanup(void) { // test define management can allocate a few things free(test_define_cache); free(test_define_cache_mask); - free(test_override_map); - free((intmax_t (**)(size_t))test_define_maps[0].defines); + free((test_define_t*)test_define_maps[TEST_DEFINE_MAP_OVERRIDE].defines); } @@ -373,7 +381,7 @@ extern const test_powerloss_t *test_powerlosses; extern size_t test_powerloss_count; const test_id_t *test_ids = (const test_id_t[]) { - {NULL, NULL, -1, NULL, NULL, 0}, + {NULL, NULL, NULL, 0, NULL, 0}, }; size_t test_id_count = 1; @@ -440,54 +448,130 @@ void test_trace(const char *fmt, ...) { } -// how many permutations are there actually in a test case -static void count_perms( +// encode our permutation into a reusable id +static void perm_printid( const struct test_suite *suite, const struct test_case *case_, - size_t perm, - const test_geometry_t *geometry, + const lfs_testbd_powercycles_t *cycles, + size_t cycle_count) { + (void)suite; + // suite[:case[:permutation[:powercycles]]]] + printf("%s:", case_->id); + for (size_t d = 0; + d < lfs_max( + suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + if (test_define_ispermutation(d)) { + leb16_print(d); + leb16_print(test_define(d)); + } + } + + // only print power-cycles if any occured + if (cycles) { + printf(":"); + for (size_t i = 0; i < cycle_count; i++) { + leb16_print(cycles[i]); + } + } +} + +static void run_powerloss_cycles( const lfs_testbd_powercycles_t *cycles, size_t cycle_count, - size_t *perms, - size_t *filtered) { - (void)cycle_count; - size_t perms_ = 0; - size_t filtered_ = 0; + const struct test_suite *suite, + const struct test_case *case_); - for (size_t k = 0; k < case_->permutations; k++) { - if (perm != (size_t)-1 && k != perm) { - continue; - } +// iterate through permutations in a test case +static void case_forperm( + const struct test_suite *suite, + const struct test_case *case_, + const test_define_t *defines, + size_t define_count, + const lfs_testbd_powercycles_t *cycles, + size_t cycle_count, + void (*cb)( + void *data, + const struct test_suite *suite, + const struct test_case *case_, + const test_powerloss_t *powerloss), + void *data) { + if (defines) { + test_define_explicit(defines, define_count); + test_define_flush(); - // define permutation - test_define_perm(suite, case_, k); - - for (size_t g = 0; g < (geometry ? 1 : test_geometry_count); g++) { - // define geometry - test_define_geometry(geometry ? geometry : &test_geometries[g]); - test_define_flush(); - - for (size_t p = 0; p < (cycles ? 1 : test_powerloss_count); p++) { + if (cycles) { + cb(data, suite, case_, &(test_powerloss_t){ + .run=run_powerloss_cycles, + .cycles=cycles, + .cycle_count=cycle_count}); + } else { + for (size_t p = 0; p < test_powerloss_count; p++) { // skip non-reentrant tests when powerloss testing - if (!cycles - && test_powerlosses[p].short_name != '0' + if (test_powerlosses[p].short_name != '0' && !(case_->flags & TEST_REENTRANT)) { continue; } - perms_ += 1; + cb(data, suite, case_, &test_powerlosses[p]); + } + } + } else { + for (size_t k = 0; k < case_->permutations; k++) { + // define permutation + test_define_perm(suite, case_, k); - if (case_->filter && !case_->filter()) { - continue; + for (size_t g = 0; g < test_geometry_count; g++) { + // define geometry + test_define_geometry(&test_geometries[g]); + test_define_flush(); + + if (cycles) { + cb(data, suite, case_, &(test_powerloss_t){ + .run=run_powerloss_cycles, + .cycles=cycles, + .cycle_count=cycle_count}); + } else { + for (size_t p = 0; p < test_powerloss_count; p++) { + // skip non-reentrant tests when powerloss testing + if (test_powerlosses[p].short_name != '0' + && !(case_->flags & TEST_REENTRANT)) { + continue; + } + + cb(data, suite, case_, &test_powerlosses[p]); + } } - - filtered_ += 1; } } } +} - *perms += perms_; - *filtered += filtered_; + +// how many permutations are there actually in a test case +struct perm_count_state { + size_t total; + size_t filtered; +}; + +void perm_count( + void *data, + const struct test_suite *suite, + const struct test_case *case_, + const test_powerloss_t *powerloss) { + struct perm_count_state *state = data; + (void)suite; + (void)case_; + (void)powerloss; + + state->total += 1; + + if (case_->filter && !case_->filter()) { + return; + } + + state->filtered += 1; } @@ -498,8 +582,7 @@ static void summary(void) { size_t suites = 0; size_t cases = 0; test_flags_t flags = 0; - size_t perms = 0; - size_t filtered = 0; + struct perm_count_state perms = {0, 0}; for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { @@ -517,12 +600,15 @@ static void summary(void) { } cases += 1; - count_perms(&test_suites[i], &test_suites[i].cases[j], - test_ids[t].perm, - test_ids[t].geometry, + case_forperm( + &test_suites[i], + &test_suites[i].cases[j], + test_ids[t].defines, + test_ids[t].define_count, test_ids[t].cycles, test_ids[t].cycle_count, - &perms, &filtered); + perm_count, + &perms); } suites += 1; @@ -531,7 +617,7 @@ static void summary(void) { } char perm_buf[64]; - sprintf(perm_buf, "%zu/%zu", filtered, perms); + sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); char flag_buf[64]; sprintf(flag_buf, "%s%s", (flags & TEST_REENTRANT) ? "r" : "", @@ -557,8 +643,7 @@ static void list_suites(void) { test_define_suite(&test_suites[i]); size_t cases = 0; - size_t perms = 0; - size_t filtered = 0; + struct perm_count_state perms = {0, 0}; for (size_t j = 0; j < test_suites[i].case_count; j++) { if (test_ids[t].case_ && strcmp( @@ -567,16 +652,19 @@ static void list_suites(void) { } cases += 1; - count_perms(&test_suites[i], &test_suites[i].cases[j], - test_ids[t].perm, - test_ids[t].geometry, + case_forperm( + &test_suites[i], + &test_suites[i].cases[j], + test_ids[t].defines, + test_ids[t].define_count, test_ids[t].cycles, test_ids[t].cycle_count, - &perms, &filtered); + perm_count, + &perms); } char perm_buf[64]; - sprintf(perm_buf, "%zu/%zu", filtered, perms); + sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); char flag_buf[64]; sprintf(flag_buf, "%s%s", (test_suites[i].flags & TEST_REENTRANT) ? "r" : "", @@ -608,18 +696,19 @@ static void list_cases(void) { continue; } - size_t perms = 0; - size_t filtered = 0; - - count_perms(&test_suites[i], &test_suites[i].cases[j], - test_ids[t].perm, - test_ids[t].geometry, + struct perm_count_state perms = {0, 0}; + case_forperm( + &test_suites[i], + &test_suites[i].cases[j], + test_ids[t].defines, + test_ids[t].define_count, test_ids[t].cycles, test_ids[t].cycle_count, - &perms, &filtered); + perm_count, + &perms); char perm_buf[64]; - sprintf(perm_buf, "%zu/%zu", filtered, perms); + sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); char flag_buf[64]; sprintf(flag_buf, "%s%s", (test_suites[i].cases[j].flags & TEST_REENTRANT) @@ -676,101 +765,102 @@ static void list_case_paths(void) { } } -struct list_define { +struct list_defines_define { const char *name; intmax_t *values; size_t value_count; size_t value_capacity; }; -static void list_defines_perms( - const struct test_suite *suite, - const struct test_case *case_, - size_t perm, - const test_geometry_t *geometry, - struct list_define **defines, - size_t *define_count, - size_t *define_capacity) { - struct list_define *defines_ = *defines; - size_t define_count_ = *define_count; - size_t define_capacity_ = *define_capacity; +struct list_defines_defines { + struct list_defines_define *defines; + size_t define_count; + size_t define_capacity; +}; - for (size_t k = 0; k < case_->permutations; k++) { - if (perm != (size_t)-1 && k != perm) { - continue; - } +static void list_defines_add( + struct list_defines_defines *defines, + size_t d) { + const char *name = test_define_name(d); + intmax_t value = test_define(d); - // define permutation - test_define_perm(suite, case_, k); - - for (size_t g = 0; g < (geometry ? 1 : test_geometry_count); g++) { - // define geometry - test_define_geometry(geometry ? geometry : &test_geometries[g]); - test_define_flush(); - - // collect defines - for (size_t d = 0; - d < lfs_max(suite->define_count, - TEST_IMPLICIT_DEFINE_COUNT); - d++) { - if (!(d < TEST_IMPLICIT_DEFINE_COUNT || ( - case_->defines - && case_->defines[k] - && case_->defines[k][d]))) { - continue; + // define already in defines? + for (size_t i = 0; i < defines->define_count; i++) { + if (strcmp(defines->defines[i].name, name) == 0) { + // value already in values? + for (size_t j = 0; j < defines->defines[i].value_count; j++) { + if (defines->defines[i].values[j] == value) { + return; } - const char *name = test_define_name(d); - intmax_t value = test_define(d); - - // define already in defines? - for (size_t i = 0; i < define_count_; i++) { - if (strcmp(defines_[i].name, name) == 0) { - // value already in values? - for (size_t j = 0; j < defines_[i].value_count; j++) { - if (defines_[i].values[j] == value) { - goto next_define; - } - } - - *(intmax_t*)mappend( - (void**)&defines_[i].values, - sizeof(intmax_t), - &defines_[i].value_count, - &defines_[i].value_capacity) = value; - - goto next_define; - } - } - - { - // new define? - struct list_define *define = mappend( - (void**)&defines_, - sizeof(struct list_define), - &define_count_, - &define_capacity_); - define->name = name; - define->values = malloc(sizeof(intmax_t)); - define->values[0] = value; - define->value_count = 1; - define->value_capacity = 1; - } - - next_define:; } + + *(intmax_t*)mappend( + (void**)&defines->defines[i].values, + sizeof(intmax_t), + &defines->defines[i].value_count, + &defines->defines[i].value_capacity) = value; + + return; } } - *defines = defines_; - *define_count = define_count_; - *define_capacity = define_capacity_; + // new define? + struct list_defines_define *define = mappend( + (void**)&defines->defines, + sizeof(struct list_defines_define), + &defines->define_count, + &defines->define_capacity); + define->name = name; + define->values = malloc(sizeof(intmax_t)); + define->values[0] = value; + define->value_count = 1; + define->value_capacity = 1; } -static void list_defines(void) { - struct list_define *defines = NULL; - size_t define_count = 0; - size_t define_capacity = 0; +void perm_list_defines( + void *data, + const struct test_suite *suite, + const struct test_case *case_, + const test_powerloss_t *powerloss) { + struct list_defines_defines *defines = data; + (void)suite; + (void)case_; + (void)powerloss; + // collect defines + for (size_t d = 0; + d < lfs_max(suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + if (!test_define_ispermutation(d)) { + continue; + } + + list_defines_add(defines, d); + } +} + +extern const test_geometry_t builtin_geometries[]; + +static void list_defines(void) { + struct list_defines_defines defines = {NULL, 0, 0}; + + // yes we do need to define a suite, this does a bit of bookeeping + // such as setting up the define cache + test_define_suite(&(const struct test_suite){0}); + + // make sure to include builtin geometries here + for (size_t g = 0; builtin_geometries[g].long_name; g++) { + test_define_geometry(&builtin_geometries[g]); + test_define_flush(); + + // add implicit defines + for (size_t d = 0; d < TEST_IMPLICIT_DEFINE_COUNT; d++) { + list_defines_add(&defines, d); + } + } + + // add permutation defines for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { if (test_ids[t].suite && strcmp( @@ -786,67 +876,119 @@ static void list_defines(void) { continue; } - list_defines_perms(&test_suites[i], &test_suites[i].cases[j], - test_ids[t].perm, - test_ids[t].geometry, - &defines, - &define_count, - &define_capacity); + case_forperm( + &test_suites[i], + &test_suites[i].cases[j], + test_ids[t].defines, + test_ids[t].define_count, + test_ids[t].cycles, + test_ids[t].cycle_count, + perm_list_defines, + &defines); } } } - for (size_t i = 0; i < define_count; i++) { - printf("%s=", defines[i].name); - for (size_t j = 0; j < defines[i].value_count; j++) { - printf("%jd", defines[i].values[j]); - if (j != defines[i].value_count-1) { + for (size_t i = 0; i < defines.define_count; i++) { + printf("%s=", defines.defines[i].name); + for (size_t j = 0; j < defines.defines[i].value_count; j++) { + printf("%jd", defines.defines[i].values[j]); + if (j != defines.defines[i].value_count-1) { printf(","); } } printf("\n"); } - for (size_t i = 0; i < define_count; i++) { - free(defines[i].values); + for (size_t i = 0; i < defines.define_count; i++) { + free(defines.defines[i].values); } - free(defines); + free(defines.defines); } -static void list_implicit(void) { - struct list_define *defines = NULL; - size_t define_count = 0; - size_t define_capacity = 0; +static void list_permutation_defines(void) { + struct list_defines_defines defines = {NULL, 0, 0}; + // add permutation defines for (size_t t = 0; t < test_id_count; t++) { - // yes we do need to define a suite, this does a bit of bookeeping - // such as setting up the define cache - test_define_suite(&(const struct test_suite){0}); - list_defines_perms( - &(const struct test_suite){0}, - &(const struct test_case){.permutations=1}, - -1, - test_ids[t].geometry, - &defines, - &define_count, - &define_capacity); + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + if (test_ids[t].suite && strcmp( + test_suites[i].name, test_ids[t].suite) != 0) { + continue; + } + + test_define_suite(&test_suites[i]); + + for (size_t j = 0; j < test_suites[i].case_count; j++) { + if (test_ids[t].case_ && strcmp( + test_suites[i].cases[j].name, test_ids[t].case_) != 0) { + continue; + } + + case_forperm( + &test_suites[i], + &test_suites[i].cases[j], + test_ids[t].defines, + test_ids[t].define_count, + test_ids[t].cycles, + test_ids[t].cycle_count, + perm_list_defines, + &defines); + } + } } - for (size_t i = 0; i < define_count; i++) { - printf("%s=", defines[i].name); - for (size_t j = 0; j < defines[i].value_count; j++) { - printf("%jd", defines[i].values[j]); - if (j != defines[i].value_count-1) { + for (size_t i = 0; i < defines.define_count; i++) { + printf("%s=", defines.defines[i].name); + for (size_t j = 0; j < defines.defines[i].value_count; j++) { + printf("%jd", defines.defines[i].values[j]); + if (j != defines.defines[i].value_count-1) { printf(","); } } printf("\n"); } - for (size_t i = 0; i < define_count; i++) { - free(defines[i].values); + for (size_t i = 0; i < defines.define_count; i++) { + free(defines.defines[i].values); } - free(defines); + free(defines.defines); +} + +static void list_implicit_defines(void) { + struct list_defines_defines defines = {NULL, 0, 0}; + + // yes we do need to define a suite, this does a bit of bookeeping + // such as setting up the define cache + test_define_suite(&(const struct test_suite){0}); + + // make sure to include builtin geometries here + extern const test_geometry_t builtin_geometries[]; + for (size_t g = 0; builtin_geometries[g].long_name; g++) { + test_define_geometry(&builtin_geometries[g]); + test_define_flush(); + + // add implicit defines + for (size_t d = 0; d < TEST_IMPLICIT_DEFINE_COUNT; d++) { + list_defines_add(&defines, d); + } + } + + for (size_t i = 0; i < defines.define_count; i++) { + printf("%s=", defines.defines[i].name); + for (size_t j = 0; j < defines.defines[i].value_count; j++) { + printf("%jd", defines.defines[i].values[j]); + if (j != defines.defines[i].value_count-1) { + printf(","); + } + } + printf("\n"); + } + + for (size_t i = 0; i < defines.define_count; i++) { + free(defines.defines[i].values); + } + free(defines.defines); } @@ -854,51 +996,35 @@ static void list_implicit(void) { // geometries to test const test_geometry_t builtin_geometries[] = { - {'d', "default", 16, 16, 512, (1024*1024)/512}, - {'e', "eeprom", 1, 1, 512, (1024*1024)/512}, - {'E', "emmc", 512, 512, 512, (1024*1024)/512}, - {'n', "nor", 1, 1, 4096, (1024*1024)/4096}, - {'N', "nand", 4096, 4096, 32768, (1024*1024)/(32*1024)}, - {0, NULL, 0, 0, 0, 0}, + {'d', "default", {{NULL}, TEST_LIT(16), TEST_LIT(512), {NULL}}}, + {'e', "eeprom", {{NULL}, TEST_LIT(1), TEST_LIT(512), {NULL}}}, + {'E', "emmc", {{NULL}, {NULL}, TEST_LIT(512), {NULL}}}, + {'n', "nor", {{NULL}, TEST_LIT(1), TEST_LIT(4096), {NULL}}}, + {'N', "nand", {{NULL}, TEST_LIT(4096), TEST_LIT(32768), {NULL}}}, + {0, NULL, {{NULL}, {NULL}, {NULL}, {NULL}}}, }; -const test_geometry_t *test_geometries = (const test_geometry_t[]){ - {'d', "default", 16, 16, 512, (1024*1024)/512}, - {'e', "eeprom", 1, 1, 512, (1024*1024)/512}, - {'E', "emmc", 512, 512, 512, (1024*1024)/512}, - {'n', "nor", 1, 1, 4096, (1024*1024)/4096}, - {'N', "nand", 4096, 4096, 32768, (1024*1024)/(32*1024)}, -}; +const test_geometry_t *test_geometries = builtin_geometries; size_t test_geometry_count = 5; static void list_geometries(void) { - printf("%-24s %7s %7s %7s %7s %11s %s\n", - "geometry", "read", "prog", "erase", "count", "size", "leb16"); - size_t i = 0; - for (; builtin_geometries[i].long_name; i++) { - uintmax_t read_size = builtin_geometries[i].read_size; - uintmax_t prog_size = builtin_geometries[i].prog_size; - uintmax_t block_size = builtin_geometries[i].block_size; - uintmax_t block_count = builtin_geometries[i].block_count; - printf("%c,%-22s %7ju %7ju %7ju %7ju %11ju ", - builtin_geometries[i].short_name, - builtin_geometries[i].long_name, - read_size, - prog_size, - block_size, - block_count, - block_size*block_count); - if (read_size != block_size || prog_size != block_size) { - if (read_size != prog_size) { - leb16_print(read_size); - } - leb16_print(prog_size); - } - leb16_print(block_size); - if (block_count*block_size != 1024*1024) { - leb16_print(block_count); - } - printf("\n"); + // yes we do need to define a suite, this does a bit of bookeeping + // such as setting up the define cache + test_define_suite(&(const struct test_suite){0}); + + printf("%-24s %7s %7s %7s %7s %11s\n", + "geometry", "read", "prog", "erase", "count", "size"); + for (size_t g = 0; builtin_geometries[g].long_name; g++) { + test_define_geometry(&builtin_geometries[g]); + test_define_flush(); + printf("%c,%-22s %7ju %7ju %7ju %7ju %11ju\n", + builtin_geometries[g].short_name, + builtin_geometries[g].long_name, + READ_SIZE, + PROG_SIZE, + BLOCK_SIZE, + BLOCK_COUNT, + BLOCK_SIZE*BLOCK_COUNT); } } @@ -906,11 +1032,10 @@ static void list_geometries(void) { // scenarios to run tests under power-loss static void run_powerloss_none( - const struct test_suite *suite, - const struct test_case *case_, - size_t perm, const lfs_testbd_powercycles_t *cycles, - size_t cycle_count) { + size_t cycle_count, + const struct test_suite *suite, + const struct test_case *case_) { (void)cycles; (void)cycle_count; (void)suite; @@ -951,13 +1076,13 @@ static void run_powerloss_none( // run the test printf("running "); - print_id(suite, case_, perm, NULL, 0); + perm_printid(suite, case_, NULL, 0); printf("\n"); case_->run(&cfg); printf("finished "); - print_id(suite, case_, perm, NULL, 0); + perm_printid(suite, case_, NULL, 0); printf("\n"); // cleanup @@ -974,11 +1099,10 @@ static void powerloss_longjmp(void *c) { } static void run_powerloss_linear( - const struct test_suite *suite, - const struct test_case *case_, - size_t perm, const lfs_testbd_powercycles_t *cycles, - size_t cycle_count) { + size_t cycle_count, + const struct test_suite *suite, + const struct test_case *case_) { (void)cycles; (void)cycle_count; (void)suite; @@ -1025,7 +1149,7 @@ static void run_powerloss_linear( // run the test, increasing power-cycles as power-loss events occur printf("running "); - print_id(suite, case_, perm, NULL, 0); + perm_printid(suite, case_, NULL, 0); printf("\n"); while (true) { @@ -1037,7 +1161,7 @@ static void run_powerloss_linear( // power-loss! printf("powerloss "); - print_id(suite, case_, perm, NULL, 0); + perm_printid(suite, case_, NULL, 0); printf(":"); for (lfs_testbd_powercycles_t j = 1; j <= i; j++) { leb16_print(j); @@ -1049,7 +1173,7 @@ static void run_powerloss_linear( } printf("finished "); - print_id(suite, case_, perm, NULL, 0); + perm_printid(suite, case_, NULL, 0); printf("\n"); // cleanup @@ -1061,11 +1185,10 @@ static void run_powerloss_linear( } static void run_powerloss_exponential( - const struct test_suite *suite, - const struct test_case *case_, - size_t perm, const lfs_testbd_powercycles_t *cycles, - size_t cycle_count) { + size_t cycle_count, + const struct test_suite *suite, + const struct test_case *case_) { (void)cycles; (void)cycle_count; (void)suite; @@ -1112,7 +1235,7 @@ static void run_powerloss_exponential( // run the test, increasing power-cycles as power-loss events occur printf("running "); - print_id(suite, case_, perm, NULL, 0); + perm_printid(suite, case_, NULL, 0); printf("\n"); while (true) { @@ -1124,7 +1247,7 @@ static void run_powerloss_exponential( // power-loss! printf("powerloss "); - print_id(suite, case_, perm, NULL, 0); + perm_printid(suite, case_, NULL, 0); printf(":"); for (lfs_testbd_powercycles_t j = 1; j <= i; j *= 2) { leb16_print(j); @@ -1136,7 +1259,7 @@ static void run_powerloss_exponential( } printf("finished "); - print_id(suite, case_, perm, NULL, 0); + perm_printid(suite, case_, NULL, 0); printf("\n"); // cleanup @@ -1148,11 +1271,10 @@ static void run_powerloss_exponential( } static void run_powerloss_cycles( - const struct test_suite *suite, - const struct test_case *case_, - size_t perm, const lfs_testbd_powercycles_t *cycles, - size_t cycle_count) { + size_t cycle_count, + const struct test_suite *suite, + const struct test_case *case_) { (void)suite; // create block device and configuration @@ -1197,7 +1319,7 @@ static void run_powerloss_cycles( // run the test, increasing power-cycles as power-loss events occur printf("running "); - print_id(suite, case_, perm, NULL, 0); + perm_printid(suite, case_, NULL, 0); printf("\n"); while (true) { @@ -1210,7 +1332,7 @@ static void run_powerloss_cycles( // power-loss! assert(i <= cycle_count); printf("powerloss "); - print_id(suite, case_, perm, cycles, i+1); + perm_printid(suite, case_, cycles, i+1); printf("\n"); i += 1; @@ -1219,7 +1341,7 @@ static void run_powerloss_cycles( } printf("finished "); - print_id(suite, case_, perm, NULL, 0); + perm_printid(suite, case_, NULL, 0); printf("\n"); // cleanup @@ -1269,13 +1391,12 @@ static void powerloss_exhaustive_branch(void *c) { } static void run_powerloss_exhaustive_layer( + struct powerloss_exhaustive_cycles *cycles, const struct test_suite *suite, const struct test_case *case_, - size_t perm, struct lfs_config *cfg, struct lfs_testbd_config *bdcfg, - size_t depth, - struct powerloss_exhaustive_cycles *cycles) { + size_t depth) { (void)suite; struct powerloss_exhaustive_state state = { @@ -1315,13 +1436,14 @@ static void run_powerloss_exhaustive_layer( *cycle = i; printf("powerloss "); - print_id(suite, case_, perm, cycles->cycles, cycles->cycle_count); + perm_printid(suite, case_, cycles->cycles, cycles->cycle_count); printf("\n"); // now recurse cfg->context = &state.branches[i]; - run_powerloss_exhaustive_layer(suite, case_, perm, - cfg, bdcfg, depth-1, cycles); + run_powerloss_exhaustive_layer(cycles, + suite, case_, + cfg, bdcfg, depth-1); // pop the cycle cycles->cycle_count -= 1; @@ -1332,11 +1454,10 @@ static void run_powerloss_exhaustive_layer( } static void run_powerloss_exhaustive( - const struct test_suite *suite, - const struct test_case *case_, - size_t perm, const lfs_testbd_powercycles_t *cycles, - size_t cycle_count) { + size_t cycle_count, + const struct test_suite *suite, + const struct test_case *case_) { (void)cycles; (void)suite; @@ -1378,14 +1499,19 @@ static void run_powerloss_exhaustive( } // run the test, increasing power-cycles as power-loss events occur - printf("running %s:%zu\n", case_->id, perm); + printf("running "); + perm_printid(suite, case_, NULL, 0); + printf("\n"); // recursively exhaust each layer of powerlosses - run_powerloss_exhaustive_layer(suite, case_, perm, - &cfg, &bdcfg, cycle_count, - &(struct powerloss_exhaustive_cycles){NULL, 0, 0}); + run_powerloss_exhaustive_layer( + &(struct powerloss_exhaustive_cycles){NULL, 0, 0}, + suite, case_, + &cfg, &bdcfg, cycle_count); - printf("finished %s:%zu\n", case_->id, perm); + printf("finished "); + perm_printid(suite, case_, NULL, 0); + printf("\n"); } @@ -1432,63 +1558,33 @@ static void list_powerlosses(void) { // global test step count size_t test_step = 0; -// run the tests -static void run_perms( +void perm_run( + void *data, const struct test_suite *suite, const struct test_case *case_, - size_t perm, - const test_geometry_t *geometry, - const lfs_testbd_powercycles_t *cycles, - size_t cycle_count) { - for (size_t k = 0; k < case_->permutations; k++) { - if (perm != (size_t)-1 && k != perm) { - continue; - } + const test_powerloss_t *powerloss) { + (void)data; - // define permutation - test_define_perm(suite, case_, k); - - for (size_t g = 0; g < (geometry ? 1 : test_geometry_count); g++) { - // define geometry - test_define_geometry(geometry ? geometry : &test_geometries[g]); - test_define_flush(); - - for (size_t p = 0; p < (cycles ? 1 : test_powerloss_count); p++) { - // skip non-reentrant tests when powerloss testing - if (!cycles - && test_powerlosses[p].short_name != '0' - && !(case_->flags & TEST_REENTRANT)) { - continue; - } - - if (!(test_step >= test_step_start - && test_step < test_step_stop - && (test_step-test_step_start) % test_step_step == 0)) { - test_step += 1; - continue; - } - test_step += 1; - - // filter? - if (case_->filter && !case_->filter()) { - printf("skipped %s:%zu\n", case_->id, k); - continue; - } - - if (cycles) { - run_powerloss_cycles( - suite, case_, k, - cycles, - cycle_count); - } else { - test_powerlosses[p].run( - suite, case_, k, - test_powerlosses[p].cycles, - test_powerlosses[p].cycle_count); - } - } - } + // skip this step? + if (!(test_step >= test_step_start + && test_step < test_step_stop + && (test_step-test_step_start) % test_step_step == 0)) { + test_step += 1; + return; } + test_step += 1; + + // filter? + if (case_->filter && !case_->filter()) { + printf("skipped "); + perm_printid(suite, case_, NULL, 0); + printf("\n"); + return; + } + + powerloss->run( + powerloss->cycles, powerloss->cycle_count, + suite, case_); } static void run(void) { @@ -1510,11 +1606,15 @@ static void run(void) { continue; } - run_perms(&test_suites[i], &test_suites[i].cases[j], - test_ids[t].perm, - test_ids[t].geometry, + case_forperm( + &test_suites[i], + &test_suites[i].cases[j], + test_ids[t].defines, + test_ids[t].define_count, test_ids[t].cycles, - test_ids[t].cycle_count); + test_ids[t].cycle_count, + perm_run, + NULL); } } } @@ -1524,25 +1624,26 @@ static void run(void) { // option handling enum opt_flags { - OPT_HELP = 'h', - OPT_SUMMARY = 'Y', - OPT_LIST_SUITES = 'l', - OPT_LIST_CASES = 'L', - OPT_LIST_SUITE_PATHS = 1, - OPT_LIST_CASE_PATHS = 2, - OPT_LIST_DEFINES = 3, - OPT_LIST_IMPLICIT = 4, - OPT_LIST_GEOMETRIES = 5, - OPT_LIST_POWERLOSSES = 6, - OPT_DEFINE = 'D', - OPT_GEOMETRY = 'g', - OPT_POWERLOSS = 'p', - OPT_STEP = 's', - OPT_DISK = 'd', - OPT_TRACE = 't', - OPT_READ_SLEEP = 7, - OPT_PROG_SLEEP = 8, - OPT_ERASE_SLEEP = 9, + OPT_HELP = 'h', + OPT_SUMMARY = 'Y', + OPT_LIST_SUITES = 'l', + OPT_LIST_CASES = 'L', + OPT_LIST_SUITE_PATHS = 1, + OPT_LIST_CASE_PATHS = 2, + OPT_LIST_DEFINES = 3, + OPT_LIST_PERMUTATION_DEFINES = 4, + OPT_LIST_IMPLICIT_DEFINES = 5, + OPT_LIST_GEOMETRIES = 6, + OPT_LIST_POWERLOSSES = 7, + OPT_DEFINE = 'D', + OPT_GEOMETRY = 'g', + OPT_POWERLOSS = 'p', + OPT_STEP = 's', + OPT_DISK = 'd', + OPT_TRACE = 't', + OPT_READ_SLEEP = 8, + OPT_PROG_SLEEP = 9, + OPT_ERASE_SLEEP = 10, }; const char *short_opts = "hYlLD:g:p:s:d:t:"; @@ -1555,7 +1656,10 @@ const struct option long_opts[] = { {"list-suite-paths", no_argument, NULL, OPT_LIST_SUITE_PATHS}, {"list-case-paths", no_argument, NULL, OPT_LIST_CASE_PATHS}, {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, - {"list-implicit", no_argument, NULL, OPT_LIST_IMPLICIT}, + {"list-permutation-defines", + no_argument, NULL, OPT_LIST_PERMUTATION_DEFINES}, + {"list-implicit-defines", + no_argument, NULL, OPT_LIST_IMPLICIT_DEFINES}, {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, {"list-powerlosses", no_argument, NULL, OPT_LIST_POWERLOSSES}, {"define", required_argument, NULL, OPT_DEFINE}, @@ -1578,6 +1682,7 @@ const char *const help_text[] = { "List the path for each test suite.", "List the path and line number for each test case.", "List all defines in this test-runner.", + "List explicit defines in this test-runner.", "List implicit defines in this test-runner.", "List the available disk geometries.", "List the available power-loss scenarios.", @@ -1682,8 +1787,11 @@ int main(int argc, char **argv) { case OPT_LIST_DEFINES: op = list_defines; break; - case OPT_LIST_IMPLICIT: - op = list_implicit; + case OPT_LIST_PERMUTATION_DEFINES: + op = list_permutation_defines; + break; + case OPT_LIST_IMPLICIT_DEFINES: + op = list_implicit_defines; break; case OPT_LIST_GEOMETRIES: op = list_geometries; @@ -1781,18 +1889,27 @@ invalid_define: } // allow implicit r=p and p=e for common geometries - geometry->read_size = sizes[0]; - geometry->prog_size - = count >= 3 ? sizes[1] - : sizes[0]; - geometry->block_size - = count >= 3 ? sizes[2] - : count >= 2 ? sizes[1] - : sizes[0]; - // if no block_count, figure out 1 MiB total size - geometry->block_count - = count >= 4 ? sizes[3] - : (1024*1024) / geometry->block_size; + memset(geometry, 0, sizeof(test_geometry_t)); + if (count >= 3) { + geometry->defines[0] + = (test_define_t)TEST_LIT(sizes[0]); + geometry->defines[1] + = (test_define_t)TEST_LIT(sizes[1]); + geometry->defines[2] + = (test_define_t)TEST_LIT(sizes[2]); + } else if (count >= 2) { + geometry->defines[1] + = (test_define_t)TEST_LIT(sizes[0]); + geometry->defines[2] + = (test_define_t)TEST_LIT(sizes[1]); + } else { + geometry->defines[2] + = (test_define_t)TEST_LIT(sizes[0]); + } + if (count >= 4) { + geometry->defines[3] + = (test_define_t)TEST_LIT(sizes[3]); + } optarg = s; goto geometry_next; } @@ -1816,18 +1933,27 @@ invalid_define: } // allow implicit r=p and p=e for common geometries - geometry->read_size = sizes[0]; - geometry->prog_size - = count >= 3 ? sizes[1] - : sizes[0]; - geometry->block_size - = count >= 3 ? sizes[2] - : count >= 2 ? sizes[1] - : sizes[0]; - // if no block_count, figure out 1 MiB total size - geometry->block_count - = count >= 4 ? sizes[3] - : (1024*1024) / geometry->block_size; + memset(geometry, 0, sizeof(test_geometry_t)); + if (count >= 3) { + geometry->defines[0] + = (test_define_t)TEST_LIT(sizes[0]); + geometry->defines[1] + = (test_define_t)TEST_LIT(sizes[1]); + geometry->defines[2] + = (test_define_t)TEST_LIT(sizes[2]); + } else if (count >= 2) { + geometry->defines[1] + = (test_define_t)TEST_LIT(sizes[0]); + geometry->defines[2] + = (test_define_t)TEST_LIT(sizes[1]); + } else { + geometry->defines[2] + = (test_define_t)TEST_LIT(sizes[0]); + } + if (count >= 4) { + geometry->defines[3] + = (test_define_t)TEST_LIT(sizes[3]); + } optarg = s; goto geometry_next; } @@ -2090,8 +2216,8 @@ getopt_done: ; // parse test identifier, if any, cannibalizing the arg in the process for (; argc > optind; optind++) { - size_t perm = -1; - test_geometry_t *geometry = NULL; + test_define_t *defines = NULL; + size_t define_count = 0; lfs_testbd_powercycles_t *cycles = NULL; size_t cycle_count = 0; @@ -2116,88 +2242,61 @@ getopt_done: ; if (case_) { // parse case - char *perm_ = strchr(case_, ':'); - if (perm_) { - *perm_ = '\0'; - perm_ += 1; + char *defines_ = strchr(case_, ':'); + if (defines_) { + *defines_ = '\0'; + defines_ += 1; } // nothing really to do for case - if (perm_) { - // parse permutation - char *geometry_ = strchr(perm_, ':'); - if (geometry_) { - *geometry_ = '\0'; - geometry_ += 1; + if (defines_) { + // parse defines + char *cycles_ = strchr(defines_, ':'); + if (cycles_) { + *cycles_ = '\0'; + cycles_ += 1; } - char *parsed = NULL; - perm = strtoumax(perm_, &parsed, 10); - if (parsed == perm_) { - fprintf(stderr, "error: " - "could not parse test permutation: %s\n", perm_); - exit(-1); - } - - if (geometry_) { - // parse geometry - char *cycles_ = strchr(geometry_, ':'); - if (cycles_) { - *cycles_ = '\0'; - cycles_ += 1; + while (true) { + char *parsed; + size_t d = leb16_parse(defines_, &parsed); + intmax_t v = leb16_parse(parsed, &parsed); + if (parsed == defines_) { + break; } + defines_ = parsed; - geometry = malloc(sizeof(test_geometry_t)); - lfs_size_t sizes[4]; - size_t count = 0; + if (d >= define_count) { + // align to power of two to avoid any superlinear growth + size_t ncount = 1 << lfs_npw2(d+1); + defines = realloc(defines, + ncount*sizeof(test_define_t)); + memset(defines+define_count, 0, + (ncount-define_count)*sizeof(test_define_t)); + define_count = ncount; + } + defines[d] = (test_define_t)TEST_LIT(v); + } - while (*geometry_ != '\0') { - uintmax_t x = leb16_parse(geometry_, &parsed); - if (parsed == geometry_ || count >= 4) { + if (cycles_) { + // parse power cycles + size_t cycle_capacity = 0; + while (*cycles_ != '\0') { + char *parsed = NULL; + *(lfs_testbd_powercycles_t*)mappend( + (void**)&cycles, + sizeof(lfs_testbd_powercycles_t), + &cycle_count, + &cycle_capacity) + = leb16_parse(cycles_, &parsed); + if (parsed == cycles_) { fprintf(stderr, "error: " - "count not parse test geometry: %s\n", - geometry_); + "could not parse test cycles: %s\n", + cycles_); exit(-1); } - - sizes[count] = x; - count += 1; - geometry_ = parsed; - } - - // allow implicit r=p and p=e for common geometries - geometry->read_size = sizes[0]; - geometry->prog_size - = count >= 3 ? sizes[1] - : sizes[0]; - geometry->block_size - = count >= 3 ? sizes[2] - : count >= 2 ? sizes[1] - : sizes[0]; - // if no block_count, figure out 1 MiB total size - geometry->block_count - = count >= 4 ? sizes[3] - : (1024*1024) / geometry->block_size; - - if (cycles_) { - // parse power cycles - size_t cycle_capacity = 0; - while (*cycles_ != '\0') { - *(lfs_testbd_powercycles_t*)mappend( - (void**)&cycles, - sizeof(lfs_testbd_powercycles_t), - &cycle_count, - &cycle_capacity) - = leb16_parse(cycles_, &parsed); - if (parsed == cycles_) { - fprintf(stderr, "error: " - "could not parse test cycles: %s\n", - cycles_); - exit(-1); - } - cycles_ = parsed; - } + cycles_ = parsed; } } } @@ -2211,8 +2310,8 @@ getopt_done: ; &test_id_capacity) = (test_id_t){ .suite = suite, .case_ = case_, - .perm = perm, - .geometry = geometry, + .defines = defines, + .define_count = define_count, .cycles = cycles, .cycle_count = cycle_count, }; @@ -2239,7 +2338,7 @@ getopt_done: ; } if (test_id_capacity) { for (size_t i = 0; i < test_id_count; i++) { - free((test_geometry_t*)test_ids[i].geometry); + free((test_geometry_t*)test_ids[i].defines); free((lfs_testbd_powercycles_t*)test_ids[i].cycles); } free((test_id_t*)test_ids); diff --git a/runners/test_runner.h b/runners/test_runner.h index 810c7df2..813dd8b0 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -24,12 +24,17 @@ void test_trace(const char *fmt, ...); // generated test configurations +struct lfs_config; + enum test_flags { TEST_REENTRANT = 0x1, }; typedef uint8_t test_flags_t; -struct lfs_config; +typedef struct test_define { + intmax_t (*cb)(void *data); + void *data; +} test_define_t; struct test_case { const char *id; @@ -38,7 +43,7 @@ struct test_case { test_flags_t flags; size_t permutations; - intmax_t (*const *const *defines)(size_t); + const test_define_t *const *defines; bool (*filter)(void); void (*run)(struct lfs_config *cfg); @@ -89,10 +94,10 @@ intmax_t test_define(size_t define); #define POWERLOSS_BEHAVIOR test_define(POWERLOSS_BEHAVIOR_i) #define TEST_IMPLICIT_DEFINES \ - TEST_DEFINE(READ_SIZE, test_geometry->read_size) \ - TEST_DEFINE(PROG_SIZE, test_geometry->prog_size) \ - TEST_DEFINE(BLOCK_SIZE, test_geometry->block_size) \ - TEST_DEFINE(BLOCK_COUNT, test_geometry->block_count) \ + TEST_DEFINE(READ_SIZE, PROG_SIZE) \ + TEST_DEFINE(PROG_SIZE, BLOCK_SIZE) \ + TEST_DEFINE(BLOCK_SIZE, 0) \ + TEST_DEFINE(BLOCK_COUNT, (1024*1024)/BLOCK_SIZE) \ TEST_DEFINE(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ TEST_DEFINE(LOOKAHEAD_SIZE, 16) \ TEST_DEFINE(BLOCK_CYCLES, -1) \ diff --git a/scripts/test.py b/scripts/test.py index b12e4e02..170307cd 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -250,19 +250,19 @@ def compile(test_paths, **args): define_cbs[v] = name f.writeln('intmax_t %s(' '__attribute__((unused)) ' - 'size_t define) {' % name) + 'void *data) {' % name) f.writeln(4*' '+'return %s;' % v) f.writeln('}') f.writeln() - f.writeln('intmax_t (*const *const ' - '__test__%s__%s__defines[])(size_t) = {' + f.writeln('const test_define_t *const ' + '__test__%s__%s__defines[] = {' % (suite.name, case.name)) for defines in case.permutations: - f.writeln(4*' '+'(intmax_t (*const[' - 'TEST_IMPLICIT_DEFINE_COUNT+%d])(size_t)){' % ( + f.writeln(4*' '+'(const test_define_t[' + 'TEST_IMPLICIT_DEFINE_COUNT+%d]){' % ( len(suite.defines))) for k, v in sorted(defines.items()): - f.writeln(8*' '+'[%-24s] = %s,' % ( + f.writeln(8*' '+'[%-24s] = {%s, NULL},' % ( k+'_i', define_cbs[v])) f.writeln(4*' '+'},') f.writeln('};') @@ -321,8 +321,8 @@ def compile(test_paths, **args): write_case_functions(f, suite, case) else: if case.defines: - f.writeln('extern intmax_t (*const *const ' - '__test__%s__%s__defines[])(size_t);' + f.writeln('extern const test_define_t *const ' + '__test__%s__%s__defines[];' % (suite.name, case.name)) if suite.if_ is not None or case.if_ is not None: f.writeln('extern bool __test__%s__%s__filter(' @@ -472,7 +472,10 @@ def list_(runner, test_ids, **args): if args.get('list_suite_paths'): cmd.append('--list-suite-paths') if args.get('list_case_paths'): cmd.append('--list-case-paths') if args.get('list_defines'): cmd.append('--list-defines') - if args.get('list_implicit'): cmd.append('--list-implicit') + if args.get('list_permutation_defines'): + cmd.append('--list-permutation-defines') + if args.get('list_implicit_defines'): + cmd.append('--list-implicit-defines') if args.get('list_geometries'): cmd.append('--list-geometries') if args.get('list_powerlosses'): cmd.append('--list-powerlosses') @@ -554,33 +557,8 @@ def find_path(runner_, id, **args): return path def find_defines(runner_, id, **args): - # query implicit defines from runner - cmd = runner_ + ['--list-implicit'] - if args.get('verbose'): - print(' '.join(shlex.quote(c) for c in cmd)) - proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=sp.PIPE if not args.get('verbose') else None, - universal_newlines=True, - errors='replace', - close_fds=False) - implicit_defines = co.OrderedDict() - pattern = re.compile('^(?P\w+)=(?P.+)') - for line in proc.stdout: - m = pattern.match(line) - if m: - define = m.group('define') - values = m.group('values').split(',') - implicit_defines[define] = set(values) - proc.wait() - if proc.returncode != 0: - if not args.get('verbose'): - for line in proc.stderr: - sys.stdout.write(line) - sys.exit(-1) - - # query case defines from runner - cmd = runner_ + ['--list-defines', id] + # query permutation defines from runner + cmd = runner_ + ['--list-permutation-defines', id] if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, @@ -596,9 +574,7 @@ def find_defines(runner_, id, **args): if m: define = m.group('define') value = m.group('value') - if (define not in implicit_defines - or value not in implicit_defines[define]): - defines[define] = value + defines[define] = value proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -998,7 +974,8 @@ def main(**args): or args.get('list_suite_paths') or args.get('list_case_paths') or args.get('list_defines') - or args.get('list_implicit') + or args.get('list_permutation_defines') + or args.get('list_implicit_defines') or args.get('list_geometries') or args.get('list_powerlosses')): return list_(**args) @@ -1039,7 +1016,9 @@ if __name__ == "__main__": help="List the path and line number for each test case.") test_parser.add_argument('--list-defines', action='store_true', help="List all defines in this test-runner.") - test_parser.add_argument('--list-implicit', action='store_true', + test_parser.add_argument('--list-permutation-defines', action='store_true', + help="List explicit defines in this test-runner.") + test_parser.add_argument('--list-implicit-defines', action='store_true', help="List implicit defines in this test-runner.") test_parser.add_argument('--list-geometries', action='store_true', help="List the available disk geometries.") From 03c1a4ee2e3df8e6450ed8cd6177d7ca4981b3f9 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 11 Sep 2022 03:10:08 -0500 Subject: [PATCH 35/81] Added permutations and ranges to test defines This is really more work for the bench runner. With this change defines can be manipulated at a rather high level at runtime. Which should be useful for generating benchmarks across various dimensions. The define grammar in the test_runner is now a bit more powerful, accepting: 1. A single value: -DN=42 2. A list of values, which get permuted: -DN=1,2,3 3. A range: -DN=range(10) 4. Some combo: -DN=1,2,range(3,0,-1) This is more complex in the test .toml defines, which can also be C expressions: 1. A single value: define=42 2. A single expression: define='42*42' 3. A list: define=[1,2,3] 4. A comma separated string: define='1,2,3' 5. A range: define='42*range(10)' 6. This mess: define=[1,2,'3,4,range(2)*range(2)+3'] --- runners/test_runner.c | 351 +++++++++++++++++++++++++++------------ runners/test_runner.h | 2 +- scripts/test.py | 80 +++++++-- tests/test_dirs.toml | 12 +- tests/test_move.toml | 4 +- tests/test_truncate.toml | 2 +- 6 files changed, 324 insertions(+), 127 deletions(-) diff --git a/runners/test_runner.c b/runners/test_runner.c index 4a31ccfc..cb7dd9f4 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -274,18 +274,17 @@ void test_define_geometry(const test_geometry_t *geometry) { // override updates typedef struct test_override { const char *name; - intmax_t define; + const intmax_t *defines; + size_t permutations; } test_override_t; const test_override_t *test_overrides = NULL; size_t test_override_count = 0; -void test_define_overrides( - const test_override_t *overrides, - size_t override_count) { - test_overrides = overrides; - test_override_count = override_count; -} +test_define_t *test_override_defines = NULL; +size_t test_override_define_count = 0; +size_t test_override_define_permutations = 1; +size_t test_override_define_capacity = 0; // suite/perm updates void test_define_suite(const struct test_suite *suite) { @@ -308,35 +307,63 @@ void test_define_suite(const struct test_suite *suite) { // map any overrides if (test_override_count > 0) { + // first figure out the total size of override permutations + size_t count = 0; + size_t permutations = 1; + for (size_t i = 0; i < test_override_count; i++) { + for (size_t d = 0; + d < lfs_max( + suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + // define name match? + const char *name = test_define_name(d); + if (name && strcmp(name, test_overrides[i].name) == 0) { + count = d+1; + permutations *= test_overrides[i].permutations; + break; + } + } + } + test_override_define_count = count; + test_override_define_permutations = permutations; + // make sure our override arrays are big enough - if (suite->define_count - > test_define_maps[TEST_DEFINE_MAP_OVERRIDE].count) { + if (count * permutations > test_override_define_capacity) { // align to power of two to avoid any superlinear growth - size_t ncount = 1 << lfs_npw2(suite->define_count); - test_define_maps[TEST_DEFINE_MAP_OVERRIDE].defines = realloc( - (test_define_t*)test_define_maps[ - TEST_DEFINE_MAP_OVERRIDE].defines, - ncount*sizeof(test_define_t)); - test_define_maps[TEST_DEFINE_MAP_OVERRIDE].count = ncount; + size_t ncapacity = 1 << lfs_npw2(count * permutations); + test_override_defines = realloc( + test_override_defines, + sizeof(test_define_t)*ncapacity); + test_override_define_capacity = ncapacity; } - for (size_t i = 0; - i < test_define_maps[TEST_DEFINE_MAP_OVERRIDE].count; - i++) { - ((test_define_t*)test_define_maps[ - TEST_DEFINE_MAP_OVERRIDE].defines)[i] - = (test_define_t){NULL}; + // zero unoverridden defines + memset(test_override_defines, 0, + sizeof(test_define_t) * count * permutations); - const char *name = test_define_name(i); - if (!name) { - continue; - } + // compute permutations + size_t p = 1; + for (size_t i = 0; i < test_override_count; i++) { + for (size_t d = 0; + d < lfs_max( + suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + // define name match? + const char *name = test_define_name(d); + if (name && strcmp(name, test_overrides[i].name) == 0) { + // scatter the define permutations based on already + // seen permutations + for (size_t j = 0; j < permutations; j++) { + test_override_defines[j*count + d] + = (test_define_t)TEST_LIT( + test_overrides[i].defines[(j/p) + % test_overrides[i].permutations]); + } - for (size_t j = 0; j < test_override_count; j++) { - if (strcmp(name, test_overrides[j].name) == 0) { - ((test_define_t*)test_define_maps[ - TEST_DEFINE_MAP_OVERRIDE].defines)[i] - = (test_define_t)TEST_LIT(test_overrides[j].define); + // keep track of how many permutations we've seen so far + p *= test_overrides[i].permutations; break; } } @@ -350,13 +377,20 @@ void test_define_perm( size_t perm) { if (case_->defines) { test_define_maps[TEST_DEFINE_MAP_PERMUTATION] = (test_define_map_t){ - case_->defines[perm], suite->define_count}; + case_->defines + perm*suite->define_count, + suite->define_count}; } else { test_define_maps[TEST_DEFINE_MAP_PERMUTATION] = (test_define_map_t){ NULL, 0}; } } +void test_define_override(size_t perm) { + test_define_maps[TEST_DEFINE_MAP_OVERRIDE] = (test_define_map_t){ + test_override_defines + perm*test_override_define_count, + test_override_define_count}; +} + void test_define_explicit( const test_define_t *defines, size_t define_count) { @@ -368,7 +402,7 @@ void test_define_cleanup(void) { // test define management can allocate a few things free(test_define_cache); free(test_define_cache_mask); - free((test_define_t*)test_define_maps[TEST_DEFINE_MAP_OVERRIDE].defines); + free(test_override_defines); } @@ -522,25 +556,30 @@ static void case_forperm( // define permutation test_define_perm(suite, case_, k); - for (size_t g = 0; g < test_geometry_count; g++) { - // define geometry - test_define_geometry(&test_geometries[g]); - test_define_flush(); + for (size_t v = 0; v < test_override_define_permutations; v++) { + // define override permutation + test_define_override(v); - if (cycles) { - cb(data, suite, case_, &(test_powerloss_t){ - .run=run_powerloss_cycles, - .cycles=cycles, - .cycle_count=cycle_count}); - } else { - for (size_t p = 0; p < test_powerloss_count; p++) { - // skip non-reentrant tests when powerloss testing - if (test_powerlosses[p].short_name != '0' - && !(case_->flags & TEST_REENTRANT)) { - continue; + for (size_t g = 0; g < test_geometry_count; g++) { + // define geometry + test_define_geometry(&test_geometries[g]); + test_define_flush(); + + if (cycles) { + cb(data, suite, case_, &(test_powerloss_t){ + .run=run_powerloss_cycles, + .cycles=cycles, + .cycle_count=cycle_count}); + } else { + for (size_t p = 0; p < test_powerloss_count; p++) { + // skip non-reentrant tests when powerloss testing + if (test_powerlosses[p].short_name != '0' + && !(case_->flags & TEST_REENTRANT)) { + continue; + } + + cb(data, suite, case_, &test_powerlosses[p]); } - - cb(data, suite, case_, &test_powerlosses[p]); } } } @@ -832,11 +871,31 @@ void perm_list_defines( d < lfs_max(suite->define_count, TEST_IMPLICIT_DEFINE_COUNT); d++) { - if (!test_define_ispermutation(d)) { - continue; + if (d < TEST_IMPLICIT_DEFINE_COUNT + || test_define_ispermutation(d)) { + list_defines_add(defines, d); } + } +} - list_defines_add(defines, d); +void perm_list_permutation_defines( + void *data, + const struct test_suite *suite, + const struct test_case *case_, + const test_powerloss_t *powerloss) { + struct list_defines_defines *defines = data; + (void)suite; + (void)case_; + (void)powerloss; + + // collect permutation_defines + for (size_t d = 0; + d < lfs_max(suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + if (test_define_ispermutation(d)) { + list_defines_add(defines, d); + } } } @@ -845,22 +904,7 @@ extern const test_geometry_t builtin_geometries[]; static void list_defines(void) { struct list_defines_defines defines = {NULL, 0, 0}; - // yes we do need to define a suite, this does a bit of bookeeping - // such as setting up the define cache - test_define_suite(&(const struct test_suite){0}); - - // make sure to include builtin geometries here - for (size_t g = 0; builtin_geometries[g].long_name; g++) { - test_define_geometry(&builtin_geometries[g]); - test_define_flush(); - - // add implicit defines - for (size_t d = 0; d < TEST_IMPLICIT_DEFINE_COUNT; d++) { - list_defines_add(&defines, d); - } - } - - // add permutation defines + // add defines for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { if (test_ids[t].suite && strcmp( @@ -932,7 +976,7 @@ static void list_permutation_defines(void) { test_ids[t].define_count, test_ids[t].cycles, test_ids[t].cycle_count, - perm_list_defines, + perm_list_permutation_defines, &defines); } } @@ -1700,10 +1744,7 @@ const char *const help_text[] = { int main(int argc, char **argv) { void (*op)(void) = run; - test_override_t *overrides = NULL; - size_t override_count = 0; - size_t override_capacity = 0; - + size_t test_override_capacity = 0; size_t test_geometry_capacity = 0; size_t test_powerloss_capacity = 0; size_t test_id_capacity = 0; @@ -1803,10 +1844,10 @@ int main(int argc, char **argv) { case OPT_DEFINE: { // allocate space test_override_t *override = mappend( - (void**)&overrides, + (void**)&test_overrides, sizeof(test_override_t), - &override_count, - &override_capacity); + &test_override_count, + &test_override_capacity); // parse into string key/intmax_t value, cannibalizing the // arg in the process @@ -1815,13 +1856,112 @@ int main(int argc, char **argv) { if (!sep) { goto invalid_define; } - override->define = strtoumax(sep+1, &parsed, 0); - if (parsed == sep+1) { - goto invalid_define; - } - - override->name = optarg; *sep = '\0'; + override->name = optarg; + optarg = sep+1; + + // parse comma-separated permutations + { + override->defines = NULL; + override->permutations = 0; + size_t override_capacity = 0; + while (true) { + optarg += strspn(optarg, " "); + + if (strncmp(optarg, "range", strlen("range")) == 0) { + // range of values + optarg += strlen("range"); + optarg += strspn(optarg, " "); + if (*optarg != '(') { + goto invalid_define; + } + optarg += 1; + + intmax_t start = strtoumax(optarg, &parsed, 0); + intmax_t stop = -1; + intmax_t step = 1; + // allow empty string for start=0 + if (parsed == optarg) { + start = 0; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != ')') { + goto invalid_define; + } + + if (*optarg == ',') { + optarg += 1; + stop = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=end + if (parsed == optarg) { + stop = -1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != ')') { + goto invalid_define; + } + + if (*optarg == ',') { + optarg += 1; + step = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=1 + if (parsed == optarg) { + step = 1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ')') { + goto invalid_define; + } + } + } else { + // single value = stop only + stop = start; + start = 0; + } + + if (*optarg != ')') { + goto invalid_define; + } + optarg += 1; + + // calculate the range of values + assert(step != 0); + for (intmax_t i = start; + (step < 0) + ? i > stop + : (uintmax_t)i < (uintmax_t)stop; + i += step) { + *(intmax_t*)mappend( + (void**)&override->defines, + sizeof(intmax_t), + &override->permutations, + &override_capacity) = i; + } + } else if (*optarg != '\0') { + // single value + intmax_t define = strtoimax(optarg, &parsed, 0); + if (parsed == optarg) { + goto invalid_define; + } + *(intmax_t*)mappend( + (void**)&override->defines, + sizeof(intmax_t), + &override->permutations, + &override_capacity) = define; + } else { + break; + } + + optarg = parsed + strspn(parsed, " "); + if (*optarg == ',') { + optarg += 1; + } + } + } + assert(override->permutations > 0); break; invalid_define: @@ -2117,10 +2257,12 @@ powerloss_next: } case OPT_STEP: { char *parsed = NULL; - size_t start = strtoumax(optarg, &parsed, 0); + test_step_start = strtoumax(optarg, &parsed, 0); + test_step_stop = -1; + test_step_step = 1; // allow empty string for start=0 - if (parsed != optarg) { - test_step_start = start; + if (parsed == optarg) { + test_step_start = 0; } optarg = parsed + strspn(parsed, " "); @@ -2130,10 +2272,10 @@ powerloss_next: if (*optarg == ',') { optarg += 1; - size_t stop = strtoumax(optarg, &parsed, 0); + test_step_stop = strtoumax(optarg, &parsed, 0); // allow empty string for stop=end - if (parsed != optarg) { - test_step_stop = stop; + if (parsed == optarg) { + test_step_stop = -1; } optarg = parsed + strspn(parsed, " "); @@ -2143,10 +2285,10 @@ powerloss_next: if (*optarg == ',') { optarg += 1; - size_t step = strtoumax(optarg, &parsed, 0); + test_step_step = strtoumax(optarg, &parsed, 0); // allow empty string for stop=1 - if (parsed != optarg) { - test_step_step = step; + if (parsed == optarg) { + test_step_step = 1; } optarg = parsed + strspn(parsed, " "); @@ -2154,6 +2296,10 @@ powerloss_next: goto step_unknown; } } + } else { + // single value = stop only + test_step_stop = test_step_start; + test_step_start = 0; } break; @@ -2317,30 +2463,31 @@ getopt_done: ; }; } - // register overrides - test_define_overrides(overrides, override_count); - // do the thing op(); // cleanup (need to be done for valgrind testing) test_define_cleanup(); - free(overrides); - + if (test_overrides) { + for (size_t i = 0; i < test_override_count; i++) { + free((void*)test_overrides[i].defines); + } + free((void*)test_overrides); + } if (test_geometry_capacity) { - free((test_geometry_t*)test_geometries); + free((void*)test_geometries); } if (test_powerloss_capacity) { for (size_t i = 0; i < test_powerloss_count; i++) { - free((lfs_testbd_powercycles_t*)test_powerlosses[i].cycles); + free((void*)test_powerlosses[i].cycles); } - free((test_powerloss_t*)test_powerlosses); + free((void*)test_powerlosses); } if (test_id_capacity) { for (size_t i = 0; i < test_id_count; i++) { - free((test_geometry_t*)test_ids[i].defines); - free((lfs_testbd_powercycles_t*)test_ids[i].cycles); + free((void*)test_ids[i].defines); + free((void*)test_ids[i].cycles); } - free((test_id_t*)test_ids); + free((void*)test_ids); } } diff --git a/runners/test_runner.h b/runners/test_runner.h index 813dd8b0..c27b9856 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -43,7 +43,7 @@ struct test_case { test_flags_t flags; size_t permutations; - const test_define_t *const *defines; + const test_define_t *defines; bool (*filter)(void); void (*run)(struct lfs_config *cfg); diff --git a/scripts/test.py b/scripts/test.py index 170307cd..fa41af3b 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -62,6 +62,7 @@ class TestCase: self.defines = set() self.permutations = [] + # defines can be a dict or a list or dicts suite_defines = config.pop('suite_defines', {}) if not isinstance(suite_defines, list): suite_defines = [suite_defines] @@ -69,15 +70,63 @@ class TestCase: if not isinstance(defines, list): defines = [defines] + def csplit(v): + # split commas but only outside of parens + parens = 0 + i_ = 0 + for i in range(len(v)): + if v[i] == ',' and parens == 0: + yield v[i_:i] + i_ = i+1 + elif v[i] in '([{': + parens += 1 + elif v[i] in '}])': + parens -= 1 + if v[i_:].strip(): + yield v[i_:] + + def parse_define(v): + # a define entry can be a list + if isinstance(v, list): + for v_ in v: + yield from parse_define(v_) + # or a string + elif isinstance(v, str): + # which can be comma-separated values, with optional + # range statements. This matches the runtime define parser in + # the runner itself. + for v_ in csplit(v): + m = re.search(r'\brange\b\s*\(' + '(?P[^,\s]*)' + '\s*(?:,\s*(?P[^,\s]*)' + '\s*(?:,\s*(?P[^,\s]*)\s*)?)?\)', + v_) + if m: + start = (int(m.group('start'), 0) + if m.group('start') else 0) + stop = (int(m.group('stop'), 0) + if m.group('stop') else None) + step = (int(m.group('step'), 0) + if m.group('step') else 1) + if m.lastindex <= 1: + start, stop = 0, start + for x in range(start, stop, step): + yield from parse_define('%s(%d)%s' % ( + v_[:m.start()], x, v_[m.end():])) + else: + yield v_ + # or a literal value + else: + yield v + # build possible permutations for suite_defines_ in suite_defines: self.defines |= suite_defines_.keys() for defines_ in defines: self.defines |= defines_.keys() - self.permutations.extend(map(dict, it.product(*( - [(k, v) for v in (vs if isinstance(vs, list) else [vs])] - for k, vs in sorted( - (suite_defines_ | defines_).items()))))) + self.permutations.extend(dict(perm) for perm in it.product(*( + [(k, v) for v in parse_define(vs)] + for k, vs in sorted((suite_defines_ | defines_).items())))) for k in config.keys(): print('%swarning:%s in %s, found unused key %r' % ( @@ -254,13 +303,12 @@ def compile(test_paths, **args): f.writeln(4*' '+'return %s;' % v) f.writeln('}') f.writeln() - f.writeln('const test_define_t *const ' - '__test__%s__%s__defines[] = {' - % (suite.name, case.name)) + f.writeln('const test_define_t ' + '__test__%s__%s__defines[][' + 'TEST_IMPLICIT_DEFINE_COUNT+%d] = {' + % (suite.name, case.name, len(suite.defines))) for defines in case.permutations: - f.writeln(4*' '+'(const test_define_t[' - 'TEST_IMPLICIT_DEFINE_COUNT+%d]){' % ( - len(suite.defines))) + f.writeln(4*' '+'{') for k, v in sorted(defines.items()): f.writeln(8*' '+'[%-24s] = {%s, NULL},' % ( k+'_i', define_cbs[v])) @@ -321,9 +369,10 @@ def compile(test_paths, **args): write_case_functions(f, suite, case) else: if case.defines: - f.writeln('extern const test_define_t *const ' - '__test__%s__%s__defines[];' - % (suite.name, case.name)) + f.writeln('extern const test_define_t ' + '__test__%s__%s__defines[][' + 'TEST_IMPLICIT_DEFINE_COUNT+%d];' + % (suite.name, case.name, len(suite.defines))) if suite.if_ is not None or case.if_ is not None: f.writeln('extern bool __test__%s__%s__filter(' 'void);' @@ -368,7 +417,8 @@ def compile(test_paths, **args): f.writeln(12*' '+'.permutations = %d,' % len(case.permutations)) if case.defines: - f.writeln(12*' '+'.defines = __test__%s__%s__defines,' + f.writeln(12*' '+'.defines ' + '= (const test_define_t*)__test__%s__%s__defines,' % (suite.name, case.name)) if suite.if_ is not None or case.if_ is not None: f.writeln(12*' '+'.filter = __test__%s__%s__filter,' @@ -1088,7 +1138,7 @@ if __name__ == "__main__": help="Output file.") # runner + test_ids overlaps test_paths, so we need to do some munging here - args = parser.parse_args() + args = parser.parse_intermixed_args() args.test_paths = [' '.join(args.runner or [])] + args.test_ids args.runner = args.runner or [RUNNER_PATH] diff --git a/tests/test_dirs.toml b/tests/test_dirs.toml index 60346c0c..07107885 100644 --- a/tests/test_dirs.toml +++ b/tests/test_dirs.toml @@ -18,7 +18,7 @@ code = ''' ''' [cases.many_dir_creation] -defines.N = [3,6,9,12,21,33,57,66,72,93,99] +defines.N = 'range(3, 100, 3)' if = 'N < BLOCK_COUNT/2' code = ''' lfs_t lfs; @@ -55,7 +55,7 @@ code = ''' ''' [cases.many_dir_removal] -defines.N = [3,6,9,12,21,33,57,66,72,93,99] +defines.N = 'range(3, 100, 11)' if = 'N < BLOCK_COUNT/2' code = ''' lfs_t lfs; @@ -112,7 +112,7 @@ code = ''' ''' [cases.many_dir_rename] -defines.N = [3,6,9,12,21,33,57,66,72,93,99] +defines.N = 'range(3, 100, 11)' if = 'N < BLOCK_COUNT/2' code = ''' lfs_t lfs; @@ -266,7 +266,7 @@ code = ''' ''' [cases.file_creation] -defines.N = [3,6,9,12,21,33,57,66,72,93,99] +defines.N = 'range(3, 100, 11)' if = 'N < BLOCK_COUNT/2' code = ''' lfs_t lfs; @@ -306,7 +306,7 @@ code = ''' ''' [cases.file_removal] -defines.N = [3,6,9,12,21,33,57,66,72,93,99] +defines.N = 'range(3, 100, 11)' if = 'N < BLOCK_COUNT/2' code = ''' lfs_t lfs; @@ -366,7 +366,7 @@ code = ''' ''' [cases.file_rename] -defines.N = [3,6,9,12,21,33,57,66,72,93,99] +defines.N = 'range(3, 100, 11)' if = 'N < BLOCK_COUNT/2' code = ''' lfs_t lfs; diff --git a/tests/test_move.toml b/tests/test_move.toml index f1825e48..dc2623e3 100644 --- a/tests/test_move.toml +++ b/tests/test_move.toml @@ -1586,7 +1586,7 @@ code = ''' # move fix in relocation [cases.move_fix_relocation] in = "lfs.c" -defines.RELOCATIONS = [0x0, 0x1, 0x2, 0x3] +defines.RELOCATIONS = 'range(4)' defines.ERASE_CYCLES = 0xffffffff code = ''' lfs_t lfs; @@ -1731,7 +1731,7 @@ code = ''' # move fix in relocation with predecessor [cases.move_fix_relocation_predecessor] in = "lfs.c" -defines.RELOCATIONS = [0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7] +defines.RELOCATIONS = 'range(8)' defines.ERASE_CYCLES = 0xffffffff code = ''' lfs_t lfs; diff --git a/tests/test_truncate.toml b/tests/test_truncate.toml index fc83ce37..80e250fe 100644 --- a/tests/test_truncate.toml +++ b/tests/test_truncate.toml @@ -285,7 +285,7 @@ code = ''' # more aggressive general truncation tests [cases.aggressive_truncate] -defines.CONFIG = [0,1,2,3,4,5] +defines.CONFIG = 'range(6)' defines.SMALLSIZE = 32 defines.MEDIUMSIZE = 2048 defines.LARGESIZE = 8192 From 23fba40f20f0fada775e7625ecfc001b2ee88187 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 12 Sep 2022 12:17:46 -0500 Subject: [PATCH 36/81] Added option for updating a CSV file with test results This is mostly for the bench runner which will contain more interesting results besides just pass/fail. --- scripts/tailpipe.py | 2 + scripts/test.py | 131 +++++++++++++++++++++++++++++++------------- scripts/tracebd.py | 4 ++ 3 files changed, 100 insertions(+), 37 deletions(-) diff --git a/scripts/tailpipe.py b/scripts/tailpipe.py index 101fd98c..ef66d32e 100755 --- a/scripts/tailpipe.py +++ b/scripts/tailpipe.py @@ -41,6 +41,8 @@ def main(path='-', *, lines=1, sleep=0.01, keep_open=False): event.set() if not keep_open: break + # don't just flood open calls + time.sleep(sleep) done = True th.Thread(target=read, daemon=True).start() diff --git a/scripts/test.py b/scripts/test.py index fa41af3b..e7091964 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -4,6 +4,7 @@ # import collections as co +import csv import errno import glob import itertools as it @@ -26,7 +27,7 @@ HEADER_PATH = 'runners/test_runner.h' def openio(path, mode='r', buffering=-1, nb=False): if path == '-': - if 'r' in mode: + if mode == 'r': return os.fdopen(os.dup(sys.stdin.fileno()), 'r', buffering) else: return os.fdopen(os.dup(sys.stdout.fileno()), 'w', buffering) @@ -475,9 +476,8 @@ def compile(test_paths, **args): f.writeln('#endif') f.writeln() -def find_runner(runner, test_ids, **args): +def find_runner(runner, **args): cmd = runner.copy() - cmd.extend(test_ids) # run under some external command? cmd[:0] = args.get('exec', []) @@ -514,8 +514,8 @@ def find_runner(runner, test_ids, **args): return cmd -def list_(runner, test_ids, **args): - cmd = find_runner(runner, test_ids, **args) +def list_(runner, test_ids=[], **args): + cmd = find_runner(runner, **args) + test_ids if args.get('summary'): cmd.append('--summary') if args.get('list_suites'): cmd.append('--list-suites') if args.get('list_cases'): cmd.append('--list-cases') @@ -534,9 +534,9 @@ def list_(runner, test_ids, **args): return sp.call(cmd) -def find_cases(runner_, **args): +def find_cases(runner_, ids=[], **args): # query from runner - cmd = runner_ + ['--list-cases'] + cmd = runner_ + ['--list-cases'] + ids if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, @@ -635,6 +635,41 @@ def find_defines(runner_, id, **args): return defines +# Thread-safe CSV writer +class TestOutput: + def __init__(self, path, head=None, tail=None): + self.f = openio(path, 'w+', 1) + self.lock = th.Lock() + self.head = head or [] + self.tail = tail or [] + self.writer = csv.DictWriter(self.f, self.head + self.tail) + self.rows = [] + + def close(self): + self.f.close() + + def __enter__(self): + return self + + def __exit__(self, *_): + self.f.close() + + def writerow(self, row): + with self.lock: + self.rows.append(row) + if all(k in self.head or k in self.tail for k in row.keys()): + # can simply append + self.writer.writerow(row) + else: + # need to rewrite the file + self.head.extend(row.keys() - (self.head + self.tail)) + self.f.truncate() + self.writer = csv.DictWriter(self.f, self.head + self.tail) + self.writer.writeheader() + for row in self.rows: + self.writer.writerow(row) + +# A test failure class TestFailure(Exception): def __init__(self, id, returncode, stdout, assert_=None): self.id = id @@ -642,10 +677,10 @@ class TestFailure(Exception): self.stdout = stdout self.assert_ = assert_ -def run_stage(name, runner_, **args): +def run_stage(name, runner_, ids, output_, **args): # get expected suite/case/perm counts expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( - find_cases(runner_, **args)) + find_cases(runner_, ids, **args)) passed_suite_perms = co.defaultdict(lambda: 0) passed_case_perms = co.defaultdict(lambda: 0) @@ -662,7 +697,7 @@ def run_stage(name, runner_, **args): locals = th.local() children = set() - def run_runner(runner_): + def run_runner(runner_, ids=[]): nonlocal passed_suite_perms nonlocal passed_case_perms nonlocal passed_perms @@ -670,7 +705,7 @@ def run_stage(name, runner_, **args): nonlocal locals # run the tests! - cmd = runner_.copy() + cmd = runner_ + ids if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) @@ -726,6 +761,14 @@ def run_stage(name, runner_, **args): passed_suite_perms[m.group('suite')] += 1 passed_case_perms[m.group('case')] += 1 passed_perms += 1 + if output_: + # get defines and write to csv + defines = find_defines( + runner_, m.group('id'), **args) + output_.writerow({ + 'case': m.group('case'), + 'test_pass': 1, + **defines}) elif op == 'skipped': locals.seen_perms += 1 elif op == 'assert': @@ -750,7 +793,7 @@ def run_stage(name, runner_, **args): last_stdout, last_assert) - def run_job(runner, start=None, step=None): + def run_job(runner_, ids=[], start=None, step=None): nonlocal failures nonlocal killed nonlocal locals @@ -758,20 +801,30 @@ def run_stage(name, runner_, **args): start = start or 0 step = step or 1 while start < total_perms: - runner_ = runner.copy() + job_runner = runner_.copy() if args.get('isolate') or args.get('valgrind'): - runner_.append('-s%s,%s,%s' % (start, start+step, step)) + job_runner.append('-s%s,%s,%s' % (start, start+step, step)) else: - runner_.append('-s%s,,%s' % (start, step)) + job_runner.append('-s%s,,%s' % (start, step)) try: # run the tests locals.seen_perms = 0 - run_runner(runner_) + run_runner(job_runner, ids) assert locals.seen_perms > 0 start += locals.seen_perms*step except TestFailure as failure: + # keep track of failures + if output_: + suite, case, _ = failure.id.split(':', 2) + # get defines and write to csv + defines = find_defines(runner_, failure.id, **args) + output_.writerow({ + 'case': ':'.join([suite, case]), + 'test_pass': 0, + **defines}) + # race condition for multiple failures? if failures and not args.get('keep_going'): break @@ -796,11 +849,11 @@ def run_stage(name, runner_, **args): if 'jobs' in args: for job in range(args['jobs']): runners.append(th.Thread( - target=run_job, args=(runner_, job, args['jobs']), + target=run_job, args=(runner_, ids, job, args['jobs']), daemon=True)) else: runners.append(th.Thread( - target=run_job, args=(runner_, None, None), + target=run_job, args=(runner_, ids, None, None), daemon=True)) def print_update(done): @@ -861,13 +914,12 @@ def run_stage(name, runner_, **args): killed) -def run(runner, test_ids, **args): +def run(runner, test_ids=[], **args): # query runner for tests - runner_ = find_runner(runner, test_ids, **args) - print('using runner: %s' - % ' '.join(shlex.quote(c) for c in runner_)) + runner_ = find_runner(runner, **args) + print('using runner: %s' % ' '.join(shlex.quote(c) for c in runner_)) expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( - find_cases(runner_, **args)) + find_cases(runner_, test_ids, **args)) print('found %d suites, %d cases, %d/%d permutations' % (len(expected_suite_perms), len(expected_case_perms), @@ -882,6 +934,9 @@ def run(runner, test_ids, **args): trace = None if args.get('trace'): trace = openio(args['trace'], 'w', 1) + output = None + if args.get('output'): + output = TestOutput(args['output'], ['case'], ['test_pass']) # measure runtime start = time.time() @@ -894,14 +949,12 @@ def run(runner, test_ids, **args): for by in (expected_case_perms.keys() if args.get('by_cases') else expected_suite_perms.keys() if args.get('by_suites') else [None]): - # rebuild runner for each stage to override test identifier if needed - stage_runner = find_runner(runner, - [by] if by is not None else test_ids, **args) - # spawn jobs for stage expected_, passed_, powerlosses_, failures_, killed = run_stage( by or 'tests', - stage_runner, + runner_, + [by] if by is not None else test_ids, + output, **args) expected += expected_ passed += passed_ @@ -916,6 +969,8 @@ def run(runner, test_ids, **args): stdout.close() if trace: trace.close() + if output: + output.close() # show summary print() @@ -975,29 +1030,29 @@ def run(runner, test_ids, **args): or args.get('gdb_case') or args.get('gdb_main')): failure = failures[0] - runner_ = find_runner(runner, [failure.id], **args) + cmd = runner_ + [failure.id] if args.get('gdb_main'): - cmd = ['gdb', + cmd[:0] = ['gdb', '-ex', 'break main', '-ex', 'run', - '--args'] + runner_ + '--args'] elif args.get('gdb_case'): path, lineno = find_path(runner_, failure.id, **args) - cmd = ['gdb', + cmd[:0] = ['gdb', '-ex', 'break %s:%d' % (path, lineno), '-ex', 'run', - '--args'] + runner_ + '--args'] elif failure.assert_ is not None: - cmd = ['gdb', + cmd[:0] = ['gdb', '-ex', 'run', '-ex', 'frame function raise', '-ex', 'up 2', - '--args'] + runner_ + '--args'] else: - cmd = ['gdb', + cmd[:0] = ['gdb', '-ex', 'run', - '--args'] + runner_ + '--args'] # exec gdb interactively if args.get('verbose'): @@ -1088,6 +1143,8 @@ if __name__ == "__main__": help="Direct trace output to this file.") test_parser.add_argument('-O', '--stdout', help="Direct stdout to this file. Note stderr is already merged here.") + test_parser.add_argument('-o', '--output', + help="CSV file to store results.") test_parser.add_argument('--read-sleep', help="Artificial read delay in seconds.") test_parser.add_argument('--prog-sleep', diff --git a/scripts/tracebd.py b/scripts/tracebd.py index daae0c08..a486c296 100755 --- a/scripts/tracebd.py +++ b/scripts/tracebd.py @@ -600,6 +600,8 @@ def main(path='-', *, time.sleep(sleep) if not keep_open: break + # don't just flood open calls + time.sleep(sleep) except KeyboardInterrupt: pass else: @@ -618,6 +620,8 @@ def main(path='-', *, event.set() if not keep_open: break + # don't just flood open calls + time.sleep(sleep) done = True th.Thread(target=parse, daemon=True).start() From acdea1880efcf0d33abd73ab968b9973583b6164 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 14 Sep 2022 13:34:59 -0500 Subject: [PATCH 37/81] Made summary.py more powerful, dropped -m from size scripts With more scripts generating CSV files this moves most CSV manipulation into summary.py, which can now handle more or less any arbitrary CSV file with arbitrary names and fields. This also includes a bunch of additional, probably unnecessary, tweaks: - summary.py/coverage.py use a custom fractional type for encoding fractions, this will also be used for test counts. - Added a smaller diff output for size scripts with the --percent flag. - Added line and hit info to coverage.py's CSV files. - Added --tree flag to stack.py to show only the call tree without other noise. - Renamed structs.py to struct.py. - Changed a few flags around for consistency between size/summary scripts. - Added `make sizes` alias. - Added `make lfs.code.csv` rules --- Makefile | 58 ++- scripts/code.py | 571 +++++++++++++++-------- scripts/coverage.py | 941 ++++++++++++++++++++++++-------------- scripts/data.py | 571 +++++++++++++++-------- scripts/stack.py | 758 ++++++++++++++++++++----------- scripts/struct.py | 522 +++++++++++++++++++++ scripts/structs.py | 348 -------------- scripts/summary.py | 1045 ++++++++++++++++++++++++++++--------------- 8 files changed, 3081 insertions(+), 1733 deletions(-) create mode 100755 scripts/struct.py delete mode 100755 scripts/structs.py diff --git a/Makefile b/Makefile index 20b1979e..d87a8792 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ override TESTFLAGS += -v override CODEFLAGS += -v override DATAFLAGS += -v override STACKFLAGS += -v -override STRUCTSFLAGS += -v +override STRUCTFLAGS += -v override COVERAGEFLAGS += -v override TESTFLAGS += -v override TESTCFLAGS += -v @@ -76,11 +76,10 @@ ifdef EXEC override TESTFLAGS += --exec="$(EXEC)" endif ifdef BUILDDIR -override TESTFLAGS += --build-dir="$(BUILDDIR:/=)" override CODEFLAGS += --build-dir="$(BUILDDIR:/=)" override DATAFLAGS += --build-dir="$(BUILDDIR:/=)" override STACKFLAGS += --build-dir="$(BUILDDIR:/=)" -override STRUCTSFLAGS += --build-dir="$(BUILDDIR:/=)" +override STRUCTFLAGS += --build-dir="$(BUILDDIR:/=)" override COVERAGEFLAGS += --build-dir="$(BUILDDIR:/=)" endif ifneq ($(NM),nm) @@ -88,7 +87,7 @@ override CODEFLAGS += --nm-tool="$(NM)" override DATAFLAGS += --nm-tool="$(NM)" endif ifneq ($(OBJDUMP),objdump) -override STRUCTSFLAGS += --objdump-tool="$(OBJDUMP)" +override STRUCTFLAGS += --objdump-tool="$(OBJDUMP)" endif @@ -132,17 +131,22 @@ data: $(OBJ) stack: $(CI) ./scripts/stack.py $^ -S $(STACKFLAGS) -.PHONY: structs -structs: $(OBJ) - ./scripts/structs.py $^ -S $(STRUCTSFLAGS) +.PHONY: struct +struct: $(OBJ) + ./scripts/struct.py $^ -S $(STRUCTFLAGS) .PHONY: coverage coverage: $(GCDA) ./scripts/coverage.py $^ -s $(COVERAGEFLAGS) -.PHONY: summary -summary: $(BUILDDIR)lfs.csv - ./scripts/summary.py -Y $^ $(SUMMARYFLAGS) +.PHONY: summary sizes +summary sizes: $(BUILDDIR)lfs.csv + $(strip ./scripts/summary.py -Y $^ \ + -f code=code_size,$\ + data=data_size,$\ + stack=stack_limit,$\ + struct=struct_size \ + $(SUMMARYFLAGS)) # rules @@ -157,11 +161,27 @@ $(BUILDDIR)lfs: $(OBJ) $(BUILDDIR)lfs.a: $(OBJ) $(AR) rcs $@ $^ -$(BUILDDIR)lfs.csv: $(OBJ) $(CI) - ./scripts/code.py $(OBJ) -q $(CODEFLAGS) -o $@ - ./scripts/data.py $(OBJ) -q -m $@ $(DATAFLAGS) -o $@ - ./scripts/stack.py $(CI) -q -m $@ $(STACKFLAGS) -o $@ - ./scripts/structs.py $(OBJ) -q -m $@ $(STRUCTSFLAGS) -o $@ +$(BUILDDIR)lfs.code.csv: $(OBJ) + ./scripts/code.py $^ -q $(CODEFLAGS) -o $@ + +$(BUILDDIR)lfs.data.csv: $(OBJ) + ./scripts/data.py $^ -q $(CODEFLAGS) -o $@ + +$(BUILDDIR)lfs.stack.csv: $(CI) + ./scripts/stack.py $^ -q $(CODEFLAGS) -o $@ + +$(BUILDDIR)lfs.struct.csv: $(OBJ) + ./scripts/struct.py $^ -q $(CODEFLAGS) -o $@ + +$(BUILDDIR)lfs.coverage.csv: $(GCDA) + ./scripts/coverage.py $^ -q $(COVERAGEFLAGS) -o $@ + +$(BUILDDIR)lfs.csv: \ + $(BUILDDIR)lfs.code.csv \ + $(BUILDDIR)lfs.data.csv \ + $(BUILDDIR)lfs.stack.csv \ + $(BUILDDIR)lfs.struct.csv + ./scripts/summary.py $^ -q $(SUMMARYFLAGS) -o $@ $(BUILDDIR)runners/test_runner: $(TEST_OBJ) $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ @@ -191,7 +211,13 @@ $(BUILDDIR)%.t.c: %.c $(TESTS) clean: rm -f $(BUILDDIR)lfs rm -f $(BUILDDIR)lfs.a - rm -f $(BUILDDIR)lfs.csv + $(strip rm -f \ + $(BUILDDIR)lfs.csv \ + $(BUILDDIR)lfs.code.csv \ + $(BUILDDIR)lfs.data.csv \ + $(BUILDDIR)lfs.stack.csv \ + $(BUILDDIR)lfs.struct.csv \ + $(BUILDDIR)lfs.coverage.csv) rm -f $(BUILDDIR)runners/test_runner rm -f $(OBJ) rm -f $(DEP) diff --git a/scripts/code.py b/scripts/code.py index 27e06ebb..8a5c39b4 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -5,71 +5,123 @@ # by Linux's Bloat-O-Meter. # -import os +import collections as co +import csv import glob import itertools as it -import subprocess as sp -import shlex +import math as m +import os import re -import csv -import collections as co +import shlex +import subprocess as sp OBJ_PATHS = ['*.o'] +NM_TOOL = ['nm'] +TYPE = 'tTrRdD' -class CodeResult(co.namedtuple('CodeResult', 'code_size')): + +# integer fields +class IntField(co.namedtuple('IntField', 'x')): __slots__ = () - def __new__(cls, code_size=0): - return super().__new__(cls, int(code_size)) + def __new__(cls, x): + if isinstance(x, IntField): + return x + if isinstance(x, str): + try: + x = int(x, 0) + except ValueError: + # also accept +-∞ and +-inf + if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): + x = float('inf') + elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): + x = float('-inf') + else: + raise + return super().__new__(cls, x) + + def __int__(self): + assert not m.isinf(self.x) + return self.x + + def __float__(self): + return float(self.x) + + def __str__(self): + if self.x == float('inf'): + return '∞' + elif self.x == float('-inf'): + return '-∞' + else: + return str(self.x) + + none = '%7s' % '-' + def table(self): + return '%7s' % (self,) + + diff_none = '%7s' % '-' + diff_table = table + + def diff_diff(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + diff = new - old + if diff == float('+inf'): + return '%7s' % '+∞' + elif diff == float('-inf'): + return '%7s' % '-∞' + else: + return '%+7d' % diff + + def ratio(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + if m.isinf(new) and m.isinf(old): + return 0.0 + elif m.isinf(new): + return float('+inf') + elif m.isinf(old): + return float('-inf') + elif not old and not new: + return 0.0 + elif not old: + return 1.0 + else: + return (new-old) / old def __add__(self, other): - return self.__class__(self.code_size + other.code_size) + return IntField(self.x + other.x) - def __sub__(self, other): - return CodeDiff(other, self) + def __mul__(self, other): + return IntField(self.x * other.x) - def __rsub__(self, other): - return self.__class__.__sub__(other, self) + def __lt__(self, other): + return self.x < other.x - def key(self, **args): - if args.get('size_sort'): - return -self.code_size - elif args.get('reverse_size_sort'): - return +self.code_size + def __gt__(self, other): + return self.__class__.__lt__(other, self) + + def __le__(self, other): + return not self.__gt__(other) + + def __ge__(self, other): + return not self.__lt__(other) + + def __truediv__(self, n): + if m.isinf(self.x): + return self else: - return None + return IntField(round(self.x / n)) - _header = '%7s' % 'size' - def __str__(self): - return '%7d' % self.code_size - -class CodeDiff(co.namedtuple('CodeDiff', 'old,new')): +# code size results +class CodeResult(co.namedtuple('CodeResult', 'file,function,code_size')): __slots__ = () + def __new__(cls, file, function, code_size): + return super().__new__(cls, file, function, IntField(code_size)) - def ratio(self): - old = self.old.code_size if self.old is not None else 0 - new = self.new.code_size if self.new is not None else 0 - return (new-old) / old if old else 1.0 - - def key(self, **args): - return ( - self.new.key(**args) if self.new is not None else 0, - -self.ratio()) - - def __bool__(self): - return bool(self.ratio()) - - _header = '%7s %7s %7s' % ('old', 'new', 'diff') - def __str__(self): - old = self.old.code_size if self.old is not None else 0 - new = self.new.code_size if self.new is not None else 0 - diff = new - old - ratio = self.ratio() - return '%7s %7s %+7d%s' % ( - old or "-", - new or "-", - diff, - ' (%+.1f%%)' % (100*ratio) if ratio else '') + def __add__(self, other): + return CodeResult(self.file, self.function, + self.code_size + other.code_size) def openio(path, mode='r'): @@ -81,20 +133,25 @@ def openio(path, mode='r'): else: return open(path, mode) -def collect(paths, **args): - results = co.defaultdict(lambda: CodeResult()) +def collect(paths, *, + nm_tool=NM_TOOL, + type=TYPE, + build_dir=None, + everything=False, + **args): + results = [] pattern = re.compile( '^(?P[0-9a-fA-F]+)' + - ' (?P[%s])' % re.escape(args['type']) + + ' (?P[%s])' % re.escape(type) + ' (?P.+?)$') for path in paths: # map to source file src_path = re.sub('\.o$', '.c', path) - if args.get('build_dir'): - src_path = re.sub('%s/*' % re.escape(args['build_dir']), '', + if build_dir: + src_path = re.sub('%s/*' % re.escape(build_dir), '', src_path) # note nm-tool may contain extra args - cmd = args['nm_tool'] + ['--size-sort', path] + cmd = nm_tool + ['--size-sort', path] if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, @@ -107,12 +164,15 @@ def collect(paths, **args): if m: func = m.group('func') # discard internal functions - if not args.get('everything') and func.startswith('__'): + if not everything and func.startswith('__'): continue # discard .8449 suffixes created by optimizer func = re.sub('\.[0-9]+', '', func) - results[(src_path, func)] += CodeResult( - int(m.group('size'), 16)) + + results.append(CodeResult( + src_path, func, + int(m.group('size'), 16))) + proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -122,12 +182,167 @@ def collect(paths, **args): return results -def main(**args): + +def fold(results, *, + by=['file', 'function'], + **_): + folding = co.OrderedDict() + for r in results: + name = tuple(getattr(r, k) for k in by) + if name not in folding: + folding[name] = [] + folding[name].append(r) + + folded = [] + for rs in folding.values(): + folded.append(sum(rs[1:], start=rs[0])) + + return folded + + +def table(results, diff_results=None, *, + by_file=False, + size_sort=False, + reverse_size_sort=False, + summary=False, + all=False, + percent=False, + **_): + all_, all = all, __builtins__.all + + # fold + results = fold(results, by=['file' if by_file else 'function']) + if diff_results is not None: + diff_results = fold(diff_results, + by=['file' if by_file else 'function']) + + table = { + r.file if by_file else r.function: r + for r in results} + diff_table = { + r.file if by_file else r.function: r + for r in diff_results or []} + + # sort, note that python's sort is stable + names = list(table.keys() | diff_table.keys()) + names.sort() + if diff_results is not None: + names.sort(key=lambda n: -IntField.ratio( + table[n].code_size if n in table else None, + diff_table[n].code_size if n in diff_table else None)) + if size_sort: + names.sort(key=lambda n: (table[n].code_size,) if n in table else (), + reverse=True) + elif reverse_size_sort: + names.sort(key=lambda n: (table[n].code_size,) if n in table else (), + reverse=False) + + # print header + print('%-36s' % ('%s%s' % ( + 'file' if by_file else 'function', + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else ''), + end='') + if diff_results is None: + print(' %s' % ('size'.rjust(len(IntField.none)))) + elif percent: + print(' %s' % ('size'.rjust(len(IntField.diff_none)))) + else: + print(' %s %s %s' % ( + 'old'.rjust(len(IntField.diff_none)), + 'new'.rjust(len(IntField.diff_none)), + 'diff'.rjust(len(IntField.diff_none)))) + + # print entries + if not summary: + for name in names: + r = table.get(name) + if diff_results is not None: + diff_r = diff_table.get(name) + ratio = IntField.ratio( + r.code_size if r else None, + diff_r.code_size if diff_r else None) + if not ratio and not all_: + continue + + print('%-36s' % name, end='') + if diff_results is None: + print(' %s' % ( + r.code_size.table() + if r else IntField.none)) + elif percent: + print(' %s%s' % ( + r.code_size.diff_table() + if r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)))) + else: + print(' %s %s %s%s' % ( + diff_r.code_size.diff_table() + if diff_r else IntField.diff_none, + r.code_size.diff_table() + if r else IntField.diff_none, + IntField.diff_diff( + r.code_size if r else None, + diff_r.code_size if diff_r else None) + if r or diff_r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)) + if ratio else '')) + + # print total + total = fold(results, by=[]) + r = total[0] if total else None + if diff_results is not None: + diff_total = fold(diff_results, by=[]) + diff_r = diff_total[0] if diff_total else None + ratio = IntField.ratio( + r.code_size if r else None, + diff_r.code_size if diff_r else None) + + print('%-36s' % 'TOTAL', end='') + if diff_results is None: + print(' %s' % ( + r.code_size.table() + if r else IntField.none)) + elif percent: + print(' %s%s' % ( + r.code_size.diff_table() + if r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)))) + else: + print(' %s %s %s%s' % ( + diff_r.code_size.diff_table() + if diff_r else IntField.diff_none, + r.code_size.diff_table() + if r else IntField.diff_none, + IntField.diff_diff( + r.code_size if r else None, + diff_r.code_size if diff_r else None) + if r or diff_r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)) + if ratio else '')) + + +def main(obj_paths, **args): # find sizes if not args.get('use', None): # find .o files paths = [] - for path in args['obj_paths']: + for path in obj_paths: if os.path.isdir(path): path = path + '/*.o' @@ -135,127 +350,61 @@ def main(**args): paths.append(path) if not paths: - print('no .obj files found in %r?' % args['obj_paths']) + print('no .obj files found in %r?' % obj_paths) sys.exit(-1) results = collect(paths, **args) else: + results = [] with openio(args['use']) as f: - r = csv.DictReader(f) - results = { - (result['file'], result['name']): CodeResult( - *(result[f] for f in CodeResult._fields)) - for result in r - if all(result.get(f) not in {None, ''} - for f in CodeResult._fields)} + reader = csv.DictReader(f) + for r in reader: + try: + results.append(CodeResult(**{ + k: v for k, v in r.items() + if k in CodeResult._fields})) + except TypeError: + pass - # find previous results? - if args.get('diff'): - try: - with openio(args['diff']) as f: - r = csv.DictReader(f) - prev_results = { - (result['file'], result['name']): CodeResult( - *(result[f] for f in CodeResult._fields)) - for result in r - if all(result.get(f) not in {None, ''} - for f in CodeResult._fields)} - except FileNotFoundError: - prev_results = [] + # fold to remove duplicates + results = fold(results) + + # sort because why not + results.sort() # write results to CSV if args.get('output'): - merged_results = co.defaultdict(lambda: {}) - other_fields = [] - - # merge? - if args.get('merge'): - try: - with openio(args['merge']) as f: - r = csv.DictReader(f) - for result in r: - file = result.pop('file', '') - func = result.pop('name', '') - for f in CodeResult._fields: - result.pop(f, None) - merged_results[(file, func)] = result - other_fields = result.keys() - except FileNotFoundError: - pass - - for (file, func), result in results.items(): - merged_results[(file, func)] |= result._asdict() - with openio(args['output'], 'w') as f: - w = csv.DictWriter(f, ['file', 'name', - *other_fields, *CodeResult._fields]) - w.writeheader() - for (file, func), result in sorted(merged_results.items()): - w.writerow({'file': file, 'name': func, **result}) + writer = csv.DictWriter(f, CodeResult._fields) + writer.writeheader() + for r in results: + writer.writerow(r._asdict()) - # print results - def print_header(by): - if by == 'total': - entry = lambda k: 'TOTAL' - elif by == 'file': - entry = lambda k: k[0] - else: - entry = lambda k: k[1] + # find previous results? + if args.get('diff'): + diff_results = [] + try: + with openio(args['diff']) as f: + reader = csv.DictReader(f) + for r in reader: + try: + diff_results.append(CodeResult(**{ + k: v for k, v in r.items() + if k in CodeResult._fields})) + except TypeError: + pass + except FileNotFoundError: + pass - if not args.get('diff'): - print('%-36s %s' % (by, CodeResult._header)) - else: - old = {entry(k) for k in results.keys()} - new = {entry(k) for k in prev_results.keys()} - print('%-36s %s' % ( - '%s (%d added, %d removed)' % (by, - sum(1 for k in new if k not in old), - sum(1 for k in old if k not in new)) - if by else '', - CodeDiff._header)) + # fold to remove duplicates + diff_results = fold(diff_results) - def print_entries(by): - if by == 'total': - entry = lambda k: 'TOTAL' - elif by == 'file': - entry = lambda k: k[0] - else: - entry = lambda k: k[1] - - entries = co.defaultdict(lambda: CodeResult()) - for k, result in results.items(): - entries[entry(k)] += result - - if not args.get('diff'): - for name, result in sorted(entries.items(), - key=lambda p: (p[1].key(**args), p)): - print('%-36s %s' % (name, result)) - else: - prev_entries = co.defaultdict(lambda: CodeResult()) - for k, result in prev_results.items(): - prev_entries[entry(k)] += result - - diff_entries = {name: entries.get(name) - prev_entries.get(name) - for name in (entries.keys() | prev_entries.keys())} - - for name, diff in sorted(diff_entries.items(), - key=lambda p: (p[1].key(**args), p)): - if diff or args.get('all'): - print('%-36s %s' % (name, diff)) - - if args.get('quiet'): - pass - elif args.get('summary'): - print_header('') - print_entries('total') - elif args.get('files'): - print_header('file') - print_entries('file') - print_entries('total') - else: - print_header('function') - print_entries('function') - print_entries('total') + # print table + if not args.get('quiet'): + table( + results, + diff_results if args.get('diff') else None, + **args) if __name__ == "__main__": @@ -263,42 +412,72 @@ if __name__ == "__main__": import sys parser = argparse.ArgumentParser( description="Find code size at the function level.") - parser.add_argument('obj_paths', nargs='*', default=OBJ_PATHS, - help="Description of where to find *.o files. May be a directory \ - or a list of paths. Defaults to %r." % OBJ_PATHS) - parser.add_argument('-v', '--verbose', action='store_true', + parser.add_argument( + 'obj_paths', + nargs='*', + default=OBJ_PATHS, + help="Description of where to find *.o files. May be a directory " + "or a list of paths. Defaults to %(default)r.") + parser.add_argument( + '-v', '--verbose', + action='store_true', help="Output commands that run behind the scenes.") - parser.add_argument('-q', '--quiet', action='store_true', + parser.add_argument( + '-q', '--quiet', + action='store_true', help="Don't show anything, useful with -o.") - parser.add_argument('-o', '--output', + parser.add_argument( + '-o', '--output', help="Specify CSV file to store results.") - parser.add_argument('-u', '--use', - help="Don't compile and find code sizes, instead use this CSV file.") - parser.add_argument('-d', '--diff', - help="Specify CSV file to diff code size against.") - parser.add_argument('-m', '--merge', - help="Merge with an existing CSV file when writing to output.") - parser.add_argument('-a', '--all', action='store_true', - help="Show all functions, not just the ones that changed.") - parser.add_argument('-A', '--everything', action='store_true', - help="Include builtin and libc specific symbols.") - parser.add_argument('-s', '--size-sort', action='store_true', + parser.add_argument( + '-u', '--use', + help="Don't parse anything, use this CSV file.") + parser.add_argument( + '-d', '--diff', + help="Specify CSV file to diff against.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") + parser.add_argument( + '-p', '--percent', + action='store_true', + help="Only show percentage change, not a full diff.") + parser.add_argument( + '-b', '--by-file', + action='store_true', + help="Group by file. Note this does not include padding " + "so sizes may differ from other tools.") + parser.add_argument( + '-s', '--size-sort', + action='store_true', help="Sort by size.") - parser.add_argument('-S', '--reverse-size-sort', action='store_true', + parser.add_argument( + '-S', '--reverse-size-sort', + action='store_true', help="Sort by size, but backwards.") - parser.add_argument('-F', '--files', action='store_true', - help="Show file-level code sizes. Note this does not include padding! " - "So sizes may differ from other tools.") - parser.add_argument('-Y', '--summary', action='store_true', - help="Only show the total code size.") - parser.add_argument('--type', default='tTrRdD', + parser.add_argument( + '-Y', '--summary', + action='store_true', + help="Only show the total size.") + parser.add_argument( + '-A', '--everything', + action='store_true', + help="Include builtin and libc specific symbols.") + parser.add_argument( + '--type', + default=TYPE, help="Type of symbols to report, this uses the same single-character " "type-names emitted by nm. Defaults to %(default)r.") - parser.add_argument('--nm-tool', default=['nm'], type=lambda x: x.split(), - help="Path to the nm tool to use.") - parser.add_argument('--build-dir', - help="Specify the relative build directory. Used to map object files \ - to the correct source files.") + parser.add_argument( + '--nm-tool', + type=lambda x: x.split(), + default=NM_TOOL, + help="Path to the nm tool to use. Defaults to %(default)r") + parser.add_argument( + '--build-dir', + help="Specify the relative build directory. Used to map object files " + "to the correct source files.") sys.exit(main(**{k: v for k, v in vars(parser.parse_args()).items() if v is not None})) diff --git a/scripts/coverage.py b/scripts/coverage.py index d30b3ffd..14fe0d2d 100755 --- a/scripts/coverage.py +++ b/scripts/coverage.py @@ -10,6 +10,7 @@ import csv import glob import itertools as it import json +import math as m import os import re import shlex @@ -20,139 +21,189 @@ import subprocess as sp GCDA_PATHS = ['*.gcda'] +GCOV_TOOL = ['gcov'] -class CoverageResult(co.namedtuple('CoverageResult', - 'coverage_line_hits,coverage_line_count,' - 'coverage_branch_hits,coverage_branch_count')): + +# integer fields +class IntField(co.namedtuple('IntField', 'x')): __slots__ = () - def __new__(cls, - coverage_line_hits=0, coverage_line_count=0, - coverage_branch_hits=0, coverage_branch_count=0): - return super().__new__(cls, - int(coverage_line_hits), - int(coverage_line_count), - int(coverage_branch_hits), - int(coverage_branch_count)) + def __new__(cls, x): + if isinstance(x, IntField): + return x + if isinstance(x, str): + try: + x = int(x, 0) + except ValueError: + # also accept +-∞ and +-inf + if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): + x = float('inf') + elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): + x = float('-inf') + else: + raise + return super().__new__(cls, x) + + def __int__(self): + assert not m.isinf(self.x) + return self.x + + def __float__(self): + return float(self.x) + + def __str__(self): + if self.x == float('inf'): + return '∞' + elif self.x == float('-inf'): + return '-∞' + else: + return str(self.x) + + none = '%7s' % '-' + def table(self): + return '%7s' % (self,) + + diff_none = '%7s' % '-' + diff_table = table + + def diff_diff(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + diff = new - old + if diff == float('+inf'): + return '%7s' % '+∞' + elif diff == float('-inf'): + return '%7s' % '-∞' + else: + return '%+7d' % diff + + def ratio(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + if m.isinf(new) and m.isinf(old): + return 0.0 + elif m.isinf(new): + return float('+inf') + elif m.isinf(old): + return float('-inf') + elif not old and not new: + return 0.0 + elif not old: + return 1.0 + else: + return (new-old) / old def __add__(self, other): - return self.__class__( - self.coverage_line_hits + other.coverage_line_hits, - self.coverage_line_count + other.coverage_line_count, - self.coverage_branch_hits + other.coverage_branch_hits, - self.coverage_branch_count + other.coverage_branch_count) + return IntField(self.x + other.x) - def __sub__(self, other): - return CoverageDiff(other, self) + def __mul__(self, other): + return IntField(self.x * other.x) - def __rsub__(self, other): - return self.__class__.__sub__(other, self) + def __lt__(self, other): + return self.x < other.x - def key(self, **args): - ratio_line = (self.coverage_line_hits/self.coverage_line_count - if self.coverage_line_count else -1) - ratio_branch = (self.coverage_branch_hits/self.coverage_branch_count - if self.coverage_branch_count else -1) + def __gt__(self, other): + return self.__class__.__lt__(other, self) - if args.get('line_sort'): - return (-ratio_line, -ratio_branch) - elif args.get('reverse_line_sort'): - return (+ratio_line, +ratio_branch) - elif args.get('branch_sort'): - return (-ratio_branch, -ratio_line) - elif args.get('reverse_branch_sort'): - return (+ratio_branch, +ratio_line) + def __le__(self, other): + return not self.__gt__(other) + + def __ge__(self, other): + return not self.__lt__(other) + + def __truediv__(self, n): + if m.isinf(self.x): + return self else: - return None + return IntField(round(self.x / n)) - _header = '%19s %19s' % ('hits/line', 'hits/branch') - def __str__(self): - line_hits = self.coverage_line_hits - line_count = self.coverage_line_count - branch_hits = self.coverage_branch_hits - branch_count = self.coverage_branch_count - return '%11s %7s %11s %7s' % ( - '%d/%d' % (line_hits, line_count) - if line_count else '-', - '%.1f%%' % (100*line_hits/line_count) - if line_count else '-', - '%d/%d' % (branch_hits, branch_count) - if branch_count else '-', - '%.1f%%' % (100*branch_hits/branch_count) - if branch_count else '-') - -class CoverageDiff(co.namedtuple('CoverageDiff', 'old,new')): +# fractional fields, a/b +class FracField(co.namedtuple('FracField', 'a,b')): __slots__ = () + def __new__(cls, a, b=None): + if isinstance(a, FracField) and b is None: + return a + if isinstance(a, str) and b is None: + a, b = a.split('/', 1) + if b is None: + b = a + return super().__new__(cls, IntField(a), IntField(b)) - def ratio_line(self): - old_line_hits = (self.old.coverage_line_hits - if self.old is not None else 0) - old_line_count = (self.old.coverage_line_count - if self.old is not None else 0) - new_line_hits = (self.new.coverage_line_hits - if self.new is not None else 0) - new_line_count = (self.new.coverage_line_count - if self.new is not None else 0) - return ((new_line_hits/new_line_count if new_line_count else 1.0) - - (old_line_hits/old_line_count if old_line_count else 1.0)) - - def ratio_branch(self): - old_branch_hits = (self.old.coverage_branch_hits - if self.old is not None else 0) - old_branch_count = (self.old.coverage_branch_count - if self.old is not None else 0) - new_branch_hits = (self.new.coverage_branch_hits - if self.new is not None else 0) - new_branch_count = (self.new.coverage_branch_count - if self.new is not None else 0) - return ((new_branch_hits/new_branch_count if new_branch_count else 1.0) - - (old_branch_hits/old_branch_count if old_branch_count else 1.0)) - - def key(self, **args): - return ( - self.new.key(**args) if self.new is not None else 0, - -self.ratio_line(), - -self.ratio_branch()) - - def __bool__(self): - return bool(self.ratio_line() or self.ratio_branch()) - - _header = '%23s %23s %23s' % ('old', 'new', 'diff') def __str__(self): - old_line_hits = (self.old.coverage_line_hits - if self.old is not None else 0) - old_line_count = (self.old.coverage_line_count - if self.old is not None else 0) - old_branch_hits = (self.old.coverage_branch_hits - if self.old is not None else 0) - old_branch_count = (self.old.coverage_branch_count - if self.old is not None else 0) - new_line_hits = (self.new.coverage_line_hits - if self.new is not None else 0) - new_line_count = (self.new.coverage_line_count - if self.new is not None else 0) - new_branch_hits = (self.new.coverage_branch_hits - if self.new is not None else 0) - new_branch_count = (self.new.coverage_branch_count - if self.new is not None else 0) - diff_line_hits = new_line_hits - old_line_hits - diff_line_count = new_line_count - old_line_count - diff_branch_hits = new_branch_hits - old_branch_hits - diff_branch_count = new_branch_count - old_branch_count - ratio_line = self.ratio_line() - ratio_branch = self.ratio_branch() - return '%11s %11s %11s %11s %11s %11s%-10s%s' % ( - '%d/%d' % (old_line_hits, old_line_count) - if old_line_count else '-', - '%d/%d' % (old_branch_hits, old_branch_count) - if old_branch_count else '-', - '%d/%d' % (new_line_hits, new_line_count) - if new_line_count else '-', - '%d/%d' % (new_branch_hits, new_branch_count) - if new_branch_count else '-', - '%+d/%+d' % (diff_line_hits, diff_line_count), - '%+d/%+d' % (diff_branch_hits, diff_branch_count), - ' (%+.1f%%)' % (100*ratio_line) if ratio_line else '', - ' (%+.1f%%)' % (100*ratio_branch) if ratio_branch else '') + return '%s/%s' % (self.a, self.b) + + none = '%11s %7s' % ('-', '-') + def table(self): + if not self.b.x: + return self.none + + t = self.a.x/self.b.x + return '%11s %7s' % ( + self, + '∞%' if t == float('+inf') + else '-∞%' if t == float('-inf') + else '%.1f%%' % (100*t)) + + diff_none = '%11s' % '-' + def diff_table(self): + if not self.b.x: + return self.diff_none + + return '%11s' % (self,) + + def diff_diff(self, other): + new_a, new_b = self if self else (IntField(0), IntField(0)) + old_a, old_b = other if other else (IntField(0), IntField(0)) + return '%11s' % ('%s/%s' % ( + new_a.diff_diff(old_a).strip(), + new_b.diff_diff(old_b).strip())) + + def ratio(self, other): + new_a, new_b = self if self else (IntField(0), IntField(0)) + old_a, old_b = other if other else (IntField(0), IntField(0)) + new = new_a.x/new_b.x if new_b.x else 1.0 + old = old_a.x/old_b.x if old_b.x else 1.0 + return new - old + + def __add__(self, other): + return FracField(self.a + other.a, self.b + other.b) + + def __mul__(self, other): + return FracField(self.a * other.a, self.b + other.b) + + def __lt__(self, other): + self_r = self.a.x/self.b.x if self.b.x else float('-inf') + other_r = other.a.x/other.b.x if other.b.x else float('-inf') + return self_r < other_r + + def __gt__(self, other): + return self.__class__.__lt__(other, self) + + def __le__(self, other): + return not self.__gt__(other) + + def __ge__(self, other): + return not self.__lt__(other) + + def __truediv__(self, n): + return FracField(self.a / n, self.b / n) + +# coverage results +class CoverageResult(co.namedtuple('CoverageResult', + 'file,function,line,' + 'coverage_hits,coverage_lines,coverage_branches')): + __slots__ = () + def __new__(cls, file, function, line, + coverage_hits, coverage_lines, coverage_branches): + return super().__new__(cls, file, function, int(IntField(line)), + IntField(coverage_hits), + FracField(coverage_lines), + FracField(coverage_branches)) + + def __add__(self, other): + return CoverageResult(self.file, self.function, self.line, + max(self.coverage_hits, other.coverage_hits), + self.coverage_lines + other.coverage_lines, + self.coverage_branches + other.coverage_branches) def openio(path, mode='r'): @@ -164,27 +215,22 @@ def openio(path, mode='r'): else: return open(path, mode) -def color(**args): - if args.get('color') == 'auto': - return sys.stdout.isatty() - elif args.get('color') == 'always': - return True - else: - return False - -def collect(paths, **args): - results = {} +def collect(paths, *, + gcov_tool=GCOV_TOOL, + build_dir=None, + everything=False, + **args): + results = [] for path in paths: # map to source file src_path = re.sub('\.t\.a\.gcda$', '.c', path) - # TODO test this - if args.get('build_dir'): - src_path = re.sub('%s/*' % re.escape(args['build_dir']), '', + if build_dir: + src_path = re.sub('%s/*' % re.escape(build_dir), '', src_path) # get coverage info through gcov's json output # note, gcov-tool may contain extra args - cmd = args['gcov_tool'] + ['-b', '-t', '--json-format', path] + cmd = GCOV_TOOL + ['-b', '-t', '--json-format', path] if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, @@ -208,49 +254,277 @@ def collect(paths, **args): for line in file['lines']: func = line.get('function_name', '(inlined)') # discard internal function (this includes injected test cases) - if not args.get('everything'): + if not everything: if func.startswith('__'): continue - results[(src_path, func, line['line_number'])] = ( + results.append(CoverageResult( + src_path, func, line['line_number'], line['count'], - CoverageResult( - coverage_line_hits=1 if line['count'] > 0 else 0, - coverage_line_count=1, - coverage_branch_hits=sum( - 1 if branch['count'] > 0 else 0 + FracField( + 1 if line['count'] > 0 else 0, + 1), + FracField( + sum(1 if branch['count'] > 0 else 0 for branch in line['branches']), - coverage_branch_count=len(line['branches']))) + len(line['branches'])))) - # merge into functions, since this is what other scripts use - func_results = co.defaultdict(lambda: CoverageResult()) - for (file, func, _), (_, result) in results.items(): - func_results[(file, func)] += result + return results - return func_results, results -def annotate(paths, results, **args): +def fold(results, *, + by=['file', 'function', 'line'], + **_): + folding = co.OrderedDict() + for r in results: + name = tuple(getattr(r, k) for k in by) + if name not in folding: + folding[name] = [] + folding[name].append(r) + + folded = [] + for rs in folding.values(): + folded.append(sum(rs[1:], start=rs[0])) + + return folded + + +def table(results, diff_results=None, *, + by_file=False, + by_line=False, + line_sort=False, + reverse_line_sort=False, + branch_sort=False, + reverse_branch_sort=False, + summary=False, + all=False, + percent=False, + **_): + all_, all = all, __builtins__.all + + # fold + results = fold(results, + by=['file', 'line'] if by_line + else ['file'] if by_file + else ['function']) + if diff_results is not None: + diff_results = fold(diff_results, + by=['file', 'line'] if by_line + else ['file'] if by_file + else ['function']) + + table = { + (r.file, r.line) if by_line + else r.file if by_file + else r.function: r + for r in results} + diff_table = { + (r.file, r.line) if by_line + else r.file if by_file + else r.function: r + for r in diff_results or []} + + # sort, note that python's sort is stable + names = list(table.keys() | diff_table.keys()) + names.sort() + if diff_results is not None: + names.sort(key=lambda n: ( + -FracField.ratio( + table[n].coverage_lines if n in table else None, + diff_table[n].coverage_lines if n in diff_table else None), + -FracField.ratio( + table[n].coverage_branches if n in table else None, + diff_table[n].coverage_branches if n in diff_table else None))) + if line_sort: + names.sort(key=lambda n: (table[n].coverage_lines,) + if n in table else (), + reverse=True) + elif reverse_line_sort: + names.sort(key=lambda n: (table[n].coverage_lines,) + if n in table else (), + reverse=False) + elif branch_sort: + names.sort(key=lambda n: (table[n].coverage_branches,) + if n in table else (), + reverse=True) + elif reverse_branch_sort: + names.sort(key=lambda n: (table[n].coverage_branches,) + if n in table else (), + reverse=False) + + # print header + print('%-36s' % ('%s%s' % ( + 'line' if by_line + else 'file' if by_file + else 'function', + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else ''), + end='') + if diff_results is None: + print(' %s %s' % ( + 'hits/line'.rjust(len(FracField.none)), + 'hits/branch'.rjust(len(FracField.none)))) + elif percent: + print(' %s %s' % ( + 'hits/line'.rjust(len(FracField.diff_none)), + 'hits/branch'.rjust(len(FracField.diff_none)))) + else: + print(' %s %s %s %s %s %s' % ( + 'oh/line'.rjust(len(FracField.diff_none)), + 'oh/branch'.rjust(len(FracField.diff_none)), + 'nh/line'.rjust(len(FracField.diff_none)), + 'nh/branch'.rjust(len(FracField.diff_none)), + 'dh/line'.rjust(len(FracField.diff_none)), + 'dh/branch'.rjust(len(FracField.diff_none)))) + + # print entries + if not summary: + for name in names: + r = table.get(name) + if diff_results is not None: + diff_r = diff_table.get(name) + line_ratio = FracField.ratio( + r.coverage_lines if r else None, + diff_r.coverage_lines if diff_r else None) + branch_ratio = FracField.ratio( + r.coverage_branches if r else None, + diff_r.coverage_branches if diff_r else None) + if not line_ratio and not branch_ratio and not all_: + continue + + print('%-36s' % ( + ':'.join('%s' % n for n in name) + if by_line else name), end='') + if diff_results is None: + print(' %s %s' % ( + r.coverage_lines.table() + if r else FracField.none, + r.coverage_branches.table() + if r else FracField.none)) + elif percent: + print(' %s %s%s' % ( + r.coverage_lines.diff_table() + if r else FracField.diff_none, + r.coverage_branches.diff_table() + if r else FracField.diff_none, + ' (%s)' % ', '.join( + '+∞%' if t == float('+inf') + else '-∞%' if t == float('-inf') + else '%+.1f%%' % (100*t) + for t in [line_ratio, branch_ratio]))) + else: + print(' %s %s %s %s %s %s%s' % ( + diff_r.coverage_lines.diff_table() + if diff_r else FracField.diff_none, + diff_r.coverage_branches.diff_table() + if diff_r else FracField.diff_none, + r.coverage_lines.diff_table() + if r else FracField.diff_none, + r.coverage_branches.diff_table() + if r else FracField.diff_none, + FracField.diff_diff( + r.coverage_lines if r else None, + diff_r.coverage_lines if diff_r else None) + if r or diff_r else FracField.diff_none, + FracField.diff_diff( + r.coverage_branches if r else None, + diff_r.coverage_branches if diff_r else None) + if r or diff_r else FracField.diff_none, + ' (%s)' % ', '.join( + '+∞%' if t == float('+inf') + else '-∞%' if t == float('-inf') + else '%+.1f%%' % (100*t) + for t in [line_ratio, branch_ratio] + if t) + if line_ratio or branch_ratio else '')) + + # print total + total = fold(results, by=[]) + r = total[0] if total else None + if diff_results is not None: + diff_total = fold(diff_results, by=[]) + diff_r = diff_total[0] if diff_total else None + line_ratio = FracField.ratio( + r.coverage_lines if r else None, + diff_r.coverage_lines if diff_r else None) + branch_ratio = FracField.ratio( + r.coverage_branches if r else None, + diff_r.coverage_branches if diff_r else None) + + print('%-36s' % 'TOTAL', end='') + if diff_results is None: + print(' %s %s' % ( + r.coverage_lines.table() + if r else FracField.none, + r.coverage_branches.table() + if r else FracField.none)) + elif percent: + print(' %s %s%s' % ( + r.coverage_lines.diff_table() + if r else FracField.diff_none, + r.coverage_branches.diff_table() + if r else FracField.diff_none, + ' (%s)' % ', '.join( + '+∞%' if t == float('+inf') + else '-∞%' if t == float('-inf') + else '%+.1f%%' % (100*t) + for t in [line_ratio, branch_ratio]))) + else: + print(' %s %s %s %s %s %s%s' % ( + diff_r.coverage_lines.diff_table() + if diff_r else FracField.diff_none, + diff_r.coverage_branches.diff_table() + if diff_r else FracField.diff_none, + r.coverage_lines.diff_table() + if r else FracField.diff_none, + r.coverage_branches.diff_table() + if r else FracField.diff_none, + FracField.diff_diff( + r.coverage_lines if r else None, + diff_r.coverage_lines if diff_r else None) + if r or diff_r else FracField.diff_none, + FracField.diff_diff( + r.coverage_branches if r else None, + diff_r.coverage_branches if diff_r else None) + if r or diff_r else FracField.diff_none, + ' (%s)' % ', '.join( + '+∞%' if t == float('+inf') + else '-∞%' if t == float('-inf') + else '%+.1f%%' % (100*t) + for t in [line_ratio, branch_ratio] + if t) + if line_ratio or branch_ratio else '')) + + +def annotate(paths, results, *, + annotate=False, + lines=False, + branches=False, + build_dir=None, + **args): for path in paths: # map to source file src_path = re.sub('\.t\.a\.gcda$', '.c', path) - # TODO test this - if args.get('build_dir'): - src_path = re.sub('%s/*' % re.escape(args['build_dir']), '', + if build_dir: + src_path = re.sub('%s/*' % re.escape(build_dir), '', src_path) # flatten to line info - line_results = {line: (hits, result) - for (_, _, line), (hits, result) in results.items()} + results = fold(results, by=['file', 'line']) + table = {r.line: r for r in results if r.file == src_path} # calculate spans to show - if not args.get('annotate'): + if not annotate: spans = [] last = None - for line, (hits, result) in sorted(line_results.items()): - if ((args.get('lines') and hits == 0) - or (args.get('branches') - and result.coverage_branch_hits - < result.coverage_branch_count)): + for line, r in sorted(table.items()): + if ((lines and int(r.coverage_hits) == 0) + or (branches + and r.coverage_branches.a + < r.coverage_branches.b)): if last is not None and line - last.stop <= args['context']: last = range( last.start, @@ -268,48 +542,55 @@ def annotate(paths, results, **args): skipped = False for i, line in enumerate(f): # skip lines not in spans? - if (not args.get('annotate') - and not any(i+1 in s for s in spans)): + if not annotate and not any(i+1 in s for s in spans): skipped = True continue if skipped: skipped = False print('%s@@ %s:%d @@%s' % ( - '\x1b[36m' if color(**args) else '', + '\x1b[36m' if args['color'] else '', src_path, i+1, - '\x1b[m' if color(**args) else '')) + '\x1b[m' if args['color'] else '')) # build line if line.endswith('\n'): line = line[:-1] - if i+1 in line_results: - hits, result = line_results[i+1] - line = '%-*s // %d hits, %d/%d branches' % ( + if i+1 in table: + r = table[i+1] + line = '%-*s // %s hits, %s branches' % ( args['width'], line, - hits, - result.coverage_branch_hits, - result.coverage_branch_count) + r.coverage_hits, + r.coverage_branches) - if color(**args): - if args.get('lines') and hits == 0: + if args['color']: + if lines and int(r.coverage_hits) == 0: line = '\x1b[1;31m%s\x1b[m' % line - elif (args.get('branches') and - result.coverage_branch_hits - < result.coverage_branch_count): + elif (branches + and r.coverage_branches.a + < r.coverage_branches.b): line = '\x1b[35m%s\x1b[m' % line print(line) -def main(**args): + +def main(gcda_paths, **args): + # figure out what color should be + if args.get('color') == 'auto': + args['color'] = sys.stdout.isatty() + elif args.get('color') == 'always': + args['color'] = True + else: + args['color'] = False + # find sizes if not args.get('use', None): # find .gcda files paths = [] - for path in args['gcda_paths']: + for path in gcda_paths: if os.path.isdir(path): path = path + '/*.gcda' @@ -317,143 +598,77 @@ def main(**args): paths.append(path) if not paths: - print('no .gcda files found in %r?' % args['gcda_paths']) + print('no .gcda files found in %r?' % gcda_paths) sys.exit(-1) - results, line_results = collect(paths, **args) + results = collect(paths, **args) else: + results = [] with openio(args['use']) as f: - r = csv.DictReader(f) - results = { - (result['file'], result['name']): CoverageResult( - *(result[f] for f in CoverageResult._fields)) - for result in r - if all(result.get(f) not in {None, ''} + reader = csv.DictReader(f) + for r in reader: + try: + results.append(CoverageResult(**{ + k: v for k, v in r.items() + if k in CoverageResult._fields})) + except TypeError: + pass - for f in CoverageResult._fields)} - paths = [] - line_results = {} + # fold to remove duplicates + results = fold(results) - # find previous results? - if args.get('diff'): - try: - with openio(args['diff']) as f: - r = csv.DictReader(f) - prev_results = { - (result['file'], result['name']): CoverageResult( - *(result[f] for f in CoverageResult._fields)) - for result in r - if all(result.get(f) not in {None, ''} - for f in CoverageResult._fields)} - except FileNotFoundError: - prev_results = [] + # sort because why not + results.sort() # write results to CSV if args.get('output'): - merged_results = co.defaultdict(lambda: {}) - other_fields = [] - - # merge? - if args.get('merge'): - try: - with openio(args['merge']) as f: - r = csv.DictReader(f) - for result in r: - file = result.pop('file', '') - func = result.pop('name', '') - for f in CoverageResult._fields: - result.pop(f, None) - merged_results[(file, func)] = result - other_fields = result.keys() - except FileNotFoundError: - pass - - for (file, func), result in results.items(): - merged_results[(file, func)] |= result._asdict() - with openio(args['output'], 'w') as f: - w = csv.DictWriter(f, ['file', 'name', - *other_fields, *CoverageResult._fields]) - w.writeheader() - for (file, func), result in sorted(merged_results.items()): - w.writerow({'file': file, 'name': func, **result}) + writer = csv.DictWriter(f, CoverageResult._fields) + writer.writeheader() + for r in results: + writer.writerow(r._asdict()) - # print results - def print_header(by): - if by == 'total': - entry = lambda k: 'TOTAL' - elif by == 'file': - entry = lambda k: k[0] + # find previous results? + if args.get('diff'): + diff_results = [] + try: + with openio(args['diff']) as f: + reader = csv.DictReader(f) + for r in reader: + try: + diff_results.append(CoverageResult(**{ + k: v for k, v in r.items() + if k in CoverageResult._fields})) + except TypeError: + pass + except FileNotFoundError: + pass + + # fold to remove duplicates + diff_results = fold(diff_results) + + if not args.get('quiet'): + if (args.get('annotate') + or args.get('lines') + or args.get('branches')): + # annotate sources + annotate( + paths, + results, + **args) else: - entry = lambda k: k[1] - - if not args.get('diff'): - print('%-36s %s' % (by, CoverageResult._header)) - else: - old = {entry(k) for k in results.keys()} - new = {entry(k) for k in prev_results.keys()} - print('%-36s %s' % ( - '%s (%d added, %d removed)' % (by, - sum(1 for k in new if k not in old), - sum(1 for k in old if k not in new)) - if by else '', - CoverageDiff._header)) - - def print_entries(by): - if by == 'total': - entry = lambda k: 'TOTAL' - elif by == 'file': - entry = lambda k: k[0] - else: - entry = lambda k: k[1] - - entries = co.defaultdict(lambda: CoverageResult()) - for k, result in results.items(): - entries[entry(k)] += result - - if not args.get('diff'): - for name, result in sorted(entries.items(), - key=lambda p: (p[1].key(**args), p)): - print('%-36s %s' % (name, result)) - else: - prev_entries = co.defaultdict(lambda: CoverageResult()) - for k, result in prev_results.items(): - prev_entries[entry(k)] += result - - diff_entries = {name: entries.get(name) - prev_entries.get(name) - for name in (entries.keys() | prev_entries.keys())} - - for name, diff in sorted(diff_entries.items(), - key=lambda p: (p[1].key(**args), p)): - if diff or args.get('all'): - print('%-36s %s' % (name, diff)) - - if args.get('quiet'): - pass - elif (args.get('annotate') - or args.get('lines') - or args.get('branches')): - annotate(paths, line_results, **args) - elif args.get('summary'): - print_header('') - print_entries('total') - elif args.get('files'): - print_header('file') - print_entries('file') - print_entries('total') - else: - print_header('function') - print_entries('function') - print_entries('total') + # print table + table( + results, + diff_results if args.get('diff') else None, + **args) # catch lack of coverage if args.get('error_on_lines') and any( - r.coverage_line_hits < r.coverage_line_count - for r in results.values()): + r.coverage_lines.a < r.coverage_lines.b for r in results): sys.exit(2) elif args.get('error_on_branches') and any( - r.coverage_branch_hits < r.coverage_branch_count - for r in results.values()): + r.coverage_branches.a < r.coverage_branches.b for r in results): sys.exit(3) @@ -462,60 +677,114 @@ if __name__ == "__main__": import sys parser = argparse.ArgumentParser( description="Find coverage info after running tests.") - parser.add_argument('gcda_paths', nargs='*', default=GCDA_PATHS, - help="Description of where to find *.gcda files. May be a directory \ - or a list of paths. Defaults to %r." % GCDA_PATHS) - parser.add_argument('-v', '--verbose', action='store_true', + parser.add_argument( + 'gcda_paths', + nargs='*', + default=GCDA_PATHS, + help="Description of where to find *.gcda files. May be a directory " + "or a list of paths. Defaults to %(default)r.") + parser.add_argument( + '-v', '--verbose', + action='store_true', help="Output commands that run behind the scenes.") - parser.add_argument('-q', '--quiet', action='store_true', + parser.add_argument( + '-q', '--quiet', + action='store_true', help="Don't show anything, useful with -o.") - parser.add_argument('-o', '--output', + parser.add_argument( + '-o', '--output', help="Specify CSV file to store results.") - parser.add_argument('-u', '--use', - help="Don't compile and find code sizes, instead use this CSV file.") - parser.add_argument('-d', '--diff', - help="Specify CSV file to diff code size against.") - parser.add_argument('-m', '--merge', - help="Merge with an existing CSV file when writing to output.") - parser.add_argument('-a', '--all', action='store_true', - help="Show all functions, not just the ones that changed.") - parser.add_argument('-A', '--everything', action='store_true', - help="Include builtin and libc specific symbols.") - parser.add_argument('-s', '--line-sort', action='store_true', + parser.add_argument( + '-u', '--use', + help="Don't parse anything, use this CSV file.") + parser.add_argument( + '-d', '--diff', + help="Specify CSV file to diff against.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") + parser.add_argument( + '-p', '--percent', + action='store_true', + help="Only show percentage change, not a full diff.") + parser.add_argument( + '-b', '--by-file', + action='store_true', + help="Group by file.") + parser.add_argument( + '--by-line', + action='store_true', + help="Group by line.") + parser.add_argument( + '-s', '--line-sort', + action='store_true', help="Sort by line coverage.") - parser.add_argument('-S', '--reverse-line-sort', action='store_true', + parser.add_argument( + '-S', '--reverse-line-sort', + action='store_true', help="Sort by line coverage, but backwards.") - parser.add_argument('--branch-sort', action='store_true', + parser.add_argument( + '--branch-sort', + action='store_true', help="Sort by branch coverage.") - parser.add_argument('--reverse-branch-sort', action='store_true', + parser.add_argument( + '--reverse-branch-sort', + action='store_true', help="Sort by branch coverage, but backwards.") - parser.add_argument('-F', '--files', action='store_true', - help="Show file-level coverage.") - parser.add_argument('-Y', '--summary', action='store_true', - help="Only show the total coverage.") - parser.add_argument('-p', '--annotate', action='store_true', + parser.add_argument( + '-Y', '--summary', + action='store_true', + help="Only show the total size.") + parser.add_argument( + '-l', '--annotate', + action='store_true', help="Show source files annotated with coverage info.") - parser.add_argument('-l', '--lines', action='store_true', + parser.add_argument( + '-L', '--lines', + action='store_true', help="Show uncovered lines.") - parser.add_argument('-b', '--branches', action='store_true', + parser.add_argument( + '-B', '--branches', + action='store_true', help="Show uncovered branches.") - parser.add_argument('-c', '--context', type=lambda x: int(x, 0), default=3, - help="Show a additional lines of context. Defaults to 3.") - parser.add_argument('-W', '--width', type=lambda x: int(x, 0), default=80, - help="Assume source is styled with this many columns. Defaults to 80.") - parser.add_argument('--color', - choices=['never', 'always', 'auto'], default='auto', + parser.add_argument( + '-c', '--context', + type=lambda x: int(x, 0), + default=3, + help="Show a additional lines of context. Defaults to %(default)r.") + parser.add_argument( + '-W', '--width', + type=lambda x: int(x, 0), + default=80, + help="Assume source is styled with this many columns. Defaults " + "to %(default)r.") + parser.add_argument( + '--color', + choices=['never', 'always', 'auto'], + default='auto', help="When to use terminal colors.") - parser.add_argument('-e', '--error-on-lines', action='store_true', + parser.add_argument( + '-e', '--error-on-lines', + action='store_true', help="Error if any lines are not covered.") - parser.add_argument('-E', '--error-on-branches', action='store_true', + parser.add_argument( + '-E', '--error-on-branches', + action='store_true', help="Error if any branches are not covered.") - parser.add_argument('--gcov-tool', default=['gcov'], + parser.add_argument( + '-A', '--everything', + action='store_true', + help="Include builtin and libc specific symbols.") + parser.add_argument( + '--gcov-tool', + default=GCOV_TOOL, type=lambda x: x.split(), - help="Path to the gcov tool to use.") - parser.add_argument('--build-dir', - help="Specify the relative build directory. Used to map object files \ - to the correct source files.") + help="Path to the gcov tool to use. Defaults to %(default)r.") + parser.add_argument( + '--build-dir', + help="Specify the relative build directory. Used to map object files " + "to the correct source files.") sys.exit(main(**{k: v for k, v in vars(parser.parse_args()).items() if v is not None})) diff --git a/scripts/data.py b/scripts/data.py index 80b0009a..353b163e 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -5,71 +5,123 @@ # by Linux's Bloat-O-Meter. # -import os +import collections as co +import csv import glob import itertools as it -import subprocess as sp -import shlex +import math as m +import os import re -import csv -import collections as co +import shlex +import subprocess as sp OBJ_PATHS = ['*.o'] +NM_TOOL = ['nm'] +TYPE = 'dDbB' -class DataResult(co.namedtuple('DataResult', 'data_size')): + +# integer fields +class IntField(co.namedtuple('IntField', 'x')): __slots__ = () - def __new__(cls, data_size=0): - return super().__new__(cls, int(data_size)) + def __new__(cls, x): + if isinstance(x, IntField): + return x + if isinstance(x, str): + try: + x = int(x, 0) + except ValueError: + # also accept +-∞ and +-inf + if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): + x = float('inf') + elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): + x = float('-inf') + else: + raise + return super().__new__(cls, x) + + def __int__(self): + assert not m.isinf(self.x) + return self.x + + def __float__(self): + return float(self.x) + + def __str__(self): + if self.x == float('inf'): + return '∞' + elif self.x == float('-inf'): + return '-∞' + else: + return str(self.x) + + none = '%7s' % '-' + def table(self): + return '%7s' % (self,) + + diff_none = '%7s' % '-' + diff_table = table + + def diff_diff(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + diff = new - old + if diff == float('+inf'): + return '%7s' % '+∞' + elif diff == float('-inf'): + return '%7s' % '-∞' + else: + return '%+7d' % diff + + def ratio(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + if m.isinf(new) and m.isinf(old): + return 0.0 + elif m.isinf(new): + return float('+inf') + elif m.isinf(old): + return float('-inf') + elif not old and not new: + return 0.0 + elif not old: + return 1.0 + else: + return (new-old) / old def __add__(self, other): - return self.__class__(self.data_size + other.data_size) + return IntField(self.x + other.x) - def __sub__(self, other): - return DataDiff(other, self) + def __mul__(self, other): + return IntField(self.x * other.x) - def __rsub__(self, other): - return self.__class__.__sub__(other, self) + def __lt__(self, other): + return self.x < other.x - def key(self, **args): - if args.get('size_sort'): - return -self.data_size - elif args.get('reverse_size_sort'): - return +self.data_size + def __gt__(self, other): + return self.__class__.__lt__(other, self) + + def __le__(self, other): + return not self.__gt__(other) + + def __ge__(self, other): + return not self.__lt__(other) + + def __truediv__(self, n): + if m.isinf(self.x): + return self else: - return None + return IntField(round(self.x / n)) - _header = '%7s' % 'size' - def __str__(self): - return '%7d' % self.data_size - -class DataDiff(co.namedtuple('DataDiff', 'old,new')): +# data size results +class DataResult(co.namedtuple('DataResult', 'file,function,data_size')): __slots__ = () + def __new__(cls, file, function, data_size): + return super().__new__(cls, file, function, IntField(data_size)) - def ratio(self): - old = self.old.data_size if self.old is not None else 0 - new = self.new.data_size if self.new is not None else 0 - return (new-old) / old if old else 1.0 - - def key(self, **args): - return ( - self.new.key(**args) if self.new is not None else 0, - -self.ratio()) - - def __bool__(self): - return bool(self.ratio()) - - _header = '%7s %7s %7s' % ('old', 'new', 'diff') - def __str__(self): - old = self.old.data_size if self.old is not None else 0 - new = self.new.data_size if self.new is not None else 0 - diff = new - old - ratio = self.ratio() - return '%7s %7s %+7d%s' % ( - old or "-", - new or "-", - diff, - ' (%+.1f%%)' % (100*ratio) if ratio else '') + def __add__(self, other): + return DataResult(self.file, self.function, + self.data_size + other.data_size) def openio(path, mode='r'): @@ -81,20 +133,25 @@ def openio(path, mode='r'): else: return open(path, mode) -def collect(paths, **args): - results = co.defaultdict(lambda: DataResult()) +def collect(paths, *, + nm_tool=NM_TOOL, + type=TYPE, + build_dir=None, + everything=False, + **args): + results = [] pattern = re.compile( '^(?P[0-9a-fA-F]+)' + - ' (?P[%s])' % re.escape(args['type']) + + ' (?P[%s])' % re.escape(type) + ' (?P.+?)$') for path in paths: # map to source file src_path = re.sub('\.o$', '.c', path) - if args.get('build_dir'): - src_path = re.sub('%s/*' % re.escape(args['build_dir']), '', + if build_dir: + src_path = re.sub('%s/*' % re.escape(build_dir), '', src_path) # note nm-tool may contain extra args - cmd = args['nm_tool'] + ['--size-sort', path] + cmd = nm_tool + ['--size-sort', path] if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, @@ -107,12 +164,15 @@ def collect(paths, **args): if m: func = m.group('func') # discard internal functions - if not args.get('everything') and func.startswith('__'): + if not everything and func.startswith('__'): continue # discard .8449 suffixes created by optimizer func = re.sub('\.[0-9]+', '', func) - results[(src_path, func)] += DataResult( - int(m.group('size'), 16)) + + results.append(DataResult( + src_path, func, + int(m.group('size'), 16))) + proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -122,12 +182,167 @@ def collect(paths, **args): return results -def main(**args): + +def fold(results, *, + by=['file', 'function'], + **_): + folding = co.OrderedDict() + for r in results: + name = tuple(getattr(r, k) for k in by) + if name not in folding: + folding[name] = [] + folding[name].append(r) + + folded = [] + for rs in folding.values(): + folded.append(sum(rs[1:], start=rs[0])) + + return folded + + +def table(results, diff_results=None, *, + by_file=False, + size_sort=False, + reverse_size_sort=False, + summary=False, + all=False, + percent=False, + **_): + all_, all = all, __builtins__.all + + # fold + results = fold(results, by=['file' if by_file else 'function']) + if diff_results is not None: + diff_results = fold(diff_results, + by=['file' if by_file else 'function']) + + table = { + r.file if by_file else r.function: r + for r in results} + diff_table = { + r.file if by_file else r.function: r + for r in diff_results or []} + + # sort, note that python's sort is stable + names = list(table.keys() | diff_table.keys()) + names.sort() + if diff_results is not None: + names.sort(key=lambda n: -IntField.ratio( + table[n].data_size if n in table else None, + diff_table[n].data_size if n in diff_table else None)) + if size_sort: + names.sort(key=lambda n: (table[n].data_size,) if n in table else (), + reverse=True) + elif reverse_size_sort: + names.sort(key=lambda n: (table[n].data_size,) if n in table else (), + reverse=False) + + # print header + print('%-36s' % ('%s%s' % ( + 'file' if by_file else 'function', + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else ''), + end='') + if diff_results is None: + print(' %s' % ('size'.rjust(len(IntField.none)))) + elif percent: + print(' %s' % ('size'.rjust(len(IntField.diff_none)))) + else: + print(' %s %s %s' % ( + 'old'.rjust(len(IntField.diff_none)), + 'new'.rjust(len(IntField.diff_none)), + 'diff'.rjust(len(IntField.diff_none)))) + + # print entries + if not summary: + for name in names: + r = table.get(name) + if diff_results is not None: + diff_r = diff_table.get(name) + ratio = IntField.ratio( + r.data_size if r else None, + diff_r.data_size if diff_r else None) + if not ratio and not all_: + continue + + print('%-36s' % name, end='') + if diff_results is None: + print(' %s' % ( + r.data_size.table() + if r else IntField.none)) + elif percent: + print(' %s%s' % ( + r.data_size.diff_table() + if r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)))) + else: + print(' %s %s %s%s' % ( + diff_r.data_size.diff_table() + if diff_r else IntField.diff_none, + r.data_size.diff_table() + if r else IntField.diff_none, + IntField.diff_diff( + r.data_size if r else None, + diff_r.data_size if diff_r else None) + if r or diff_r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)) + if ratio else '')) + + # print total + total = fold(results, by=[]) + r = total[0] if total else None + if diff_results is not None: + diff_total = fold(diff_results, by=[]) + diff_r = diff_total[0] if diff_total else None + ratio = IntField.ratio( + r.data_size if r else None, + diff_r.data_size if diff_r else None) + + print('%-36s' % 'TOTAL', end='') + if diff_results is None: + print(' %s' % ( + r.data_size.table() + if r else IntField.none)) + elif percent: + print(' %s%s' % ( + r.data_size.diff_table() + if r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)))) + else: + print(' %s %s %s%s' % ( + diff_r.data_size.diff_table() + if diff_r else IntField.diff_none, + r.data_size.diff_table() + if r else IntField.diff_none, + IntField.diff_diff( + r.data_size if r else None, + diff_r.data_size if diff_r else None) + if r or diff_r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)) + if ratio else '')) + + +def main(obj_paths, **args): # find sizes if not args.get('use', None): # find .o files paths = [] - for path in args['obj_paths']: + for path in obj_paths: if os.path.isdir(path): path = path + '/*.o' @@ -135,127 +350,61 @@ def main(**args): paths.append(path) if not paths: - print('no .obj files found in %r?' % args['obj_paths']) + print('no .obj files found in %r?' % obj_paths) sys.exit(-1) results = collect(paths, **args) else: + results = [] with openio(args['use']) as f: - r = csv.DictReader(f) - results = { - (result['file'], result['name']): DataResult( - *(result[f] for f in DataResult._fields)) - for result in r - if all(result.get(f) not in {None, ''} - for f in DataResult._fields)} + reader = csv.DictReader(f) + for r in reader: + try: + results.append(DataResult(**{ + k: v for k, v in r.items() + if k in DataResult._fields})) + except TypeError: + pass - # find previous results? - if args.get('diff'): - try: - with openio(args['diff']) as f: - r = csv.DictReader(f) - prev_results = { - (result['file'], result['name']): DataResult( - *(result[f] for f in DataResult._fields)) - for result in r - if all(result.get(f) not in {None, ''} - for f in DataResult._fields)} - except FileNotFoundError: - prev_results = [] + # fold to remove duplicates + results = fold(results) + + # sort because why not + results.sort() # write results to CSV if args.get('output'): - merged_results = co.defaultdict(lambda: {}) - other_fields = [] - - # merge? - if args.get('merge'): - try: - with openio(args['merge']) as f: - r = csv.DictReader(f) - for result in r: - file = result.pop('file', '') - func = result.pop('name', '') - for f in DataResult._fields: - result.pop(f, None) - merged_results[(file, func)] = result - other_fields = result.keys() - except FileNotFoundError: - pass - - for (file, func), result in results.items(): - merged_results[(file, func)] |= result._asdict() - with openio(args['output'], 'w') as f: - w = csv.DictWriter(f, ['file', 'name', - *other_fields, *DataResult._fields]) - w.writeheader() - for (file, func), result in sorted(merged_results.items()): - w.writerow({'file': file, 'name': func, **result}) + writer = csv.DictWriter(f, DataResult._fields) + writer.writeheader() + for r in results: + writer.writerow(r._asdict()) - # print results - def print_header(by): - if by == 'total': - entry = lambda k: 'TOTAL' - elif by == 'file': - entry = lambda k: k[0] - else: - entry = lambda k: k[1] + # find previous results? + if args.get('diff'): + diff_results = [] + try: + with openio(args['diff']) as f: + reader = csv.DictReader(f) + for r in reader: + try: + diff_results.append(DataResult(**{ + k: v for k, v in r.items() + if k in DataResult._fields})) + except TypeError: + pass + except FileNotFoundError: + pass - if not args.get('diff'): - print('%-36s %s' % (by, DataResult._header)) - else: - old = {entry(k) for k in results.keys()} - new = {entry(k) for k in prev_results.keys()} - print('%-36s %s' % ( - '%s (%d added, %d removed)' % (by, - sum(1 for k in new if k not in old), - sum(1 for k in old if k not in new)) - if by else '', - DataDiff._header)) + # fold to remove duplicates + diff_results = fold(diff_results) - def print_entries(by): - if by == 'total': - entry = lambda k: 'TOTAL' - elif by == 'file': - entry = lambda k: k[0] - else: - entry = lambda k: k[1] - - entries = co.defaultdict(lambda: DataResult()) - for k, result in results.items(): - entries[entry(k)] += result - - if not args.get('diff'): - for name, result in sorted(entries.items(), - key=lambda p: (p[1].key(**args), p)): - print('%-36s %s' % (name, result)) - else: - prev_entries = co.defaultdict(lambda: DataResult()) - for k, result in prev_results.items(): - prev_entries[entry(k)] += result - - diff_entries = {name: entries.get(name) - prev_entries.get(name) - for name in (entries.keys() | prev_entries.keys())} - - for name, diff in sorted(diff_entries.items(), - key=lambda p: (p[1].key(**args), p)): - if diff or args.get('all'): - print('%-36s %s' % (name, diff)) - - if args.get('quiet'): - pass - elif args.get('summary'): - print_header('') - print_entries('total') - elif args.get('files'): - print_header('file') - print_entries('file') - print_entries('total') - else: - print_header('function') - print_entries('function') - print_entries('total') + # print table + if not args.get('quiet'): + table( + results, + diff_results if args.get('diff') else None, + **args) if __name__ == "__main__": @@ -263,42 +412,72 @@ if __name__ == "__main__": import sys parser = argparse.ArgumentParser( description="Find data size at the function level.") - parser.add_argument('obj_paths', nargs='*', default=OBJ_PATHS, - help="Description of where to find *.o files. May be a directory \ - or a list of paths. Defaults to %r." % OBJ_PATHS) - parser.add_argument('-v', '--verbose', action='store_true', + parser.add_argument( + 'obj_paths', + nargs='*', + default=OBJ_PATHS, + help="Description of where to find *.o files. May be a directory " + "or a list of paths. Defaults to %(default)r.") + parser.add_argument( + '-v', '--verbose', + action='store_true', help="Output commands that run behind the scenes.") - parser.add_argument('-q', '--quiet', action='store_true', + parser.add_argument( + '-q', '--quiet', + action='store_true', help="Don't show anything, useful with -o.") - parser.add_argument('-o', '--output', + parser.add_argument( + '-o', '--output', help="Specify CSV file to store results.") - parser.add_argument('-u', '--use', - help="Don't compile and find data sizes, instead use this CSV file.") - parser.add_argument('-d', '--diff', - help="Specify CSV file to diff data size against.") - parser.add_argument('-m', '--merge', - help="Merge with an existing CSV file when writing to output.") - parser.add_argument('-a', '--all', action='store_true', - help="Show all functions, not just the ones that changed.") - parser.add_argument('-A', '--everything', action='store_true', - help="Include builtin and libc specific symbols.") - parser.add_argument('-s', '--size-sort', action='store_true', + parser.add_argument( + '-u', '--use', + help="Don't parse anything, use this CSV file.") + parser.add_argument( + '-d', '--diff', + help="Specify CSV file to diff against.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") + parser.add_argument( + '-p', '--percent', + action='store_true', + help="Only show percentage change, not a full diff.") + parser.add_argument( + '-b', '--by-file', + action='store_true', + help="Group by file. Note this does not include padding " + "so sizes may differ from other tools.") + parser.add_argument( + '-s', '--size-sort', + action='store_true', help="Sort by size.") - parser.add_argument('-S', '--reverse-size-sort', action='store_true', + parser.add_argument( + '-S', '--reverse-size-sort', + action='store_true', help="Sort by size, but backwards.") - parser.add_argument('-F', '--files', action='store_true', - help="Show file-level data sizes. Note this does not include padding! " - "So sizes may differ from other tools.") - parser.add_argument('-Y', '--summary', action='store_true', - help="Only show the total data size.") - parser.add_argument('--type', default='dDbB', + parser.add_argument( + '-Y', '--summary', + action='store_true', + help="Only show the total size.") + parser.add_argument( + '-A', '--everything', + action='store_true', + help="Include builtin and libc specific symbols.") + parser.add_argument( + '--type', + default=TYPE, help="Type of symbols to report, this uses the same single-character " "type-names emitted by nm. Defaults to %(default)r.") - parser.add_argument('--nm-tool', default=['nm'], type=lambda x: x.split(), - help="Path to the nm tool to use.") - parser.add_argument('--build-dir', - help="Specify the relative build directory. Used to map object files \ - to the correct source files.") + parser.add_argument( + '--nm-tool', + type=lambda x: x.split(), + default=NM_TOOL, + help="Path to the nm tool to use. Defaults to %(default)r") + parser.add_argument( + '--build-dir', + help="Specify the relative build directory. Used to map object files " + "to the correct source files.") sys.exit(main(**{k: v for k, v in vars(parser.parse_args()).items() if v is not None})) diff --git a/scripts/stack.py b/scripts/stack.py index 22169192..194f831c 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -4,17 +4,124 @@ # report as infinite stack usage. # -import os +import collections as co +import csv import glob import itertools as it -import re -import csv -import collections as co import math as m +import os +import re CI_PATHS = ['*.ci'] + +# integer fields +class IntField(co.namedtuple('IntField', 'x')): + __slots__ = () + def __new__(cls, x): + if isinstance(x, IntField): + return x + if isinstance(x, str): + try: + x = int(x, 0) + except ValueError: + # also accept +-∞ and +-inf + if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): + x = float('inf') + elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): + x = float('-inf') + else: + raise + return super().__new__(cls, x) + + def __int__(self): + assert not m.isinf(self.x) + return self.x + + def __float__(self): + return float(self.x) + + def __str__(self): + if self.x == float('inf'): + return '∞' + elif self.x == float('-inf'): + return '-∞' + else: + return str(self.x) + + none = '%7s' % '-' + def table(self): + return '%7s' % (self,) + + diff_none = '%7s' % '-' + diff_table = table + + def diff_diff(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + diff = new - old + if diff == float('+inf'): + return '%7s' % '+∞' + elif diff == float('-inf'): + return '%7s' % '-∞' + else: + return '%+7d' % diff + + def ratio(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + if m.isinf(new) and m.isinf(old): + return 0.0 + elif m.isinf(new): + return float('+inf') + elif m.isinf(old): + return float('-inf') + elif not old and not new: + return 0.0 + elif not old: + return 1.0 + else: + return (new-old) / old + + def __add__(self, other): + return IntField(self.x + other.x) + + def __mul__(self, other): + return IntField(self.x * other.x) + + def __lt__(self, other): + return self.x < other.x + + def __gt__(self, other): + return self.__class__.__lt__(other, self) + + def __le__(self, other): + return not self.__gt__(other) + + def __ge__(self, other): + return not self.__lt__(other) + + def __truediv__(self, n): + if m.isinf(self.x): + return self + else: + return IntField(round(self.x / n)) + +# size results +class StackResult(co.namedtuple('StackResult', + 'file,function,stack_frame,stack_limit')): + __slots__ = () + def __new__(cls, file, function, stack_frame, stack_limit): + return super().__new__(cls, file, function, + IntField(stack_frame), IntField(stack_limit)) + + def __add__(self, other): + return StackResult(self.file, self.function, + self.stack_frame + other.stack_frame, + max(self.stack_limit, other.stack_limit)) + + def openio(path, mode='r'): if path == '-': if 'r' in mode: @@ -24,91 +131,10 @@ def openio(path, mode='r'): else: return open(path, mode) -class StackResult(co.namedtuple('StackResult', 'stack_frame,stack_limit')): - __slots__ = () - def __new__(cls, stack_frame=0, stack_limit=0): - return super().__new__(cls, - int(stack_frame), - float(stack_limit)) - def __add__(self, other): - return self.__class__( - self.stack_frame + other.stack_frame, - max(self.stack_limit, other.stack_limit)) - - def __sub__(self, other): - return StackDiff(other, self) - - def __rsub__(self, other): - return self.__class__.__sub__(other, self) - - def key(self, **args): - if args.get('limit_sort'): - return -self.stack_limit - elif args.get('reverse_limit_sort'): - return +self.stack_limit - elif args.get('frame_sort'): - return -self.stack_frame - elif args.get('reverse_frame_sort'): - return +self.stack_frame - else: - return None - - _header = '%7s %7s' % ('frame', 'limit') - def __str__(self): - return '%7d %7s' % ( - self.stack_frame, - '∞' if m.isinf(self.stack_limit) else int(self.stack_limit)) - -class StackDiff(co.namedtuple('StackDiff', 'old,new')): - __slots__ = () - - def ratio(self): - old_limit = self.old.stack_limit if self.old is not None else 0 - new_limit = self.new.stack_limit if self.new is not None else 0 - return (0.0 if m.isinf(new_limit) and m.isinf(old_limit) - else +float('inf') if m.isinf(new_limit) - else -float('inf') if m.isinf(old_limit) - else 0.0 if not old_limit and not new_limit - else 1.0 if not old_limit - else (new_limit-old_limit) / old_limit) - - def key(self, **args): - return ( - self.new.key(**args) if self.new is not None else 0, - -self.ratio()) - - def __bool__(self): - return bool(self.ratio()) - - _header = '%15s %15s %15s' % ('old', 'new', 'diff') - def __str__(self): - old_frame = self.old.stack_frame if self.old is not None else 0 - old_limit = self.old.stack_limit if self.old is not None else 0 - new_frame = self.new.stack_frame if self.new is not None else 0 - new_limit = self.new.stack_limit if self.new is not None else 0 - diff_frame = new_frame - old_frame - diff_limit = (0 if m.isinf(new_limit) and m.isinf(old_limit) - else new_limit - old_limit) - ratio = self.ratio() - return '%7s %7s %7s %7s %+7d %7s%s' % ( - old_frame if self.old is not None else '-', - ('∞' if m.isinf(old_limit) else int(old_limit)) - if self.old is not None else '-', - new_frame if self.new is not None else '-', - ('∞' if m.isinf(new_limit) else int(new_limit)) - if self.new is not None else '-', - diff_frame, - '+∞' if diff_limit > 0 and m.isinf(diff_limit) - else '-∞' if diff_limit < 0 and m.isinf(diff_limit) - else '%+d' % diff_limit, - '' if not ratio - else ' (+∞%)' if ratio > 0 and m.isinf(ratio) - else ' (-∞%)' if ratio < 0 and m.isinf(ratio) - else ' (%+.1f%%)' % (100*ratio)) - - -def collect(paths, **args): +def collect(paths, *, + everything=False, + **args): # parse the vcg format k_pattern = re.compile('([a-z]+)\s*:', re.DOTALL) v_pattern = re.compile('(?:"(.*?)"|([a-z]+))', re.DOTALL) @@ -154,9 +180,11 @@ def collect(paths, **args): m = f_pattern.match(info['label']) if m: function, file, size, type = m.groups() - if not args.get('quiet') and type != 'static': + if (not args.get('quiet') + and 'static' not in type + and 'bounded' not in type): print('warning: found non-static stack for %s (%s)' - % (function, type)) + % (function, type, size)) _, _, _, targets = callgraph[info['title']] callgraph[info['title']] = ( file, function, int(size), targets) @@ -167,7 +195,7 @@ def collect(paths, **args): else: continue - if not args.get('everything'): + if not everything: for source, (s_file, s_function, _, _) in list(callgraph.items()): # discard internal functions if s_file.startswith('<') or s_file.startswith('/usr/include'): @@ -200,22 +228,266 @@ def collect(paths, **args): return calls # build results - results = {} - result_calls = {} + results = [] + calls = {} for source, (s_file, s_function, frame, targets) in callgraph.items(): limit = find_limit(source) - calls = find_calls(targets) - results[(s_file, s_function)] = StackResult(frame, limit) - result_calls[(s_file, s_function)] = calls + cs = find_calls(targets) + results.append(StackResult(s_file, s_function, frame, limit)) + calls[(s_file, s_function)] = cs - return results, result_calls + return results, calls -def main(**args): + +def fold(results, *, + by=['file', 'function'], + **_): + folding = co.OrderedDict() + for r in results: + name = tuple(getattr(r, k) for k in by) + if name not in folding: + folding[name] = [] + folding[name].append(r) + + folded = [] + for rs in folding.values(): + folded.append(sum(rs[1:], start=rs[0])) + + return folded + +def fold_calls(calls, *, + by=['file', 'function'], + **_): + def by_(name): + file, function = name + return (((file,) if 'file' in by else ()) + + ((function,) if 'function' in by else ())) + + folded = {} + for name, cs in calls.items(): + name = by_(name) + if name not in folded: + folded[name] = set() + folded[name] |= {by_(c) for c in cs} + + return folded + + +def table(results, calls, diff_results=None, *, + by_file=False, + limit_sort=False, + reverse_limit_sort=False, + frame_sort=False, + reverse_frame_sort=False, + summary=False, + all=False, + percent=False, + tree=False, + depth=None, + **_): + all_, all = all, __builtins__.all + + # tree doesn't really make sense with depth=0, assume depth=inf + if depth is None: + depth = float('inf') if tree else 0 + + # fold + results = fold(results, by=['file' if by_file else 'function']) + calls = fold_calls(calls, by=['file' if by_file else 'function']) + if diff_results is not None: + diff_results = fold(diff_results, + by=['file' if by_file else 'function']) + + table = { + r.file if by_file else r.function: r + for r in results} + diff_table = { + r.file if by_file else r.function: r + for r in diff_results or []} + + # sort, note that python's sort is stable + names = list(table.keys() | diff_table.keys()) + names.sort() + if diff_results is not None: + names.sort(key=lambda n: -IntField.ratio( + table[n].stack_frame if n in table else None, + diff_table[n].stack_frame if n in diff_table else None)) + if limit_sort: + names.sort(key=lambda n: (table[n].stack_limit,) if n in table else (), + reverse=True) + elif reverse_limit_sort: + names.sort(key=lambda n: (table[n].stack_limit,) if n in table else (), + reverse=False) + elif frame_sort: + names.sort(key=lambda n: (table[n].stack_frame,) if n in table else (), + reverse=True) + elif reverse_frame_sort: + names.sort(key=lambda n: (table[n].stack_frame,) if n in table else (), + reverse=False) + + # adjust the name width based on the expected call depth, note that we + # can't always find the depth due to recursion + width = 36 + (4*depth if not m.isinf(depth) else 0) + + # print header + if not tree: + print('%-*s' % (width, '%s%s' % ( + 'file' if by_file else 'function', + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else ''), + end='') + if diff_results is None: + print(' %s %s' % ( + 'frame'.rjust(len(IntField.none)), + 'limit'.rjust(len(IntField.none)))) + elif percent: + print(' %s %s' % ( + 'frame'.rjust(len(IntField.diff_none)), + 'limit'.rjust(len(IntField.diff_none)))) + else: + print(' %s %s %s %s %s %s' % ( + 'oframe'.rjust(len(IntField.diff_none)), + 'olimit'.rjust(len(IntField.diff_none)), + 'nframe'.rjust(len(IntField.diff_none)), + 'nlimit'.rjust(len(IntField.diff_none)), + 'dframe'.rjust(len(IntField.diff_none)), + 'dlimit'.rjust(len(IntField.diff_none)))) + + # print entries + if not summary: + # print the tree recursively + def table_calls(names_, depth, + prefixes=('', '', '', '')): + for i, name in enumerate(names_): + r = table.get(name) + if diff_results is not None: + diff_r = diff_table.get(name) + ratio = IntField.ratio( + r.stack_limit if r else None, + diff_r.stack_limit if diff_r else None) + if not ratio and not all_: + continue + + is_last = (i == len(names_)-1) + print('%-*s' % (width, prefixes[0+is_last] + name), end='') + if tree: + print() + elif diff_results is None: + print(' %s %s' % ( + r.stack_frame.table() + if r else IntField.none, + r.stack_limit.table() + if r else IntField.none)) + elif percent: + print(' %s %s%s' % ( + r.stack_frame.diff_table() + if r else IntField.diff_none, + r.stack_limit.diff_table() + if r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)))) + else: + print(' %s %s %s %s %s %s%s' % ( + diff_r.stack_frame.diff_table() + if diff_r else IntField.diff_none, + diff_r.stack_limit.diff_table() + if diff_r else IntField.diff_none, + r.stack_frame.diff_table() + if r else IntField.diff_none, + r.stack_limit.diff_table() + if r else IntField.diff_none, + IntField.diff_diff( + r.stack_frame if r else None, + diff_r.stack_frame if diff_r else None) + if r or diff_r else IntField.diff_none, + IntField.diff_diff( + r.stack_limit if r else None, + diff_r.stack_limit if diff_r else None) + if r or diff_r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)) + if ratio else '')) + + # recurse? + if depth > 0: + cs = calls.get((name,), set()) + table_calls( + [n for n in names if (n,) in cs], + depth-1, + ( prefixes[2+is_last] + "|-> ", + prefixes[2+is_last] + "'-> ", + prefixes[2+is_last] + "| ", + prefixes[2+is_last] + " ")) + + + table_calls(names, depth) + + # print total + if not tree: + total = fold(results, by=[]) + r = total[0] if total else None + if diff_results is not None: + diff_total = fold(diff_results, by=[]) + diff_r = diff_total[0] if diff_total else None + ratio = IntField.ratio( + r.stack_limit if r else None, + diff_r.stack_limit if diff_r else None) + + print('%-*s' % (width, 'TOTAL'), end='') + if diff_results is None: + print(' %s %s' % ( + r.stack_frame.table() + if r else IntField.none, + r.stack_limit.table() + if r else IntField.none)) + elif percent: + print(' %s %s%s' % ( + r.stack_frame.diff_table() + if r else IntField.diff_none, + r.stack_limit.diff_table() + if r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)))) + else: + print(' %s %s %s %s %s %s%s' % ( + diff_r.stack_frame.diff_table() + if diff_r else IntField.diff_none, + diff_r.stack_limit.diff_table() + if diff_r else IntField.diff_none, + r.stack_frame.diff_table() + if r else IntField.diff_none, + r.stack_limit.diff_table() + if r else IntField.diff_none, + IntField.diff_diff( + r.stack_frame if r else None, + diff_r.stack_frame if diff_r else None) + if r or diff_r else IntField.diff_none, + IntField.diff_diff( + r.stack_limit if r else None, + diff_r.stack_limit if diff_r else None) + if r or diff_r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)) + if ratio else '')) + + +def main(ci_paths, **args): # find sizes if not args.get('use', None): # find .ci files paths = [] - for path in args['ci_paths']: + for path in ci_paths: if os.path.isdir(path): path = path + '/*.ci' @@ -223,160 +495,68 @@ def main(**args): paths.append(path) if not paths: - print('no .ci files found in %r?' % args['ci_paths']) + print('no .ci files found in %r?' % ci_paths) sys.exit(-1) - results, result_calls = collect(paths, **args) + results, calls = collect(paths, **args) else: + results = [] with openio(args['use']) as f: - r = csv.DictReader(f) - results = { - (result['file'], result['name']): StackResult( - *(result[f] for f in StackResult._fields)) - for result in r - if all(result.get(f) not in {None, ''} - for f in StackResult._fields)} + reader = csv.DictReader(f) + for r in reader: + try: + results.append(StackResult(**{ + k: v for k, v in r.items() + if k in StackResult._fields})) + except TypeError: + pass - result_calls = {} + calls = {} - # find previous results? - if args.get('diff'): - try: - with openio(args['diff']) as f: - r = csv.DictReader(f) - prev_results = { - (result['file'], result['name']): StackResult( - *(result[f] for f in StackResult._fields)) - for result in r - if all(result.get(f) not in {None, ''} - for f in StackResult._fields)} - except FileNotFoundError: - prev_results = [] + # fold to remove duplicates + results = fold(results) + + # sort because why not + results.sort() # write results to CSV if args.get('output'): - merged_results = co.defaultdict(lambda: {}) - other_fields = [] - - # merge? - if args.get('merge'): - try: - with openio(args['merge']) as f: - r = csv.DictReader(f) - for result in r: - file = result.pop('file', '') - func = result.pop('name', '') - for f in StackResult._fields: - result.pop(f, None) - merged_results[(file, func)] = result - other_fields = result.keys() - except FileNotFoundError: - pass - - for (file, func), result in results.items(): - merged_results[(file, func)] |= result._asdict() - with openio(args['output'], 'w') as f: - w = csv.DictWriter(f, ['file', 'name', - *other_fields, *StackResult._fields]) - w.writeheader() - for (file, func), result in sorted(merged_results.items()): - w.writerow({'file': file, 'name': func, **result}) + writer = csv.DictWriter(f, StackResult._fields) + writer.writeheader() + for r in results: + writer.writerow(r._asdict()) - # print results - def print_header(by): - if by == 'total': - entry = lambda k: 'TOTAL' - elif by == 'file': - entry = lambda k: k[0] - else: - entry = lambda k: k[1] + # find previous results? + if args.get('diff'): + diff_results = [] + try: + with openio(args['diff']) as f: + reader = csv.DictReader(f) + for r in reader: + try: + diff_results.append(StackResult(**{ + k: v for k, v in r.items() + if k in StackResult._fields})) + except TypeError: + pass + except FileNotFoundError: + pass - if not args.get('diff'): - print('%-36s %s' % (by, StackResult._header)) - else: - old = {entry(k) for k in results.keys()} - new = {entry(k) for k in prev_results.keys()} - print('%-36s %s' % ( - '%s (%d added, %d removed)' % (by, - sum(1 for k in new if k not in old), - sum(1 for k in old if k not in new)) - if by else '', - StackDiff._header)) + # fold to remove duplicates + diff_results = fold(diff_results) - def print_entries(by): - # print optional tree of dependencies - def print_calls(entries, entry_calls, depth, - filter=lambda _: True, - prefixes=('', '', '', '')): - filtered_entries = { - name: result for name, result in entries.items() - if filter(name)} - for i, (name, result) in enumerate(sorted(filtered_entries.items(), - key=lambda p: (p[1].key(**args), p))): - last = (i == len(filtered_entries)-1) - print('%-36s %s' % (prefixes[0+last] + name, result)) + # print table + if not args.get('quiet'): + table( + results, + calls, + diff_results if args.get('diff') else None, + **args) - if depth > 0 and by != 'total': - calls = entry_calls.get(name, set()) - print_calls(entries, entry_calls, depth-1, - lambda name: name in calls, - ( prefixes[2+last] + "|-> ", - prefixes[2+last] + "'-> ", - prefixes[2+last] + "| ", - prefixes[2+last] + " ")) - - if by == 'total': - entry = lambda k: 'TOTAL' - elif by == 'file': - entry = lambda k: k[0] - else: - entry = lambda k: k[1] - - entries = co.defaultdict(lambda: StackResult()) - for k, result in results.items(): - entries[entry(k)] += result - - entry_calls = co.defaultdict(lambda: set()) - for k, calls in result_calls.items(): - entry_calls[entry(k)] |= {entry(c) for c in calls} - - if not args.get('diff'): - print_calls( - entries, - entry_calls, - args.get('depth', 0)) - else: - prev_entries = co.defaultdict(lambda: StackResult()) - for k, result in prev_results.items(): - prev_entries[entry(k)] += result - - diff_entries = {name: entries.get(name) - prev_entries.get(name) - for name in (entries.keys() | prev_entries.keys())} - - print_calls( - {name: diff for name, diff in diff_entries.items() - if diff or args.get('all')}, - entry_calls, - args.get('depth', 0)) - - if args.get('quiet'): - pass - elif args.get('summary'): - print_header('') - print_entries('total') - elif args.get('files'): - print_header('file') - print_entries('file') - print_entries('total') - else: - print_header('function') - print_entries('function') - print_entries('total') - - # catch recursion + # error on recursion if args.get('error_on_recursion') and any( - m.isinf(limit) for _, _, _, limit, _ in results): + m.isinf(float(r.stack_limit)) for r in results): sys.exit(2) @@ -385,45 +565,83 @@ if __name__ == "__main__": import sys parser = argparse.ArgumentParser( description="Find stack usage at the function level.") - parser.add_argument('ci_paths', nargs='*', default=CI_PATHS, - help="Description of where to find *.ci files. May be a directory \ - or a list of paths. Defaults to %r." % CI_PATHS) - parser.add_argument('-v', '--verbose', action='store_true', + parser.add_argument( + 'ci_paths', + nargs='*', + default=CI_PATHS, + help="Description of where to find *.ci files. May be a directory " + "or a list of paths. Defaults to %r." % CI_PATHS) + parser.add_argument( + '-v', '--verbose', + action='store_true', help="Output commands that run behind the scenes.") - parser.add_argument('-q', '--quiet', action='store_true', + parser.add_argument( + '-q', '--quiet', + action='store_true', help="Don't show anything, useful with -o.") - parser.add_argument('-o', '--output', + parser.add_argument( + '-o', '--output', help="Specify CSV file to store results.") - parser.add_argument('-u', '--use', - help="Don't parse callgraph files, instead use this CSV file.") - parser.add_argument('-d', '--diff', + parser.add_argument( + '-u', '--use', + help="Don't parse anything, use this CSV file.") + parser.add_argument( + '-d', '--diff', help="Specify CSV file to diff against.") - parser.add_argument('-m', '--merge', - help="Merge with an existing CSV file when writing to output.") - parser.add_argument('-a', '--all', action='store_true', - help="Show all functions, not just the ones that changed.") - parser.add_argument('-A', '--everything', action='store_true', - help="Include builtin and libc specific symbols.") - parser.add_argument('--frame-sort', action='store_true', - help="Sort by stack frame size.") - parser.add_argument('--reverse-frame-sort', action='store_true', - help="Sort by stack frame size, but backwards.") - parser.add_argument('-s', '--limit-sort', action='store_true', + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") + parser.add_argument( + '-p', '--percent', + action='store_true', + help="Only show percentage change, not a full diff.") + parser.add_argument( + '-t', '--tree', + action='store_true', + help="Only show the function call tree.") + parser.add_argument( + '-b', '--by-file', + action='store_true', + help="Group by file.") + parser.add_argument( + '-s', '--limit-sort', + action='store_true', help="Sort by stack limit.") - parser.add_argument('-S', '--reverse-limit-sort', action='store_true', + parser.add_argument( + '-S', '--reverse-limit-sort', + action='store_true', help="Sort by stack limit, but backwards.") - parser.add_argument('-L', '--depth', default=0, type=lambda x: int(x, 0), - nargs='?', const=float('inf'), - help="Depth of dependencies to show.") - parser.add_argument('-F', '--files', action='store_true', - help="Show file-level calls.") - parser.add_argument('-Y', '--summary', action='store_true', - help="Only show the total stack size.") - parser.add_argument('-e', '--error-on-recursion', action='store_true', + parser.add_argument( + '--frame-sort', + action='store_true', + help="Sort by stack frame.") + parser.add_argument( + '--reverse-frame-sort', + action='store_true', + help="Sort by stack frame, but backwards.") + parser.add_argument( + '-Y', '--summary', + action='store_true', + help="Only show the total size.") + parser.add_argument( + '-L', '--depth', + nargs='?', + type=lambda x: int(x, 0), + const=float('inf'), + help="Depth of function calls to show.") + parser.add_argument( + '-e', '--error-on-recursion', + action='store_true', help="Error if any functions are recursive.") - parser.add_argument('--build-dir', - help="Specify the relative build directory. Used to map object files \ - to the correct source files.") + parser.add_argument( + '-A', '--everything', + action='store_true', + help="Include builtin and libc specific symbols.") + parser.add_argument( + '--build-dir', + help="Specify the relative build directory. Used to map object files " + "to the correct source files.") sys.exit(main(**{k: v for k, v in vars(parser.parse_args()).items() if v is not None})) diff --git a/scripts/struct.py b/scripts/struct.py new file mode 100755 index 00000000..73ad9829 --- /dev/null +++ b/scripts/struct.py @@ -0,0 +1,522 @@ +#!/usr/bin/env python3 +# +# Script to find struct sizes. +# + +import collections as co +import csv +import glob +import itertools as it +import math as m +import os +import re +import shlex +import subprocess as sp + + +OBJ_PATHS = ['*.o'] +OBJDUMP_TOOL = ['objdump'] + + +# integer fields +class IntField(co.namedtuple('IntField', 'x')): + __slots__ = () + def __new__(cls, x): + if isinstance(x, IntField): + return x + if isinstance(x, str): + try: + x = int(x, 0) + except ValueError: + # also accept +-∞ and +-inf + if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): + x = float('inf') + elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): + x = float('-inf') + else: + raise + return super().__new__(cls, x) + + def __int__(self): + assert not m.isinf(self.x) + return self.x + + def __float__(self): + return float(self.x) + + def __str__(self): + if self.x == float('inf'): + return '∞' + elif self.x == float('-inf'): + return '-∞' + else: + return str(self.x) + + none = '%7s' % '-' + def table(self): + return '%7s' % (self,) + + diff_none = '%7s' % '-' + diff_table = table + + def diff_diff(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + diff = new - old + if diff == float('+inf'): + return '%7s' % '+∞' + elif diff == float('-inf'): + return '%7s' % '-∞' + else: + return '%+7d' % diff + + def ratio(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + if m.isinf(new) and m.isinf(old): + return 0.0 + elif m.isinf(new): + return float('+inf') + elif m.isinf(old): + return float('-inf') + elif not old and not new: + return 0.0 + elif not old: + return 1.0 + else: + return (new-old) / old + + def __add__(self, other): + return IntField(self.x + other.x) + + def __mul__(self, other): + return IntField(self.x * other.x) + + def __lt__(self, other): + return self.x < other.x + + def __gt__(self, other): + return self.__class__.__lt__(other, self) + + def __le__(self, other): + return not self.__gt__(other) + + def __ge__(self, other): + return not self.__lt__(other) + + def __truediv__(self, n): + if m.isinf(self.x): + return self + else: + return IntField(round(self.x / n)) + +# struct size results +class StructResult(co.namedtuple('StructResult', 'file,struct,struct_size')): + __slots__ = () + def __new__(cls, file, struct, struct_size): + return super().__new__(cls, file, struct, IntField(struct_size)) + + def __add__(self, other): + return StructResult(self.file, self.struct, + self.struct_size + other.struct_size) + + +def openio(path, mode='r'): + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) + +def collect(paths, *, + objdump_tool=OBJDUMP_TOOL, + build_dir=None, + everything=False, + **args): + decl_pattern = re.compile( + '^\s+(?P[0-9]+)' + '\s+(?P[0-9]+)' + '\s+.*' + '\s+(?P[^\s]+)$') + struct_pattern = re.compile( + '^(?:.*DW_TAG_(?P[a-z_]+).*' + '|^.*DW_AT_name.*:\s*(?P[^:\s]+)\s*' + '|^.*DW_AT_decl_file.*:\s*(?P[0-9]+)\s*' + '|^.*DW_AT_byte_size.*:\s*(?P[0-9]+)\s*)$') + + results = [] + for path in paths: + # find decl, we want to filter by structs in .h files + decls = {} + # note objdump-tool may contain extra args + cmd = objdump_tool + ['--dwarf=rawline', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace') + for line in proc.stdout: + # find file numbers + m = decl_pattern.match(line) + if m: + decls[int(m.group('no'))] = m.group('file') + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + # collect structs as we parse dwarf info + found = False + name = None + decl = None + size = None + + # note objdump-tool may contain extra args + cmd = objdump_tool + ['--dwarf=info', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace') + for line in proc.stdout: + # state machine here to find structs + m = struct_pattern.match(line) + if m: + if m.group('tag'): + if (name is not None + and decl is not None + and size is not None): + file = decls.get(decl, '?') + # map to source file + file = re.sub('\.o$', '.c', file) + if build_dir: + file = re.sub( + '%s/*' % re.escape(build_dir), '', + file) + # only include structs declared in header files in the + # current directory, ignore internal-only structs ( + # these are represented in other measurements) + if everything or file.endswith('.h'): + results.append(StructResult(file, name, size)) + + found = (m.group('tag') == 'structure_type') + name = None + decl = None + size = None + elif found and m.group('name'): + name = m.group('name') + elif found and name and m.group('decl'): + decl = int(m.group('decl')) + elif found and name and m.group('size'): + size = int(m.group('size')) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return results + + +def fold(results, *, + by=['file', 'struct'], + **_): + folding = co.OrderedDict() + for r in results: + name = tuple(getattr(r, k) for k in by) + if name not in folding: + folding[name] = [] + folding[name].append(r) + + folded = [] + for rs in folding.values(): + folded.append(sum(rs[1:], start=rs[0])) + + return folded + + +def table(results, diff_results=None, *, + by_file=False, + size_sort=False, + reverse_size_sort=False, + summary=False, + all=False, + percent=False, + **_): + all_, all = all, __builtins__.all + + # fold + results = fold(results, by=['file' if by_file else 'struct']) + if diff_results is not None: + diff_results = fold(diff_results, + by=['file' if by_file else 'struct']) + + table = { + r.file if by_file else r.struct: r + for r in results} + diff_table = { + r.file if by_file else r.struct: r + for r in diff_results or []} + + # sort, note that python's sort is stable + names = list(table.keys() | diff_table.keys()) + names.sort() + if diff_results is not None: + names.sort(key=lambda n: -IntField.ratio( + table[n].struct_size if n in table else None, + diff_table[n].struct_size if n in diff_table else None)) + if size_sort: + names.sort(key=lambda n: (table[n].struct_size,) if n in table else (), + reverse=True) + elif reverse_size_sort: + names.sort(key=lambda n: (table[n].struct_size,) if n in table else (), + reverse=False) + + # print header + print('%-36s' % ('%s%s' % ( + 'file' if by_file else 'struct', + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else ''), + end='') + if diff_results is None: + print(' %s' % ('size'.rjust(len(IntField.none)))) + elif percent: + print(' %s' % ('size'.rjust(len(IntField.diff_none)))) + else: + print(' %s %s %s' % ( + 'old'.rjust(len(IntField.diff_none)), + 'new'.rjust(len(IntField.diff_none)), + 'diff'.rjust(len(IntField.diff_none)))) + + # print entries + if not summary: + for name in names: + r = table.get(name) + if diff_results is not None: + diff_r = diff_table.get(name) + ratio = IntField.ratio( + r.struct_size if r else None, + diff_r.struct_size if diff_r else None) + if not ratio and not all_: + continue + + print('%-36s' % name, end='') + if diff_results is None: + print(' %s' % ( + r.struct_size.table() + if r else IntField.none)) + elif percent: + print(' %s%s' % ( + r.struct_size.diff_table() + if r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)))) + else: + print(' %s %s %s%s' % ( + diff_r.struct_size.diff_table() + if diff_r else IntField.diff_none, + r.struct_size.diff_table() + if r else IntField.diff_none, + IntField.diff_diff( + r.struct_size if r else None, + diff_r.struct_size if diff_r else None) + if r or diff_r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)) + if ratio else '')) + + # print total + total = fold(results, by=[]) + r = total[0] if total else None + if diff_results is not None: + diff_total = fold(diff_results, by=[]) + diff_r = diff_total[0] if diff_total else None + ratio = IntField.ratio( + r.struct_size if r else None, + diff_r.struct_size if diff_r else None) + + print('%-36s' % 'TOTAL', end='') + if diff_results is None: + print(' %s' % ( + r.struct_size.table() + if r else IntField.none)) + elif percent: + print(' %s%s' % ( + r.struct_size.diff_table() + if r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)))) + else: + print(' %s %s %s%s' % ( + diff_r.struct_size.diff_table() + if diff_r else IntField.diff_none, + r.struct_size.diff_table() + if r else IntField.diff_none, + IntField.diff_diff( + r.struct_size if r else None, + diff_r.struct_size if diff_r else None) + if r or diff_r else IntField.diff_none, + ' (%s)' % ( + '+∞%' if ratio == float('+inf') + else '-∞%' if ratio == float('-inf') + else '%+.1f%%' % (100*ratio)) + if ratio else '')) + + +def main(obj_paths, **args): + # find sizes + if not args.get('use', None): + # find .o files + paths = [] + for path in obj_paths: + if os.path.isdir(path): + path = path + '/*.o' + + for path in glob.glob(path): + paths.append(path) + + if not paths: + print('no .obj files found in %r?' % obj_paths) + sys.exit(-1) + + results = collect(paths, **args) + else: + results = [] + with openio(args['use']) as f: + reader = csv.DictReader(f) + for r in reader: + try: + results.append(StructResult(**{ + k: v for k, v in r.items() + if k in StructResult._fields})) + except TypeError: + pass + + # fold to remove duplicates + results = fold(results) + + # sort because why not + results.sort() + + # write results to CSV + if args.get('output'): + with openio(args['output'], 'w') as f: + writer = csv.DictWriter(f, StructResult._fields) + writer.writeheader() + for r in results: + writer.writerow(r._asdict()) + + # find previous results? + if args.get('diff'): + diff_results = [] + try: + with openio(args['diff']) as f: + reader = csv.DictReader(f) + for r in reader: + try: + diff_results.append(StructResult(**{ + k: v for k, v in r.items() + if k in StructResult._fields})) + except TypeError: + pass + except FileNotFoundError: + pass + + # fold to remove duplicates + diff_results = fold(diff_results) + + # print table + if not args.get('quiet'): + table( + results, + diff_results if args.get('diff') else None, + **args) + + +if __name__ == "__main__": + import argparse + import sys + parser = argparse.ArgumentParser( + description="Find struct sizes.") + parser.add_argument( + 'obj_paths', + nargs='*', + default=OBJ_PATHS, + help="Description of where to find *.o files. May be a directory " + "or a list of paths. Defaults to %(default)r.") + parser.add_argument( + '-v', '--verbose', + action='store_true', + help="Output commands that run behind the scenes.") + parser.add_argument( + '-q', '--quiet', + action='store_true', + help="Don't show anything, useful with -o.") + parser.add_argument( + '-o', '--output', + help="Specify CSV file to store results.") + parser.add_argument( + '-u', '--use', + help="Don't parse anything, use this CSV file.") + parser.add_argument( + '-d', '--diff', + help="Specify CSV file to diff against.") + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") + parser.add_argument( + '-p', '--percent', + action='store_true', + help="Only show percentage change, not a full diff.") + parser.add_argument( + '-b', '--by-file', + action='store_true', + help="Group by file. Note this does not include padding " + "so sizes may differ from other tools.") + parser.add_argument( + '-s', '--size-sort', + action='store_true', + help="Sort by size.") + parser.add_argument( + '-S', '--reverse-size-sort', + action='store_true', + help="Sort by size, but backwards.") + parser.add_argument( + '-Y', '--summary', + action='store_true', + help="Only show the total size.") + parser.add_argument( + '-A', '--everything', + action='store_true', + help="Include builtin and libc specific symbols.") + parser.add_argument( + '--objdump-tool', + type=lambda x: x.split(), + default=OBJDUMP_TOOL, + help="Path to the objdump tool to use.") + parser.add_argument( + '--build-dir', + help="Specify the relative build directory. Used to map object files " + "to the correct source files.") + sys.exit(main(**{k: v + for k, v in vars(parser.parse_args()).items() + if v is not None})) diff --git a/scripts/structs.py b/scripts/structs.py deleted file mode 100755 index 28284fe1..00000000 --- a/scripts/structs.py +++ /dev/null @@ -1,348 +0,0 @@ -#!/usr/bin/env python3 -# -# Script to find struct sizes. -# - -import os -import glob -import itertools as it -import subprocess as sp -import shlex -import re -import csv -import collections as co - - -OBJ_PATHS = ['*.o'] - -def openio(path, mode='r'): - if path == '-': - if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), 'r') - else: - return os.fdopen(os.dup(sys.stdout.fileno()), 'w') - else: - return open(path, mode) - -class StructsResult(co.namedtuple('StructsResult', 'struct_size')): - __slots__ = () - def __new__(cls, struct_size=0): - return super().__new__(cls, int(struct_size)) - - def __add__(self, other): - return self.__class__(self.struct_size + other.struct_size) - - def __sub__(self, other): - return StructsDiff(other, self) - - def __rsub__(self, other): - return self.__class__.__sub__(other, self) - - def key(self, **args): - if args.get('size_sort'): - return -self.struct_size - elif args.get('reverse_size_sort'): - return +self.struct_size - else: - return None - - _header = '%7s' % 'size' - def __str__(self): - return '%7d' % self.struct_size - -class StructsDiff(co.namedtuple('StructsDiff', 'old,new')): - __slots__ = () - - def ratio(self): - old = self.old.struct_size if self.old is not None else 0 - new = self.new.struct_size if self.new is not None else 0 - return (new-old) / old if old else 1.0 - - def key(self, **args): - return ( - self.new.key(**args) if self.new is not None else 0, - -self.ratio()) - - def __bool__(self): - return bool(self.ratio()) - - _header = '%7s %7s %7s' % ('old', 'new', 'diff') - def __str__(self): - old = self.old.struct_size if self.old is not None else 0 - new = self.new.struct_size if self.new is not None else 0 - diff = new - old - ratio = self.ratio() - return '%7s %7s %+7d%s' % ( - old or "-", - new or "-", - diff, - ' (%+.1f%%)' % (100*ratio) if ratio else '') - -def collect(paths, **args): - decl_pattern = re.compile( - '^\s+(?P[0-9]+)' - '\s+(?P[0-9]+)' - '\s+.*' - '\s+(?P[^\s]+)$') - struct_pattern = re.compile( - '^(?:.*DW_TAG_(?P[a-z_]+).*' - '|^.*DW_AT_name.*:\s*(?P[^:\s]+)\s*' - '|^.*DW_AT_decl_file.*:\s*(?P[0-9]+)\s*' - '|^.*DW_AT_byte_size.*:\s*(?P[0-9]+)\s*)$') - - results = {} - for path in paths: - # find decl, we want to filter by structs in .h files - decls = {} - # note objdump-tool may contain extra args - cmd = args['objdump_tool'] + ['--dwarf=rawline', path] - if args.get('verbose'): - print(' '.join(shlex.quote(c) for c in cmd)) - proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=sp.PIPE if not args.get('verbose') else None, - universal_newlines=True, - errors='replace') - for line in proc.stdout: - # find file numbers - m = decl_pattern.match(line) - if m: - decls[int(m.group('no'))] = m.group('file') - proc.wait() - if proc.returncode != 0: - if not args.get('verbose'): - for line in proc.stderr: - sys.stdout.write(line) - sys.exit(-1) - - # collect structs as we parse dwarf info - found = False - name = None - decl = None - size = None - - # note objdump-tool may contain extra args - cmd = args['objdump_tool'] + ['--dwarf=info', path] - if args.get('verbose'): - print(' '.join(shlex.quote(c) for c in cmd)) - proc = sp.Popen(cmd, - stdout=sp.PIPE, - stderr=sp.PIPE if not args.get('verbose') else None, - universal_newlines=True, - errors='replace') - for line in proc.stdout: - # state machine here to find structs - m = struct_pattern.match(line) - if m: - if m.group('tag'): - if (name is not None - and decl is not None - and size is not None): - file = decls.get(decl, '?') - # map to source file - file = re.sub('\.o$', '.c', file) - if args.get('build_dir'): - file = re.sub( - '%s/*' % re.escape(args['build_dir']), '', - file) - # only include structs declared in header files in the - # current directory, ignore internal-only structs ( - # these are represented in other measurements) - if args.get('everything') or file.endswith('.h'): - results[(file, name)] = StructsResult(size) - found = (m.group('tag') == 'structure_type') - name = None - decl = None - size = None - elif found and m.group('name'): - name = m.group('name') - elif found and name and m.group('decl'): - decl = int(m.group('decl')) - elif found and name and m.group('size'): - size = int(m.group('size')) - proc.wait() - if proc.returncode != 0: - if not args.get('verbose'): - for line in proc.stderr: - sys.stdout.write(line) - sys.exit(-1) - - return results - - -def main(**args): - # find sizes - if not args.get('use', None): - # find .o files - paths = [] - for path in args['obj_paths']: - if os.path.isdir(path): - path = path + '/*.o' - - for path in glob.glob(path): - paths.append(path) - - if not paths: - print('no .obj files found in %r?' % args['obj_paths']) - sys.exit(-1) - - results = collect(paths, **args) - else: - with openio(args['use']) as f: - r = csv.DictReader(f) - results = { - (result['file'], result['name']): StructsResult( - *(result[f] for f in StructsResult._fields)) - for result in r - if all(result.get(f) not in {None, ''} - for f in StructsResult._fields)} - - # find previous results? - if args.get('diff'): - try: - with openio(args['diff']) as f: - r = csv.DictReader(f) - prev_results = { - (result['file'], result['name']): StructsResult( - *(result[f] for f in StructsResult._fields)) - for result in r - if all(result.get(f) not in {None, ''} - for f in StructsResult._fields)} - except FileNotFoundError: - prev_results = [] - - # write results to CSV - if args.get('output'): - merged_results = co.defaultdict(lambda: {}) - other_fields = [] - - # merge? - if args.get('merge'): - try: - with openio(args['merge']) as f: - r = csv.DictReader(f) - for result in r: - file = result.pop('file', '') - func = result.pop('name', '') - for f in StructsResult._fields: - result.pop(f, None) - merged_results[(file, func)] = result - other_fields = result.keys() - except FileNotFoundError: - pass - - for (file, func), result in results.items(): - merged_results[(file, func)] |= result._asdict() - - with openio(args['output'], 'w') as f: - w = csv.DictWriter(f, ['file', 'name', - *other_fields, *StructsResult._fields]) - w.writeheader() - for (file, func), result in sorted(merged_results.items()): - w.writerow({'file': file, 'name': func, **result}) - - # print results - def print_header(by): - if by == 'total': - entry = lambda k: 'TOTAL' - elif by == 'file': - entry = lambda k: k[0] - else: - entry = lambda k: k[1] - - if not args.get('diff'): - print('%-36s %s' % (by, StructsResult._header)) - else: - old = {entry(k) for k in results.keys()} - new = {entry(k) for k in prev_results.keys()} - print('%-36s %s' % ( - '%s (%d added, %d removed)' % (by, - sum(1 for k in new if k not in old), - sum(1 for k in old if k not in new)) - if by else '', - StructsDiff._header)) - - def print_entries(by): - if by == 'total': - entry = lambda k: 'TOTAL' - elif by == 'file': - entry = lambda k: k[0] - else: - entry = lambda k: k[1] - - entries = co.defaultdict(lambda: StructsResult()) - for k, result in results.items(): - entries[entry(k)] += result - - if not args.get('diff'): - for name, result in sorted(entries.items(), - key=lambda p: (p[1].key(**args), p)): - print('%-36s %s' % (name, result)) - else: - prev_entries = co.defaultdict(lambda: StructsResult()) - for k, result in prev_results.items(): - prev_entries[entry(k)] += result - - diff_entries = {name: entries.get(name) - prev_entries.get(name) - for name in (entries.keys() | prev_entries.keys())} - - for name, diff in sorted(diff_entries.items(), - key=lambda p: (p[1].key(**args), p)): - if diff or args.get('all'): - print('%-36s %s' % (name, diff)) - - if args.get('quiet'): - pass - elif args.get('summary'): - print_header('') - print_entries('total') - elif args.get('files'): - print_header('file') - print_entries('file') - print_entries('total') - else: - print_header('struct') - print_entries('struct') - print_entries('total') - - -if __name__ == "__main__": - import argparse - import sys - parser = argparse.ArgumentParser( - description="Find struct sizes.") - parser.add_argument('obj_paths', nargs='*', default=OBJ_PATHS, - help="Description of where to find *.o files. May be a directory \ - or a list of paths. Defaults to %r." % OBJ_PATHS) - parser.add_argument('-v', '--verbose', action='store_true', - help="Output commands that run behind the scenes.") - parser.add_argument('-q', '--quiet', action='store_true', - help="Don't show anything, useful with -o.") - parser.add_argument('-o', '--output', - help="Specify CSV file to store results.") - parser.add_argument('-u', '--use', - help="Don't compile and find struct sizes, instead use this CSV file.") - parser.add_argument('-d', '--diff', - help="Specify CSV file to diff struct size against.") - parser.add_argument('-m', '--merge', - help="Merge with an existing CSV file when writing to output.") - parser.add_argument('-a', '--all', action='store_true', - help="Show all structs, not just the ones that changed.") - parser.add_argument('-A', '--everything', action='store_true', - help="Include builtin and libc specific symbols.") - parser.add_argument('-s', '--size-sort', action='store_true', - help="Sort by size.") - parser.add_argument('-S', '--reverse-size-sort', action='store_true', - help="Sort by size, but backwards.") - parser.add_argument('-F', '--files', action='store_true', - help="Show file-level struct sizes.") - parser.add_argument('-Y', '--summary', action='store_true', - help="Only show the total struct size.") - parser.add_argument('--objdump-tool', default=['objdump'], type=lambda x: x.split(), - help="Path to the objdump tool to use.") - parser.add_argument('--build-dir', - help="Specify the relative build directory. Used to map object files \ - to the correct source files.") - sys.exit(main(**{k: v - for k, v in vars(parser.parse_args()).items() - if v is not None})) diff --git a/scripts/summary.py b/scripts/summary.py index c5a48f2d..27daca28 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -6,199 +6,38 @@ import collections as co import csv import functools as ft +import glob import math as m import os import re -# each result is a type generated by another script -RESULTS = [] -FIELDS = 'code,data,stack,structs' -def result(cls): - RESULTS.append(cls) - return cls -@result -class CodeResult(co.namedtuple('CodeResult', 'code_size')): - __slots__ = () - def __new__(cls, code_size=0): - return super().__new__(cls, int(code_size)) +CSV_PATHS = ['*.csv'] - def __add__(self, other): - return self.__class__(self.code_size + other.code_size) - - def __sub__(self, other): - old = other.code_size if other is not None else 0 - new = self.code_size if self is not None else 0 - return (new-old) / old if old else 1.0 - - def __rsub__(self, other): - return self.__class__.__sub__(other, self) - - def key(self): - return -self.code_size - - _header = '%7s' % 'code' - _nil = '%7s' % '-' - def __str__(self): - return '%7s' % self.code_size - -@result -class DataResult(co.namedtuple('DataResult', 'data_size')): - __slots__ = () - def __new__(cls, data_size=0): - return super().__new__(cls, int(data_size)) - - def __add__(self, other): - return self.__class__(self.data_size + other.data_size) - - def __sub__(self, other): - old = other.data_size if other is not None else 0 - new = self.data_size if self is not None else 0 - return (new-old) / old if old else 1.0 - - def __rsub__(self, other): - return self.__class__.__sub__(other, self) - - def key(self): - return -self.data_size - - _header = '%7s' % 'data' - _nil = '%7s' % '-' - def __str__(self): - return '%7s' % self.data_size - -@result -class StackResult(co.namedtuple('StackResult', 'stack_limit')): - __slots__ = () - def __new__(cls, stack_limit=0): - return super().__new__(cls, float(stack_limit)) - - def __add__(self, other): - return self.__class__(max(self.stack_limit, other.stack_limit)) - - def __sub__(self, other): - old_limit = other.stack_limit if other is not None else 0 - new_limit = self.stack_limit if self is not None else 0 - return (0.0 if m.isinf(new_limit) and m.isinf(old_limit) - else +float('inf') if m.isinf(new_limit) - else -float('inf') if m.isinf(old_limit) - else 0.0 if not old_limit and not new_limit - else 1.0 if not old_limit - else (new_limit-old_limit) / old_limit) - - def __rsub__(self, other): - return self.__class__.__sub__(other, self) - - def key(self): - return -self.stack_limit - - _header = '%7s' % 'stack' - _nil = '%7s' % '-' - def __str__(self): - return '%7s' % ( - '∞' if m.isinf(self.stack_limit) - else int(self.stack_limit)) - -@result -class StructsResult(co.namedtuple('StructsResult', 'struct_size')): - __slots__ = () - def __new__(cls, struct_size=0): - return super().__new__(cls, int(struct_size)) - - def __add__(self, other): - return self.__class__(self.struct_size + other.struct_size) - - def __sub__(self, other): - old = other.struct_size if other is not None else 0 - new = self.struct_size if self is not None else 0 - return (new-old) / old if old else 1.0 - - def __rsub__(self, other): - return self.__class__.__sub__(other, self) - - def key(self): - return -self.struct_size - - _header = '%7s' % 'structs' - _nil = '%7s' % '-' - def __str__(self): - return '%7s' % self.struct_size - -@result -class CoverageLineResult(co.namedtuple('CoverageResult', - 'coverage_line_hits,coverage_line_count')): - __slots__ = () - def __new__(cls, coverage_line_hits=0, coverage_line_count=0): - return super().__new__(cls, - int(coverage_line_hits), - int(coverage_line_count)) - - def __add__(self, other): - return self.__class__( - self.coverage_line_hits + other.coverage_line_hits, - self.coverage_line_count + other.coverage_line_count) - - def __sub__(self, other): - old_hits = other.coverage_line_hits if other is not None else 0 - old_count = other.coverage_line_count if other is not None else 0 - new_hits = self.coverage_line_hits if self is not None else 0 - new_count = self.coverage_line_count if self is not None else 0 - return ((new_hits/new_count if new_count else 1.0) - - (old_hits/old_count if old_count else 1.0)) - - def __rsub__(self, other): - return self.__class__.__sub__(other, self) - - def key(self): - return -(self.coverage_line_hits/self.coverage_line_count - if self.coverage_line_count else -1) - - _header = '%19s' % 'coverage/line' - _nil = '%11s %7s' % ('-', '-') - def __str__(self): - return '%11s %7s' % ( - '%d/%d' % (self.coverage_line_hits, self.coverage_line_count) - if self.coverage_line_count else '-', - '%.1f%%' % (100*self.coverage_line_hits/self.coverage_line_count) - if self.coverage_line_count else '-') - -@result -class CoverageBranchResult(co.namedtuple('CoverageResult', - 'coverage_branch_hits,coverage_branch_count')): - __slots__ = () - def __new__(cls, coverage_branch_hits=0, coverage_branch_count=0): - return super().__new__(cls, - int(coverage_branch_hits), - int(coverage_branch_count)) - - def __add__(self, other): - return self.__class__( - self.coverage_branch_hits + other.coverage_branch_hits, - self.coverage_branch_count + other.coverage_branch_count) - - def __sub__(self, other): - old_hits = other.coverage_branch_hits if other is not None else 0 - old_count = other.coverage_branch_count if other is not None else 0 - new_hits = self.coverage_branch_hits if self is not None else 0 - new_count = self.coverage_branch_count if self is not None else 0 - return ((new_hits/new_count if new_count else 1.0) - - (old_hits/old_count if old_count else 1.0)) - - def __rsub__(self, other): - return self.__class__.__sub__(other, self) - - def key(self): - return -(self.coverage_branch_hits/self.coverage_branch_count - if self.coverage_branch_count else -1) - - _header = '%19s' % 'coverage/branch' - _nil = '%11s %7s' % ('-', '-') - def __str__(self): - return '%11s %7s' % ( - '%d/%d' % (self.coverage_branch_hits, self.coverage_branch_count) - if self.coverage_branch_count else '-', - '%.1f%%' % (100*self.coverage_branch_hits/self.coverage_branch_count) - if self.coverage_branch_count else '-') +# Defaults are common fields generated by other littlefs scripts +MERGES = { + 'add': ( + ['code_size', 'data_size', 'stack_frame', 'struct_size', + 'coverage_lines', 'coverage_branches'], + lambda xs: sum(xs[1:], start=xs[0]) + ), + 'mul': ( + [], + lambda xs: m.prod(xs[1:], start=xs[0]) + ), + 'min': ( + [], + min + ), + 'max': ( + ['stack_limit', 'coverage_hits'], + max + ), + 'avg': ( + [], + lambda xs: sum(xs[1:], start=xs[0]) / len(xs) + ), +} def openio(path, mode='r'): @@ -210,212 +49,676 @@ def openio(path, mode='r'): else: return open(path, mode) -def main(**args): - # find results - results = co.defaultdict(lambda: {}) - for path in args.get('csv_paths', '-'): + +# integer fields +class IntField(co.namedtuple('IntField', 'x')): + __slots__ = () + def __new__(cls, x): + if isinstance(x, IntField): + return x + if isinstance(x, str): + try: + x = int(x, 0) + except ValueError: + # also accept +-∞ and +-inf + if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): + x = float('inf') + elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): + x = float('-inf') + else: + raise + return super().__new__(cls, x) + + def __int__(self): + assert not m.isinf(self.x) + return self.x + + def __float__(self): + return float(self.x) + + def __str__(self): + if self.x == float('inf'): + return '∞' + elif self.x == float('-inf'): + return '-∞' + else: + return str(self.x) + + none = '%7s' % '-' + def table(self): + return '%7s' % (self,) + + diff_none = '%7s' % '-' + diff_table = table + + def diff_diff(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + diff = new - old + if diff == float('+inf'): + return '%7s' % '+∞' + elif diff == float('-inf'): + return '%7s' % '-∞' + else: + return '%+7d' % diff + + def ratio(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + if m.isinf(new) and m.isinf(old): + return 0.0 + elif m.isinf(new): + return float('+inf') + elif m.isinf(old): + return float('-inf') + elif not old and not new: + return 0.0 + elif not old: + return 1.0 + else: + return (new-old) / old + + def __add__(self, other): + return IntField(self.x + other.x) + + def __mul__(self, other): + return IntField(self.x * other.x) + + def __lt__(self, other): + return self.x < other.x + + def __gt__(self, other): + return self.__class__.__lt__(other, self) + + def __le__(self, other): + return not self.__gt__(other) + + def __ge__(self, other): + return not self.__lt__(other) + + def __truediv__(self, n): + if m.isinf(self.x): + return self + else: + return IntField(round(self.x / n)) + +# float fields +class FloatField(co.namedtuple('FloatField', 'x')): + __slots__ = () + def __new__(cls, x): + if isinstance(x, FloatField): + return x + if isinstance(x, str): + try: + x = float(x) + except ValueError: + # also accept +-∞ and +-inf + if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): + x = float('inf') + elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): + x = float('-inf') + else: + raise + return super().__new__(cls, x) + + def __float__(self): + return float(self.x) + + def __str__(self): + if self.x == float('inf'): + return '∞' + elif self.x == float('-inf'): + return '-∞' + else: + return '%.1f' % self.x + + none = IntField.none + table = IntField.table + diff_none = IntField.diff_none + diff_table = IntField.diff_table + diff_diff = IntField.diff_diff + ratio = IntField.ratio + __add__ = IntField.__add__ + __mul__ = IntField.__mul__ + __lt__ = IntField.__lt__ + __gt__ = IntField.__gt__ + __le__ = IntField.__le__ + __ge__ = IntField.__ge__ + + def __truediv__(self, n): + if m.isinf(self.x): + return self + else: + return FloatField(self.x / n) + +# fractional fields, a/b +class FracField(co.namedtuple('FracField', 'a,b')): + __slots__ = () + def __new__(cls, a, b=None): + if isinstance(a, FracField) and b is None: + return a + if isinstance(a, str) and b is None: + a, b = a.split('/', 1) + if b is None: + b = a + return super().__new__(cls, IntField(a), IntField(b)) + + def __str__(self): + return '%s/%s' % (self.a, self.b) + + none = '%11s %7s' % ('-', '-') + def table(self): + if not self.b.x: + return self.none + + t = self.a.x/self.b.x + return '%11s %7s' % ( + self, + '∞%' if t == float('+inf') + else '-∞%' if t == float('-inf') + else '%.1f%%' % (100*t)) + + diff_none = '%11s' % '-' + def diff_table(self): + if not self.b.x: + return self.diff_none + + return '%11s' % (self,) + + def diff_diff(self, other): + new_a, new_b = self if self else (IntField(0), IntField(0)) + old_a, old_b = other if other else (IntField(0), IntField(0)) + return '%11s' % ('%s/%s' % ( + new_a.diff_diff(old_a).strip(), + new_b.diff_diff(old_b).strip())) + + def ratio(self, other): + new_a, new_b = self if self else (IntField(0), IntField(0)) + old_a, old_b = other if other else (IntField(0), IntField(0)) + new = new_a.x/new_b.x if new_b.x else 1.0 + old = old_a.x/old_b.x if old_b.x else 1.0 + return new - old + + def __add__(self, other): + return FracField(self.a + other.a, self.b + other.b) + + def __mul__(self, other): + return FracField(self.a * other.a, self.b + other.b) + + def __lt__(self, other): + self_r = self.a.x/self.b.x if self.b.x else float('-inf') + other_r = other.a.x/other.b.x if other.b.x else float('-inf') + return self_r < other_r + + def __gt__(self, other): + return self.__class__.__lt__(other, self) + + def __le__(self, other): + return not self.__gt__(other) + + def __ge__(self, other): + return not self.__lt__(other) + + def __truediv__(self, n): + return FracField(self.a / n, self.b / n) + + +def homogenize(results, *, + fields=None, + merges=None, + renames=None, + types=None, + **_): + # rename fields? + if renames is not None: + results_ = [] + for r in results: + results_.append({renames.get(k, k): v for k, v in r.items()}) + results = results_ + + # find all fields + if not fields: + fields = co.OrderedDict() + for r in results: + # also remove None fields, these can get introduced by + # csv.DictReader when header and rows mismatch + fields.update((k, v) for k, v in r.items() if k is not None) + fields = list(fields.keys()) + + # go ahead and clean up none values, these can have a few forms + results_ = [] + for r in results: + results_.append({ + k: r[k] for k in fields + if r.get(k) is not None and not( + isinstance(r[k], str) + and re.match('^\s*[+-]?\s*$', r[k]))}) + + # find best type for all fields + def try_(x, type): + try: + type(x) + return True + except ValueError: + return False + + if types is None: + types = {} + for k in fields: + if merges is not None and merges.get(k): + for type in [IntField, FloatField, FracField]: + if all(k not in r or try_(r[k], type) for r in results_): + types[k] = type + break + else: + print("no type matches field %r?" % k) + sys.exit(-1) + + # homogenize types + for k in fields: + if k in types: + for r in results_: + if k in r: + r[k] = types[k](r[k]) + + return fields, types, results_ + + +def fold(results, *, + fields=None, + merges=None, + by=None, + **_): + folding = co.OrderedDict() + if by is None: + by = [k for k in fields if k not in merges] + + for r in results: + name = tuple(r.get(k) for k in by) + if name not in folding: + folding[name] = {k: [] for k in fields if k in merges} + for k in fields: + # drop all fields fields without a type + if k in merges and k in r: + folding[name][k].append(r[k]) + + # merge fields, we need the count at this point for averages + folded = [] + types = {} + for name, r in folding.items(): + r_ = {} + for k, vs in r.items(): + if vs: + _, merge = MERGES[merges[k]] + r_[k] = merge(vs) + + # drop all rows without any fields + # and drop all empty keys + if r_: + folded.append(dict( + {k: n for k, n in zip(by, name) if n}, + **r_)) + + fields_ = by + [k for k in fields if k in merges] + return fields_, folded + + +def table(results, diff_results=None, *, + fields=None, + types=None, + merges=None, + by=None, + sort=None, + reverse_sort=None, + summary=False, + all=False, + percent=False, + **_): + all_, all = all, __builtins__.all + + # fold + if by is not None: + fields, results = fold(results, fields=fields, merges=merges, by=by) + if diff_results is not None: + _, diff_results = fold(diff_results, + fields=fields, merges=merges, by=by) + + table = { + tuple(r.get(k,'') for k in fields if k not in merges): r + for r in results} + diff_table = { + tuple(r.get(k,'') for k in fields if k not in merges): r + for r in diff_results or []} + + # sort, note that python's sort is stable + names = list(table.keys() | diff_table.keys()) + names.sort() + if diff_results is not None: + names.sort(key=lambda n: [ + -types[k].ratio( + table.get(n,{}).get(k), + diff_table.get(n,{}).get(k)) + for k in fields if k in merges]) + if sort: + names.sort(key=lambda n: tuple( + (table[n][k],) if k in table.get(n,{}) else () + for k in sort), + reverse=True) + elif reverse_sort: + names.sort(key=lambda n: tuple( + (table[n][k],) if k in table.get(n,{}) else () + for k in reverse_sort), + reverse=False) + + # print header + print('%-36s' % ('%s%s' % ( + ','.join(k for k in fields if k not in merges), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else ''), + end='') + if diff_results is None: + print(' %s' % ( + ' '.join(k.rjust(len(types[k].none)) + for k in fields if k in merges))) + elif percent: + print(' %s' % ( + ' '.join(k.rjust(len(types[k].diff_none)) + for k in fields if k in merges))) + else: + print(' %s %s %s' % ( + ' '.join(('o'+k).rjust(len(types[k].diff_none)) + for k in fields if k in merges), + ' '.join(('n'+k).rjust(len(types[k].diff_none)) + for k in fields if k in merges), + ' '.join(('d'+k).rjust(len(types[k].diff_none)) + for k in fields if k in merges))) + + # print entries + if not summary: + for name in names: + r = table.get(name, {}) + if diff_results is not None: + diff_r = diff_table.get(name, {}) + ratios = [types[k].ratio(r.get(k), diff_r.get(k)) + for k in fields if k in merges] + if not any(ratios) and not all_: + continue + + print('%-36s' % ','.join(name), end='') + if diff_results is None: + print(' %s' % ( + ' '.join(r[k].table() + if k in r else types[k].none + for k in fields if k in merges))) + elif percent: + print(' %s%s' % ( + ' '.join(r[k].diff_table() + if k in r else types[k].diff_none + for k in fields if k in merges), + ' (%s)' % ', '.join( + '+∞%' if t == float('+inf') + else '-∞%' if t == float('-inf') + else '%+.1f%%' % (100*t) + for t in ratios))) + else: + print(' %s %s %s%s' % ( + ' '.join(diff_r[k].diff_table() + if k in diff_r else types[k].diff_none + for k in fields if k in merges), + ' '.join(r[k].diff_table() + if k in r else types[k].diff_none + for k in fields if k in merges), + ' '.join(types[k].diff_diff(r.get(k), diff_r.get(k)) + if k in r or k in diff_r else types[k].diff_none + for k in fields if k in merges), + ' (%s)' % ', '.join( + '+∞%' if t == float('+inf') + else '-∞%' if t == float('-inf') + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '')) + + # print total + _, total = fold(results, fields=fields, merges=merges, by=[]) + r = total[0] if total else {} + if diff_results is not None: + _, diff_total = fold(diff_results, + fields=fields, merges=merges, by=[]) + diff_r = diff_total[0] if diff_total else {} + ratios = [types[k].ratio(r.get(k), diff_r.get(k)) + for k in fields if k in merges] + + print('%-36s' % 'TOTAL', end='') + if diff_results is None: + print(' %s' % ( + ' '.join(r[k].table() + if k in r else types[k].none + for k in fields if k in merges))) + elif percent: + print(' %s%s' % ( + ' '.join(r[k].diff_table() + if k in r else types[k].diff_none + for k in fields if k in merges), + ' (%s)' % ', '.join( + '+∞%' if t == float('+inf') + else '-∞%' if t == float('-inf') + else '%+.1f%%' % (100*t) + for t in ratios))) + else: + print(' %s %s %s%s' % ( + ' '.join(diff_r[k].diff_table() + if k in diff_r else types[k].diff_none + for k in fields if k in merges), + ' '.join(r[k].diff_table() + if k in r else types[k].diff_none + for k in fields if k in merges), + ' '.join(types[k].diff_diff(r.get(k), diff_r.get(k)) + if k in r or k in diff_r else types[k].diff_none + for k in fields if k in merges), + ' (%s)' % ', '.join( + '+∞%' if t == float('+inf') + else '-∞%' if t == float('-inf') + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '')) + + +def main(csv_paths, *, fields=None, by=None, **args): + # figure out what fields to use + renames = {} + + if fields is not None: + fields_ = [] + for name in fields: + if '=' in name: + a, b = name.split('=', 1) + renames[b] = a + name = a + fields_.append(name) + fields = fields_ + + if by is not None: + by_ = [] + for name in by: + if '=' in name: + a, b = name.split('=', 1) + renames[b] = a + name = a + by_.append(name) + by = by_ + + # include 'by' fields in fields, it doesn't make sense to not + if fields is not None and by is not None: + fields[:0] = [k for k in by if k not in fields] + + # use preconfigured merge operations unless any merge operation is + # explictly specified + merge_args = (args + if any(args.get(m) for m in MERGES.keys()) + else {m: k for m, (k, _) in MERGES.items()}) + merges = {} + for m in MERGES.keys(): + for k in merge_args.get(m, []): + if k in merges: + print("conflicting merge type for field %r?" % k) + sys.exit(-1) + merges[k] = m + # allow renames to apply to merges + for m in MERGES.keys(): + for k in merge_args.get(m, []): + if renames.get(k, k) not in merges: + merges[renames.get(k, k)] = m + # ignore merges that conflict with 'by' fields + if by is not None: + for k in by: + if k in merges: + del merges[k] + + # find CSV files + paths = [] + for path in csv_paths: + if os.path.isdir(path): + path = path + '/*.csv' + + for path in glob.glob(path): + paths.append(path) + + if not paths: + print('no .csv files found in %r?' % csv_paths) + sys.exit(-1) + + results = [] + for path in paths: try: with openio(path) as f: - r = csv.DictReader(f) - for result in r: - file = result.pop('file', '') - name = result.pop('name', '') - for Result in RESULTS: - if all(result.get(f) not in {None, ''} - for f in Result._fields): - results[(file, name)][Result.__name__] = ( - results[(file, name)].get( - Result.__name__, Result()) - + Result(*(result[f] - for f in Result._fields))) + reader = csv.DictReader(f) + for r in reader: + results.append(r) except FileNotFoundError: pass - # find previous results? - if args.get('diff'): - prev_results = co.defaultdict(lambda: {}) - for path in args.get('csv_paths', '-'): - try: - with openio(args['diff']) as f: - r = csv.DictReader(f) - for result in r: - file = result.pop('file', '') - name = result.pop('name', '') - for Result in RESULTS: - if all(result.get(f) not in {None, ''} - for f in Result._fields): - prev_results[(file, name)][Result.__name__] = ( - prev_results[(file, name)].get( - Result.__name__, Result()) - + Result(*(result[f] - for f in Result._fields))) - except FileNotFoundError: - pass + # homogenize + fields, types, results = homogenize(results, + fields=fields, merges=merges, renames=renames) - # filter our result types by results that are present - if 'all' in args['fields']: - filtered_results = RESULTS - else: - filtered_results = [ - Result for Result in RESULTS - if (any(f.startswith(r) - for r in args['fields'] - for f in Result._fields) - or any(Result._header.strip().startswith(r) - for r in args['fields']))] + # fold to remove duplicates + fields, results = fold(results, + fields=fields, merges=merges) - # figure out a sort key - if args.get('sort'): - key_Result = next( - Result for Result in RESULTS - if (any(f.startswith(args['sort']) - for f in Result._fields) - or Result._header.strip().startswith(args['sort']))) - key = lambda result: result.get(key_Result.__name__, key_Result()).key() - reverse = False - elif args.get('reverse_sort'): - key_Result = next( - Result for Result in RESULTS - if (any(f.startswith(args['reverse_sort']) - for f in Result._fields) - or Result._header.strip().startswith(args['reverse_sort']))) - key = lambda result: result.get(key_Result.__name__, key_Result()).key() - reverse = True - else: - key = lambda _: None - reverse = False - - # write merged results to CSV + # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - w = csv.DictWriter(f, sum( - (Result._fields for Result in filtered_results), - ('file', 'name'))) - w.writeheader() - for (file, name), result in sorted(results.items()): - w.writerow(ft.reduce(dict.__or__, - (r._asdict() for r in result.values()), - {'file': file, 'name': name})) + writer = csv.DictWriter(f, fields) + writer.writeheader() + for r in results: + writer.writerow(r) - # print results - def print_header(by): - if by == 'total': - entry = lambda k: 'TOTAL' - elif by == 'file': - entry = lambda k: k[0] - else: - entry = lambda k: k[1] + # find previous results? + if args.get('diff'): + diff_results = [] + try: + with openio(args['diff']) as f: + reader = csv.DictReader(f) + for r in reader: + diff_results.append(r) + except FileNotFoundError: + pass - if not args.get('diff'): - print('%-36s %s' % (by, - ' '.join(Result._header for Result in filtered_results))) - else: - old = {entry(k) for k in results.keys()} - new = {entry(k) for k in prev_results.keys()} - print('%-36s %s' % ( - '%s (%d added, %d removed)' % (by, - sum(1 for k in new if k not in old), - sum(1 for k in old if k not in new)) - if by else '', - ' '.join('%s%-10s' % (Result._header, '') - for Result in filtered_results))) + # homogenize + _, _, diff_results = homogenize(diff_results, + fields=fields, merges=merges, renames=renames, types=types) - def print_entries(by): - if by == 'total': - entry = lambda k: 'TOTAL' - elif by == 'file': - entry = lambda k: k[0] - else: - entry = lambda k: k[1] + # fold to remove duplicates + _, diff_results = fold(diff_results, + fields=fields, merges=merges) - entries = co.defaultdict(lambda: {}) - for k, result in results.items(): - entries[entry(k)] |= { - r.__class__.__name__: entries[entry(k)].get( - r.__class__.__name__, r.__class__()) + r - for r in result.values()} - - if not args.get('diff'): - for name, result in sorted(entries.items(), - key=lambda p: (key(p[1]), p), - reverse=reverse): - print('%-36s %s' % (name, ' '.join( - str(result.get(Result.__name__, Result._nil)) - for Result in filtered_results))) - else: - prev_entries = co.defaultdict(lambda: {}) - for k, result in prev_results.items(): - prev_entries[entry(k)] |= { - r.__class__.__name__: prev_entries[entry(k)].get( - r.__class__.__name__, r.__class__()) + r - for r in result.values()} - - diff_entries = { - name: (prev_entries.get(name), entries.get(name)) - for name in (entries.keys() | prev_entries.keys())} - - for name, (old, new) in sorted(diff_entries.items(), - key=lambda p: (key(p[1][1]), p)): - fields = [] - changed = False - for Result in filtered_results: - o = old.get(Result.__name__) if old is not None else None - n = new.get(Result.__name__) if new is not None else None - ratio = n - o if n is not None or o is not None else 0 - changed = changed or ratio - fields.append('%s%-10s' % ( - n if n is not None else Result._nil, - '' if not ratio - else ' (+∞%)' if ratio > 0 and m.isinf(ratio) - else ' (-∞%)' if ratio < 0 and m.isinf(ratio) - else ' (%+.1f%%)' % (100*ratio))) - if changed or args.get('all'): - print('%-36s %s' % (name, ' '.join(fields))) - - if args.get('quiet'): - pass - elif args.get('summary'): - print_header('') - print_entries('total') - elif args.get('files'): - print_header('file') - print_entries('file') - print_entries('total') - else: - print_header('name') - print_entries('name') - print_entries('total') + # print table + if not args.get('quiet'): + table( + results, + diff_results if args.get('diff') else None, + fields=fields, + types=types, + merges=merges, + by=by, + **args) if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Summarize measurements") - parser.add_argument('csv_paths', nargs='*', default='-', - help="Description of where to find *.csv files. May be a directory \ - or list of paths.") - parser.add_argument('-q', '--quiet', action='store_true', + description="Summarize measurements in CSV files.") + parser.add_argument( + 'csv_paths', + nargs='*', + default=CSV_PATHS, + help="Description of where to find *.csv files. May be a directory " + "or list of paths. Defaults to %(default)r.") + parser.add_argument( + '-q', '--quiet', + action='store_true', help="Don't show anything, useful with -o.") - parser.add_argument('-o', '--output', + parser.add_argument( + '-o', '--output', help="Specify CSV file to store results.") - parser.add_argument('-d', '--diff', + parser.add_argument( + '-d', '--diff', help="Specify CSV file to diff against.") - parser.add_argument('-a', '--all', action='store_true', - help="Show all objects, not just the ones that changed.") - parser.add_argument('-f', '--fields', - type=lambda x: set(re.split('\s*,\s*', x)), - default=FIELDS, - help="Comma separated list of fields to print, by default all fields \ - that are found in the CSV files are printed. \"all\" prints all \ - fields this script knows. Defaults to %r." % FIELDS) - parser.add_argument('-s', '--sort', - help="Sort by this field.") - parser.add_argument('-S', '--reverse-sort', - help="Sort by this field, but backwards.") - parser.add_argument('-F', '--files', action='store_true', - help="Show file-level calls.") - parser.add_argument('-Y', '--summary', action='store_true', + parser.add_argument( + '-a', '--all', + action='store_true', + help="Show all, not just the ones that changed.") + parser.add_argument( + '-p', '--percent', + action='store_true', + help="Only show percentage change, not a full diff.") + parser.add_argument( + '-f', '--fields', + type=lambda x: [x.strip() for x in x.split(',')], + help="Only show these fields. Can rename fields " + "with old_name=new_name.") + parser.add_argument( + '-b', '--by', + type=lambda x: [x.strip() for x in x.split(',')], + help="Group by these fields. Can rename fields " + "with old_name=new_name.") + parser.add_argument( + '--add', + type=lambda x: [x.strip() for x in x.split(',')], + help="Add these fields when merging.") + parser.add_argument( + '--mul', + type=lambda x: [x.strip() for x in x.split(',')], + help="Multiply these fields when merging.") + parser.add_argument( + '--min', + type=lambda x: [x.strip() for x in x.split(',')], + help="Take the minimum of these fields when merging.") + parser.add_argument( + '--max', + type=lambda x: [x.strip() for x in x.split(',')], + help="Take the maximum of these fields when merging.") + parser.add_argument( + '--avg', + type=lambda x: [x.strip() for x in x.split(',')], + help="Average these fields when merging.") + parser.add_argument( + '-s', '--sort', + type=lambda x: [x.strip() for x in x.split(',')], + help="Sort by these fields.") + parser.add_argument( + '-S', '--reverse-sort', + type=lambda x: [x.strip() for x in x.split(',')], + help="Sort by these fields, but backwards.") + parser.add_argument( + '-Y', '--summary', + action='store_true', help="Only show the totals.") sys.exit(main(**{k: v for k, v in vars(parser.parse_args()).items() From 1fcd82d5d8a007ad9a8e1e77e6f7955bf53760e5 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 16 Sep 2022 03:55:34 -0500 Subject: [PATCH 38/81] Made test.py output parsable by summary.py Also fixed an issue with truncation that resulted in a bunch of null bytes being injected into the CSV output. --- scripts/summary.py | 3 ++- scripts/test.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/summary.py b/scripts/summary.py index 27daca28..eccd565a 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -18,7 +18,8 @@ CSV_PATHS = ['*.csv'] MERGES = { 'add': ( ['code_size', 'data_size', 'stack_frame', 'struct_size', - 'coverage_lines', 'coverage_branches'], + 'coverage_lines', 'coverage_branches', + 'test_passed'], lambda xs: sum(xs[1:], start=xs[0]) ), 'mul': ( diff --git a/scripts/test.py b/scripts/test.py index e7091964..fcad9e0f 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -663,6 +663,7 @@ class TestOutput: else: # need to rewrite the file self.head.extend(row.keys() - (self.head + self.tail)) + self.f.seek(0) self.f.truncate() self.writer = csv.DictWriter(self.f, self.head + self.tail) self.writer.writeheader() @@ -767,7 +768,7 @@ def run_stage(name, runner_, ids, output_, **args): runner_, m.group('id'), **args) output_.writerow({ 'case': m.group('case'), - 'test_pass': 1, + 'test_passed': '1/1', **defines}) elif op == 'skipped': locals.seen_perms += 1 @@ -822,7 +823,7 @@ def run_stage(name, runner_, ids, output_, **args): defines = find_defines(runner_, failure.id, **args) output_.writerow({ 'case': ':'.join([suite, case]), - 'test_pass': 0, + 'test_passed': '0/1', **defines}) # race condition for multiple failures? @@ -936,7 +937,7 @@ def run(runner, test_ids=[], **args): trace = openio(args['trace'], 'w', 1) output = None if args.get('output'): - output = TestOutput(args['output'], ['case'], ['test_pass']) + output = TestOutput(args['output'], ['case'], ['test_passed']) # measure runtime start = time.time() From 11d6d1251e4ac377f91e1b4cd89f43c717a80bdb Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 16 Sep 2022 20:40:44 -0500 Subject: [PATCH 39/81] Dropped namespacing of test cases The main benefit is small test ids everywhere, though this is with the downside of needing longer names to properly prefix and avoid collisions. But this fits into the rest of the scripts with globally unique names a bit better. This is a C project after all. The other small benefit is test generators may have an easier time since per-case symbols can expect to be unique. --- runners/test_runner.c | 246 +++++++++++++++++------------------ runners/test_runner.h | 2 - scripts/test.py | 189 +++++++++++++++++++-------- tests/test_alloc.toml | 24 ++-- tests/test_attrs.toml | 8 +- tests/test_badblocks.toml | 8 +- tests/test_bd.toml | 10 +- tests/test_dirs.toml | 28 ++-- tests/test_entries.toml | 16 +-- tests/test_evil.toml | 16 +-- tests/test_exhaustion.toml | 10 +- tests/test_files.toml | 20 +-- tests/test_interspersed.toml | 8 +- tests/test_move.toml | 34 ++--- tests/test_orphans.toml | 4 +- tests/test_paths.toml | 26 ++-- tests/test_relocations.toml | 8 +- tests/test_seek.toml | 12 +- tests/test_superblocks.toml | 14 +- tests/test_truncate.toml | 14 +- 20 files changed, 386 insertions(+), 311 deletions(-) diff --git a/runners/test_runner.c b/runners/test_runner.c index cb7dd9f4..55f237dc 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -110,8 +110,7 @@ typedef struct test_powerloss { } test_powerloss_t; typedef struct test_id { - const char *suite; - const char *case_; + const char *name; const test_define_t *defines; size_t define_count; const lfs_testbd_powercycles_t *cycles; @@ -415,7 +414,7 @@ extern const test_powerloss_t *test_powerlosses; extern size_t test_powerloss_count; const test_id_t *test_ids = (const test_id_t[]) { - {NULL, NULL, NULL, 0, NULL, 0}, + {NULL, NULL, 0, NULL, 0}, }; size_t test_id_count = 1; @@ -489,8 +488,8 @@ static void perm_printid( const lfs_testbd_powercycles_t *cycles, size_t cycle_count) { (void)suite; - // suite[:case[:permutation[:powercycles]]]] - printf("%s:", case_->id); + // case[:permutation[:powercycles]]] + printf("%s:", case_->name); for (size_t d = 0; d < lfs_max( suite->define_count, @@ -625,16 +624,15 @@ static void summary(void) { for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { - if (test_ids[t].suite && strcmp( - test_suites[i].name, test_ids[t].suite) != 0) { - continue; - } - test_define_suite(&test_suites[i]); for (size_t j = 0; j < test_suites[i].case_count; j++) { - if (test_ids[t].case_ && strcmp( - test_suites[i].cases[j].name, test_ids[t].case_) != 0) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { continue; } @@ -674,19 +672,18 @@ static void list_suites(void) { for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { - if (test_ids[t].suite && strcmp( - test_suites[i].name, test_ids[t].suite) != 0) { - continue; - } - test_define_suite(&test_suites[i]); size_t cases = 0; struct perm_count_state perms = {0, 0}; for (size_t j = 0; j < test_suites[i].case_count; j++) { - if (test_ids[t].case_ && strcmp( - test_suites[i].cases[j].name, test_ids[t].case_) != 0) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { continue; } @@ -702,6 +699,11 @@ static void list_suites(void) { &perms); } + // no tests found? + if (!cases) { + continue; + } + char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); char flag_buf[64]; @@ -709,7 +711,7 @@ static void list_suites(void) { (test_suites[i].flags & TEST_REENTRANT) ? "r" : "", (!test_suites[i].flags) ? "-" : ""); printf("%-36s %7s %7zu %11s\n", - test_suites[i].id, + test_suites[i].name, flag_buf, cases, perm_buf); @@ -722,16 +724,15 @@ static void list_cases(void) { for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { - if (test_ids[t].suite && strcmp( - test_suites[i].name, test_ids[t].suite) != 0) { - continue; - } - test_define_suite(&test_suites[i]); for (size_t j = 0; j < test_suites[i].case_count; j++) { - if (test_ids[t].case_ && strcmp( - test_suites[i].cases[j].name, test_ids[t].case_) != 0) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { continue; } @@ -755,7 +756,7 @@ static void list_cases(void) { (!test_suites[i].cases[j].flags) ? "-" : ""); printf("%-36s %7s %11s\n", - test_suites[i].cases[j].id, + test_suites[i].cases[j].name, flag_buf, perm_buf); } @@ -768,13 +769,26 @@ static void list_suite_paths(void) { for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { - if (test_ids[t].suite && strcmp( - test_suites[i].name, test_ids[t].suite) != 0) { + size_t cases = 0; + + for (size_t j = 0; j < test_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { + continue; + } + } + + // no tests found? + if (!cases) { continue; } printf("%-36s %s\n", - test_suites[i].id, + test_suites[i].name, test_suites[i].path); } } @@ -785,19 +799,18 @@ static void list_case_paths(void) { for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { - if (test_ids[t].suite && strcmp( - test_suites[i].name, test_ids[t].suite) != 0) { - continue; - } - for (size_t j = 0; j < test_suites[i].case_count; j++) { - if (test_ids[t].case_ && strcmp( - test_suites[i].cases[j].name, test_ids[t].case_) != 0) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { continue; } printf("%-36s %s\n", - test_suites[i].cases[j].id, + test_suites[i].cases[j].name, test_suites[i].cases[j].path); } } @@ -907,16 +920,15 @@ static void list_defines(void) { // add defines for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { - if (test_ids[t].suite && strcmp( - test_suites[i].name, test_ids[t].suite) != 0) { - continue; - } - test_define_suite(&test_suites[i]); for (size_t j = 0; j < test_suites[i].case_count; j++) { - if (test_ids[t].case_ && strcmp( - test_suites[i].cases[j].name, test_ids[t].case_) != 0) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { continue; } @@ -956,16 +968,15 @@ static void list_permutation_defines(void) { // add permutation defines for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { - if (test_ids[t].suite && strcmp( - test_suites[i].name, test_ids[t].suite) != 0) { - continue; - } - test_define_suite(&test_suites[i]); for (size_t j = 0; j < test_suites[i].case_count; j++) { - if (test_ids[t].case_ && strcmp( - test_suites[i].cases[j].name, test_ids[t].case_) != 0) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { continue; } @@ -1637,16 +1648,15 @@ static void run(void) { for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { - if (test_ids[t].suite && strcmp( - test_suites[i].name, test_ids[t].suite) != 0) { - continue; - } - test_define_suite(&test_suites[i]); for (size_t j = 0; j < test_suites[i].case_count; j++) { - if (test_ids[t].case_ && strcmp( - test_suites[i].cases[j].name, test_ids[t].case_) != 0) { + // does neither suite nor case name match? + if (test_ids[t].name && !( + strcmp(test_ids[t].name, + test_suites[i].name) == 0 + || strcmp(test_ids[t].name, + test_suites[i].cases[j].name) == 0)) { continue; } @@ -2367,83 +2377,72 @@ getopt_done: ; lfs_testbd_powercycles_t *cycles = NULL; size_t cycle_count = 0; - // parse suite - char *suite = argv[optind]; - char *case_ = strchr(suite, ':'); - if (case_) { - *case_ = '\0'; - case_ += 1; + // parse name, can be suite or case + char *name = argv[optind]; + char *defines_ = strchr(name, ':'); + if (defines_) { + *defines_ = '\0'; + defines_ += 1; } // remove optional path and .toml suffix - char *slash = strrchr(suite, '/'); + char *slash = strrchr(name, '/'); if (slash) { - suite = slash+1; + name = slash+1; } - size_t suite_len = strlen(suite); - if (suite_len > 5 && strcmp(&suite[suite_len-5], ".toml") == 0) { - suite[suite_len-5] = '\0'; + size_t name_len = strlen(name); + if (name_len > 5 && strcmp(&name[name_len-5], ".toml") == 0) { + name[name_len-5] = '\0'; } - if (case_) { - // parse case - char *defines_ = strchr(case_, ':'); - if (defines_) { - *defines_ = '\0'; - defines_ += 1; + if (defines_) { + // parse defines + char *cycles_ = strchr(defines_, ':'); + if (cycles_) { + *cycles_ = '\0'; + cycles_ += 1; } - // nothing really to do for case - - if (defines_) { - // parse defines - char *cycles_ = strchr(defines_, ':'); - if (cycles_) { - *cycles_ = '\0'; - cycles_ += 1; + while (true) { + char *parsed; + size_t d = leb16_parse(defines_, &parsed); + intmax_t v = leb16_parse(parsed, &parsed); + if (parsed == defines_) { + break; } + defines_ = parsed; - while (true) { - char *parsed; - size_t d = leb16_parse(defines_, &parsed); - intmax_t v = leb16_parse(parsed, &parsed); - if (parsed == defines_) { - break; - } - defines_ = parsed; - - if (d >= define_count) { - // align to power of two to avoid any superlinear growth - size_t ncount = 1 << lfs_npw2(d+1); - defines = realloc(defines, - ncount*sizeof(test_define_t)); - memset(defines+define_count, 0, - (ncount-define_count)*sizeof(test_define_t)); - define_count = ncount; - } - defines[d] = (test_define_t)TEST_LIT(v); + if (d >= define_count) { + // align to power of two to avoid any superlinear growth + size_t ncount = 1 << lfs_npw2(d+1); + defines = realloc(defines, + ncount*sizeof(test_define_t)); + memset(defines+define_count, 0, + (ncount-define_count)*sizeof(test_define_t)); + define_count = ncount; } + defines[d] = (test_define_t)TEST_LIT(v); + } - if (cycles_) { - // parse power cycles - size_t cycle_capacity = 0; - while (*cycles_ != '\0') { - char *parsed = NULL; - *(lfs_testbd_powercycles_t*)mappend( - (void**)&cycles, - sizeof(lfs_testbd_powercycles_t), - &cycle_count, - &cycle_capacity) - = leb16_parse(cycles_, &parsed); - if (parsed == cycles_) { - fprintf(stderr, "error: " - "could not parse test cycles: %s\n", - cycles_); - exit(-1); - } - cycles_ = parsed; + if (cycles_) { + // parse power cycles + size_t cycle_capacity = 0; + while (*cycles_ != '\0') { + char *parsed = NULL; + *(lfs_testbd_powercycles_t*)mappend( + (void**)&cycles, + sizeof(lfs_testbd_powercycles_t), + &cycle_count, + &cycle_capacity) + = leb16_parse(cycles_, &parsed); + if (parsed == cycles_) { + fprintf(stderr, "error: " + "could not parse test cycles: %s\n", + cycles_); + exit(-1); } + cycles_ = parsed; } } } @@ -2454,8 +2453,7 @@ getopt_done: ; sizeof(test_id_t), &test_id_count, &test_id_capacity) = (test_id_t){ - .suite = suite, - .case_ = case_, + .name = name, .defines = defines, .define_count = define_count, .cycles = cycles, diff --git a/runners/test_runner.h b/runners/test_runner.h index c27b9856..e5986acc 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -37,7 +37,6 @@ typedef struct test_define { } test_define_t; struct test_case { - const char *id; const char *name; const char *path; test_flags_t flags; @@ -50,7 +49,6 @@ struct test_case { }; struct test_suite { - const char *id; const char *name; const char *path; test_flags_t flags; diff --git a/scripts/test.py b/scripts/test.py index fcad9e0f..3f4ff564 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -133,13 +133,10 @@ class TestCase: print('%swarning:%s in %s, found unused key %r' % ( '\x1b[01;33m' if args['color'] else '', '\x1b[m' if args['color'] else '', - self.id(), + self.name, k), file=sys.stderr) - def id(self): - return '%s:%s' % (self.suite, self.name) - class TestSuite: # create a TestSuite object from a toml file @@ -221,13 +218,10 @@ class TestSuite: print('%swarning:%s in %s, found unused key %r' % ( '\x1b[01;33m' if args['color'] else '', '\x1b[m' if args['color'] else '', - self.id(), + self.name, k), file=sys.stderr) - def id(self): - return self.name - def compile(test_paths, **args): @@ -244,17 +238,45 @@ def compile(test_paths, **args): print('no test suites found in %r?' % test_paths) sys.exit(-1) + # load the suites + suites = [TestSuite(path, args) for path in paths] + suites.sort(key=lambda s: s.name) + + # check for name conflicts, these will cause ambiguity problems later + # when running tests + seen = {} + for suite in suites: + if suite.name in seen: + print('%swarning:%s conflicting suite %r, %s and %s' % ( + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + suite.name, + suite.path, + seen[suite.name].path), + file=sys.stderr) + seen[suite.name] = suite + + for case in suite.cases: + # only allow conflicts if a case and its suite share a name + if case.name in seen and not ( + isinstance(seen[case.name], TestSuite) + and seen[case.name].cases == [case]): + print('%swarning:%s conflicting case %r, %s and %s' % ( + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + case.name, + case.path, + seen[case.name].path), + file=sys.stderr) + seen[case.name] = case + + # we can only compile one test suite at a time if not args.get('source'): - if len(paths) > 1: + if len(suites) > 1: print('more than one test suite for compilation? (%r)' % test_paths) sys.exit(-1) - # load our suite - suite = TestSuite(paths[0], args) - else: - # load all suites - suites = [TestSuite(path, args) for path in paths] - suites.sort(key=lambda s: s.name) + suite = suites[0] # write generated test source if 'output' in args: @@ -332,7 +354,7 @@ def compile(test_paths, **args): f.writeln('void __test__%s__%s__run(' '__attribute__((unused)) struct lfs_config *cfg) {' % (suite.name, case.name)) - f.writeln(4*' '+'// test case %s' % case.id()) + f.writeln(4*' '+'// test case %s' % case.name) if case.code_lineno is not None: f.writeln(4*' '+'#line %d "%s"' % (case.code_lineno, suite.path)) @@ -384,10 +406,14 @@ def compile(test_paths, **args): f.writeln() # create suite struct - f.writeln('__attribute__((section("_test_suites")))') + # + # note we place this in the custom test_suites section with + # minimum alignment, otherwise GCC ups the alignment to + # 32-bytes for some reason + f.writeln('__attribute__((section("_test_suites"), ' + 'aligned(1)))') f.writeln('const struct test_suite __test__%s__suite = {' % suite.name) - f.writeln(4*' '+'.id = "%s",' % suite.id()) f.writeln(4*' '+'.name = "%s",' % suite.name) f.writeln(4*' '+'.path = "%s",' % suite.path) f.writeln(4*' '+'.flags = %s,' @@ -408,7 +434,6 @@ def compile(test_paths, **args): for case in suite.cases: # create case structs f.writeln(8*' '+'{') - f.writeln(12*' '+'.id = "%s",' % case.id()) f.writeln(12*' '+'.name = "%s",' % case.name) f.writeln(12*' '+'.path = "%s",' % case.path) f.writeln(12*' '+'.flags = %s,' @@ -534,8 +559,13 @@ def list_(runner, test_ids=[], **args): return sp.call(cmd) -def find_cases(runner_, ids=[], **args): - # query from runner +def find_perms(runner_, ids=[], **args): + case_suites = {} + expected_case_perms = co.defaultdict(lambda: 0) + expected_perms = 0 + total_perms = 0 + + # query cases from the runner cmd = runner_ + ['--list-cases'] + ids if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) @@ -545,21 +575,17 @@ def find_cases(runner_, ids=[], **args): universal_newlines=True, errors='replace', close_fds=False) - expected_suite_perms = co.defaultdict(lambda: 0) - expected_case_perms = co.defaultdict(lambda: 0) - expected_perms = 0 - total_perms = 0 pattern = re.compile( - '^(?P(?P(?P[^:]+):[^\s:]+)[^\s]*)\s+' - '[^\s]+\s+(?P\d+)/(?P\d+)') + '^(?P[^\s]+)' + '\s+(?P[^\s]+)' + '\s+(?P\d+)/(?P\d+)') # skip the first line for line in it.islice(proc.stdout, 1, None): m = pattern.match(line) if m: filtered = int(m.group('filtered')) perms = int(m.group('perms')) - expected_suite_perms[m.group('suite')] += filtered - expected_case_perms[m.group('id')] += filtered + expected_case_perms[m.group('case')] += filtered expected_perms += filtered total_perms += perms proc.wait() @@ -569,13 +595,50 @@ def find_cases(runner_, ids=[], **args): sys.stdout.write(line) sys.exit(-1) + # get which suite each case belongs to via paths + cmd = runner_ + ['--list-case-paths'] + ids + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + pattern = re.compile( + '^(?P[^\s]+)' + '\s+(?P[^:]+):(?P\d+)') + # skip the first line + for line in it.islice(proc.stdout, 1, None): + m = pattern.match(line) + if m: + path = m.group('path') + # strip path/suffix here + suite = os.path.basename(path) + if suite.endswith('.toml'): + suite = suite[:-len('.toml')] + case_suites[m.group('case')] = suite + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + # figure out expected suite perms + expected_suite_perms = co.defaultdict(lambda: 0) + for case, suite in case_suites.items(): + expected_suite_perms[suite] += expected_case_perms[case] + return ( + case_suites, expected_suite_perms, expected_case_perms, expected_perms, total_perms) def find_path(runner_, id, **args): + path = None # query from runner cmd = runner_ + ['--list-case-paths', id] if args.get('verbose'): @@ -586,10 +649,9 @@ def find_path(runner_, id, **args): universal_newlines=True, errors='replace', close_fds=False) - path = None pattern = re.compile( - '^(?P(?P(?P[^:]+):[^\s:]+)[^\s]*)\s+' - '(?P[^:]+):(?P\d+)') + '^(?P[^\s]+)' + '\s+(?P[^:]+):(?P\d+)') # skip the first line for line in it.islice(proc.stdout, 1, None): m = pattern.match(line) @@ -680,8 +742,11 @@ class TestFailure(Exception): def run_stage(name, runner_, ids, output_, **args): # get expected suite/case/perm counts - expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( - find_cases(runner_, ids, **args)) + (case_suites, + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) = find_perms(runner_, ids, **args) passed_suite_perms = co.defaultdict(lambda: 0) passed_case_perms = co.defaultdict(lambda: 0) @@ -692,9 +757,10 @@ def run_stage(name, runner_, ids, output_, **args): pattern = re.compile('^(?:' '(?Prunning|finished|skipped|powerloss) ' - '(?P(?P(?P[^:]+):[^\s:]+)[^\s]*)' + '(?P(?P[^:]+)[^\s]*)' '|' '(?P[^:]+):(?P\d+):(?Passert):' - ' *(?P.*)' ')$') + ' *(?P.*)' + ')$') locals = th.local() children = set() @@ -759,15 +825,18 @@ def run_stage(name, runner_, ids, output_, **args): last_id = m.group('id') powerlosses += 1 elif op == 'finished': - passed_suite_perms[m.group('suite')] += 1 - passed_case_perms[m.group('case')] += 1 + case = m.group('case') + suite = case_suites[case] + passed_suite_perms[suite] += 1 + passed_case_perms[case] += 1 passed_perms += 1 if output_: # get defines and write to csv defines = find_defines( runner_, m.group('id'), **args) output_.writerow({ - 'case': m.group('case'), + 'suite': suite, + 'case': case, 'test_passed': '1/1', **defines}) elif op == 'skipped': @@ -818,11 +887,13 @@ def run_stage(name, runner_, ids, output_, **args): except TestFailure as failure: # keep track of failures if output_: - suite, case, _ = failure.id.split(':', 2) + case, _ = failure.id.split(':', 1) + suite = case_suites[case] # get defines and write to csv defines = find_defines(runner_, failure.id, **args) output_.writerow({ - 'case': ':'.join([suite, case]), + 'suite': suite, + 'case': case, 'test_passed': '0/1', **defines}) @@ -919,13 +990,16 @@ def run(runner, test_ids=[], **args): # query runner for tests runner_ = find_runner(runner, **args) print('using runner: %s' % ' '.join(shlex.quote(c) for c in runner_)) - expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( - find_cases(runner_, test_ids, **args)) - print('found %d suites, %d cases, %d/%d permutations' - % (len(expected_suite_perms), - len(expected_case_perms), - expected_perms, - total_perms)) + (_, + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) = find_perms(runner_, test_ids, **args) + print('found %d suites, %d cases, %d/%d permutations' % ( + len(expected_suite_perms), + len(expected_case_perms), + expected_perms, + total_perms)) print() # truncate and open logs here so they aren't disconnected between tests @@ -937,7 +1011,7 @@ def run(runner, test_ids=[], **args): trace = openio(args['trace'], 'w', 1) output = None if args.get('output'): - output = TestOutput(args['output'], ['case'], ['test_passed']) + output = TestOutput(args['output'], ['suite', 'case'], ['test_passed']) # measure runtime start = time.time() @@ -951,12 +1025,17 @@ def run(runner, test_ids=[], **args): else expected_suite_perms.keys() if args.get('by_suites') else [None]): # spawn jobs for stage - expected_, passed_, powerlosses_, failures_, killed = run_stage( - by or 'tests', - runner_, - [by] if by is not None else test_ids, - output, - **args) + (expected_, + passed_, + powerlosses_, + failures_, + killed) = run_stage( + by or 'tests', + runner_, + [by] if by is not None else test_ids, + output, + **args) + # collect passes/failures expected += expected_ passed += passed_ powerlosses += powerlosses_ diff --git a/tests/test_alloc.toml b/tests/test_alloc.toml index 4e43db33..64b805fa 100644 --- a/tests/test_alloc.toml +++ b/tests/test_alloc.toml @@ -3,7 +3,7 @@ if = 'BLOCK_CYCLES == -1' # parallel allocation test -[cases.parallel_allocation] +[cases.test_alloc_parallel] defines.FILES = 3 defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / FILES)' code = ''' @@ -52,7 +52,7 @@ code = ''' ''' # serial allocation test -[cases.serial_allocation] +[cases.test_alloc_serial] defines.FILES = 3 defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / FILES)' code = ''' @@ -99,7 +99,7 @@ code = ''' ''' # parallel allocation reuse test -[cases.parallel_allocation_reuse] +[cases.test_alloc_parallel_reuse] defines.FILES = 3 defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / FILES)' defines.CYCLES = [1, 10] @@ -161,7 +161,7 @@ code = ''' ''' # serial allocation reuse test -[cases.serial_allocation_reuse] +[cases.test_alloc_serial_reuse] defines.FILES = 3 defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / FILES)' defines.CYCLES = [1, 10] @@ -221,7 +221,7 @@ code = ''' ''' # exhaustion test -[cases.exhaustion] +[cases.test_alloc_exhaustion] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -261,7 +261,7 @@ code = ''' ''' # exhaustion wraparound test -[cases.exhaustion_wraparound] +[cases.test_alloc_exhaustion_wraparound] defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-4)) / 3)' code = ''' lfs_t lfs; @@ -313,7 +313,7 @@ code = ''' ''' # dir exhaustion test -[cases.dir_exhaustion] +[cases.test_alloc_dir_exhaustion] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -367,7 +367,7 @@ code = ''' ''' # what if we have a bad block during an allocation scan? -[cases.bad_block_allocation] +[cases.test_alloc_bad_blocks] in = "lfs.c" defines.ERASE_CYCLES = 0xffffffff defines.BADBLOCK_BEHAVIOR = 'LFS_TESTBD_BADBLOCK_READERROR' @@ -459,7 +459,7 @@ code = ''' # should be removed and replaced with generalized tests. # chained dir exhaustion test -[cases.chained_dir_exhaustion] +[cases.test_alloc_chained_dir_exhaustion] if = 'BLOCK_SIZE == 512' defines.BLOCK_COUNT = 1024 code = ''' @@ -537,7 +537,7 @@ code = ''' ''' # split dir test -[cases.split_dir] +[cases.test_alloc_split_dir] if = 'BLOCK_SIZE == 512' defines.BLOCK_COUNT = 1024 code = ''' @@ -586,7 +586,7 @@ code = ''' ''' # outdated lookahead test -[cases.outdated_lookahead] +[cases.test_alloc_outdated_lookahead] if = 'BLOCK_SIZE == 512' defines.BLOCK_COUNT = 1024 code = ''' @@ -654,7 +654,7 @@ code = ''' ''' # outdated lookahead and split dir test -[cases.outdated_lookahead_split_dir] +[cases.test_alloc_outdated_lookahead_split_dir] if = 'BLOCK_SIZE == 512' defines.BLOCK_COUNT = 1024 code = ''' diff --git a/tests/test_attrs.toml b/tests/test_attrs.toml index 719ea682..3c69001c 100644 --- a/tests/test_attrs.toml +++ b/tests/test_attrs.toml @@ -1,4 +1,4 @@ -[cases.get_set_attrs] +[cases.test_attrs_get_set] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -79,7 +79,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.get_set_root_attrs] +[cases.test_attrs_get_set_root] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -159,7 +159,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.get_set_file_attrs] +[cases.test_attrs_get_set_file] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -269,7 +269,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.deferred_file_attrs] +[cases.test_attrs_deferred_file] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; diff --git a/tests/test_badblocks.toml b/tests/test_badblocks.toml index c5cab47c..012d8765 100644 --- a/tests/test_badblocks.toml +++ b/tests/test_badblocks.toml @@ -1,7 +1,7 @@ # bad blocks with block cycles should be tested in test_relocations if = '(int32_t)BLOCK_CYCLES == -1' -[cases.single_bad_blocks] +[cases.test_badblocks_single] defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] @@ -81,7 +81,7 @@ code = ''' } ''' -[cases.region_corruption] # (causes cascading failures) +[cases.test_badblocks_region_corruption] # (causes cascading failures) defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] @@ -160,7 +160,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.alternating_corruption] # (causes cascading failures) +[cases.test_badblocks_alternating_corruption] # (causes cascading failures) defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] @@ -240,7 +240,7 @@ code = ''' ''' # other corner cases -[cases.bad_superblocks] # (corrupt 1 or 0) +[cases.test_badblocks_superblocks] # (corrupt 1 or 0) defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] defines.BADBLOCK_BEHAVIOR = [ diff --git a/tests/test_bd.toml b/tests/test_bd.toml index 3cbc178f..8c6510df 100644 --- a/tests/test_bd.toml +++ b/tests/test_bd.toml @@ -4,7 +4,7 @@ # Note we use 251, a prime, in places to avoid aliasing powers of 2. # -[cases.one_block] +[cases.test_bd_one_block] defines.READ = ['READ_SIZE', 'BLOCK_SIZE'] defines.PROG = ['PROG_SIZE', 'BLOCK_SIZE'] code = ''' @@ -29,7 +29,7 @@ code = ''' } ''' -[cases.two_block] +[cases.test_bd_two_block] defines.READ = ['READ_SIZE', 'BLOCK_SIZE'] defines.PROG = ['PROG_SIZE', 'BLOCK_SIZE'] code = ''' @@ -87,7 +87,7 @@ code = ''' } ''' -[cases.last_block] +[cases.test_bd_last_block] defines.READ = ['READ_SIZE', 'BLOCK_SIZE'] defines.PROG = ['PROG_SIZE', 'BLOCK_SIZE'] code = ''' @@ -145,7 +145,7 @@ code = ''' } ''' -[cases.powers_of_two] +[cases.test_bd_powers_of_two] defines.READ = ['READ_SIZE', 'BLOCK_SIZE'] defines.PROG = ['PROG_SIZE', 'BLOCK_SIZE'] code = ''' @@ -191,7 +191,7 @@ code = ''' } ''' -[cases.fibonacci] +[cases.test_bd_fibonacci] defines.READ = ['READ_SIZE', 'BLOCK_SIZE'] defines.PROG = ['PROG_SIZE', 'BLOCK_SIZE'] code = ''' diff --git a/tests/test_dirs.toml b/tests/test_dirs.toml index 07107885..3774a557 100644 --- a/tests/test_dirs.toml +++ b/tests/test_dirs.toml @@ -1,4 +1,4 @@ -[cases.root] +[cases.test_dirs_root] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -17,7 +17,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.many_dir_creation] +[cases.test_dirs_many_creation] defines.N = 'range(3, 100, 3)' if = 'N < BLOCK_COUNT/2' code = ''' @@ -54,7 +54,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.many_dir_removal] +[cases.test_dirs_many_removal] defines.N = 'range(3, 100, 11)' if = 'N < BLOCK_COUNT/2' code = ''' @@ -111,7 +111,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.many_dir_rename] +[cases.test_dirs_many_rename] defines.N = 'range(3, 100, 11)' if = 'N < BLOCK_COUNT/2' code = ''' @@ -177,7 +177,7 @@ code = ''' lfs_unmount(&lfs); ''' -[cases.reentrant_many_dir] +[cases.test_dirs_many_reentrant] defines.N = [5, 11] reentrant = true code = ''' @@ -265,7 +265,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.file_creation] +[cases.test_dirs_file_creation] defines.N = 'range(3, 100, 11)' if = 'N < BLOCK_COUNT/2' code = ''' @@ -305,7 +305,7 @@ code = ''' lfs_unmount(&lfs); ''' -[cases.file_removal] +[cases.test_dirs_file_removal] defines.N = 'range(3, 100, 11)' if = 'N < BLOCK_COUNT/2' code = ''' @@ -365,7 +365,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.file_rename] +[cases.test_dirs_file_rename] defines.N = 'range(3, 100, 11)' if = 'N < BLOCK_COUNT/2' code = ''' @@ -434,7 +434,7 @@ code = ''' lfs_unmount(&lfs); ''' -[cases.reentrant_files] +[cases.test_dirs_file_reentrant] defines.N = [5, 25] if = 'N < BLOCK_COUNT/2' reentrant = true @@ -524,7 +524,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.nested_dirs] +[cases.test_dirs_nested] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -652,7 +652,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.recursive_remove] +[cases.test_dirs_recursive_remove] defines.N = [10, 100] if = 'N < BLOCK_COUNT/2' code = ''' @@ -716,7 +716,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.other_errors] +[cases.test_dirs_other_errors] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -794,7 +794,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.directory_seek] +[cases.test_dirs_seek] defines.COUNT = [4, 128, 132] if = 'COUNT < BLOCK_COUNT/2' code = ''' @@ -862,7 +862,7 @@ code = ''' } ''' -[cases.root_seek] +[cases.test_dirs_toot_seek] defines.COUNT = [4, 128, 132] if = 'COUNT < BLOCK_COUNT/2' code = ''' diff --git a/tests/test_entries.toml b/tests/test_entries.toml index 6c1f1d7f..7aa551e0 100644 --- a/tests/test_entries.toml +++ b/tests/test_entries.toml @@ -5,7 +5,7 @@ defines.CACHE_SIZE = 512 if = 'CACHE_SIZE % PROG_SIZE == 0 && CACHE_SIZE == 512' -[cases.entry_grow] +[cases.test_entries_grow] code = ''' uint8_t wbuffer[1024]; uint8_t rbuffer[1024]; @@ -98,7 +98,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.entry_shrink] +[cases.test_entries_shrink] code = ''' uint8_t wbuffer[1024]; uint8_t rbuffer[1024]; @@ -191,7 +191,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.entry_spill] +[cases.test_entries_spill] code = ''' uint8_t wbuffer[1024]; uint8_t rbuffer[1024]; @@ -268,7 +268,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.entry_push_spill] +[cases.test_entries_push_spill] code = ''' uint8_t wbuffer[1024]; uint8_t rbuffer[1024]; @@ -361,7 +361,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.entry_push_spill_two] +[cases.test_entries_push_spill_two] code = ''' uint8_t wbuffer[1024]; uint8_t rbuffer[1024]; @@ -469,7 +469,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.entry_drop] +[cases.test_entries_drop] code = ''' uint8_t wbuffer[1024]; uint8_t rbuffer[1024]; @@ -572,7 +572,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.create_too_big] +[cases.test_entries_create_too_big] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -600,7 +600,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.resize_too_big] +[cases.test_entries_resize_too_big] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; diff --git a/tests/test_evil.toml b/tests/test_evil.toml index 78a5034d..4acd5ef0 100644 --- a/tests/test_evil.toml +++ b/tests/test_evil.toml @@ -3,7 +3,7 @@ # invalid pointer tests (outside of block_count) -[cases.invalid_tail_pointer] +[cases.test_evil_invalid_tail_pointer] defines.TAIL_TYPE = ['LFS_TYPE_HARDTAIL', 'LFS_TYPE_SOFTTAIL'] defines.INVALSET = [0x3, 0x1, 0x2] in = "lfs.c" @@ -27,7 +27,7 @@ code = ''' lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT; ''' -[cases.invalid_dir_pointer] +[cases.test_evil_invalid_dir_pointer] defines.INVALSET = [0x3, 0x1, 0x2] in = "lfs.c" code = ''' @@ -78,7 +78,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.invalid_file_pointer] +[cases.test_evil_invalid_file_pointer] in = "lfs.c" defines.SIZE = [10, 1000, 100000] # faked file size code = ''' @@ -130,7 +130,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.invalid_ctz_pointer] # invalid pointer in CTZ skip-list test +[cases.test_evil_invalid_ctz_pointer] # invalid pointer in CTZ skip-list test defines.SIZE = ['2*BLOCK_SIZE', '3*BLOCK_SIZE', '4*BLOCK_SIZE'] in = "lfs.c" code = ''' @@ -196,7 +196,7 @@ code = ''' ''' -[cases.invalid_gstate_pointer] +[cases.test_evil_invalid_gstate_pointer] defines.INVALSET = [0x3, 0x1, 0x2] in = "lfs.c" code = ''' @@ -224,7 +224,7 @@ code = ''' # cycle detection/recovery tests -[cases.mdir_loop] # metadata-pair threaded-list loop test +[cases.test_evil_mdir_loop] # metadata-pair threaded-list loop test in = "lfs.c" code = ''' // create littlefs @@ -244,7 +244,7 @@ code = ''' lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT; ''' -[cases.mdir_loop_2] # metadata-pair threaded-list 2-length loop test +[cases.test_evil_mdir_loop2] # metadata-pair threaded-list 2-length loop test in = "lfs.c" code = ''' // create littlefs with child dir @@ -275,7 +275,7 @@ code = ''' lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT; ''' -[cases.mdir_loop_child] # metadata-pair threaded-list 1-length child loop test +[cases.test_evil_mdir_loop_child] # metadata-pair threaded-list 1-length child loop test in = "lfs.c" code = ''' // create littlefs with child dir diff --git a/tests/test_exhaustion.toml b/tests/test_exhaustion.toml index 1914d628..fdcef24c 100644 --- a/tests/test_exhaustion.toml +++ b/tests/test_exhaustion.toml @@ -1,5 +1,5 @@ # test running a filesystem to exhaustion -[cases.exhaustion] +[cases.test_exhaustion_normal] defines.ERASE_CYCLES = 10 defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' @@ -92,7 +92,7 @@ exhausted: # test running a filesystem to exhaustion # which also requires expanding superblocks -[cases.exhaustion_superblocks] +[cases.test_exhaustion_superblocks] defines.ERASE_CYCLES = 10 defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' @@ -186,7 +186,7 @@ exhausted: # check for. # wear-level test running a filesystem to exhaustion -[cases.wear_leveling_exhaustion] +[cases.test_exhuastion_wear_leveling] defines.ERASE_CYCLES = 20 defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' @@ -286,7 +286,7 @@ exhausted: ''' # wear-level test + expanding superblock -[cases.wear_leveling_exhaustion_superblocks] +[cases.test_exhaustion_wear_leveling_superblocks] defines.ERASE_CYCLES = 20 defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' @@ -383,7 +383,7 @@ exhausted: ''' # test that we wear blocks roughly evenly -[cases.wear_leveling_distribution] +[cases.test_exhaustion_wear_distribution] defines.ERASE_CYCLES = 0xffffffff defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = [5, 4, 3, 2, 1] diff --git a/tests/test_files.toml b/tests/test_files.toml index 026e47ff..89ce1517 100644 --- a/tests/test_files.toml +++ b/tests/test_files.toml @@ -1,5 +1,5 @@ -[cases.simple_file] +[cases.test_files_simple] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -22,7 +22,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.large_files] +[cases.test_files_large] defines.SIZE = [32, 8192, 262144, 0, 7, 8193] defines.CHUNKSIZE = [31, 16, 33, 1, 1023] code = ''' @@ -63,7 +63,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.rewriting_files] +[cases.test_files_rewrite] defines.SIZE1 = [32, 8192, 131072, 0, 7, 8193] defines.SIZE2 = [32, 8192, 131072, 0, 7, 8193] defines.CHUNKSIZE = [31, 16, 1] @@ -148,7 +148,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.appending_files] +[cases.test_files_append] defines.SIZE1 = [32, 8192, 131072, 0, 7, 8193] defines.SIZE2 = [32, 8192, 131072, 0, 7, 8193] defines.CHUNKSIZE = [31, 16, 1] @@ -228,7 +228,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.truncating_files] +[cases.test_files_truncate] defines.SIZE1 = [32, 8192, 131072, 0, 7, 8193] defines.SIZE2 = [32, 8192, 131072, 0, 7, 8193] defines.CHUNKSIZE = [31, 16, 1] @@ -300,7 +300,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.reentrant_file_writing] +[cases.test_files_reentrant_write] defines.SIZE = [32, 0, 7, 2049] defines.CHUNKSIZE = [31, 16, 65] reentrant = true @@ -351,7 +351,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.reentrant_file_writing_sync] +[cases.test_files_reentrant_write_sync] defines = [ # append (O(n)) {MODE='LFS_O_APPEND', SIZE=[32, 0, 7, 2049], CHUNKSIZE=[31, 16, 65]}, @@ -424,7 +424,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.many_files] +[cases.test_files_many] defines.N = 300 code = ''' lfs_t lfs; @@ -452,7 +452,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.many_files_power_cycle] +[cases.test_files_many_power_cycle] defines.N = 300 code = ''' lfs_t lfs; @@ -482,7 +482,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.many_files_power_loss] +[cases.test_files_many_power_loss] defines.N = 300 reentrant = true code = ''' diff --git a/tests/test_interspersed.toml b/tests/test_interspersed.toml index 92d96d83..d7143f61 100644 --- a/tests/test_interspersed.toml +++ b/tests/test_interspersed.toml @@ -1,5 +1,5 @@ -[cases.interspersed_files] +[cases.test_interspersed_files] defines.SIZE = [10, 100] defines.FILES = [4, 10, 26] code = ''' @@ -66,7 +66,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.interspersed_remove_files] +[cases.test_interspersed_remove_files] defines.SIZE = [10, 100] defines.FILES = [4, 10, 26] code = ''' @@ -127,7 +127,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.remove_inconveniently] +[cases.test_interspersed_remove_inconveniently] defines.SIZE = [10, 100] code = ''' lfs_t lfs; @@ -191,7 +191,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.reentrant_interspersed_files] +[cases.test_interspersed_reentrant_files] defines.SIZE = [10, 100] defines.FILES = [4, 10, 26] reentrant = true diff --git a/tests/test_move.toml b/tests/test_move.toml index dc2623e3..6c89766c 100644 --- a/tests/test_move.toml +++ b/tests/test_move.toml @@ -1,4 +1,4 @@ -[cases.move_file] +[cases.test_move_file] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -60,7 +60,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.nop_move] # yes this is legal +[cases.test_move_nop] # yes this is legal code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -78,7 +78,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.move_file_corrupt_source] +[cases.test_move_file_corrupt_source] in = "lfs.c" code = ''' lfs_t lfs; @@ -158,7 +158,7 @@ code = ''' ''' # move file corrupt source and dest -[cases.move_file_corrupt_source_dest] +[cases.test_move_file_corrupt_source_dest] in = "lfs.c" if = 'PROG_SIZE <= 0x3fe' # only works with one crc per commit code = ''' @@ -254,7 +254,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.move_file_after_corrupt] +[cases.test_move_file_after_corrupt] in = "lfs.c" if = 'PROG_SIZE <= 0x3fe' # only works with one crc per commit code = ''' @@ -355,7 +355,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.reentrant_move_file] +[cases.test_move_reentrant_file] reentrant = true code = ''' lfs_t lfs; @@ -472,7 +472,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.move_dir] +[cases.test_move_dir] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -540,7 +540,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.move_dir_corrupt_source] +[cases.test_move_dir_corrupt_source] in = "lfs.c" code = ''' lfs_t lfs; @@ -626,7 +626,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.move_dir_corrupt_source_dest] +[cases.test_move_dir_corrupt_source_dest] in = "lfs.c" if = 'PROG_SIZE <= 0x3fe' # only works with one crc per commit code = ''' @@ -729,7 +729,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.move_dir_after_corrupt] +[cases.test_move_dir_after_corrupt] in = "lfs.c" if = 'PROG_SIZE <= 0x3fe' # only works with one crc per commit code = ''' @@ -837,7 +837,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.reentrant_move_dir] +[cases.test_reentrant_dir] reentrant = true code = ''' lfs_t lfs; @@ -958,7 +958,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.move_state_stealing] +[cases.test_move_state_stealing] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -1028,7 +1028,7 @@ code = ''' # Other specific corner cases # create + delete in same commit with neighbors -[cases.create_delete_same] +[cases.test_move_create_delete_same] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -1179,7 +1179,7 @@ code = ''' ''' # create + delete + delete in same commit with neighbors -[cases.create_delete_delete_same] +[cases.test_move_create_delete_delete_same] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -1341,7 +1341,7 @@ code = ''' ''' # create + delete in different dirs with neighbors -[cases.create_delete_different] +[cases.test_move_create_delete_different] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -1584,7 +1584,7 @@ code = ''' ''' # move fix in relocation -[cases.move_fix_relocation] +[cases.test_move_fix_relocation] in = "lfs.c" defines.RELOCATIONS = 'range(4)' defines.ERASE_CYCLES = 0xffffffff @@ -1729,7 +1729,7 @@ code = ''' ''' # move fix in relocation with predecessor -[cases.move_fix_relocation_predecessor] +[cases.test_move_fix_relocation_predecessor] in = "lfs.c" defines.RELOCATIONS = 'range(8)' defines.ERASE_CYCLES = 0xffffffff diff --git a/tests/test_orphans.toml b/tests/test_orphans.toml index fd9b521c..0f200d73 100644 --- a/tests/test_orphans.toml +++ b/tests/test_orphans.toml @@ -1,4 +1,4 @@ -[cases.orphan] +[cases.test_orphans_normal] in = "lfs.c" if = 'PROG_SIZE <= 0x3fe' # only works with one crc per commit code = ''' @@ -60,7 +60,7 @@ code = ''' ''' # reentrant testing for orphans, basically just spam mkdir/remove -[cases.reentrant_orphan] +[cases.test_orphans_reentrant] reentrant = true # TODO fix this case, caused by non-DAG trees if = '!(DEPTH == 3 && CACHE_SIZE != 64)' diff --git a/tests/test_paths.toml b/tests/test_paths.toml index 310364d8..97a519ea 100644 --- a/tests/test_paths.toml +++ b/tests/test_paths.toml @@ -1,6 +1,6 @@ # simple path test -[cases.path] +[cases.test_paths_normal] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -25,7 +25,7 @@ code = ''' ''' # redundant slashes -[cases.redundant_slashes] +[cases.test_paths_redundant_slashes] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -52,7 +52,7 @@ code = ''' ''' # dot path test -[cases.dot_path] +[cases.test_paths_dot] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -81,7 +81,7 @@ code = ''' ''' # dot dot path test -[cases.dot_dot_path] +[cases.test_paths_dot_dot] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -114,7 +114,7 @@ code = ''' ''' # trailing dot path test -[cases.trailing_dot_path] +[cases.test_paths_trailing_dot] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -139,7 +139,7 @@ code = ''' ''' # leading dot path test -[cases.leading_dot_path] +[cases.test_paths_leading_dot] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -154,7 +154,7 @@ code = ''' ''' # root dot dot path test -[cases.root_dot_dot_path] +[cases.test_paths_root_dot_dot] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -181,7 +181,7 @@ code = ''' ''' # invalid path tests -[cases.invalid_path] +[cases.test_paths_invalid] code = ''' lfs_t lfs; lfs_format(&lfs, cfg); @@ -206,7 +206,7 @@ code = ''' ''' # root operations -[cases.root] +[cases.test_paths_root] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -226,7 +226,7 @@ code = ''' ''' # root representations -[cases.root_reprs] +[cases.test_paths_root_reprs] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -254,7 +254,7 @@ code = ''' ''' # superblock conflict test -[cases.superblock_conflict] +[cases.test_paths_superblock_conflict] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -273,7 +273,7 @@ code = ''' ''' # max path test -[cases.max_path] +[cases.test_paths_max] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -301,7 +301,7 @@ code = ''' ''' # really big path test -[cases.really_big_path] +[cases.test_paths_really_big] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; diff --git a/tests/test_relocations.toml b/tests/test_relocations.toml index f177c730..350ac95b 100644 --- a/tests/test_relocations.toml +++ b/tests/test_relocations.toml @@ -1,5 +1,5 @@ # specific corner cases worth explicitly testing for -[cases.dangling_split_dir] +[cases.test_relocations_dangling_split_dir] defines.ITERATIONS = 20 defines.COUNT = 10 defines.BLOCK_CYCLES = [8, 1] @@ -77,7 +77,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.outdated_head] +[cases.test_relocations_outdated_head] defines.ITERATIONS = 20 defines.COUNT = 10 defines.BLOCK_CYCLES = [8, 1] @@ -168,7 +168,7 @@ code = ''' # reentrant testing for relocations, this is the same as the # orphan testing, except here we also set block_cycles so that # almost every tree operation needs a relocation -[cases.reentrant_relocations] +[cases.test_relocations_reentrant] reentrant = true # TODO fix this case, caused by non-DAG trees if = '!(DEPTH == 3 && CACHE_SIZE != 64)' @@ -236,7 +236,7 @@ code = ''' ''' # reentrant testing for relocations, but now with random renames! -[cases.reentrant_relocations_renames] +[cases.test_relocations_reentrant_renames] reentrant = true # TODO fix this case, caused by non-DAG trees if = '!(DEPTH == 3 && CACHE_SIZE != 64)' diff --git a/tests/test_seek.toml b/tests/test_seek.toml index 383c1ba1..b976057b 100644 --- a/tests/test_seek.toml +++ b/tests/test_seek.toml @@ -1,6 +1,6 @@ # simple file seek -[cases.seek] +[cases.test_seek_read] defines = [ {COUNT=132, SKIP=4}, {COUNT=132, SKIP=128}, @@ -73,7 +73,7 @@ code = ''' ''' # simple file seek and write -[cases.seek_write] +[cases.test_seek_write] defines = [ {COUNT=132, SKIP=4}, {COUNT=132, SKIP=128}, @@ -138,7 +138,7 @@ code = ''' ''' # boundary seek and writes -[cases.boundary_seek_write] +[cases.test_seek_boundary_write] defines.COUNT = 132 code = ''' lfs_t lfs; @@ -195,7 +195,7 @@ code = ''' ''' # out of bounds seek -[cases.out_of_bounds_seek] +[cases.test_seek_out_of_bounds] defines = [ {COUNT=132, SKIP=4}, {COUNT=132, SKIP=128}, @@ -254,7 +254,7 @@ code = ''' ''' # inline write and seek -[cases.inline_write_seek] +[cases.test_seek_inline_write] defines.SIZE = [2, 4, 128, 132] code = ''' lfs_t lfs; @@ -325,7 +325,7 @@ code = ''' ''' # file seek and write with power-loss -[cases.reentrant_seek_write] +[cases.test_seek_reentrant_write] # must be power-of-2 for quadratic probing to be exhaustive defines.COUNT = [4, 64, 128] reentrant = true diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index d511675f..d45d7887 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -1,12 +1,12 @@ # simple formatting test -[cases.format] +[cases.test_superblocks_format] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; ''' # mount/unmount -[cases.mount] +[cases.test_superblocks_mount] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -15,7 +15,7 @@ code = ''' ''' # reentrant format -[cases.reentrant_format] +[cases.test_superblocks_reentrant_format] reentrant = true code = ''' lfs_t lfs; @@ -28,14 +28,14 @@ code = ''' ''' # invalid mount -[cases.invalid_mount] +[cases.test_superblocks_invalid_mount] code = ''' lfs_t lfs; lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT; ''' # expanding superblock -[cases.expanding_superblock] +[cases.test_superblocks_expand] defines.LFS_BLOCK_CYCLES = [32, 33, 1] defines.N = [10, 100, 1000] code = ''' @@ -69,7 +69,7 @@ code = ''' ''' # expanding superblock with power cycle -[cases.expanding_superblock_power_cycle] +[cases.test_superblocks_expand_power_cycle] defines.LFS_BLOCK_CYCLES = [32, 33, 1] defines.N = [10, 100, 1000] code = ''' @@ -107,7 +107,7 @@ code = ''' ''' # reentrant expanding superblock -[cases.reentrant_expanding_superblock] +[cases.test_superblocks_reentrant_expand] defines.LFS_BLOCK_CYCLES = [2, 1] defines.N = 24 reentrant = true diff --git a/tests/test_truncate.toml b/tests/test_truncate.toml index 80e250fe..a0da50e8 100644 --- a/tests/test_truncate.toml +++ b/tests/test_truncate.toml @@ -1,5 +1,5 @@ # simple truncate -[cases.truncate] +[cases.test_truncate_simple] defines.MEDIUMSIZE = [32, 2048] defines.LARGESIZE = 8192 code = ''' @@ -47,7 +47,7 @@ code = ''' ''' # truncate and read -[cases.truncate_read] +[cases.test_truncate_read] defines.MEDIUMSIZE = [32, 2048] defines.LARGESIZE = 8192 code = ''' @@ -102,7 +102,7 @@ code = ''' ''' # write, truncate, and read -[cases.write_truncate_read] +[cases.test_truncate_write_read] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -158,7 +158,7 @@ code = ''' ''' # truncate and write -[cases.truncate_write] +[cases.test_truncate_write] defines.MEDIUMSIZE = [32, 2048] defines.LARGESIZE = 8192 code = ''' @@ -213,7 +213,7 @@ code = ''' ''' # truncate write under powerloss -[cases.reentrant_truncate_write] +[cases.test_truncate_reentrant_write] defines.SMALLSIZE = [4, 512] defines.MEDIUMSIZE = [32, 1024] defines.LARGESIZE = 2048 @@ -284,7 +284,7 @@ code = ''' ''' # more aggressive general truncation tests -[cases.aggressive_truncate] +[cases.test_truncate_aggressive] defines.CONFIG = 'range(6)' defines.SMALLSIZE = 32 defines.MEDIUMSIZE = 2048 @@ -428,7 +428,7 @@ code = ''' ''' # noop truncate -[cases.nop_truncate] +[cases.test_truncate_nop] defines.MEDIUMSIZE = [32, 2048] code = ''' lfs_t lfs; From 20ec0be875bbb1735c7f2c8b2780354ead9b7ee2 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 19 Sep 2022 14:29:41 -0500 Subject: [PATCH 40/81] Cleaned up a number of small tweaks in the scripts - Added the littlefs license note to the scripts. - Adopted parse_intermixed_args everywhere for more consistent arg handling. - Removed argparse's implicit help text formatting as it does not work with perse_intermixed_args and breaks sometimes. - Used string concatenation for argparse everywhere, uses backslashed line continuations only works with argparse because it strips redundant whitespace. - Consistent argparse formatting. - Consistent openio mode handling. - Consistent color argument handling. - Adopted functools.lru_cache in tracebd.py. - Moved unicode printing behind --subscripts in traceby.py, making all scripts ascii by default. - Renamed pretty_asserts.py -> prettyasserts.py. - Renamed struct.py -> struct_.py, the original name conflicts with Python's built in struct module in horrible ways. --- Makefile | 8 +- scripts/code.py | 19 +- scripts/coverage.py | 26 ++- scripts/data.py | 17 +- .../{pretty_asserts.py => prettyasserts.py} | 36 +++- scripts/stack.py | 12 +- scripts/{struct.py => struct_.py} | 14 +- scripts/summary.py | 15 +- scripts/tailpipe.py | 10 +- scripts/test.py | 204 ++++++++++++------ scripts/tracebd.py | 108 +++++----- 11 files changed, 307 insertions(+), 162 deletions(-) rename scripts/{pretty_asserts.py => prettyasserts.py} (94%) rename scripts/{struct.py => struct_.py} (97%) diff --git a/Makefile b/Makefile index d87a8792..ce9a5539 100644 --- a/Makefile +++ b/Makefile @@ -133,7 +133,7 @@ stack: $(CI) .PHONY: struct struct: $(OBJ) - ./scripts/struct.py $^ -S $(STRUCTFLAGS) + ./scripts/struct_.py $^ -S $(STRUCTFLAGS) .PHONY: coverage coverage: $(GCDA) @@ -171,7 +171,7 @@ $(BUILDDIR)lfs.stack.csv: $(CI) ./scripts/stack.py $^ -q $(CODEFLAGS) -o $@ $(BUILDDIR)lfs.struct.csv: $(OBJ) - ./scripts/struct.py $^ -q $(CODEFLAGS) -o $@ + ./scripts/struct_.py $^ -q $(CODEFLAGS) -o $@ $(BUILDDIR)lfs.coverage.csv: $(GCDA) ./scripts/coverage.py $^ -q $(COVERAGEFLAGS) -o $@ @@ -195,10 +195,10 @@ $(BUILDDIR)%.s: %.c $(CC) -S $(CFLAGS) $< -o $@ $(BUILDDIR)%.a.c: %.c - ./scripts/pretty_asserts.py -p LFS_ASSERT $< -o $@ + ./scripts/prettyasserts.py -p LFS_ASSERT $< -o $@ $(BUILDDIR)%.a.c: $(BUILDDIR)%.c - ./scripts/pretty_asserts.py -p LFS_ASSERT $< -o $@ + ./scripts/prettyasserts.py -p LFS_ASSERT $< -o $@ $(BUILDDIR)%.t.c: %.toml ./scripts/test.py -c $< $(TESTCFLAGS) -o $@ diff --git a/scripts/code.py b/scripts/code.py index 8a5c39b4..4adc0c94 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -1,9 +1,16 @@ #!/usr/bin/env python3 # -# Script to find code size at the function level. Basically just a bit wrapper +# Script to find code size at the function level. Basically just a big wrapper # around nm with some extra conveniences for comparing builds. Heavily inspired # by Linux's Bloat-O-Meter. # +# Example: +# ./scripts/code.py lfs.o lfs_util.o -S +# +# Copyright (c) 2022, The littlefs authors. +# Copyright (c) 2020, Arm Limited. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause +# import collections as co import csv @@ -126,7 +133,7 @@ class CodeResult(co.namedtuple('CodeResult', 'file,function,code_size')): def openio(path, mode='r'): if path == '-': - if 'r' in mode: + if mode == 'r': return os.fdopen(os.dup(sys.stdin.fileno()), 'r') else: return os.fdopen(os.dup(sys.stdout.fileno()), 'w') @@ -417,7 +424,7 @@ if __name__ == "__main__": nargs='*', default=OBJ_PATHS, help="Description of where to find *.o files. May be a directory " - "or a list of paths. Defaults to %(default)r.") + "or a list of paths. Defaults to %r." % OBJ_PATHS) parser.add_argument( '-v', '--verbose', action='store_true', @@ -468,16 +475,16 @@ if __name__ == "__main__": '--type', default=TYPE, help="Type of symbols to report, this uses the same single-character " - "type-names emitted by nm. Defaults to %(default)r.") + "type-names emitted by nm. Defaults to %r." % TYPE) parser.add_argument( '--nm-tool', type=lambda x: x.split(), default=NM_TOOL, - help="Path to the nm tool to use. Defaults to %(default)r") + help="Path to the nm tool to use. Defaults to %r." % NM_TOOL) parser.add_argument( '--build-dir', help="Specify the relative build directory. Used to map object files " "to the correct source files.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_args()).items() + for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) diff --git a/scripts/coverage.py b/scripts/coverage.py index 14fe0d2d..5f0e11a8 100755 --- a/scripts/coverage.py +++ b/scripts/coverage.py @@ -1,8 +1,13 @@ #!/usr/bin/env python3 # -# Script to find test coverage. Basically just a big wrapper around gcov with -# some extra conveniences for comparing builds. Heavily inspired by Linux's -# Bloat-O-Meter. +# Script to find coverage info after running tests. +# +# Example: +# ./scripts/coverage.py lfs.t.a.gcda lfs_util.t.a.gcda -s +# +# Copyright (c) 2022, The littlefs authors. +# Copyright (c) 2020, Arm Limited. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause # import collections as co @@ -208,7 +213,7 @@ class CoverageResult(co.namedtuple('CoverageResult', def openio(path, mode='r'): if path == '-': - if 'r' in mode: + if mode == 'r': return os.fdopen(os.dup(sys.stdin.fileno()), 'r') else: return os.fdopen(os.dup(sys.stdout.fileno()), 'w') @@ -682,7 +687,7 @@ if __name__ == "__main__": nargs='*', default=GCDA_PATHS, help="Description of where to find *.gcda files. May be a directory " - "or a list of paths. Defaults to %(default)r.") + "or a list of paths. Defaults to %r." % GCDA_PATHS) parser.add_argument( '-v', '--verbose', action='store_true', @@ -752,18 +757,17 @@ if __name__ == "__main__": '-c', '--context', type=lambda x: int(x, 0), default=3, - help="Show a additional lines of context. Defaults to %(default)r.") + help="Show a additional lines of context. Defaults to 3.") parser.add_argument( '-W', '--width', type=lambda x: int(x, 0), default=80, - help="Assume source is styled with this many columns. Defaults " - "to %(default)r.") + help="Assume source is styled with this many columns. Defaults to 80.") parser.add_argument( '--color', choices=['never', 'always', 'auto'], default='auto', - help="When to use terminal colors.") + help="When to use terminal colors. Defaults to 'auto'.") parser.add_argument( '-e', '--error-on-lines', action='store_true', @@ -780,11 +784,11 @@ if __name__ == "__main__": '--gcov-tool', default=GCOV_TOOL, type=lambda x: x.split(), - help="Path to the gcov tool to use. Defaults to %(default)r.") + help="Path to the gcov tool to use. Defaults to %r." % GCOV_TOOL) parser.add_argument( '--build-dir', help="Specify the relative build directory. Used to map object files " "to the correct source files.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_args()).items() + for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) diff --git a/scripts/data.py b/scripts/data.py index 353b163e..d42f5319 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -4,6 +4,13 @@ # around nm with some extra conveniences for comparing builds. Heavily inspired # by Linux's Bloat-O-Meter. # +# Example: +# ./scripts/data.py lfs.o lfs_util.o -S +# +# Copyright (c) 2022, The littlefs authors. +# Copyright (c) 2020, Arm Limited. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause +# import collections as co import csv @@ -126,7 +133,7 @@ class DataResult(co.namedtuple('DataResult', 'file,function,data_size')): def openio(path, mode='r'): if path == '-': - if 'r' in mode: + if mode == 'r': return os.fdopen(os.dup(sys.stdin.fileno()), 'r') else: return os.fdopen(os.dup(sys.stdout.fileno()), 'w') @@ -417,7 +424,7 @@ if __name__ == "__main__": nargs='*', default=OBJ_PATHS, help="Description of where to find *.o files. May be a directory " - "or a list of paths. Defaults to %(default)r.") + "or a list of paths. Defaults to %r." % OBJ_PATHS) parser.add_argument( '-v', '--verbose', action='store_true', @@ -468,16 +475,16 @@ if __name__ == "__main__": '--type', default=TYPE, help="Type of symbols to report, this uses the same single-character " - "type-names emitted by nm. Defaults to %(default)r.") + "type-names emitted by nm. Defaults to %r." % TYPE) parser.add_argument( '--nm-tool', type=lambda x: x.split(), default=NM_TOOL, - help="Path to the nm tool to use. Defaults to %(default)r") + help="Path to the nm tool to use. Defaults to %r." % NM_TOOL) parser.add_argument( '--build-dir', help="Specify the relative build directory. Used to map object files " "to the correct source files.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_args()).items() + for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) diff --git a/scripts/pretty_asserts.py b/scripts/prettyasserts.py similarity index 94% rename from scripts/pretty_asserts.py rename to scripts/prettyasserts.py index 8afa6545..73a43a11 100755 --- a/scripts/pretty_asserts.py +++ b/scripts/prettyasserts.py @@ -1,4 +1,14 @@ #!/usr/bin/env python3 +# +# Preprocessor that makes asserts easier to debug. +# +# Example: +# ./scripts/prettyasserts.py -p LFS_ASSERT lfs.c -o lfs.a.c +# +# Copyright (c) 2022, The littlefs authors. +# Copyright (c) 2020, Arm Limited. All rights reserved. +# SPDX-License-Identifier: BSD-3-Clause +# import re import sys @@ -34,7 +44,7 @@ LEXEMES = { def openio(path, mode='r'): if path == '-': - if 'r' in mode: + if mode == 'r': return os.fdopen(os.dup(sys.stdin.fileno()), 'r') else: return os.fdopen(os.dup(sys.stdout.fileno()), 'w') @@ -414,17 +424,25 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Preprocessor that makes asserts easy to debug.") - parser.add_argument('input', + description="Preprocessor that makes asserts easier to debug.") + parser.add_argument( + 'input', help="Input C file.") - parser.add_argument('-o', '--output', required=True, + parser.add_argument( + '-o', '--output', + required=True, help="Output C file.") - parser.add_argument('-p', '--pattern', action='append', + parser.add_argument( + '-p', '--pattern', + action='append', help="Regex patterns to search for starting an assert statement. This" " implicitly includes \"assert\" and \"=>\".") - parser.add_argument('-l', '--limit', - default=LIMIT, type=lambda x: int(x, 0), - help="Maximum number of characters to display in strcmp and memcmp.") + parser.add_argument( + '-l', '--limit', + type=lambda x: int(x, 0), + default=LIMIT, + help="Maximum number of characters to display in strcmp and memcmp. " + "Defaults to %r." % LIMIT) sys.exit(main(**{k: v - for k, v in vars(parser.parse_args()).items() + for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) diff --git a/scripts/stack.py b/scripts/stack.py index 194f831c..36ef3dca 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -3,6 +3,12 @@ # Script to find stack usage at the function level. Will detect recursion and # report as infinite stack usage. # +# Example: +# ./scripts/stack.py lfs.ci lfs_util.ci -S +# +# Copyright (c) 2022, The littlefs authors. +# SPDX-License-Identifier: BSD-3-Clause +# import collections as co import csv @@ -124,7 +130,7 @@ class StackResult(co.namedtuple('StackResult', def openio(path, mode='r'): if path == '-': - if 'r' in mode: + if mode == 'r': return os.fdopen(os.dup(sys.stdin.fileno()), 'r') else: return os.fdopen(os.dup(sys.stdout.fileno()), 'w') @@ -425,7 +431,7 @@ def table(results, calls, diff_results=None, *, prefixes[2+is_last] + "'-> ", prefixes[2+is_last] + "| ", prefixes[2+is_last] + " ")) - + table_calls(names, depth) @@ -643,5 +649,5 @@ if __name__ == "__main__": help="Specify the relative build directory. Used to map object files " "to the correct source files.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_args()).items() + for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) diff --git a/scripts/struct.py b/scripts/struct_.py similarity index 97% rename from scripts/struct.py rename to scripts/struct_.py index 73ad9829..49994977 100755 --- a/scripts/struct.py +++ b/scripts/struct_.py @@ -2,6 +2,12 @@ # # Script to find struct sizes. # +# Example: +# ./scripts/struct_.py lfs.o lfs_util.o -S +# +# Copyright (c) 2022, The littlefs authors. +# SPDX-License-Identifier: BSD-3-Clause +# import collections as co import csv @@ -123,7 +129,7 @@ class StructResult(co.namedtuple('StructResult', 'file,struct,struct_size')): def openio(path, mode='r'): if path == '-': - if 'r' in mode: + if mode == 'r': return os.fdopen(os.dup(sys.stdin.fileno()), 'r') else: return os.fdopen(os.dup(sys.stdout.fileno()), 'w') @@ -461,7 +467,7 @@ if __name__ == "__main__": nargs='*', default=OBJ_PATHS, help="Description of where to find *.o files. May be a directory " - "or a list of paths. Defaults to %(default)r.") + "or a list of paths. Defaults to %r." % OBJ_PATHS) parser.add_argument( '-v', '--verbose', action='store_true', @@ -512,11 +518,11 @@ if __name__ == "__main__": '--objdump-tool', type=lambda x: x.split(), default=OBJDUMP_TOOL, - help="Path to the objdump tool to use.") + help="Path to the objdump tool to use. Defaults to %r." % OBJDUMP_TOOL) parser.add_argument( '--build-dir', help="Specify the relative build directory. Used to map object files " "to the correct source files.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_args()).items() + for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) diff --git a/scripts/summary.py b/scripts/summary.py index eccd565a..680a7150 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -2,6 +2,15 @@ # # Script to summarize the outputs of other scripts. Operates on CSV files. # +# Example: +# ./scripts/code.py lfs.o lfs_util.o -q -o lfs.code.csv +# ./scripts/data.py lfs.o lfs_util.o -q -o lfs.data.csv +# ./scripts/summary.py lfs.code.csv lfs.data.csv -q -o lfs.csv +# ./scripts/summary.py -Y lfs.csv -f code=code_size,data=data_size +# +# Copyright (c) 2022, The littlefs authors. +# SPDX-License-Identifier: BSD-3-Clause +# import collections as co import csv @@ -43,7 +52,7 @@ MERGES = { def openio(path, mode='r'): if path == '-': - if 'r' in mode: + if mode == 'r': return os.fdopen(os.dup(sys.stdin.fileno()), 'r') else: return os.fdopen(os.dup(sys.stdout.fileno()), 'w') @@ -660,7 +669,7 @@ if __name__ == "__main__": nargs='*', default=CSV_PATHS, help="Description of where to find *.csv files. May be a directory " - "or list of paths. Defaults to %(default)r.") + "or list of paths. Defaults to %r." % CSV_PATHS) parser.add_argument( '-q', '--quiet', action='store_true', @@ -722,5 +731,5 @@ if __name__ == "__main__": action='store_true', help="Only show the totals.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_args()).items() + for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) diff --git a/scripts/tailpipe.py b/scripts/tailpipe.py index ef66d32e..c9e742ca 100755 --- a/scripts/tailpipe.py +++ b/scripts/tailpipe.py @@ -2,6 +2,12 @@ # # Efficiently displays the last n lines of a file/pipe. # +# Example: +# ./scripts/tailpipe.py trace -n5 +# +# Copyright (c) 2022, The littlefs authors. +# SPDX-License-Identifier: BSD-3-Clause +# import os import sys @@ -11,7 +17,7 @@ import time def openio(path, mode='r'): if path == '-': - if 'r' in mode: + if mode == 'r': return os.fdopen(os.dup(sys.stdin.fileno()), 'r') else: return os.fdopen(os.dup(sys.stdout.fileno()), 'w') @@ -111,5 +117,5 @@ if __name__ == "__main__": help="Reopen the pipe on EOF, useful when multiple " "processes are writing.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_args()).items() + for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) diff --git a/scripts/test.py b/scripts/test.py index 3f4ff564..42bc308a 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -2,6 +2,12 @@ # # Script to compile and runs tests. # +# Example: +# ./scripts/test.py runners/test_runner -b +# +# Copyright (c) 2022, The littlefs authors. +# SPDX-License-Identifier: BSD-3-Clause +# import collections as co import csv @@ -114,7 +120,7 @@ class TestCase: for x in range(start, stop, step): yield from parse_define('%s(%d)%s' % ( v_[:m.start()], x, v_[m.end():])) - else: + else: yield v_ # or a literal value else: @@ -337,7 +343,7 @@ def compile(test_paths, **args): k+'_i', define_cbs[v])) f.writeln(4*' '+'},') f.writeln('};') - f.writeln() + f.writeln() # create case filter function if suite.if_ is not None or case.if_ is not None: @@ -505,7 +511,8 @@ def find_runner(runner, **args): cmd = runner.copy() # run under some external command? - cmd[:0] = args.get('exec', []) + if args.get('exec'): + cmd[:0] = args['exec'] # run under valgrind? if args.get('valgrind'): @@ -914,7 +921,7 @@ def run_stage(name, runner_, ids, output_, **args): for child in children.copy(): child.kill() break - + # parallel jobs? runners = [] @@ -984,7 +991,7 @@ def run_stage(name, runner_, ids, output_, **args): powerlosses, failures, killed) - + def run(runner, test_ids=[], **args): # query runner for tests @@ -1176,102 +1183,173 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( description="Build and run tests.", conflict_handler='ignore') - parser.add_argument('-v', '--verbose', action='store_true', + parser.add_argument( + '-v', '--verbose', + action='store_true', help="Output commands that run behind the scenes.") - parser.add_argument('--color', - choices=['never', 'always', 'auto'], default='auto', - help="When to use terminal colors.") + parser.add_argument( + '--color', + choices=['never', 'always', 'auto'], + default='auto', + help="When to use terminal colors. Defaults to 'auto'.") # test flags test_parser = parser.add_argument_group('test options') - test_parser.add_argument('runner', nargs='?', + test_parser.add_argument( + 'runner', + nargs='?', type=lambda x: x.split(), help="Test runner to use for testing. Defaults to %r." % RUNNER_PATH) - test_parser.add_argument('test_ids', nargs='*', + test_parser.add_argument( + 'test_ids', + nargs='*', help="Description of tests to run.") - test_parser.add_argument('-Y', '--summary', action='store_true', + test_parser.add_argument( + '-Y', '--summary', + action='store_true', help="Show quick summary.") - test_parser.add_argument('-l', '--list-suites', action='store_true', + test_parser.add_argument( + '-l', '--list-suites', + action='store_true', help="List test suites.") - test_parser.add_argument('-L', '--list-cases', action='store_true', + test_parser.add_argument( + '-L', '--list-cases', + action='store_true', help="List test cases.") - test_parser.add_argument('--list-suite-paths', action='store_true', + test_parser.add_argument( + '--list-suite-paths', + action='store_true', help="List the path for each test suite.") - test_parser.add_argument('--list-case-paths', action='store_true', + test_parser.add_argument( + '--list-case-paths', + action='store_true', help="List the path and line number for each test case.") - test_parser.add_argument('--list-defines', action='store_true', + test_parser.add_argument( + '--list-defines', + action='store_true', help="List all defines in this test-runner.") - test_parser.add_argument('--list-permutation-defines', action='store_true', + test_parser.add_argument( + '--list-permutation-defines', + action='store_true', help="List explicit defines in this test-runner.") - test_parser.add_argument('--list-implicit-defines', action='store_true', + test_parser.add_argument( + '--list-implicit-defines', + action='store_true', help="List implicit defines in this test-runner.") - test_parser.add_argument('--list-geometries', action='store_true', + test_parser.add_argument( + '--list-geometries', + action='store_true', help="List the available disk geometries.") - test_parser.add_argument('--list-powerlosses', action='store_true', + test_parser.add_argument( + '--list-powerlosses', + action='store_true', help="List the available power-loss scenarios.") - test_parser.add_argument('-D', '--define', action='append', + test_parser.add_argument( + '-D', '--define', + action='append', help="Override a test define.") - test_parser.add_argument('-g', '--geometry', - help="Comma-separated list of disk geometries to test. \ - Defaults to d,e,E,n,N.") - test_parser.add_argument('-p', '--powerloss', - help="Comma-separated list of power-loss scenarios to test. \ - Defaults to 0,l.") - test_parser.add_argument('-d', '--disk', + test_parser.add_argument( + '-g', '--geometry', + help="Comma-separated list of disk geometries to test. " + "Defaults to d,e,E,n,N.") + test_parser.add_argument( + '-p', '--powerloss', + help="Comma-separated list of power-loss scenarios to test. " + "Defaults to 0,l.") + test_parser.add_argument( + '-d', '--disk', help="Direct block device operations to this file.") - test_parser.add_argument('-t', '--trace', + test_parser.add_argument( + '-t', '--trace', help="Direct trace output to this file.") - test_parser.add_argument('-O', '--stdout', + test_parser.add_argument( + '-O', '--stdout', help="Direct stdout to this file. Note stderr is already merged here.") - test_parser.add_argument('-o', '--output', + test_parser.add_argument( + '-o', '--output', help="CSV file to store results.") - test_parser.add_argument('--read-sleep', + test_parser.add_argument( + '--read-sleep', help="Artificial read delay in seconds.") - test_parser.add_argument('--prog-sleep', + test_parser.add_argument( + '--prog-sleep', help="Artificial prog delay in seconds.") - test_parser.add_argument('--erase-sleep', + test_parser.add_argument( + '--erase-sleep', help="Artificial erase delay in seconds.") - test_parser.add_argument('-j', '--jobs', nargs='?', type=int, + test_parser.add_argument( + '-j', '--jobs', + nargs='?', + type=lambda x: int(x, 0), const=len(os.sched_getaffinity(0)), help="Number of parallel runners to run.") - test_parser.add_argument('-k', '--keep-going', action='store_true', + test_parser.add_argument( + '-k', '--keep-going', + action='store_true', help="Don't stop on first error.") - test_parser.add_argument('-i', '--isolate', action='store_true', + test_parser.add_argument( + '-i', '--isolate', + action='store_true', help="Run each test permutation in a separate process.") - test_parser.add_argument('-b', '--by-suites', action='store_true', + test_parser.add_argument( + '-b', '--by-suites', + action='store_true', help="Step through tests by suite.") - test_parser.add_argument('-B', '--by-cases', action='store_true', + test_parser.add_argument( + '-B', '--by-cases', + action='store_true', help="Step through tests by case.") - test_parser.add_argument('--context', type=lambda x: int(x, 0), - help="Show this many lines of stdout on test failure. \ - Defaults to 5.") - test_parser.add_argument('--gdb', action='store_true', + test_parser.add_argument( + '--context', + type=lambda x: int(x, 0), + default=5, + help="Show this many lines of stdout on test failure. " + "Defaults to 5.") + test_parser.add_argument( + '--gdb', + action='store_true', help="Drop into gdb on test failure.") - test_parser.add_argument('--gdb-case', action='store_true', - help="Drop into gdb on test failure but stop at the beginning \ - of the failing test case.") - test_parser.add_argument('--gdb-main', action='store_true', - help="Drop into gdb on test failure but stop at the beginning \ - of main.") - test_parser.add_argument('--exec', default=[], type=lambda e: e.split(), + test_parser.add_argument( + '--gdb-case', + action='store_true', + help="Drop into gdb on test failure but stop at the beginning " + "of the failing test case.") + test_parser.add_argument( + '--gdb-main', + action='store_true', + help="Drop into gdb on test failure but stop at the beginning " + "of main.") + test_parser.add_argument( + '--exec', + type=lambda e: e.split(), help="Run under another executable.") - test_parser.add_argument('--valgrind', action='store_true', - help="Run under Valgrind to find memory errors. Implicitly sets \ - --isolate.") + test_parser.add_argument( + '--valgrind', + action='store_true', + help="Run under Valgrind to find memory errors. Implicitly sets " + "--isolate.") # compilation flags comp_parser = parser.add_argument_group('compilation options') - comp_parser.add_argument('test_paths', nargs='*', - help="Description of *.toml files to compile. May be a directory \ - or a list of paths.") - comp_parser.add_argument('-c', '--compile', action='store_true', + comp_parser.add_argument( + 'test_paths', + nargs='*', + help="Description of *.toml files to compile. May be a directory " + "or a list of paths.") + comp_parser.add_argument( + '-c', '--compile', + action='store_true', help="Compile a test suite or source file.") - comp_parser.add_argument('-s', '--source', + comp_parser.add_argument( + '-s', '--source', help="Source file to compile, possibly injecting internal tests.") - comp_parser.add_argument('--include', default=HEADER_PATH, - help="Inject this header file into every compiled test file. \ - Defaults to %r." % HEADER_PATH) - comp_parser.add_argument('-o', '--output', + comp_parser.add_argument( + '--include', + default=HEADER_PATH, + help="Inject this header file into every compiled test file. " + "Defaults to %r." % HEADER_PATH) + comp_parser.add_argument( + '-o', '--output', help="Output file.") # runner + test_ids overlaps test_paths, so we need to do some munging here diff --git a/scripts/tracebd.py b/scripts/tracebd.py index a486c296..ff0bbbea 100755 --- a/scripts/tracebd.py +++ b/scripts/tracebd.py @@ -2,8 +2,15 @@ # # Display operations on block devices based on trace output # +# Example: +# ./scripts/tracebd.py trace +# +# Copyright (c) 2022, The littlefs authors. +# SPDX-License-Identifier: BSD-3-Clause +# import collections as co +import functools as ft import itertools as it import math as m import os @@ -15,7 +22,7 @@ import time def openio(path, mode='r'): if path == '-': - if 'r' in mode: + if mode == 'r': return os.fdopen(os.dup(sys.stdin.fileno()), 'r') else: return os.fdopen(os.dup(sys.stdout.fileno()), 'w') @@ -23,11 +30,11 @@ def openio(path, mode='r'): return open(path, mode) # space filling Hilbert-curve +# +# note we memoize the last curve since this is a bit expensive +# +@ft.lru_cache(1) def hilbert_curve(width, height): - # memoize the last curve - if getattr(hilbert_curve, 'last', (None,))[0] == (width, height): - return hilbert_curve.last[1] - # based on generalized Hilbert curves: # https://github.com/jakubcerveny/gilbert # @@ -83,16 +90,14 @@ def hilbert_curve(width, height): else: curve = hilbert_(0, 0, 0, +height, +width, 0) - curve = list(curve) - hilbert_curve.last = ((width, height), curve) - return curve + return list(curve) # space filling Z-curve/Lebesgue-curve +# +# note we memoize the last curve since this is a bit expensive +# +@ft.lru_cache(1) def lebesgue_curve(width, height): - # memoize the last curve - if getattr(lebesgue_curve, 'last', (None,))[0] == (width, height): - return lebesgue_curve.last[1] - # we create a truncated Z-curve by simply filtering out the points # that are outside our region curve = [] @@ -104,7 +109,6 @@ def lebesgue_curve(width, height): if x < width and y < height: curve.append((x, y)) - lebesgue_curve.last = ((width, height), curve) return curve @@ -151,29 +155,30 @@ class Block: def __add__(self, other): return Block( - max(self.wear, other.wear), + max(self.wear, other.wear), self.readed | other.readed, self.proged | other.proged, self.erased | other.erased) - def draw(self, - ascii=False, + def draw(self, *, + subscripts=False, chars=None, wear_chars=None, - color='always', + color=True, read=True, prog=True, erase=True, wear=False, max_wear=None, - block_cycles=None): + block_cycles=None, + **_): if not chars: chars = '.rpe' c = chars[0] f = [] if wear: - if not wear_chars and ascii: wear_chars = '0123456789' - elif not wear_chars: wear_chars = '.₁₂₃₄₅₆789' + if not wear_chars and subscripts: wear_chars = '.₁₂₃₄₅₆789' + elif not wear_chars: wear_chars = '0123456789' if block_cycles: w = self.wear / block_cycles @@ -183,8 +188,7 @@ class Block: c = wear_chars[min( int(w*(len(wear_chars)-1)), len(wear_chars)-1)] - if color == 'wear' or ( - color == 'always' and not read and not prog and not erase): + if color: if w*9 >= 9: f.append('\x1b[1;31m') elif w*9 >= 7: f.append('\x1b[35m') @@ -192,12 +196,12 @@ class Block: elif prog and self.proged: c = chars[2] elif read and self.readed: c = chars[1] - if color == 'ops' or color == 'always': + if color: if erase and self.erased: f.append('\x1b[44m') elif prog and self.proged: f.append('\x1b[45m') elif read and self.readed: f.append('\x1b[42m') - if color in ['always', 'wear', 'ops'] and f: + if color: return '%s%c\x1b[m' % (''.join(f), c) else: return c @@ -318,16 +322,13 @@ def main(path='-', *, prog=False, erase=False, wear=False, - reset=False, - ascii=False, - chars=None, - wear_chars=None, color='auto', block=(None,None), off=(None,None), block_size=None, block_count=None, block_cycles=None, + reset=False, width=None, height=1, scale=None, @@ -336,13 +337,20 @@ def main(path='-', *, sleep=None, hilbert=False, lebesgue=False, - keep_open=False): + keep_open=False, + **args): + # exclusive wear or read/prog/erase by default if not read and not prog and not erase and not wear: read = True prog = True erase = True + # figure out what color should be if color == 'auto': - color = 'always' if sys.stdout.isatty() else 'never' + color = sys.stdout.isatty() + elif color == 'always': + color = True + else: + color = False block_start = block[0] block_stop = block[1] if len(block) > 1 else block[0]+1 @@ -438,7 +446,7 @@ def main(path='-', *, with lock: if reset: bd.reset() - + # ignore the new values if block_stop/off_stop is explicit bd.smoosh( size=(size if off_stop is None @@ -513,16 +521,14 @@ def main(path='-', *, def draw(b): return b.draw( - ascii=ascii, - chars=chars, - wear_chars=wear_chars, - color=color, read=read, prog=prog, erase=erase, wear=wear, + color=color, max_wear=max_wear, - block_cycles=block_cycles) + block_cycles=block_cycles, + **args) # fold via a curve? if height > 1: @@ -562,7 +568,7 @@ def main(path='-', *, def print_line(): nonlocal last_rows if not lines: - return + return # give ourself a canvas while last_rows < len(history)*height: @@ -672,15 +678,8 @@ if __name__ == "__main__": action='store_true', help="Render wear.") parser.add_argument( - '-R', - '--reset', - action='store_true', - help="Reset wear on block device initialization.") - parser.add_argument( - '-A', - '--ascii', - action='store_true', - help="Don't use unicode characters.") + '--subscripts', + help="Use unicode subscripts for showing wear.") parser.add_argument( '--chars', help="Characters to use for noop, read, prog, erase operations.") @@ -689,8 +688,9 @@ if __name__ == "__main__": help="Characters to use to show wear.") parser.add_argument( '--color', - choices=['never', 'always', 'auto', 'ops', 'wear'], - help="When to use terminal colors, defaults to auto.") + choices=['never', 'always', 'auto'], + default='auto', + help="When to use terminal colors. Defaults to 'auto'.") parser.add_argument( '-b', '--block', @@ -715,6 +715,11 @@ if __name__ == "__main__": '--block-cycles', type=lambda x: int(x, 0), help="Assumed maximum number of erase cycles when measuring wear.") + parser.add_argument( + '-R', + '--reset', + action='store_true', + help="Reset wear on block device initialization.") parser.add_argument( '-W', '--width', @@ -735,13 +740,12 @@ if __name__ == "__main__": '-n', '--lines', type=lambda x: int(x, 0), - help="Number of lines to show, with 0 indicating no limit. " - "Defaults to 0.") + help="Number of lines to show.") parser.add_argument( '-c', '--coalesce', type=lambda x: int(x, 0), - help="Number of operations to coalesce together. Defaults to 1.") + help="Number of operations to coalesce together.") parser.add_argument( '-s', '--sleep', @@ -765,5 +769,5 @@ if __name__ == "__main__": help="Reopen the pipe on EOF, useful when multiple " "processes are writing.") sys.exit(main(**{k: v - for k, v in vars(parser.parse_args()).items() + for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) From 4fe0738ff42cdf6a0cfb468e9d681531044e3819 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 20 Sep 2022 02:01:59 -0500 Subject: [PATCH 41/81] Added bench.py and bench_runner.c for benchmarking These are really just different flavors of test.py and test_runner.c without support for power-loss testing, but with support for measuring the cumulative number of bytes read, programmed, and erased. Note that the existing define parameterization should work perfectly fine for running benchmarks across various dimensions: ./scripts/bench.py \ runners/bench_runner \ bench_file_read \ -gnor \ -DSIZE='range(0,131072,1024)' Also added a couple basic benchmarks as a starting point. --- .gitignore | 2 + Makefile | 75 +- bd/{lfs_testbd.c => lfs_emubd.c} | 265 +++-- bd/{lfs_testbd.h => lfs_emubd.h} | 145 ++- benches/bench_dir.toml | 284 +++++ benches/bench_file.toml | 109 ++ benches/bench_superblock.toml | 56 + runners/bench_runner.c | 1795 ++++++++++++++++++++++++++++++ runners/bench_runner.h | 119 ++ runners/test_runner.c | 265 ++--- runners/test_runner.h | 53 +- scripts/bench.py | 1355 ++++++++++++++++++++++ scripts/summary.py | 3 +- scripts/tailpipe.py | 2 +- scripts/test.py | 8 +- scripts/tracebd.py | 4 +- tests/test_alloc.toml | 6 +- tests/test_badblocks.toml | 52 +- tests/test_exhaustion.toml | 40 +- tests/test_move.toml | 20 +- 20 files changed, 4253 insertions(+), 405 deletions(-) rename bd/{lfs_testbd.c => lfs_emubd.c} (57%) rename bd/{lfs_testbd.h => lfs_emubd.h} (52%) create mode 100644 benches/bench_dir.toml create mode 100644 benches/bench_file.toml create mode 100644 benches/bench_superblock.toml create mode 100644 runners/bench_runner.c create mode 100644 runners/bench_runner.h create mode 100755 scripts/bench.py diff --git a/.gitignore b/.gitignore index 58ff85d3..8f1b90e6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ *.ci *.csv *.t.c +*.b.c *.a.c *.gcno *.gcda @@ -17,3 +18,4 @@ tests/*.toml.* scripts/__pycache__ .gdb_history runners/test_runner +runners/bench_runner diff --git a/Makefile b/Makefile index ce9a5539..3eaa92d6 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,8 @@ $(if $(findstring n,$(MAKEFLAGS)),, $(shell mkdir -p \ $(BUILDDIR) \ $(BUILDDIR)bd \ $(BUILDDIR)runners \ - $(BUILDDIR)tests)) + $(BUILDDIR)tests \ + $(BUILDDIR)benches)) endif # overridable target/src/tools/flags/etc @@ -45,6 +46,18 @@ TEST_CI := $(TEST_TAC:%.t.a.c=%.t.a.ci) TEST_GCNO := $(TEST_TAC:%.t.a.c=%.t.a.gcno) TEST_GCDA := $(TEST_TAC:%.t.a.c=%.t.a.gcda) +BENCHES ?= $(wildcard benches/*.toml) +BENCH_SRC ?= $(SRC) \ + $(filter-out $(wildcard bd/*.*.c),$(wildcard bd/*.c)) \ + runners/bench_runner.c +BENCH_BC := $(BENCHES:%.toml=$(BUILDDIR)%.b.c) $(BENCH_SRC:%.c=$(BUILDDIR)%.b.c) +BENCH_BAC := $(BENCH_BC:%.b.c=%.b.a.c) +BENCH_OBJ := $(BENCH_BAC:%.b.a.c=%.b.a.o) +BENCH_DEP := $(BENCH_BAC:%.b.a.c=%.b.a.d) +BENCH_CI := $(BENCH_BAC:%.b.a.c=%.b.a.ci) +BENCH_GCNO := $(BENCH_BAC:%.b.a.c=%.b.a.gcno) +BENCH_GCDA := $(BENCH_BAC:%.b.a.c=%.b.a.gcda) + ifdef DEBUG override CFLAGS += -O0 else @@ -60,27 +73,31 @@ override CFLAGS += -Wextra -Wshadow -Wjump-misses-init -Wundef override CFLAGS += -ftrack-macro-expansion=0 override TESTFLAGS += -b +override BENCHFLAGS += -b # forward -j flag override TESTFLAGS += $(filter -j%,$(MAKEFLAGS)) +override BENCHFLAGS += $(filter -j%,$(MAKEFLAGS)) ifdef VERBOSE -override TESTFLAGS += -v -override CODEFLAGS += -v -override DATAFLAGS += -v -override STACKFLAGS += -v -override STRUCTFLAGS += -v -override COVERAGEFLAGS += -v -override TESTFLAGS += -v -override TESTCFLAGS += -v +override CODEFLAGS += -v +override DATAFLAGS += -v +override STACKFLAGS += -v +override STRUCTFLAGS += -v +override COVERAGEFLAGS += -v +override TESTFLAGS += -v +override TESTCFLAGS += -v +override BENCHFLAGS += -v +override BENCHCFLAGS += -v endif ifdef EXEC -override TESTFLAGS += --exec="$(EXEC)" +override TESTFLAGS += --exec="$(EXEC)" +override BENCHFLAGS += --exec="$(EXEC)" endif ifdef BUILDDIR -override CODEFLAGS += --build-dir="$(BUILDDIR:/=)" -override DATAFLAGS += --build-dir="$(BUILDDIR:/=)" -override STACKFLAGS += --build-dir="$(BUILDDIR:/=)" -override STRUCTFLAGS += --build-dir="$(BUILDDIR:/=)" -override COVERAGEFLAGS += --build-dir="$(BUILDDIR:/=)" +override CODEFLAGS += --build-dir="$(BUILDDIR:/=)" +override DATAFLAGS += --build-dir="$(BUILDDIR:/=)" +override STACKFLAGS += --build-dir="$(BUILDDIR:/=)" +override STRUCTFLAGS += --build-dir="$(BUILDDIR:/=)" +override COVERAGEFLAGS += --build-dir="$(BUILDDIR:/=)" endif ifneq ($(NM),nm) override CODEFLAGS += --nm-tool="$(NM)" @@ -119,6 +136,17 @@ test: test-runner test-list: test-runner ./scripts/test.py $(BUILDDIR)runners/test_runner $(TESTFLAGS) -l +.PHONY: bench-runner build-bench +bench-runner build-bench: $(BUILDDIR)runners/bench_runner + +.PHONY: bench +bench: bench-runner + ./scripts/bench.py $(BUILDDIR)runners/bench_runner $(BENCHFLAGS) + +.PHONY: bench-list +bench-list: bench-runner + ./scripts/bench.py $(BUILDDIR)runners/bench_runner $(BENCHFLAGS) -l + .PHONY: code code: $(OBJ) ./scripts/code.py $^ -S $(CODEFLAGS) @@ -186,6 +214,9 @@ $(BUILDDIR)lfs.csv: \ $(BUILDDIR)runners/test_runner: $(TEST_OBJ) $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ +$(BUILDDIR)runners/bench_runner: $(BENCH_OBJ) + $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ + # our main build rule generates .o, .d, and .ci files, the latter # used for stack analysis $(BUILDDIR)%.o $(BUILDDIR)%.ci: %.c @@ -206,6 +237,12 @@ $(BUILDDIR)%.t.c: %.toml $(BUILDDIR)%.t.c: %.c $(TESTS) ./scripts/test.py -c $(TESTS) -s $< $(TESTCFLAGS) -o $@ +$(BUILDDIR)%.b.c: %.toml + ./scripts/bench.py -c $< $(BENCHCFLAGS) -o $@ + +$(BUILDDIR)%.b.c: %.c $(BENCHES) + ./scripts/bench.py -c $(BENCHES) -s $< $(BENCHCFLAGS) -o $@ + # clean everything .PHONY: clean clean: @@ -219,6 +256,7 @@ clean: $(BUILDDIR)lfs.struct.csv \ $(BUILDDIR)lfs.coverage.csv) rm -f $(BUILDDIR)runners/test_runner + rm -f $(BUILDDIR)runners/bench_runner rm -f $(OBJ) rm -f $(DEP) rm -f $(ASM) @@ -230,3 +268,10 @@ clean: rm -f $(TEST_CI) rm -f $(TEST_GCNO) rm -f $(TEST_GCDA) + rm -f $(BENCH_BC) + rm -f $(BENCH_BAC) + rm -f $(BENCH_OBJ) + rm -f $(BENCH_DEP) + rm -f $(BENCH_CI) + rm -f $(BENCH_GCNO) + rm -f $(BENCH_GCDA) diff --git a/bd/lfs_testbd.c b/bd/lfs_emubd.c similarity index 57% rename from bd/lfs_testbd.c rename to bd/lfs_emubd.c index 61063af8..8a9da7a7 100644 --- a/bd/lfs_testbd.c +++ b/bd/lfs_emubd.c @@ -1,5 +1,5 @@ /* - * Testing block device, wraps filebd and rambd while providing a bunch + * Emulating block device, wraps filebd and rambd while providing a bunch * of hooks for testing littlefs in various conditions. * * Copyright (c) 2022, The littlefs authors. @@ -11,7 +11,7 @@ #define _POSIX_C_SOURCE 199309L #endif -#include "bd/lfs_testbd.h" +#include "bd/lfs_emubd.h" #include #include @@ -29,14 +29,14 @@ // Note we can only modify a block if we have exclusive access to it (rc == 1) // -static lfs_testbd_block_t *lfs_testbd_incblock(lfs_testbd_block_t *block) { +static lfs_emubd_block_t *lfs_emubd_incblock(lfs_emubd_block_t *block) { if (block) { block->rc += 1; } return block; } -static void lfs_testbd_decblock(lfs_testbd_block_t *block) { +static void lfs_emubd_decblock(lfs_emubd_block_t *block) { if (block) { block->rc -= 1; if (block->rc == 0) { @@ -45,34 +45,34 @@ static void lfs_testbd_decblock(lfs_testbd_block_t *block) { } } -static lfs_testbd_block_t *lfs_testbd_mutblock( +static lfs_emubd_block_t *lfs_emubd_mutblock( const struct lfs_config *cfg, - lfs_testbd_block_t **block) { - lfs_testbd_block_t *block_ = *block; + lfs_emubd_block_t **block) { + lfs_emubd_block_t *block_ = *block; if (block_ && block_->rc == 1) { // rc == 1? can modify return block_; } else if (block_) { // rc > 1? need to create a copy - lfs_testbd_block_t *nblock = malloc( - sizeof(lfs_testbd_block_t) + cfg->block_size); + lfs_emubd_block_t *nblock = malloc( + sizeof(lfs_emubd_block_t) + cfg->block_size); if (!nblock) { return NULL; } memcpy(nblock, block_, - sizeof(lfs_testbd_block_t) + cfg->block_size); + sizeof(lfs_emubd_block_t) + cfg->block_size); nblock->rc = 1; - lfs_testbd_decblock(block_); + lfs_emubd_decblock(block_); *block = nblock; return nblock; } else { // no block? need to allocate - lfs_testbd_block_t *nblock = malloc( - sizeof(lfs_testbd_block_t) + cfg->block_size); + lfs_emubd_block_t *nblock = malloc( + sizeof(lfs_emubd_block_t) + cfg->block_size); if (!nblock) { return NULL; } @@ -81,7 +81,7 @@ static lfs_testbd_block_t *lfs_testbd_mutblock( nblock->wear = 0; // zero for consistency - lfs_testbd_t *bd = cfg->context; + lfs_emubd_t *bd = cfg->context; memset(nblock->data, (bd->cfg->erase_value != -1) ? bd->cfg->erase_value : 0, cfg->block_size); @@ -92,11 +92,11 @@ static lfs_testbd_block_t *lfs_testbd_mutblock( } -// testbd create/destroy +// emubd create/destroy -int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, - const struct lfs_testbd_config *bdcfg) { - LFS_TESTBD_TRACE("lfs_testbd_createcfg(%p {.context=%p, " +int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, + const struct lfs_emubd_config *bdcfg) { + LFS_EMUBD_TRACE("lfs_emubd_createcfg(%p {.context=%p, " ".read=%p, .prog=%p, .erase=%p, .sync=%p, " ".read_size=%"PRIu32", .prog_size=%"PRIu32", " ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " @@ -113,25 +113,28 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, bdcfg->badblock_behavior, bdcfg->power_cycles, bdcfg->powerloss_behavior, (void*)(uintptr_t)bdcfg->powerloss_cb, bdcfg->powerloss_data, bdcfg->track_branches); - lfs_testbd_t *bd = cfg->context; + lfs_emubd_t *bd = cfg->context; bd->cfg = bdcfg; // allocate our block array, all blocks start as uninitialized - bd->blocks = malloc(cfg->block_count * sizeof(lfs_testbd_block_t*)); + bd->blocks = malloc(cfg->block_count * sizeof(lfs_emubd_block_t*)); if (!bd->blocks) { - LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } - memset(bd->blocks, 0, cfg->block_count * sizeof(lfs_testbd_block_t*)); + memset(bd->blocks, 0, cfg->block_count * sizeof(lfs_emubd_block_t*)); // setup testing things + bd->read = 0; + bd->prog = 0; + bd->erased = 0; bd->power_cycles = bd->cfg->power_cycles; bd->disk = NULL; if (bd->cfg->disk_path) { - bd->disk = malloc(sizeof(lfs_testbd_disk_t)); + bd->disk = malloc(sizeof(lfs_emubd_disk_t)); if (!bd->disk) { - LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } bd->disk->rc = 1; @@ -146,7 +149,7 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, #endif if (bd->disk->fd < 0) { int err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_create -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", err); return err; } @@ -155,7 +158,7 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, if (bd->cfg->erase_value != -1) { bd->disk->scratch = malloc(cfg->block_size); if (!bd->disk->scratch) { - LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } memset(bd->disk->scratch, @@ -164,12 +167,12 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, } } - LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", 0); return 0; } -int lfs_testbd_create(const struct lfs_config *cfg, const char *path) { - LFS_TESTBD_TRACE("lfs_testbd_create(%p {.context=%p, " +int lfs_emubd_create(const struct lfs_config *cfg, const char *path) { + LFS_EMUBD_TRACE("lfs_emubd_create(%p {.context=%p, " ".read=%p, .prog=%p, .erase=%p, .sync=%p, " ".read_size=%"PRIu32", .prog_size=%"PRIu32", " ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " @@ -179,19 +182,19 @@ int lfs_testbd_create(const struct lfs_config *cfg, const char *path) { (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, path); - static const struct lfs_testbd_config defaults = {.erase_value=-1}; - int err = lfs_testbd_createcfg(cfg, path, &defaults); - LFS_TESTBD_TRACE("lfs_testbd_create -> %d", err); + static const struct lfs_emubd_config defaults = {.erase_value=-1}; + int err = lfs_emubd_createcfg(cfg, path, &defaults); + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", err); return err; } -int lfs_testbd_destroy(const struct lfs_config *cfg) { - LFS_TESTBD_TRACE("lfs_testbd_destroy(%p)", (void*)cfg); - lfs_testbd_t *bd = cfg->context; +int lfs_emubd_destroy(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_destroy(%p)", (void*)cfg); + lfs_emubd_t *bd = cfg->context; // decrement reference counts for (lfs_block_t i = 0; i < cfg->block_count; i++) { - lfs_testbd_decblock(bd->blocks[i]); + lfs_emubd_decblock(bd->blocks[i]); } free(bd->blocks); @@ -205,7 +208,7 @@ int lfs_testbd_destroy(const struct lfs_config *cfg) { } } - LFS_TESTBD_TRACE("lfs_testbd_destroy -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_destroy -> %d", 0); return 0; } @@ -213,12 +216,12 @@ int lfs_testbd_destroy(const struct lfs_config *cfg) { // block device API -int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, +int lfs_emubd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { - LFS_TESTBD_TRACE("lfs_testbd_read(%p, " + LFS_EMUBD_TRACE("lfs_emubd_read(%p, " "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")", (void*)cfg, block, off, buffer, size); - lfs_testbd_t *bd = cfg->context; + lfs_emubd_t *bd = cfg->context; // check if read is valid LFS_ASSERT(block < cfg->block_count); @@ -227,12 +230,12 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, LFS_ASSERT(off+size <= cfg->block_size); // get the block - const lfs_testbd_block_t *b = bd->blocks[block]; + const lfs_emubd_block_t *b = bd->blocks[block]; if (b) { // block bad? if (bd->cfg->erase_cycles && b->wear >= bd->cfg->erase_cycles && - bd->cfg->badblock_behavior == LFS_TESTBD_BADBLOCK_READERROR) { - LFS_TESTBD_TRACE("lfs_testbd_read -> %d", LFS_ERR_CORRUPT); + bd->cfg->badblock_behavior == LFS_EMUBD_BADBLOCK_READERROR) { + LFS_EMUBD_TRACE("lfs_emubd_read -> %d", LFS_ERR_CORRUPT); return LFS_ERR_CORRUPT; } @@ -243,8 +246,10 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, memset(buffer, (bd->cfg->erase_value != -1) ? bd->cfg->erase_value : 0, size); - } + } + // track reads + bd->read += size; if (bd->cfg->read_sleep) { int err = nanosleep(&(struct timespec){ .tv_sec=bd->cfg->read_sleep/1000000000, @@ -252,21 +257,21 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, NULL); if (err) { err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_read -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_read -> %d", err); return err; } } - LFS_TESTBD_TRACE("lfs_testbd_read -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_read -> %d", 0); return 0; } -int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, +int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size) { - LFS_TESTBD_TRACE("lfs_testbd_prog(%p, " + LFS_EMUBD_TRACE("lfs_emubd_prog(%p, " "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")", (void*)cfg, block, off, buffer, size); - lfs_testbd_t *bd = cfg->context; + lfs_emubd_t *bd = cfg->context; // check if write is valid LFS_ASSERT(block < cfg->block_count); @@ -275,23 +280,23 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, LFS_ASSERT(off+size <= cfg->block_size); // get the block - lfs_testbd_block_t *b = lfs_testbd_mutblock(cfg, &bd->blocks[block]); + lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); if (!b) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } // block bad? if (bd->cfg->erase_cycles && b->wear >= bd->cfg->erase_cycles) { if (bd->cfg->badblock_behavior == - LFS_TESTBD_BADBLOCK_PROGERROR) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_CORRUPT); + LFS_EMUBD_BADBLOCK_PROGERROR) { + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", LFS_ERR_CORRUPT); return LFS_ERR_CORRUPT; } else if (bd->cfg->badblock_behavior == - LFS_TESTBD_BADBLOCK_PROGNOOP || + LFS_EMUBD_BADBLOCK_PROGNOOP || bd->cfg->badblock_behavior == - LFS_TESTBD_BADBLOCK_ERASENOOP) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0); + LFS_EMUBD_BADBLOCK_ERASENOOP) { + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", 0); return 0; } } @@ -313,18 +318,20 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, SEEK_SET); if (res1 < 0) { int err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", err); return err; } ssize_t res2 = write(bd->disk->fd, buffer, size); if (res2 < 0) { int err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", err); return err; } } + // track progs + bd->prog += size; if (bd->cfg->prog_sleep) { int err = nanosleep(&(struct timespec){ .tv_sec=bd->cfg->prog_sleep/1000000000, @@ -332,7 +339,7 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, NULL); if (err) { err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", err); return err; } } @@ -346,21 +353,21 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, } } - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", 0); return 0; } -int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { - LFS_TESTBD_TRACE("lfs_testbd_erase(%p, 0x%"PRIx32")", (void*)cfg, block); - lfs_testbd_t *bd = cfg->context; +int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block) { + LFS_EMUBD_TRACE("lfs_emubd_erase(%p, 0x%"PRIx32")", (void*)cfg, block); + lfs_emubd_t *bd = cfg->context; // check if erase is valid LFS_ASSERT(block < cfg->block_count); // get the block - lfs_testbd_block_t *b = lfs_testbd_mutblock(cfg, &bd->blocks[block]); + lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); if (!b) { - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } @@ -368,12 +375,12 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { if (bd->cfg->erase_cycles) { if (b->wear >= bd->cfg->erase_cycles) { if (bd->cfg->badblock_behavior == - LFS_TESTBD_BADBLOCK_ERASEERROR) { - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", LFS_ERR_CORRUPT); + LFS_EMUBD_BADBLOCK_ERASEERROR) { + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", LFS_ERR_CORRUPT); return LFS_ERR_CORRUPT; } else if (bd->cfg->badblock_behavior == - LFS_TESTBD_BADBLOCK_ERASENOOP) { - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", 0); + LFS_EMUBD_BADBLOCK_ERASENOOP) { + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", 0); return 0; } } else { @@ -393,7 +400,7 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { SEEK_SET); if (res1 < 0) { int err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", err); return err; } @@ -402,12 +409,14 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { cfg->block_size); if (res2 < 0) { int err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", err); return err; } } } + // track erases + bd->erased += cfg->block_size; if (bd->cfg->erase_sleep) { int err = nanosleep(&(struct timespec){ .tv_sec=bd->cfg->erase_sleep/1000000000, @@ -415,7 +424,7 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { NULL); if (err) { err = -errno; - LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err); + LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", err); return err; } } @@ -429,98 +438,142 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) { } } - LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", 0); return 0; } -int lfs_testbd_sync(const struct lfs_config *cfg) { - LFS_TESTBD_TRACE("lfs_testbd_sync(%p)", (void*)cfg); +int lfs_emubd_sync(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_sync(%p)", (void*)cfg); // do nothing (void)cfg; - LFS_TESTBD_TRACE("lfs_testbd_sync -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_sync -> %d", 0); return 0; } +/// Additional extended API for driving test features /// -// simulated wear operations +lfs_emubd_sio_t lfs_emubd_getread(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_getread(%p)", (void*)cfg); + lfs_emubd_t *bd = cfg->context; + LFS_EMUBD_TRACE("lfs_emubd_getread -> %"PRIu64, bd->read); + return bd->read; +} -lfs_testbd_swear_t lfs_testbd_getwear(const struct lfs_config *cfg, +lfs_emubd_sio_t lfs_emubd_getprog(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_getprog(%p)", (void*)cfg); + lfs_emubd_t *bd = cfg->context; + LFS_EMUBD_TRACE("lfs_emubd_getprog -> %"PRIu64, bd->prog); + return bd->prog; +} + +lfs_emubd_sio_t lfs_emubd_geterased(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_geterased(%p)", (void*)cfg); + lfs_emubd_t *bd = cfg->context; + LFS_EMUBD_TRACE("lfs_emubd_geterased -> %"PRIu64, bd->erased); + return bd->erased; +} + +int lfs_emubd_setread(const struct lfs_config *cfg, lfs_emubd_io_t read) { + LFS_EMUBD_TRACE("lfs_emubd_setread(%p, %"PRIu64")", (void*)cfg, read); + lfs_emubd_t *bd = cfg->context; + bd->read = read; + LFS_EMUBD_TRACE("lfs_emubd_setread -> %d", 0); + return 0; +} + +int lfs_emubd_setprog(const struct lfs_config *cfg, lfs_emubd_io_t prog) { + LFS_EMUBD_TRACE("lfs_emubd_setprog(%p, %"PRIu64")", (void*)cfg, prog); + lfs_emubd_t *bd = cfg->context; + bd->prog = prog; + LFS_EMUBD_TRACE("lfs_emubd_setprog -> %d", 0); + return 0; +} + +int lfs_emubd_seterased(const struct lfs_config *cfg, lfs_emubd_io_t erased) { + LFS_EMUBD_TRACE("lfs_emubd_seterased(%p, %"PRIu64")", (void*)cfg, erased); + lfs_emubd_t *bd = cfg->context; + bd->erased = erased; + LFS_EMUBD_TRACE("lfs_emubd_seterased -> %d", 0); + return 0; +} + +lfs_emubd_swear_t lfs_emubd_getwear(const struct lfs_config *cfg, lfs_block_t block) { - LFS_TESTBD_TRACE("lfs_testbd_getwear(%p, %"PRIu32")", (void*)cfg, block); - lfs_testbd_t *bd = cfg->context; + LFS_EMUBD_TRACE("lfs_emubd_getwear(%p, %"PRIu32")", (void*)cfg, block); + lfs_emubd_t *bd = cfg->context; // check if block is valid LFS_ASSERT(block < cfg->block_count); // get the wear - lfs_testbd_wear_t wear; - const lfs_testbd_block_t *b = bd->blocks[block]; + lfs_emubd_wear_t wear; + const lfs_emubd_block_t *b = bd->blocks[block]; if (b) { wear = b->wear; } else { wear = 0; } - LFS_TESTBD_TRACE("lfs_testbd_getwear -> %"PRIu32, wear); + LFS_EMUBD_TRACE("lfs_emubd_getwear -> %"PRIu32, wear); return wear; } -int lfs_testbd_setwear(const struct lfs_config *cfg, - lfs_block_t block, lfs_testbd_wear_t wear) { - LFS_TESTBD_TRACE("lfs_testbd_setwear(%p, %"PRIu32")", (void*)cfg, block); - lfs_testbd_t *bd = cfg->context; +int lfs_emubd_setwear(const struct lfs_config *cfg, + lfs_block_t block, lfs_emubd_wear_t wear) { + LFS_EMUBD_TRACE("lfs_emubd_setwear(%p, %"PRIu32")", (void*)cfg, block); + lfs_emubd_t *bd = cfg->context; // check if block is valid LFS_ASSERT(block < cfg->block_count); // set the wear - lfs_testbd_block_t *b = lfs_testbd_mutblock(cfg, &bd->blocks[block]); + lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); if (!b) { - LFS_TESTBD_TRACE("lfs_testbd_setwear -> %"PRIu32, LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_setwear -> %"PRIu32, LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } b->wear = wear; - LFS_TESTBD_TRACE("lfs_testbd_setwear -> %"PRIu32, 0); + LFS_EMUBD_TRACE("lfs_emubd_setwear -> %"PRIu32, 0); return 0; } -lfs_testbd_spowercycles_t lfs_testbd_getpowercycles( +lfs_emubd_spowercycles_t lfs_emubd_getpowercycles( const struct lfs_config *cfg) { - LFS_TESTBD_TRACE("lfs_testbd_getpowercycles(%p)", (void*)cfg); - lfs_testbd_t *bd = cfg->context; + LFS_EMUBD_TRACE("lfs_emubd_getpowercycles(%p)", (void*)cfg); + lfs_emubd_t *bd = cfg->context; - LFS_TESTBD_TRACE("lfs_testbd_getpowercycles -> %"PRIi32, bd->power_cycles); + LFS_EMUBD_TRACE("lfs_emubd_getpowercycles -> %"PRIi32, bd->power_cycles); return bd->power_cycles; } -int lfs_testbd_setpowercycles(const struct lfs_config *cfg, - lfs_testbd_powercycles_t power_cycles) { - LFS_TESTBD_TRACE("lfs_testbd_setpowercycles(%p, %"PRIi32")", +int lfs_emubd_setpowercycles(const struct lfs_config *cfg, + lfs_emubd_powercycles_t power_cycles) { + LFS_EMUBD_TRACE("lfs_emubd_setpowercycles(%p, %"PRIi32")", (void*)cfg, power_cycles); - lfs_testbd_t *bd = cfg->context; + lfs_emubd_t *bd = cfg->context; bd->power_cycles = power_cycles; - LFS_TESTBD_TRACE("lfs_testbd_getpowercycles -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_getpowercycles -> %d", 0); return 0; } -int lfs_testbd_copy(const struct lfs_config *cfg, lfs_testbd_t *copy) { - LFS_TESTBD_TRACE("lfs_testbd_copy(%p, %p)", (void*)cfg, (void*)copy); - lfs_testbd_t *bd = cfg->context; +int lfs_emubd_copy(const struct lfs_config *cfg, lfs_emubd_t *copy) { + LFS_EMUBD_TRACE("lfs_emubd_copy(%p, %p)", (void*)cfg, (void*)copy); + lfs_emubd_t *bd = cfg->context; // lazily copy over our block array - copy->blocks = malloc(cfg->block_count * sizeof(lfs_testbd_block_t*)); + copy->blocks = malloc(cfg->block_count * sizeof(lfs_emubd_block_t*)); if (!copy->blocks) { - LFS_TESTBD_TRACE("lfs_testbd_copy -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_copy -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } for (size_t i = 0; i < cfg->block_count; i++) { - copy->blocks[i] = lfs_testbd_incblock(bd->blocks[i]); + copy->blocks[i] = lfs_emubd_incblock(bd->blocks[i]); } // other state @@ -531,7 +584,7 @@ int lfs_testbd_copy(const struct lfs_config *cfg, lfs_testbd_t *copy) { } copy->cfg = bd->cfg; - LFS_TESTBD_TRACE("lfs_testbd_copy -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_copy -> %d", 0); return 0; } diff --git a/bd/lfs_testbd.h b/bd/lfs_emubd.h similarity index 52% rename from bd/lfs_testbd.h rename to bd/lfs_emubd.h index b0c9d005..8aff161f 100644 --- a/bd/lfs_testbd.h +++ b/bd/lfs_emubd.h @@ -1,13 +1,13 @@ /* - * Testing block device, wraps filebd and rambd while providing a bunch + * Emulating block device, wraps filebd and rambd while providing a bunch * of hooks for testing littlefs in various conditions. * * Copyright (c) 2022, The littlefs authors. * Copyright (c) 2017, Arm Limited. All rights reserved. * SPDX-License-Identifier: BSD-3-Clause */ -#ifndef LFS_TESTBD_H -#define LFS_TESTBD_H +#ifndef LFS_EMUBD_H +#define LFS_EMUBD_H #include "lfs.h" #include "lfs_util.h" @@ -21,11 +21,11 @@ extern "C" // Block device specific tracing -#ifndef LFS_TESTBD_TRACE -#ifdef LFS_TESTBD_YES_TRACE -#define LFS_TESTBD_TRACE(...) LFS_TRACE(__VA_ARGS__) +#ifndef LFS_EMUBD_TRACE +#ifdef LFS_EMUBD_YES_TRACE +#define LFS_EMUBD_TRACE(...) LFS_TRACE(__VA_ARGS__) #else -#define LFS_TESTBD_TRACE(...) +#define LFS_EMUBD_TRACE(...) #endif #endif @@ -35,34 +35,38 @@ extern "C" // // Not that read-noop is not allowed. Read _must_ return a consistent (but // may be arbitrary) value on every read. -typedef enum lfs_testbd_badblock_behavior { - LFS_TESTBD_BADBLOCK_PROGERROR, - LFS_TESTBD_BADBLOCK_ERASEERROR, - LFS_TESTBD_BADBLOCK_READERROR, - LFS_TESTBD_BADBLOCK_PROGNOOP, - LFS_TESTBD_BADBLOCK_ERASENOOP, -} lfs_testbd_badblock_behavior_t; +typedef enum lfs_emubd_badblock_behavior { + LFS_EMUBD_BADBLOCK_PROGERROR, + LFS_EMUBD_BADBLOCK_ERASEERROR, + LFS_EMUBD_BADBLOCK_READERROR, + LFS_EMUBD_BADBLOCK_PROGNOOP, + LFS_EMUBD_BADBLOCK_ERASENOOP, +} lfs_emubd_badblock_behavior_t; // Mode determining how power-loss behaves during testing. For now this // only supports a noop behavior, leaving the data on-disk untouched. -typedef enum lfs_testbd_powerloss_behavior { - LFS_TESTBD_POWERLOSS_NOOP, -} lfs_testbd_powerloss_behavior_t; +typedef enum lfs_emubd_powerloss_behavior { + LFS_EMUBD_POWERLOSS_NOOP, +} lfs_emubd_powerloss_behavior_t; + +// Type for measuring read/program/erase operations +typedef uint64_t lfs_emubd_io_t; +typedef int64_t lfs_emubd_sio_t; // Type for measuring wear -typedef uint32_t lfs_testbd_wear_t; -typedef int32_t lfs_testbd_swear_t; +typedef uint32_t lfs_emubd_wear_t; +typedef int32_t lfs_emubd_swear_t; // Type for tracking power-cycles -typedef uint32_t lfs_testbd_powercycles_t; -typedef int32_t lfs_testbd_spowercycles_t; +typedef uint32_t lfs_emubd_powercycles_t; +typedef int32_t lfs_emubd_spowercycles_t; // Type for delays in nanoseconds -typedef uint64_t lfs_testbd_sleep_t; -typedef int64_t lfs_testbd_ssleep_t; +typedef uint64_t lfs_emubd_sleep_t; +typedef int64_t lfs_emubd_ssleep_t; -// testbd config, this is required for testing -struct lfs_testbd_config { +// emubd config, this is required for testing +struct lfs_emubd_config { // 8-bit erase value to use for simulating erases. -1 does not simulate // erases, which can speed up testing by avoiding the extra block-device // operations to store the erase value. @@ -73,15 +77,15 @@ struct lfs_testbd_config { uint32_t erase_cycles; // The mode determining how bad-blocks fail - lfs_testbd_badblock_behavior_t badblock_behavior; + lfs_emubd_badblock_behavior_t badblock_behavior; // Number of write operations (erase/prog) before triggering a power-loss. // power_cycles=0 disables this. The exact behavior of power-loss is // controlled by a combination of powerloss_behavior and powerloss_cb. - lfs_testbd_powercycles_t power_cycles; + lfs_emubd_powercycles_t power_cycles; // The mode determining how power-loss affects disk - lfs_testbd_powerloss_behavior_t powerloss_behavior; + lfs_emubd_powerloss_behavior_t powerloss_behavior; // Function to call to emulate power-loss. The exact behavior of power-loss // is up to the runner to provide. @@ -100,98 +104,119 @@ struct lfs_testbd_config { // Artificial delay in nanoseconds, there is no purpose for this other // than slowing down the simulation. - lfs_testbd_sleep_t read_sleep; + lfs_emubd_sleep_t read_sleep; // Artificial delay in nanoseconds, there is no purpose for this other // than slowing down the simulation. - lfs_testbd_sleep_t prog_sleep; + lfs_emubd_sleep_t prog_sleep; // Artificial delay in nanoseconds, there is no purpose for this other // than slowing down the simulation. - lfs_testbd_sleep_t erase_sleep; + lfs_emubd_sleep_t erase_sleep; }; // A reference counted block -typedef struct lfs_testbd_block { +typedef struct lfs_emubd_block { uint32_t rc; - lfs_testbd_wear_t wear; + lfs_emubd_wear_t wear; uint8_t data[]; -} lfs_testbd_block_t; +} lfs_emubd_block_t; // Disk mirror -typedef struct lfs_testbd_disk { +typedef struct lfs_emubd_disk { uint32_t rc; int fd; uint8_t *scratch; -} lfs_testbd_disk_t; +} lfs_emubd_disk_t; -// testbd state -typedef struct lfs_testbd { +// emubd state +typedef struct lfs_emubd { // array of copy-on-write blocks - lfs_testbd_block_t **blocks; + lfs_emubd_block_t **blocks; // some other test state - uint32_t power_cycles; - lfs_testbd_disk_t *disk; + lfs_emubd_io_t read; + lfs_emubd_io_t prog; + lfs_emubd_io_t erased; + lfs_emubd_powercycles_t power_cycles; + lfs_emubd_disk_t *disk; - const struct lfs_testbd_config *cfg; -} lfs_testbd_t; + const struct lfs_emubd_config *cfg; +} lfs_emubd_t; /// Block device API /// -// Create a test block device using the geometry in lfs_config +// Create an emulating block device using the geometry in lfs_config // // Note that filebd is used if a path is provided, if path is NULL -// testbd will use rambd which can be much faster. -int lfs_testbd_create(const struct lfs_config *cfg, const char *path); -int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path, - const struct lfs_testbd_config *bdcfg); +// emubd will use rambd which can be much faster. +int lfs_emubd_create(const struct lfs_config *cfg, const char *path); +int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, + const struct lfs_emubd_config *bdcfg); // Clean up memory associated with block device -int lfs_testbd_destroy(const struct lfs_config *cfg); +int lfs_emubd_destroy(const struct lfs_config *cfg); // Read a block -int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block, +int lfs_emubd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size); // Program a block // // The block must have previously been erased. -int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block, +int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size); // Erase a block // // A block must be erased before being programmed. The // state of an erased block is undefined. -int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block); +int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block); // Sync the block device -int lfs_testbd_sync(const struct lfs_config *cfg); +int lfs_emubd_sync(const struct lfs_config *cfg); /// Additional extended API for driving test features /// +// Get total amount of bytes read +lfs_emubd_sio_t lfs_emubd_getread(const struct lfs_config *cfg); + +// Get total amount of bytes programmed +lfs_emubd_sio_t lfs_emubd_getprog(const struct lfs_config *cfg); + +// Get total amount of bytes erased +lfs_emubd_sio_t lfs_emubd_geterased(const struct lfs_config *cfg); + +// Manually set amount of bytes read +int lfs_emubd_setread(const struct lfs_config *cfg, lfs_emubd_io_t read); + +// Manually set amount of bytes programmed +int lfs_emubd_setprog(const struct lfs_config *cfg, lfs_emubd_io_t prog); + +// Manually set amount of bytes erased +int lfs_emubd_seterased(const struct lfs_config *cfg, lfs_emubd_io_t erased); + // Get simulated wear on a given block -lfs_testbd_swear_t lfs_testbd_getwear(const struct lfs_config *cfg, +lfs_emubd_swear_t lfs_emubd_getwear(const struct lfs_config *cfg, lfs_block_t block); // Manually set simulated wear on a given block -int lfs_testbd_setwear(const struct lfs_config *cfg, - lfs_block_t block, lfs_testbd_wear_t wear); +int lfs_emubd_setwear(const struct lfs_config *cfg, + lfs_block_t block, lfs_emubd_wear_t wear); // Get the remaining power-cycles -lfs_testbd_spowercycles_t lfs_testbd_getpowercycles( +lfs_emubd_spowercycles_t lfs_emubd_getpowercycles( const struct lfs_config *cfg); // Manually set the remaining power-cycles -int lfs_testbd_setpowercycles(const struct lfs_config *cfg, - lfs_testbd_powercycles_t power_cycles); +int lfs_emubd_setpowercycles(const struct lfs_config *cfg, + lfs_emubd_powercycles_t power_cycles); // Create a copy-on-write copy of the state of this block device -int lfs_testbd_copy(const struct lfs_config *cfg, lfs_testbd_t *copy); +int lfs_emubd_copy(const struct lfs_config *cfg, lfs_emubd_t *copy); #ifdef __cplusplus diff --git a/benches/bench_dir.toml b/benches/bench_dir.toml new file mode 100644 index 00000000..937f70d2 --- /dev/null +++ b/benches/bench_dir.toml @@ -0,0 +1,284 @@ + + +# deterministic prng +code = ''' +static uint32_t xorshift32(uint32_t *state) { + uint32_t x = *state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *state = x; + return x; +} +''' + +[cases.bench_dir_open] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.N = 1024 +defines.FILE_SIZE = 8 +defines.CHUNK_SIZE = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + // first create the files + char name[256]; + uint8_t buffer[CHUNK_SIZE]; + for (lfs_size_t i = 0; i < N; i++) { + sprintf(name, "file%08x", i); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint32_t file_prng = i; + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + buffer[k] = xorshift32(&file_prng); + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + } + + // then read the files + BENCH_START(); + uint32_t prng = 42; + for (lfs_size_t i = 0; i < N; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (N-1-i) + : xorshift32(&prng) % N; + sprintf(name, "file%08x", i_); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, LFS_O_RDONLY) => 0; + + uint32_t file_prng = i_; + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + lfs_file_read(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + assert(buffer[k] == xorshift32(&file_prng)); + } + } + + lfs_file_close(&lfs, &file) => 0; + } + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_dir_creat] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.N = 1024 +defines.FILE_SIZE = 8 +defines.CHUNK_SIZE = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + BENCH_START(); + uint32_t prng = 42; + char name[256]; + uint8_t buffer[CHUNK_SIZE]; + for (lfs_size_t i = 0; i < N; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (N-1-i) + : xorshift32(&prng) % N; + sprintf(name, "file%08x", i_); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + + uint32_t file_prng = i_; + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + buffer[k] = xorshift32(&file_prng); + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + } + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_dir_remove] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.N = 1024 +defines.FILE_SIZE = 8 +defines.CHUNK_SIZE = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + // first create the files + char name[256]; + uint8_t buffer[CHUNK_SIZE]; + for (lfs_size_t i = 0; i < N; i++) { + sprintf(name, "file%08x", i); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint32_t file_prng = i; + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + buffer[k] = xorshift32(&file_prng); + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + } + + // then remove the files + BENCH_START(); + uint32_t prng = 42; + for (lfs_size_t i = 0; i < N; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (N-1-i) + : xorshift32(&prng) % N; + sprintf(name, "file%08x", i_); + int err = lfs_remove(&lfs, name); + assert(!err || err == LFS_ERR_NOENT); + } + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_dir_read] +defines.N = 1024 +defines.FILE_SIZE = 8 +defines.CHUNK_SIZE = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + // first create the files + char name[256]; + uint8_t buffer[CHUNK_SIZE]; + for (lfs_size_t i = 0; i < N; i++) { + sprintf(name, "file%08x", i); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint32_t file_prng = i; + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + buffer[k] = xorshift32(&file_prng); + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + } + + // then read the directory + BENCH_START(); + lfs_dir_t dir; + lfs_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; + lfs_dir_read(&lfs, &dir, &info) => 1; + assert(info.type == LFS_TYPE_DIR); + assert(strcmp(info.name, ".") == 0); + lfs_dir_read(&lfs, &dir, &info) => 1; + assert(info.type == LFS_TYPE_DIR); + assert(strcmp(info.name, "..") == 0); + for (int i = 0; i < N; i++) { + sprintf(name, "file%08x", i); + lfs_dir_read(&lfs, &dir, &info) => 1; + assert(info.type == LFS_TYPE_REG); + assert(strcmp(info.name, name) == 0); + } + lfs_dir_read(&lfs, &dir, &info) => 0; + lfs_dir_close(&lfs, &dir) => 0; + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_dir_mkdir] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.N = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + BENCH_START(); + uint32_t prng = 42; + char name[256]; + for (lfs_size_t i = 0; i < N; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (N-1-i) + : xorshift32(&prng) % N; + printf("hm %d\n", i); + sprintf(name, "dir%08x", i_); + int err = lfs_mkdir(&lfs, name); + assert(!err || err == LFS_ERR_EXIST); + } + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_dir_rmdir] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.N = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + + // first create the dirs + char name[256]; + for (lfs_size_t i = 0; i < N; i++) { + sprintf(name, "dir%08x", i); + lfs_mkdir(&lfs, name) => 0; + } + + // then remove the dirs + BENCH_START(); + uint32_t prng = 42; + for (lfs_size_t i = 0; i < N; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (N-1-i) + : xorshift32(&prng) % N; + sprintf(name, "dir%08x", i_); + int err = lfs_remove(&lfs, name); + assert(!err || err == LFS_ERR_NOENT); + } + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + + diff --git a/benches/bench_file.toml b/benches/bench_file.toml new file mode 100644 index 00000000..7e556428 --- /dev/null +++ b/benches/bench_file.toml @@ -0,0 +1,109 @@ + + +# deterministic prng +code = ''' +static uint32_t xorshift32(uint32_t *state) { + uint32_t x = *state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *state = x; + return x; +} +''' + +[cases.bench_file_read] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.SIZE = '128*1024' +defines.CHUNK_SIZE = 64 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_size_t chunks = (SIZE+CHUNK_SIZE-1)/CHUNK_SIZE; + + // first write the file + lfs_file_t file; + uint8_t buffer[CHUNK_SIZE]; + lfs_file_open(&lfs, &file, "file", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + for (lfs_size_t i = 0; i < chunks; i++) { + uint32_t chunk_prng = i; + for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) { + buffer[j] = xorshift32(&chunk_prng); + } + + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + lfs_file_close(&lfs, &file) => 0; + + // then read the file + BENCH_START(); + lfs_file_open(&lfs, &file, "file", LFS_O_RDONLY) => 0; + + uint32_t prng = 42; + for (lfs_size_t i = 0; i < chunks; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (chunks-1-i) + : xorshift32(&prng) % chunks; + lfs_file_seek(&lfs, &file, i_*CHUNK_SIZE, LFS_SEEK_SET) + => i_*CHUNK_SIZE; + lfs_file_read(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + + uint32_t chunk_prng = i_; + for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) { + assert(buffer[j] == xorshift32(&chunk_prng)); + } + } + + lfs_file_close(&lfs, &file) => 0; + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_file_write] +# 0 = in-order +# 1 = reversed-order +# 2 = random-order +defines.ORDER = [0, 1, 2] +defines.SIZE = '128*1024' +defines.CHUNK_SIZE = 64 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + lfs_size_t chunks = (SIZE+CHUNK_SIZE-1)/CHUNK_SIZE; + + BENCH_START(); + lfs_file_t file; + lfs_file_open(&lfs, &file, "file", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + uint8_t buffer[CHUNK_SIZE]; + uint32_t prng = 42; + for (lfs_size_t i = 0; i < chunks; i++) { + lfs_off_t i_ + = (ORDER == 0) ? i + : (ORDER == 1) ? (chunks-1-i) + : xorshift32(&prng) % chunks; + uint32_t chunk_prng = i_; + for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) { + buffer[j] = xorshift32(&chunk_prng); + } + + lfs_file_seek(&lfs, &file, i_*CHUNK_SIZE, LFS_SEEK_SET) + => i_*CHUNK_SIZE; + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' diff --git a/benches/bench_superblock.toml b/benches/bench_superblock.toml new file mode 100644 index 00000000..37659d47 --- /dev/null +++ b/benches/bench_superblock.toml @@ -0,0 +1,56 @@ +[cases.bench_superblocks_found] +# support benchmarking with files +defines.N = [0, 1024] +defines.FILE_SIZE = 8 +defines.CHUNK_SIZE = 8 +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + // create files? + lfs_mount(&lfs, cfg) => 0; + char name[256]; + uint8_t buffer[CHUNK_SIZE]; + for (lfs_size_t i = 0; i < N; i++) { + sprintf(name, "file%08x", i); + lfs_file_t file; + lfs_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + + for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) { + for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) { + buffer[k] = i+j+k; + } + lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE; + } + + lfs_file_close(&lfs, &file) => 0; + } + lfs_unmount(&lfs) => 0; + + BENCH_START(); + lfs_mount(&lfs, cfg) => 0; + BENCH_STOP(); + + lfs_unmount(&lfs) => 0; +''' + +[cases.bench_superblocks_missing] +code = ''' + lfs_t lfs; + + BENCH_START(); + int err = lfs_mount(&lfs, cfg); + assert(err != 0); + BENCH_STOP(); +''' + +[cases.bench_superblocks_format] +code = ''' + lfs_t lfs; + + BENCH_START(); + lfs_format(&lfs, cfg) => 0; + BENCH_STOP(); +''' + diff --git a/runners/bench_runner.c b/runners/bench_runner.c new file mode 100644 index 00000000..39a38f38 --- /dev/null +++ b/runners/bench_runner.c @@ -0,0 +1,1795 @@ + +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 199309L +#endif + +#include "runners/bench_runner.h" +#include "bd/lfs_emubd.h" + +#include +#include +#include +#include +#include +#include +#include +#include + + +// some helpers + +// append to an array with amortized doubling +void *mappend(void **p, + size_t size, + size_t *count, + size_t *capacity) { + uint8_t *p_ = *p; + size_t count_ = *count; + size_t capacity_ = *capacity; + + count_ += 1; + if (count_ > capacity_) { + capacity_ = (2*capacity_ < 4) ? 4 : 2*capacity_; + + p_ = realloc(p_, capacity_*size); + if (!p_) { + return NULL; + } + } + + *p = p_; + *count = count_; + *capacity = capacity_; + return &p_[(count_-1)*size]; +} + +// a quick self-terminating text-safe varint scheme +static void leb16_print(uintmax_t x) { + while (true) { + char nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0); + printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10); + if (x <= 0xf) { + break; + } + x >>= 4; + } +} + +static uintmax_t leb16_parse(const char *s, char **tail) { + uintmax_t x = 0; + size_t i = 0; + while (true) { + uintmax_t nibble = s[i]; + if (nibble >= '0' && nibble <= '9') { + nibble = nibble - '0'; + } else if (nibble >= 'a' && nibble <= 'v') { + nibble = nibble - 'a' + 10; + } else { + // invalid? + if (tail) { + *tail = (char*)s; + } + return 0; + } + + x |= (nibble & 0xf) << (4*i); + i += 1; + if (!(nibble & 0x10)) { + break; + } + } + + if (tail) { + *tail = (char*)s + i; + } + return x; +} + + + +// bench_runner types + +typedef struct bench_geometry { + char short_name; + const char *long_name; + + bench_define_t defines[BENCH_GEOMETRY_DEFINE_COUNT]; +} bench_geometry_t; + +typedef struct bench_id { + const char *name; + const bench_define_t *defines; + size_t define_count; +} bench_id_t; + + +// bench suites are linked into a custom ld section +extern struct bench_suite __start__bench_suites; +extern struct bench_suite __stop__bench_suites; + +const struct bench_suite *bench_suites = &__start__bench_suites; +#define BENCH_SUITE_COUNT \ + ((size_t)(&__stop__bench_suites - &__start__bench_suites)) + + +// bench define management +typedef struct bench_define_map { + const bench_define_t *defines; + size_t count; +} bench_define_map_t; + +typedef struct bench_define_names { + const char *const *names; + size_t count; +} bench_define_names_t; + +intmax_t bench_define_lit(void *data) { + return (intmax_t)data; +} + +#define BENCH_CONST(x) {bench_define_lit, (void*)(uintptr_t)(x)} +#define BENCH_LIT(x) ((bench_define_t)BENCH_CONST(x)) + + +#define BENCH_DEF(k, v) \ + intmax_t bench_define_##k(void *data) { \ + (void)data; \ + return v; \ + } + + BENCH_IMPLICIT_DEFINES +#undef BENCH_DEF + +#define BENCH_DEFINE_MAP_EXPLICIT 0 +#define BENCH_DEFINE_MAP_OVERRIDE 1 +#define BENCH_DEFINE_MAP_PERMUTATION 2 +#define BENCH_DEFINE_MAP_GEOMETRY 3 +#define BENCH_DEFINE_MAP_IMPLICIT 4 +#define BENCH_DEFINE_MAP_COUNT 5 + +bench_define_map_t bench_define_maps[BENCH_DEFINE_MAP_COUNT] = { + [BENCH_DEFINE_MAP_IMPLICIT] = { + (const bench_define_t[BENCH_IMPLICIT_DEFINE_COUNT]) { + #define BENCH_DEF(k, v) \ + [k##_i] = {bench_define_##k, NULL}, + + BENCH_IMPLICIT_DEFINES + #undef BENCH_DEF + }, + BENCH_IMPLICIT_DEFINE_COUNT, + }, +}; + +#define BENCH_DEFINE_NAMES_SUITE 0 +#define BENCH_DEFINE_NAMES_IMPLICIT 1 +#define BENCH_DEFINE_NAMES_COUNT 2 + +bench_define_names_t bench_define_names[BENCH_DEFINE_NAMES_COUNT] = { + [BENCH_DEFINE_NAMES_IMPLICIT] = { + (const char *const[BENCH_IMPLICIT_DEFINE_COUNT]){ + #define BENCH_DEF(k, v) \ + [k##_i] = #k, + + BENCH_IMPLICIT_DEFINES + #undef BENCH_DEF + }, + BENCH_IMPLICIT_DEFINE_COUNT, + }, +}; + +intmax_t *bench_define_cache; +size_t bench_define_cache_count; +unsigned *bench_define_cache_mask; + +const char *bench_define_name(size_t define) { + // lookup in our bench names + for (size_t i = 0; i < BENCH_DEFINE_NAMES_COUNT; i++) { + if (define < bench_define_names[i].count + && bench_define_names[i].names + && bench_define_names[i].names[define]) { + return bench_define_names[i].names[define]; + } + } + + return NULL; +} + +bool bench_define_ispermutation(size_t define) { + // is this define specific to the permutation? + for (size_t i = 0; i < BENCH_DEFINE_MAP_IMPLICIT; i++) { + if (define < bench_define_maps[i].count + && bench_define_maps[i].defines[define].cb) { + return true; + } + } + + return false; +} + +intmax_t bench_define(size_t define) { + // is the define in our cache? + if (define < bench_define_cache_count + && (bench_define_cache_mask[define/(8*sizeof(unsigned))] + & (1 << (define%(8*sizeof(unsigned)))))) { + return bench_define_cache[define]; + } + + // lookup in our bench defines + for (size_t i = 0; i < BENCH_DEFINE_MAP_COUNT; i++) { + if (define < bench_define_maps[i].count + && bench_define_maps[i].defines[define].cb) { + intmax_t v = bench_define_maps[i].defines[define].cb( + bench_define_maps[i].defines[define].data); + + // insert into cache! + bench_define_cache[define] = v; + bench_define_cache_mask[define / (8*sizeof(unsigned))] + |= 1 << (define%(8*sizeof(unsigned))); + + return v; + } + } + + return 0; + + // not found? + const char *name = bench_define_name(define); + fprintf(stderr, "error: undefined define %s (%zd)\n", + name ? name : "(unknown)", + define); + assert(false); + exit(-1); +} + +void bench_define_flush(void) { + // clear cache between permutations + memset(bench_define_cache_mask, 0, + sizeof(unsigned)*( + (bench_define_cache_count+(8*sizeof(unsigned))-1) + / (8*sizeof(unsigned)))); +} + +// geometry updates +const bench_geometry_t *bench_geometry = NULL; + +void bench_define_geometry(const bench_geometry_t *geometry) { + bench_define_maps[BENCH_DEFINE_MAP_GEOMETRY] = (bench_define_map_t){ + geometry->defines, BENCH_GEOMETRY_DEFINE_COUNT}; +} + +// override updates +typedef struct bench_override { + const char *name; + const intmax_t *defines; + size_t permutations; +} bench_override_t; + +const bench_override_t *bench_overrides = NULL; +size_t bench_override_count = 0; + +bench_define_t *bench_override_defines = NULL; +size_t bench_override_define_count = 0; +size_t bench_override_define_permutations = 1; +size_t bench_override_define_capacity = 0; + +// suite/perm updates +void bench_define_suite(const struct bench_suite *suite) { + bench_define_names[BENCH_DEFINE_NAMES_SUITE] = (bench_define_names_t){ + suite->define_names, suite->define_count}; + + // make sure our cache is large enough + if (lfs_max(suite->define_count, BENCH_IMPLICIT_DEFINE_COUNT) + > bench_define_cache_count) { + // align to power of two to avoid any superlinear growth + size_t ncount = 1 << lfs_npw2( + lfs_max(suite->define_count, BENCH_IMPLICIT_DEFINE_COUNT)); + bench_define_cache = realloc(bench_define_cache, ncount*sizeof(intmax_t)); + bench_define_cache_mask = realloc(bench_define_cache_mask, + sizeof(unsigned)*( + (ncount+(8*sizeof(unsigned))-1) + / (8*sizeof(unsigned)))); + bench_define_cache_count = ncount; + } + + // map any overrides + if (bench_override_count > 0) { + // first figure out the total size of override permutations + size_t count = 0; + size_t permutations = 1; + for (size_t i = 0; i < bench_override_count; i++) { + for (size_t d = 0; + d < lfs_max( + suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + // define name match? + const char *name = bench_define_name(d); + if (name && strcmp(name, bench_overrides[i].name) == 0) { + count = lfs_max(count, d+1); + permutations *= bench_overrides[i].permutations; + break; + } + } + } + bench_override_define_count = count; + bench_override_define_permutations = permutations; + + // make sure our override arrays are big enough + if (count * permutations > bench_override_define_capacity) { + // align to power of two to avoid any superlinear growth + size_t ncapacity = 1 << lfs_npw2(count * permutations); + bench_override_defines = realloc( + bench_override_defines, + sizeof(bench_define_t)*ncapacity); + bench_override_define_capacity = ncapacity; + } + + // zero unoverridden defines + memset(bench_override_defines, 0, + sizeof(bench_define_t) * count * permutations); + + // compute permutations + size_t p = 1; + for (size_t i = 0; i < bench_override_count; i++) { + for (size_t d = 0; + d < lfs_max( + suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + // define name match? + const char *name = bench_define_name(d); + if (name && strcmp(name, bench_overrides[i].name) == 0) { + // scatter the define permutations based on already + // seen permutations + for (size_t j = 0; j < permutations; j++) { + bench_override_defines[j*count + d] = BENCH_LIT( + bench_overrides[i].defines[(j/p) + % bench_overrides[i].permutations]); + } + + // keep track of how many permutations we've seen so far + p *= bench_overrides[i].permutations; + break; + } + } + } + } +} + +void bench_define_perm( + const struct bench_suite *suite, + const struct bench_case *case_, + size_t perm) { + if (case_->defines) { + bench_define_maps[BENCH_DEFINE_MAP_PERMUTATION] = (bench_define_map_t){ + case_->defines + perm*suite->define_count, + suite->define_count}; + } else { + bench_define_maps[BENCH_DEFINE_MAP_PERMUTATION] = (bench_define_map_t){ + NULL, 0}; + } +} + +void bench_define_override(size_t perm) { + bench_define_maps[BENCH_DEFINE_MAP_OVERRIDE] = (bench_define_map_t){ + bench_override_defines + perm*bench_override_define_count, + bench_override_define_count}; +} + +void bench_define_explicit( + const bench_define_t *defines, + size_t define_count) { + bench_define_maps[BENCH_DEFINE_MAP_EXPLICIT] = (bench_define_map_t){ + defines, define_count}; +} + +void bench_define_cleanup(void) { + // bench define management can allocate a few things + free(bench_define_cache); + free(bench_define_cache_mask); + free(bench_override_defines); +} + + + +// bench state +extern const bench_geometry_t *bench_geometries; +extern size_t bench_geometry_count; + +const bench_id_t *bench_ids = (const bench_id_t[]) { + {NULL, NULL, 0}, +}; +size_t bench_id_count = 1; + +size_t bench_step_start = 0; +size_t bench_step_stop = -1; +size_t bench_step_step = 1; + +const char *bench_disk_path = NULL; +const char *bench_trace_path = NULL; +FILE *bench_trace_file = NULL; +uint32_t bench_trace_cycles = 0; +lfs_emubd_sleep_t bench_read_sleep = 0.0; +lfs_emubd_sleep_t bench_prog_sleep = 0.0; +lfs_emubd_sleep_t bench_erase_sleep = 0.0; + + +// trace printing +void bench_trace(const char *fmt, ...) { + if (bench_trace_path) { + if (!bench_trace_file) { + // Tracing output is heavy and trying to open every trace + // call is slow, so we only try to open the trace file every + // so often. Note this doesn't affect successfully opened files + if (bench_trace_cycles % 128 != 0) { + bench_trace_cycles += 1; + return; + } + bench_trace_cycles += 1; + + int fd; + if (strcmp(bench_trace_path, "-") == 0) { + fd = dup(1); + if (fd < 0) { + return; + } + } else { + fd = open( + bench_trace_path, + O_WRONLY | O_CREAT | O_APPEND | O_NONBLOCK, + 0666); + if (fd < 0) { + return; + } + int err = fcntl(fd, F_SETFL, O_WRONLY | O_CREAT | O_APPEND); + assert(!err); + } + + FILE *f = fdopen(fd, "a"); + assert(f); + int err = setvbuf(f, NULL, _IOLBF, BUFSIZ); + assert(!err); + bench_trace_file = f; + } + + va_list va; + va_start(va, fmt); + int res = vfprintf(bench_trace_file, fmt, va); + if (res < 0) { + fclose(bench_trace_file); + bench_trace_file = NULL; + } + va_end(va); + } +} + + +// bench recording state +static struct lfs_config *bench_cfg = NULL; +static lfs_emubd_io_t bench_last_read = 0; +static lfs_emubd_io_t bench_last_prog = 0; +static lfs_emubd_io_t bench_last_erased = 0; +lfs_emubd_io_t bench_read = 0; +lfs_emubd_io_t bench_prog = 0; +lfs_emubd_io_t bench_erased = 0; + +void bench_reset(void) { + bench_read = 0; + bench_prog = 0; + bench_erased = 0; + bench_last_read = 0; + bench_last_prog = 0; + bench_last_erased = 0; +} + +void bench_start(void) { + assert(bench_cfg); + lfs_emubd_sio_t read = lfs_emubd_getread(bench_cfg); + assert(read >= 0); + lfs_emubd_sio_t prog = lfs_emubd_getprog(bench_cfg); + assert(prog >= 0); + lfs_emubd_sio_t erased = lfs_emubd_geterased(bench_cfg); + assert(erased >= 0); + + bench_last_read = read; + bench_last_prog = prog; + bench_last_erased = erased; +} + +void bench_stop(void) { + assert(bench_cfg); + lfs_emubd_sio_t read = lfs_emubd_getread(bench_cfg); + assert(read >= 0); + lfs_emubd_sio_t prog = lfs_emubd_getprog(bench_cfg); + assert(prog >= 0); + lfs_emubd_sio_t erased = lfs_emubd_geterased(bench_cfg); + assert(erased >= 0); + + bench_read += read - bench_last_read; + bench_prog += prog - bench_last_prog; + bench_erased += erased - bench_last_erased; +} + + +// encode our permutation into a reusable id +static void perm_printid( + const struct bench_suite *suite, + const struct bench_case *case_) { + (void)suite; + // case[:permutation] + printf("%s:", case_->name); + for (size_t d = 0; + d < lfs_max( + suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + if (bench_define_ispermutation(d)) { + leb16_print(d); + leb16_print(BENCH_DEFINE(d)); + } + } +} + +// iterate through permutations in a bench case +static void case_forperm( + const struct bench_suite *suite, + const struct bench_case *case_, + const bench_define_t *defines, + size_t define_count, + void (*cb)( + void *data, + const struct bench_suite *suite, + const struct bench_case *case_), + void *data) { + if (defines) { + bench_define_explicit(defines, define_count); + bench_define_flush(); + + cb(data, suite, case_); + } else { + for (size_t k = 0; k < case_->permutations; k++) { + // define permutation + bench_define_perm(suite, case_, k); + + for (size_t v = 0; v < bench_override_define_permutations; v++) { + // define override permutation + bench_define_override(v); + + for (size_t g = 0; g < bench_geometry_count; g++) { + // define geometry + bench_define_geometry(&bench_geometries[g]); + bench_define_flush(); + + cb(data, suite, case_); + } + } + } + } +} + + +// how many permutations are there actually in a bench case +struct perm_count_state { + size_t total; + size_t filtered; +}; + +void perm_count( + void *data, + const struct bench_suite *suite, + const struct bench_case *case_) { + struct perm_count_state *state = data; + (void)suite; + (void)case_; + + state->total += 1; + + if (case_->filter && !case_->filter()) { + return; + } + + state->filtered += 1; +} + + +// operations we can do +static void summary(void) { + printf("%-36s %7s %7s %7s %11s\n", + "", "flags", "suites", "cases", "perms"); + size_t suites = 0; + size_t cases = 0; + bench_flags_t flags = 0; + struct perm_count_state perms = {0, 0}; + + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + cases += 1; + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_count, + &perms); + } + + suites += 1; + flags |= bench_suites[i].flags; + } + } + + char perm_buf[64]; + sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); + char flag_buf[64]; + sprintf(flag_buf, "%s%s", + (flags & BENCH_REENTRANT) ? "r" : "", + (!flags) ? "-" : ""); + printf("%-36s %7s %7zu %7zu %11s\n", + "TOTAL", + flag_buf, + suites, + cases, + perm_buf); +} + +static void list_suites(void) { + printf("%-36s %7s %7s %11s\n", "suite", "flags", "cases", "perms"); + + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + size_t cases = 0; + struct perm_count_state perms = {0, 0}; + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + cases += 1; + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_count, + &perms); + } + + // no benches found? + if (!cases) { + continue; + } + + char perm_buf[64]; + sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); + char flag_buf[64]; + sprintf(flag_buf, "%s%s", + (bench_suites[i].flags & BENCH_REENTRANT) ? "r" : "", + (!bench_suites[i].flags) ? "-" : ""); + printf("%-36s %7s %7zu %11s\n", + bench_suites[i].name, + flag_buf, + cases, + perm_buf); + } + } +} + +static void list_cases(void) { + printf("%-36s %7s %11s\n", "case", "flags", "perms"); + + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + struct perm_count_state perms = {0, 0}; + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_count, + &perms); + + char perm_buf[64]; + sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total); + char flag_buf[64]; + sprintf(flag_buf, "%s%s", + (bench_suites[i].cases[j].flags & BENCH_REENTRANT) + ? "r" : "", + (!bench_suites[i].cases[j].flags) + ? "-" : ""); + printf("%-36s %7s %11s\n", + bench_suites[i].cases[j].name, + flag_buf, + perm_buf); + } + } + } +} + +static void list_suite_paths(void) { + printf("%-36s %s\n", "suite", "path"); + + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + size_t cases = 0; + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + } + + // no benches found? + if (!cases) { + continue; + } + + printf("%-36s %s\n", + bench_suites[i].name, + bench_suites[i].path); + } + } +} + +static void list_case_paths(void) { + printf("%-36s %s\n", "case", "path"); + + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + printf("%-36s %s\n", + bench_suites[i].cases[j].name, + bench_suites[i].cases[j].path); + } + } + } +} + +struct list_defines_define { + const char *name; + intmax_t *values; + size_t value_count; + size_t value_capacity; +}; + +struct list_defines_defines { + struct list_defines_define *defines; + size_t define_count; + size_t define_capacity; +}; + +static void list_defines_add( + struct list_defines_defines *defines, + size_t d) { + const char *name = bench_define_name(d); + intmax_t value = BENCH_DEFINE(d); + + // define already in defines? + for (size_t i = 0; i < defines->define_count; i++) { + if (strcmp(defines->defines[i].name, name) == 0) { + // value already in values? + for (size_t j = 0; j < defines->defines[i].value_count; j++) { + if (defines->defines[i].values[j] == value) { + return; + } + } + + *(intmax_t*)mappend( + (void**)&defines->defines[i].values, + sizeof(intmax_t), + &defines->defines[i].value_count, + &defines->defines[i].value_capacity) = value; + + return; + } + } + + // new define? + struct list_defines_define *define = mappend( + (void**)&defines->defines, + sizeof(struct list_defines_define), + &defines->define_count, + &defines->define_capacity); + define->name = name; + define->values = malloc(sizeof(intmax_t)); + define->values[0] = value; + define->value_count = 1; + define->value_capacity = 1; +} + +void perm_list_defines( + void *data, + const struct bench_suite *suite, + const struct bench_case *case_) { + struct list_defines_defines *defines = data; + (void)suite; + (void)case_; + + // collect defines + for (size_t d = 0; + d < lfs_max(suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + if (d < BENCH_IMPLICIT_DEFINE_COUNT + || bench_define_ispermutation(d)) { + list_defines_add(defines, d); + } + } +} + +void perm_list_permutation_defines( + void *data, + const struct bench_suite *suite, + const struct bench_case *case_) { + struct list_defines_defines *defines = data; + (void)suite; + (void)case_; + + // collect permutation_defines + for (size_t d = 0; + d < lfs_max(suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + if (bench_define_ispermutation(d)) { + list_defines_add(defines, d); + } + } +} + +extern const bench_geometry_t builtin_geometries[]; + +static void list_defines(void) { + struct list_defines_defines defines = {NULL, 0, 0}; + + // add defines + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_list_defines, + &defines); + } + } + } + + for (size_t i = 0; i < defines.define_count; i++) { + printf("%s=", defines.defines[i].name); + for (size_t j = 0; j < defines.defines[i].value_count; j++) { + printf("%jd", defines.defines[i].values[j]); + if (j != defines.defines[i].value_count-1) { + printf(","); + } + } + printf("\n"); + } + + for (size_t i = 0; i < defines.define_count; i++) { + free(defines.defines[i].values); + } + free(defines.defines); +} + +static void list_permutation_defines(void) { + struct list_defines_defines defines = {NULL, 0, 0}; + + // add permutation defines + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_list_permutation_defines, + &defines); + } + } + } + + for (size_t i = 0; i < defines.define_count; i++) { + printf("%s=", defines.defines[i].name); + for (size_t j = 0; j < defines.defines[i].value_count; j++) { + printf("%jd", defines.defines[i].values[j]); + if (j != defines.defines[i].value_count-1) { + printf(","); + } + } + printf("\n"); + } + + for (size_t i = 0; i < defines.define_count; i++) { + free(defines.defines[i].values); + } + free(defines.defines); +} + +static void list_implicit_defines(void) { + struct list_defines_defines defines = {NULL, 0, 0}; + + // yes we do need to define a suite, this does a bit of bookeeping + // such as setting up the define cache + bench_define_suite(&(const struct bench_suite){0}); + + // make sure to include builtin geometries here + extern const bench_geometry_t builtin_geometries[]; + for (size_t g = 0; builtin_geometries[g].long_name; g++) { + bench_define_geometry(&builtin_geometries[g]); + bench_define_flush(); + + // add implicit defines + for (size_t d = 0; d < BENCH_IMPLICIT_DEFINE_COUNT; d++) { + list_defines_add(&defines, d); + } + } + + for (size_t i = 0; i < defines.define_count; i++) { + printf("%s=", defines.defines[i].name); + for (size_t j = 0; j < defines.defines[i].value_count; j++) { + printf("%jd", defines.defines[i].values[j]); + if (j != defines.defines[i].value_count-1) { + printf(","); + } + } + printf("\n"); + } + + for (size_t i = 0; i < defines.define_count; i++) { + free(defines.defines[i].values); + } + free(defines.defines); +} + + + +// geometries to bench + +const bench_geometry_t builtin_geometries[] = { + {'d', "default", {{NULL}, BENCH_CONST(16), BENCH_CONST(512), {NULL}}}, + {'e', "eeprom", {{NULL}, BENCH_CONST(1), BENCH_CONST(512), {NULL}}}, + {'E', "emmc", {{NULL}, {NULL}, BENCH_CONST(512), {NULL}}}, + {'n', "nor", {{NULL}, BENCH_CONST(1), BENCH_CONST(4096), {NULL}}}, + {'N', "nand", {{NULL}, BENCH_CONST(4096), BENCH_CONST(32768), {NULL}}}, + {0, NULL, {{NULL}, {NULL}, {NULL}, {NULL}}}, +}; + +const bench_geometry_t *bench_geometries = builtin_geometries; +size_t bench_geometry_count = 5; + +static void list_geometries(void) { + // yes we do need to define a suite, this does a bit of bookeeping + // such as setting up the define cache + bench_define_suite(&(const struct bench_suite){0}); + + printf("%-24s %7s %7s %7s %7s %11s\n", + "geometry", "read", "prog", "erase", "count", "size"); + for (size_t g = 0; builtin_geometries[g].long_name; g++) { + bench_define_geometry(&builtin_geometries[g]); + bench_define_flush(); + printf("%c,%-22s %7ju %7ju %7ju %7ju %11ju\n", + builtin_geometries[g].short_name, + builtin_geometries[g].long_name, + READ_SIZE, + PROG_SIZE, + BLOCK_SIZE, + BLOCK_COUNT, + BLOCK_SIZE*BLOCK_COUNT); + } +} + + + +// global bench step count +size_t bench_step = 0; + +void perm_run( + void *data, + const struct bench_suite *suite, + const struct bench_case *case_) { + (void)data; + + // skip this step? + if (!(bench_step >= bench_step_start + && bench_step < bench_step_stop + && (bench_step-bench_step_start) % bench_step_step == 0)) { + bench_step += 1; + return; + } + bench_step += 1; + + // filter? + if (case_->filter && !case_->filter()) { + printf("skipped "); + perm_printid(suite, case_); + printf("\n"); + return; + } + + // create block device and configuration + lfs_emubd_t bd; + + struct lfs_config cfg = { + .context = &bd, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .block_size = BLOCK_SIZE, + .block_count = BLOCK_COUNT, + .block_cycles = BLOCK_CYCLES, + .cache_size = CACHE_SIZE, + .lookahead_size = LOOKAHEAD_SIZE, + }; + + struct lfs_emubd_config bdcfg = { + .erase_value = ERASE_VALUE, + .erase_cycles = ERASE_CYCLES, + .badblock_behavior = BADBLOCK_BEHAVIOR, + .disk_path = bench_disk_path, + .read_sleep = bench_read_sleep, + .prog_sleep = bench_prog_sleep, + .erase_sleep = bench_erase_sleep, + }; + + int err = lfs_emubd_createcfg(&cfg, bench_disk_path, &bdcfg); + if (err) { + fprintf(stderr, "error: could not create block device: %d\n", err); + exit(-1); + } + + // run the bench + bench_cfg = &cfg; + bench_reset(); + printf("running "); + perm_printid(suite, case_); + printf("\n"); + + case_->run(&cfg); + + printf("finished "); + perm_printid(suite, case_); + printf(" %"PRIu64" %"PRIu64" %"PRIu64, + bench_read, + bench_prog, + bench_erased); + printf("\n"); + + // cleanup + err = lfs_emubd_destroy(&cfg); + if (err) { + fprintf(stderr, "error: could not destroy block device: %d\n", err); + exit(-1); + } +} + +static void run(void) { + // ignore disconnected pipes + signal(SIGPIPE, SIG_IGN); + + for (size_t t = 0; t < bench_id_count; t++) { + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + bench_define_suite(&bench_suites[i]); + + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + // does neither suite nor case name match? + if (bench_ids[t].name && !( + strcmp(bench_ids[t].name, + bench_suites[i].name) == 0 + || strcmp(bench_ids[t].name, + bench_suites[i].cases[j].name) == 0)) { + continue; + } + + case_forperm( + &bench_suites[i], + &bench_suites[i].cases[j], + bench_ids[t].defines, + bench_ids[t].define_count, + perm_run, + NULL); + } + } + } +} + + + +// option handling +enum opt_flags { + OPT_HELP = 'h', + OPT_SUMMARY = 'Y', + OPT_LIST_SUITES = 'l', + OPT_LIST_CASES = 'L', + OPT_LIST_SUITE_PATHS = 1, + OPT_LIST_CASE_PATHS = 2, + OPT_LIST_DEFINES = 3, + OPT_LIST_PERMUTATION_DEFINES = 4, + OPT_LIST_IMPLICIT_DEFINES = 5, + OPT_LIST_GEOMETRIES = 6, + OPT_DEFINE = 'D', + OPT_GEOMETRY = 'g', + OPT_STEP = 's', + OPT_DISK = 'd', + OPT_TRACE = 't', + OPT_READ_SLEEP = 7, + OPT_PROG_SLEEP = 8, + OPT_ERASE_SLEEP = 9, +}; + +const char *short_opts = "hYlLD:g:s:d:t:"; + +const struct option long_opts[] = { + {"help", no_argument, NULL, OPT_HELP}, + {"summary", no_argument, NULL, OPT_SUMMARY}, + {"list-suites", no_argument, NULL, OPT_LIST_SUITES}, + {"list-cases", no_argument, NULL, OPT_LIST_CASES}, + {"list-suite-paths", no_argument, NULL, OPT_LIST_SUITE_PATHS}, + {"list-case-paths", no_argument, NULL, OPT_LIST_CASE_PATHS}, + {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, + {"list-permutation-defines", + no_argument, NULL, OPT_LIST_PERMUTATION_DEFINES}, + {"list-implicit-defines", + no_argument, NULL, OPT_LIST_IMPLICIT_DEFINES}, + {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, + {"define", required_argument, NULL, OPT_DEFINE}, + {"geometry", required_argument, NULL, OPT_GEOMETRY}, + {"step", required_argument, NULL, OPT_STEP}, + {"disk", required_argument, NULL, OPT_DISK}, + {"trace", required_argument, NULL, OPT_TRACE}, + {"read-sleep", required_argument, NULL, OPT_READ_SLEEP}, + {"prog-sleep", required_argument, NULL, OPT_PROG_SLEEP}, + {"erase-sleep", required_argument, NULL, OPT_ERASE_SLEEP}, + {NULL, 0, NULL, 0}, +}; + +const char *const help_text[] = { + "Show this help message.", + "Show quick summary.", + "List bench suites.", + "List bench cases.", + "List the path for each bench suite.", + "List the path and line number for each bench case.", + "List all defines in this bench-runner.", + "List explicit defines in this bench-runner.", + "List implicit defines in this bench-runner.", + "List the available disk geometries.", + "Override a bench define.", + "Comma-separated list of disk geometries to bench. Defaults to d,e,E,n,N.", + "Comma-separated range of bench permutations to run (start,stop,step).", + "Redirect block device operations to this file.", + "Redirect trace output to this file.", + "Artificial read delay in seconds.", + "Artificial prog delay in seconds.", + "Artificial erase delay in seconds.", +}; + +int main(int argc, char **argv) { + void (*op)(void) = run; + + size_t bench_override_capacity = 0; + size_t bench_geometry_capacity = 0; + size_t bench_id_capacity = 0; + + // parse options + while (true) { + int c = getopt_long(argc, argv, short_opts, long_opts, NULL); + switch (c) { + // generate help message + case OPT_HELP: { + printf("usage: %s [options] [bench_id]\n", argv[0]); + printf("\n"); + + printf("options:\n"); + size_t i = 0; + while (long_opts[i].name) { + size_t indent; + if (long_opts[i].has_arg == no_argument) { + if (long_opts[i].val >= '0' && long_opts[i].val < 'z') { + indent = printf(" -%c, --%s ", + long_opts[i].val, + long_opts[i].name); + } else { + indent = printf(" --%s ", + long_opts[i].name); + } + } else { + if (long_opts[i].val >= '0' && long_opts[i].val < 'z') { + indent = printf(" -%c %s, --%s %s ", + long_opts[i].val, + long_opts[i].name, + long_opts[i].name, + long_opts[i].name); + } else { + indent = printf(" --%s %s ", + long_opts[i].name, + long_opts[i].name); + } + } + + // a quick, hacky, byte-level method for text wrapping + size_t len = strlen(help_text[i]); + size_t j = 0; + if (indent < 24) { + printf("%*s %.80s\n", + (int)(24-1-indent), + "", + &help_text[i][j]); + j += 80; + } else { + printf("\n"); + } + + while (j < len) { + printf("%24s%.80s\n", "", &help_text[i][j]); + j += 80; + } + + i += 1; + } + + printf("\n"); + exit(0); + } + // summary/list flags + case OPT_SUMMARY: + op = summary; + break; + case OPT_LIST_SUITES: + op = list_suites; + break; + case OPT_LIST_CASES: + op = list_cases; + break; + case OPT_LIST_SUITE_PATHS: + op = list_suite_paths; + break; + case OPT_LIST_CASE_PATHS: + op = list_case_paths; + break; + case OPT_LIST_DEFINES: + op = list_defines; + break; + case OPT_LIST_PERMUTATION_DEFINES: + op = list_permutation_defines; + break; + case OPT_LIST_IMPLICIT_DEFINES: + op = list_implicit_defines; + break; + case OPT_LIST_GEOMETRIES: + op = list_geometries; + break; + // configuration + case OPT_DEFINE: { + // allocate space + bench_override_t *override = mappend( + (void**)&bench_overrides, + sizeof(bench_override_t), + &bench_override_count, + &bench_override_capacity); + + // parse into string key/intmax_t value, cannibalizing the + // arg in the process + char *sep = strchr(optarg, '='); + char *parsed = NULL; + if (!sep) { + goto invalid_define; + } + *sep = '\0'; + override->name = optarg; + optarg = sep+1; + + // parse comma-separated permutations + { + override->defines = NULL; + override->permutations = 0; + size_t override_capacity = 0; + while (true) { + optarg += strspn(optarg, " "); + + if (strncmp(optarg, "range", strlen("range")) == 0) { + // range of values + optarg += strlen("range"); + optarg += strspn(optarg, " "); + if (*optarg != '(') { + goto invalid_define; + } + optarg += 1; + + intmax_t start = strtoumax(optarg, &parsed, 0); + intmax_t stop = -1; + intmax_t step = 1; + // allow empty string for start=0 + if (parsed == optarg) { + start = 0; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != ')') { + goto invalid_define; + } + + if (*optarg == ',') { + optarg += 1; + stop = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=end + if (parsed == optarg) { + stop = -1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != ')') { + goto invalid_define; + } + + if (*optarg == ',') { + optarg += 1; + step = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=1 + if (parsed == optarg) { + step = 1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ')') { + goto invalid_define; + } + } + } else { + // single value = stop only + stop = start; + start = 0; + } + + if (*optarg != ')') { + goto invalid_define; + } + optarg += 1; + + // calculate the range of values + assert(step != 0); + for (intmax_t i = start; + (step < 0) + ? i > stop + : (uintmax_t)i < (uintmax_t)stop; + i += step) { + *(intmax_t*)mappend( + (void**)&override->defines, + sizeof(intmax_t), + &override->permutations, + &override_capacity) = i; + } + } else if (*optarg != '\0') { + // single value + intmax_t define = strtoimax(optarg, &parsed, 0); + if (parsed == optarg) { + goto invalid_define; + } + optarg = parsed + strspn(parsed, " "); + *(intmax_t*)mappend( + (void**)&override->defines, + sizeof(intmax_t), + &override->permutations, + &override_capacity) = define; + } else { + break; + } + + if (*optarg == ',') { + optarg += 1; + } + } + } + assert(override->permutations > 0); + break; + +invalid_define: + fprintf(stderr, "error: invalid define: %s\n", optarg); + exit(-1); + } + case OPT_GEOMETRY: { + // reset our geometry scenarios + if (bench_geometry_capacity > 0) { + free((bench_geometry_t*)bench_geometries); + } + bench_geometries = NULL; + bench_geometry_count = 0; + bench_geometry_capacity = 0; + + // parse the comma separated list of disk geometries + while (*optarg) { + // allocate space + bench_geometry_t *geometry = mappend( + (void**)&bench_geometries, + sizeof(bench_geometry_t), + &bench_geometry_count, + &bench_geometry_capacity); + + // parse the disk geometry + optarg += strspn(optarg, " "); + + // named disk geometry + size_t len = strcspn(optarg, " ,"); + for (size_t i = 0; builtin_geometries[i].long_name; i++) { + if ((len == 1 + && *optarg == builtin_geometries[i].short_name) + || (len == strlen( + builtin_geometries[i].long_name) + && memcmp(optarg, + builtin_geometries[i].long_name, + len) == 0)) { + *geometry = builtin_geometries[i]; + optarg += len; + goto geometry_next; + } + } + + // comma-separated read/prog/erase/count + if (*optarg == '{') { + lfs_size_t sizes[4]; + size_t count = 0; + + char *s = optarg + 1; + while (count < 4) { + char *parsed = NULL; + sizes[count] = strtoumax(s, &parsed, 0); + count += 1; + + s = parsed + strspn(parsed, " "); + if (*s == ',') { + s += 1; + continue; + } else if (*s == '}') { + s += 1; + break; + } else { + goto geometry_unknown; + } + } + + // allow implicit r=p and p=e for common geometries + memset(geometry, 0, sizeof(bench_geometry_t)); + if (count >= 3) { + geometry->defines[READ_SIZE_i] + = BENCH_LIT(sizes[0]); + geometry->defines[PROG_SIZE_i] + = BENCH_LIT(sizes[1]); + geometry->defines[BLOCK_SIZE_i] + = BENCH_LIT(sizes[2]); + } else if (count >= 2) { + geometry->defines[PROG_SIZE_i] + = BENCH_LIT(sizes[0]); + geometry->defines[BLOCK_SIZE_i] + = BENCH_LIT(sizes[1]); + } else { + geometry->defines[BLOCK_SIZE_i] + = BENCH_LIT(sizes[0]); + } + if (count >= 4) { + geometry->defines[BLOCK_COUNT_i] + = BENCH_LIT(sizes[3]); + } + optarg = s; + goto geometry_next; + } + + // leb16-encoded read/prog/erase/count + if (*optarg == ':') { + lfs_size_t sizes[4]; + size_t count = 0; + + char *s = optarg + 1; + while (true) { + char *parsed = NULL; + uintmax_t x = leb16_parse(s, &parsed); + if (parsed == s || count >= 4) { + break; + } + + sizes[count] = x; + count += 1; + s = parsed; + } + + // allow implicit r=p and p=e for common geometries + memset(geometry, 0, sizeof(bench_geometry_t)); + if (count >= 3) { + geometry->defines[READ_SIZE_i] + = BENCH_LIT(sizes[0]); + geometry->defines[PROG_SIZE_i] + = BENCH_LIT(sizes[1]); + geometry->defines[BLOCK_SIZE_i] + = BENCH_LIT(sizes[2]); + } else if (count >= 2) { + geometry->defines[PROG_SIZE_i] + = BENCH_LIT(sizes[0]); + geometry->defines[BLOCK_SIZE_i] + = BENCH_LIT(sizes[1]); + } else { + geometry->defines[BLOCK_SIZE_i] + = BENCH_LIT(sizes[0]); + } + if (count >= 4) { + geometry->defines[BLOCK_COUNT_i] + = BENCH_LIT(sizes[3]); + } + optarg = s; + goto geometry_next; + } + +geometry_unknown: + // unknown scenario? + fprintf(stderr, "error: unknown disk geometry: %s\n", + optarg); + exit(-1); + +geometry_next: + optarg += strspn(optarg, " "); + if (*optarg == ',') { + optarg += 1; + } else if (*optarg == '\0') { + break; + } else { + goto geometry_unknown; + } + } + break; + } + case OPT_STEP: { + char *parsed = NULL; + bench_step_start = strtoumax(optarg, &parsed, 0); + bench_step_stop = -1; + bench_step_step = 1; + // allow empty string for start=0 + if (parsed == optarg) { + bench_step_start = 0; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != '\0') { + goto step_unknown; + } + + if (*optarg == ',') { + optarg += 1; + bench_step_stop = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=end + if (parsed == optarg) { + bench_step_stop = -1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != '\0') { + goto step_unknown; + } + + if (*optarg == ',') { + optarg += 1; + bench_step_step = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=1 + if (parsed == optarg) { + bench_step_step = 1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != '\0') { + goto step_unknown; + } + } + } else { + // single value = stop only + bench_step_stop = bench_step_start; + bench_step_start = 0; + } + + break; +step_unknown: + fprintf(stderr, "error: invalid step: %s\n", optarg); + exit(-1); + } + case OPT_DISK: + bench_disk_path = optarg; + break; + case OPT_TRACE: + bench_trace_path = optarg; + break; + case OPT_READ_SLEEP: { + char *parsed = NULL; + double read_sleep = strtod(optarg, &parsed); + if (parsed == optarg) { + fprintf(stderr, "error: invalid read-sleep: %s\n", optarg); + exit(-1); + } + bench_read_sleep = read_sleep*1.0e9; + break; + } + case OPT_PROG_SLEEP: { + char *parsed = NULL; + double prog_sleep = strtod(optarg, &parsed); + if (parsed == optarg) { + fprintf(stderr, "error: invalid prog-sleep: %s\n", optarg); + exit(-1); + } + bench_prog_sleep = prog_sleep*1.0e9; + break; + } + case OPT_ERASE_SLEEP: { + char *parsed = NULL; + double erase_sleep = strtod(optarg, &parsed); + if (parsed == optarg) { + fprintf(stderr, "error: invalid erase-sleep: %s\n", optarg); + exit(-1); + } + bench_erase_sleep = erase_sleep*1.0e9; + break; + } + // done parsing + case -1: + goto getopt_done; + // unknown arg, getopt prints a message for us + default: + exit(-1); + } + } +getopt_done: ; + + if (argc > optind) { + // reset our bench identifier list + bench_ids = NULL; + bench_id_count = 0; + bench_id_capacity = 0; + } + + // parse bench identifier, if any, cannibalizing the arg in the process + for (; argc > optind; optind++) { + bench_define_t *defines = NULL; + size_t define_count = 0; + + // parse name, can be suite or case + char *name = argv[optind]; + char *defines_ = strchr(name, ':'); + if (defines_) { + *defines_ = '\0'; + defines_ += 1; + } + + // remove optional path and .toml suffix + char *slash = strrchr(name, '/'); + if (slash) { + name = slash+1; + } + + size_t name_len = strlen(name); + if (name_len > 5 && strcmp(&name[name_len-5], ".toml") == 0) { + name[name_len-5] = '\0'; + } + + if (defines_) { + // parse defines + while (true) { + char *parsed; + size_t d = leb16_parse(defines_, &parsed); + intmax_t v = leb16_parse(parsed, &parsed); + if (parsed == defines_) { + break; + } + defines_ = parsed; + + if (d >= define_count) { + // align to power of two to avoid any superlinear growth + size_t ncount = 1 << lfs_npw2(d+1); + defines = realloc(defines, + ncount*sizeof(bench_define_t)); + memset(defines+define_count, 0, + (ncount-define_count)*sizeof(bench_define_t)); + define_count = ncount; + } + defines[d] = BENCH_LIT(v); + } + } + + // append to identifier list + *(bench_id_t*)mappend( + (void**)&bench_ids, + sizeof(bench_id_t), + &bench_id_count, + &bench_id_capacity) = (bench_id_t){ + .name = name, + .defines = defines, + .define_count = define_count, + }; + } + + // do the thing + op(); + + // cleanup (need to be done for valgrind benching) + bench_define_cleanup(); + if (bench_overrides) { + for (size_t i = 0; i < bench_override_count; i++) { + free((void*)bench_overrides[i].defines); + } + free((void*)bench_overrides); + } + if (bench_geometry_capacity) { + free((void*)bench_geometries); + } + if (bench_id_capacity) { + for (size_t i = 0; i < bench_id_count; i++) { + free((void*)bench_ids[i].defines); + } + free((void*)bench_ids); + } +} diff --git a/runners/bench_runner.h b/runners/bench_runner.h new file mode 100644 index 00000000..a33c31d8 --- /dev/null +++ b/runners/bench_runner.h @@ -0,0 +1,119 @@ +#ifndef BENCH_RUNNER_H +#define BENCH_RUNNER_H + + +// override LFS_TRACE +void bench_trace(const char *fmt, ...); + +#define LFS_TRACE_(fmt, ...) \ + bench_trace("%s:%d:trace: " fmt "%s\n", \ + __FILE__, \ + __LINE__, \ + __VA_ARGS__) +#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") +#define LFS_EMUBD_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") + +// provide BENCH_START/BENCH_STOP macros +void bench_start(void); +void bench_stop(void); + +#define BENCH_START() bench_start() +#define BENCH_STOP() bench_stop() + + +// note these are indirectly included in any generated files +#include "bd/lfs_emubd.h" +#include + +// give source a chance to define feature macros +#undef _FEATURES_H +#undef _STDIO_H + + +// generated bench configurations +struct lfs_config; + +enum bench_flags { + BENCH_REENTRANT = 0x1, +}; +typedef uint8_t bench_flags_t; + +typedef struct bench_define { + intmax_t (*cb)(void *data); + void *data; +} bench_define_t; + +struct bench_case { + const char *name; + const char *path; + bench_flags_t flags; + size_t permutations; + + const bench_define_t *defines; + + bool (*filter)(void); + void (*run)(struct lfs_config *cfg); +}; + +struct bench_suite { + const char *name; + const char *path; + bench_flags_t flags; + + const char *const *define_names; + size_t define_count; + + const struct bench_case *cases; + size_t case_count; +}; + + +// access generated bench defines +intmax_t bench_define(size_t define); + +#define BENCH_DEFINE(i) bench_define(i) + +// a few preconfigured defines that control how benches run + +#define READ_SIZE_i 0 +#define PROG_SIZE_i 1 +#define BLOCK_SIZE_i 2 +#define BLOCK_COUNT_i 3 +#define CACHE_SIZE_i 4 +#define LOOKAHEAD_SIZE_i 5 +#define BLOCK_CYCLES_i 6 +#define ERASE_VALUE_i 7 +#define ERASE_CYCLES_i 8 +#define BADBLOCK_BEHAVIOR_i 9 +#define POWERLOSS_BEHAVIOR_i 10 + +#define READ_SIZE bench_define(READ_SIZE_i) +#define PROG_SIZE bench_define(PROG_SIZE_i) +#define BLOCK_SIZE bench_define(BLOCK_SIZE_i) +#define BLOCK_COUNT bench_define(BLOCK_COUNT_i) +#define CACHE_SIZE bench_define(CACHE_SIZE_i) +#define LOOKAHEAD_SIZE bench_define(LOOKAHEAD_SIZE_i) +#define BLOCK_CYCLES bench_define(BLOCK_CYCLES_i) +#define ERASE_VALUE bench_define(ERASE_VALUE_i) +#define ERASE_CYCLES bench_define(ERASE_CYCLES_i) +#define BADBLOCK_BEHAVIOR bench_define(BADBLOCK_BEHAVIOR_i) +#define POWERLOSS_BEHAVIOR bench_define(POWERLOSS_BEHAVIOR_i) + +#define BENCH_IMPLICIT_DEFINES \ + BENCH_DEF(READ_SIZE, PROG_SIZE) \ + BENCH_DEF(PROG_SIZE, BLOCK_SIZE) \ + BENCH_DEF(BLOCK_SIZE, 0) \ + BENCH_DEF(BLOCK_COUNT, (1024*1024)/BLOCK_SIZE) \ + BENCH_DEF(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ + BENCH_DEF(LOOKAHEAD_SIZE, 16) \ + BENCH_DEF(BLOCK_CYCLES, -1) \ + BENCH_DEF(ERASE_VALUE, 0xff) \ + BENCH_DEF(ERASE_CYCLES, 0) \ + BENCH_DEF(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR) \ + BENCH_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP) + +#define BENCH_GEOMETRY_DEFINE_COUNT 4 +#define BENCH_IMPLICIT_DEFINE_COUNT 11 + + +#endif diff --git a/runners/test_runner.c b/runners/test_runner.c index 55f237dc..8333e388 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -4,7 +4,7 @@ #endif #include "runners/test_runner.h" -#include "bd/lfs_testbd.h" +#include "bd/lfs_emubd.h" #include #include @@ -46,7 +46,7 @@ void *mappend(void **p, // a quick self-terminating text-safe varint scheme static void leb16_print(uintmax_t x) { while (true) { - lfs_testbd_powercycles_t nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0); + char nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0); printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10); if (x <= 0xf) { break; @@ -101,11 +101,11 @@ typedef struct test_powerloss { const char *long_name; void (*run)( - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, const struct test_suite *suite, const struct test_case *case_); - const lfs_testbd_powercycles_t *cycles; + const lfs_emubd_powercycles_t *cycles; size_t cycle_count; } test_powerloss_t; @@ -113,7 +113,7 @@ typedef struct test_id { const char *name; const test_define_t *defines; size_t define_count; - const lfs_testbd_powercycles_t *cycles; + const lfs_emubd_powercycles_t *cycles; size_t cycle_count; } test_id_t; @@ -141,17 +141,19 @@ typedef struct test_define_names { intmax_t test_define_lit(void *data) { return (intmax_t)data; } -#define TEST_LIT(x) {test_define_lit, (void*)(uintptr_t)(x)} + +#define TEST_CONST(x) {test_define_lit, (void*)(uintptr_t)(x)} +#define TEST_LIT(x) ((test_define_t)TEST_CONST(x)) -#define TEST_DEFINE(k, v) \ +#define TEST_DEF(k, v) \ intmax_t test_define_##k(void *data) { \ (void)data; \ return v; \ } TEST_IMPLICIT_DEFINES -#undef TEST_DEFINE +#undef TEST_DEF #define TEST_DEFINE_MAP_EXPLICIT 0 #define TEST_DEFINE_MAP_OVERRIDE 1 @@ -163,11 +165,11 @@ intmax_t test_define_lit(void *data) { test_define_map_t test_define_maps[TEST_DEFINE_MAP_COUNT] = { [TEST_DEFINE_MAP_IMPLICIT] = { (const test_define_t[TEST_IMPLICIT_DEFINE_COUNT]) { - #define TEST_DEFINE(k, v) \ + #define TEST_DEF(k, v) \ [k##_i] = {test_define_##k, NULL}, TEST_IMPLICIT_DEFINES - #undef TEST_DEFINE + #undef TEST_DEF }, TEST_IMPLICIT_DEFINE_COUNT, }, @@ -180,11 +182,11 @@ test_define_map_t test_define_maps[TEST_DEFINE_MAP_COUNT] = { test_define_names_t test_define_names[TEST_DEFINE_NAMES_COUNT] = { [TEST_DEFINE_NAMES_IMPLICIT] = { (const char *const[TEST_IMPLICIT_DEFINE_COUNT]){ - #define TEST_DEFINE(k, v) \ + #define TEST_DEF(k, v) \ [k##_i] = #k, TEST_IMPLICIT_DEFINES - #undef TEST_DEFINE + #undef TEST_DEF }, TEST_IMPLICIT_DEFINE_COUNT, }, @@ -318,7 +320,7 @@ void test_define_suite(const struct test_suite *suite) { // define name match? const char *name = test_define_name(d); if (name && strcmp(name, test_overrides[i].name) == 0) { - count = d+1; + count = lfs_max(count, d+1); permutations *= test_overrides[i].permutations; break; } @@ -355,10 +357,9 @@ void test_define_suite(const struct test_suite *suite) { // scatter the define permutations based on already // seen permutations for (size_t j = 0; j < permutations; j++) { - test_override_defines[j*count + d] - = (test_define_t)TEST_LIT( - test_overrides[i].defines[(j/p) - % test_overrides[i].permutations]); + test_override_defines[j*count + d] = TEST_LIT( + test_overrides[i].defines[(j/p) + % test_overrides[i].permutations]); } // keep track of how many permutations we've seen so far @@ -426,9 +427,9 @@ const char *test_disk_path = NULL; const char *test_trace_path = NULL; FILE *test_trace_file = NULL; uint32_t test_trace_cycles = 0; -lfs_testbd_sleep_t test_read_sleep = 0.0; -lfs_testbd_sleep_t test_prog_sleep = 0.0; -lfs_testbd_sleep_t test_erase_sleep = 0.0; +lfs_emubd_sleep_t test_read_sleep = 0.0; +lfs_emubd_sleep_t test_prog_sleep = 0.0; +lfs_emubd_sleep_t test_erase_sleep = 0.0; // trace printing @@ -485,10 +486,10 @@ void test_trace(const char *fmt, ...) { static void perm_printid( const struct test_suite *suite, const struct test_case *case_, - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count) { (void)suite; - // case[:permutation[:powercycles]]] + // case[:permutation[:powercycles]] printf("%s:", case_->name); for (size_t d = 0; d < lfs_max( @@ -497,7 +498,7 @@ static void perm_printid( d++) { if (test_define_ispermutation(d)) { leb16_print(d); - leb16_print(test_define(d)); + leb16_print(TEST_DEFINE(d)); } } @@ -511,7 +512,7 @@ static void perm_printid( } static void run_powerloss_cycles( - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, const struct test_suite *suite, const struct test_case *case_); @@ -522,7 +523,7 @@ static void case_forperm( const struct test_case *case_, const test_define_t *defines, size_t define_count, - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, void (*cb)( void *data, @@ -834,7 +835,7 @@ static void list_defines_add( struct list_defines_defines *defines, size_t d) { const char *name = test_define_name(d); - intmax_t value = test_define(d); + intmax_t value = TEST_DEFINE(d); // define already in defines? for (size_t i = 0; i < defines->define_count; i++) { @@ -1051,11 +1052,11 @@ static void list_implicit_defines(void) { // geometries to test const test_geometry_t builtin_geometries[] = { - {'d', "default", {{NULL}, TEST_LIT(16), TEST_LIT(512), {NULL}}}, - {'e', "eeprom", {{NULL}, TEST_LIT(1), TEST_LIT(512), {NULL}}}, - {'E', "emmc", {{NULL}, {NULL}, TEST_LIT(512), {NULL}}}, - {'n', "nor", {{NULL}, TEST_LIT(1), TEST_LIT(4096), {NULL}}}, - {'N', "nand", {{NULL}, TEST_LIT(4096), TEST_LIT(32768), {NULL}}}, + {'d', "default", {{NULL}, TEST_CONST(16), TEST_CONST(512), {NULL}}}, + {'e', "eeprom", {{NULL}, TEST_CONST(1), TEST_CONST(512), {NULL}}}, + {'E', "emmc", {{NULL}, {NULL}, TEST_CONST(512), {NULL}}}, + {'n', "nor", {{NULL}, TEST_CONST(1), TEST_CONST(4096), {NULL}}}, + {'N', "nand", {{NULL}, TEST_CONST(4096), TEST_CONST(32768), {NULL}}}, {0, NULL, {{NULL}, {NULL}, {NULL}, {NULL}}}, }; @@ -1087,7 +1088,7 @@ static void list_geometries(void) { // scenarios to run tests under power-loss static void run_powerloss_none( - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, const struct test_suite *suite, const struct test_case *case_) { @@ -1096,14 +1097,14 @@ static void run_powerloss_none( (void)suite; // create block device and configuration - lfs_testbd_t bd; + lfs_emubd_t bd; struct lfs_config cfg = { .context = &bd, - .read = lfs_testbd_read, - .prog = lfs_testbd_prog, - .erase = lfs_testbd_erase, - .sync = lfs_testbd_sync, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, .block_size = BLOCK_SIZE, @@ -1113,7 +1114,7 @@ static void run_powerloss_none( .lookahead_size = LOOKAHEAD_SIZE, }; - struct lfs_testbd_config bdcfg = { + struct lfs_emubd_config bdcfg = { .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1123,7 +1124,7 @@ static void run_powerloss_none( .erase_sleep = test_erase_sleep, }; - int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1141,7 +1142,7 @@ static void run_powerloss_none( printf("\n"); // cleanup - err = lfs_testbd_destroy(&cfg); + err = lfs_emubd_destroy(&cfg); if (err) { fprintf(stderr, "error: could not destroy block device: %d\n", err); exit(-1); @@ -1154,7 +1155,7 @@ static void powerloss_longjmp(void *c) { } static void run_powerloss_linear( - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, const struct test_suite *suite, const struct test_case *case_) { @@ -1163,16 +1164,16 @@ static void run_powerloss_linear( (void)suite; // create block device and configuration - lfs_testbd_t bd; + lfs_emubd_t bd; jmp_buf powerloss_jmp; - volatile lfs_testbd_powercycles_t i = 1; + volatile lfs_emubd_powercycles_t i = 1; struct lfs_config cfg = { .context = &bd, - .read = lfs_testbd_read, - .prog = lfs_testbd_prog, - .erase = lfs_testbd_erase, - .sync = lfs_testbd_sync, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, .block_size = BLOCK_SIZE, @@ -1182,7 +1183,7 @@ static void run_powerloss_linear( .lookahead_size = LOOKAHEAD_SIZE, }; - struct lfs_testbd_config bdcfg = { + struct lfs_emubd_config bdcfg = { .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1196,7 +1197,7 @@ static void run_powerloss_linear( .powerloss_data = &powerloss_jmp, }; - int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1218,13 +1219,13 @@ static void run_powerloss_linear( printf("powerloss "); perm_printid(suite, case_, NULL, 0); printf(":"); - for (lfs_testbd_powercycles_t j = 1; j <= i; j++) { + for (lfs_emubd_powercycles_t j = 1; j <= i; j++) { leb16_print(j); } printf("\n"); i += 1; - lfs_testbd_setpowercycles(&cfg, i); + lfs_emubd_setpowercycles(&cfg, i); } printf("finished "); @@ -1232,7 +1233,7 @@ static void run_powerloss_linear( printf("\n"); // cleanup - err = lfs_testbd_destroy(&cfg); + err = lfs_emubd_destroy(&cfg); if (err) { fprintf(stderr, "error: could not destroy block device: %d\n", err); exit(-1); @@ -1240,7 +1241,7 @@ static void run_powerloss_linear( } static void run_powerloss_exponential( - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, const struct test_suite *suite, const struct test_case *case_) { @@ -1249,16 +1250,16 @@ static void run_powerloss_exponential( (void)suite; // create block device and configuration - lfs_testbd_t bd; + lfs_emubd_t bd; jmp_buf powerloss_jmp; - volatile lfs_testbd_powercycles_t i = 1; + volatile lfs_emubd_powercycles_t i = 1; struct lfs_config cfg = { .context = &bd, - .read = lfs_testbd_read, - .prog = lfs_testbd_prog, - .erase = lfs_testbd_erase, - .sync = lfs_testbd_sync, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, .block_size = BLOCK_SIZE, @@ -1268,7 +1269,7 @@ static void run_powerloss_exponential( .lookahead_size = LOOKAHEAD_SIZE, }; - struct lfs_testbd_config bdcfg = { + struct lfs_emubd_config bdcfg = { .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1282,7 +1283,7 @@ static void run_powerloss_exponential( .powerloss_data = &powerloss_jmp, }; - int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1304,13 +1305,13 @@ static void run_powerloss_exponential( printf("powerloss "); perm_printid(suite, case_, NULL, 0); printf(":"); - for (lfs_testbd_powercycles_t j = 1; j <= i; j *= 2) { + for (lfs_emubd_powercycles_t j = 1; j <= i; j *= 2) { leb16_print(j); } printf("\n"); i *= 2; - lfs_testbd_setpowercycles(&cfg, i); + lfs_emubd_setpowercycles(&cfg, i); } printf("finished "); @@ -1318,7 +1319,7 @@ static void run_powerloss_exponential( printf("\n"); // cleanup - err = lfs_testbd_destroy(&cfg); + err = lfs_emubd_destroy(&cfg); if (err) { fprintf(stderr, "error: could not destroy block device: %d\n", err); exit(-1); @@ -1326,23 +1327,23 @@ static void run_powerloss_exponential( } static void run_powerloss_cycles( - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, const struct test_suite *suite, const struct test_case *case_) { (void)suite; // create block device and configuration - lfs_testbd_t bd; + lfs_emubd_t bd; jmp_buf powerloss_jmp; volatile size_t i = 0; struct lfs_config cfg = { .context = &bd, - .read = lfs_testbd_read, - .prog = lfs_testbd_prog, - .erase = lfs_testbd_erase, - .sync = lfs_testbd_sync, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, .block_size = BLOCK_SIZE, @@ -1352,7 +1353,7 @@ static void run_powerloss_cycles( .lookahead_size = LOOKAHEAD_SIZE, }; - struct lfs_testbd_config bdcfg = { + struct lfs_emubd_config bdcfg = { .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1366,7 +1367,7 @@ static void run_powerloss_cycles( .powerloss_data = &powerloss_jmp, }; - int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1391,7 +1392,7 @@ static void run_powerloss_cycles( printf("\n"); i += 1; - lfs_testbd_setpowercycles(&cfg, + lfs_emubd_setpowercycles(&cfg, (i < cycle_count) ? cycles[i] : 0); } @@ -1400,7 +1401,7 @@ static void run_powerloss_cycles( printf("\n"); // cleanup - err = lfs_testbd_destroy(&cfg); + err = lfs_emubd_destroy(&cfg); if (err) { fprintf(stderr, "error: could not destroy block device: %d\n", err); exit(-1); @@ -1410,13 +1411,13 @@ static void run_powerloss_cycles( struct powerloss_exhaustive_state { struct lfs_config *cfg; - lfs_testbd_t *branches; + lfs_emubd_t *branches; size_t branch_count; size_t branch_capacity; }; struct powerloss_exhaustive_cycles { - lfs_testbd_powercycles_t *cycles; + lfs_emubd_powercycles_t *cycles; size_t cycle_count; size_t cycle_capacity; }; @@ -1424,9 +1425,9 @@ struct powerloss_exhaustive_cycles { static void powerloss_exhaustive_branch(void *c) { struct powerloss_exhaustive_state *state = c; // append to branches - lfs_testbd_t *branch = mappend( + lfs_emubd_t *branch = mappend( (void**)&state->branches, - sizeof(lfs_testbd_t), + sizeof(lfs_emubd_t), &state->branch_count, &state->branch_capacity); if (!branch) { @@ -1435,14 +1436,14 @@ static void powerloss_exhaustive_branch(void *c) { } // create copy-on-write copy - int err = lfs_testbd_copy(state->cfg, branch); + int err = lfs_emubd_copy(state->cfg, branch); if (err) { fprintf(stderr, "error: exhaustive: could not create bd copy\n"); exit(-1); } // also trigger on next power cycle - lfs_testbd_setpowercycles(state->cfg, 1); + lfs_emubd_setpowercycles(state->cfg, 1); } static void run_powerloss_exhaustive_layer( @@ -1450,7 +1451,7 @@ static void run_powerloss_exhaustive_layer( const struct test_suite *suite, const struct test_case *case_, struct lfs_config *cfg, - struct lfs_testbd_config *bdcfg, + struct lfs_emubd_config *bdcfg, size_t depth) { (void)suite; @@ -1463,14 +1464,14 @@ static void run_powerloss_exhaustive_layer( // run through the test without additional powerlosses, collecting possible // branches as we do so - lfs_testbd_setpowercycles(state.cfg, depth > 0 ? 1 : 0); + lfs_emubd_setpowercycles(state.cfg, depth > 0 ? 1 : 0); bdcfg->powerloss_data = &state; // run the tests case_->run(cfg); // aggressively clean up memory here to try to keep our memory usage low - int err = lfs_testbd_destroy(cfg); + int err = lfs_emubd_destroy(cfg); if (err) { fprintf(stderr, "error: could not destroy block device: %d\n", err); exit(-1); @@ -1479,9 +1480,9 @@ static void run_powerloss_exhaustive_layer( // recurse into each branch for (size_t i = 0; i < state.branch_count; i++) { // first push and print the branch - lfs_testbd_powercycles_t *cycle = mappend( + lfs_emubd_powercycles_t *cycle = mappend( (void**)&cycles->cycles, - sizeof(lfs_testbd_powercycles_t), + sizeof(lfs_emubd_powercycles_t), &cycles->cycle_count, &cycles->cycle_capacity); if (!cycle) { @@ -1509,7 +1510,7 @@ static void run_powerloss_exhaustive_layer( } static void run_powerloss_exhaustive( - const lfs_testbd_powercycles_t *cycles, + const lfs_emubd_powercycles_t *cycles, size_t cycle_count, const struct test_suite *suite, const struct test_case *case_) { @@ -1517,14 +1518,14 @@ static void run_powerloss_exhaustive( (void)suite; // create block device and configuration - lfs_testbd_t bd; + lfs_emubd_t bd; struct lfs_config cfg = { .context = &bd, - .read = lfs_testbd_read, - .prog = lfs_testbd_prog, - .erase = lfs_testbd_erase, - .sync = lfs_testbd_sync, + .read = lfs_emubd_read, + .prog = lfs_emubd_prog, + .erase = lfs_emubd_erase, + .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, .block_size = BLOCK_SIZE, @@ -1534,7 +1535,7 @@ static void run_powerloss_exhaustive( .lookahead_size = LOOKAHEAD_SIZE, }; - struct lfs_testbd_config bdcfg = { + struct lfs_emubd_config bdcfg = { .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1547,7 +1548,7 @@ static void run_powerloss_exhaustive( .powerloss_data = NULL, }; - int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1956,6 +1957,7 @@ int main(int argc, char **argv) { if (parsed == optarg) { goto invalid_define; } + optarg = parsed + strspn(parsed, " "); *(intmax_t*)mappend( (void**)&override->defines, sizeof(intmax_t), @@ -1965,7 +1967,6 @@ int main(int argc, char **argv) { break; } - optarg = parsed + strspn(parsed, " "); if (*optarg == ',') { optarg += 1; } @@ -2041,24 +2042,24 @@ invalid_define: // allow implicit r=p and p=e for common geometries memset(geometry, 0, sizeof(test_geometry_t)); if (count >= 3) { - geometry->defines[0] - = (test_define_t)TEST_LIT(sizes[0]); - geometry->defines[1] - = (test_define_t)TEST_LIT(sizes[1]); - geometry->defines[2] - = (test_define_t)TEST_LIT(sizes[2]); + geometry->defines[READ_SIZE_i] + = TEST_LIT(sizes[0]); + geometry->defines[PROG_SIZE_i] + = TEST_LIT(sizes[1]); + geometry->defines[BLOCK_SIZE_i] + = TEST_LIT(sizes[2]); } else if (count >= 2) { - geometry->defines[1] - = (test_define_t)TEST_LIT(sizes[0]); - geometry->defines[2] - = (test_define_t)TEST_LIT(sizes[1]); + geometry->defines[PROG_SIZE_i] + = TEST_LIT(sizes[0]); + geometry->defines[BLOCK_SIZE_i] + = TEST_LIT(sizes[1]); } else { - geometry->defines[2] - = (test_define_t)TEST_LIT(sizes[0]); + geometry->defines[BLOCK_SIZE_i] + = TEST_LIT(sizes[0]); } if (count >= 4) { - geometry->defines[3] - = (test_define_t)TEST_LIT(sizes[3]); + geometry->defines[BLOCK_COUNT_i] + = TEST_LIT(sizes[3]); } optarg = s; goto geometry_next; @@ -2085,24 +2086,24 @@ invalid_define: // allow implicit r=p and p=e for common geometries memset(geometry, 0, sizeof(test_geometry_t)); if (count >= 3) { - geometry->defines[0] - = (test_define_t)TEST_LIT(sizes[0]); - geometry->defines[1] - = (test_define_t)TEST_LIT(sizes[1]); - geometry->defines[2] - = (test_define_t)TEST_LIT(sizes[2]); + geometry->defines[READ_SIZE_i] + = TEST_LIT(sizes[0]); + geometry->defines[PROG_SIZE_i] + = TEST_LIT(sizes[1]); + geometry->defines[BLOCK_SIZE_i] + = TEST_LIT(sizes[2]); } else if (count >= 2) { - geometry->defines[1] - = (test_define_t)TEST_LIT(sizes[0]); - geometry->defines[2] - = (test_define_t)TEST_LIT(sizes[1]); + geometry->defines[PROG_SIZE_i] + = TEST_LIT(sizes[0]); + geometry->defines[BLOCK_SIZE_i] + = TEST_LIT(sizes[1]); } else { - geometry->defines[2] - = (test_define_t)TEST_LIT(sizes[0]); + geometry->defines[BLOCK_SIZE_i] + = TEST_LIT(sizes[0]); } if (count >= 4) { - geometry->defines[3] - = (test_define_t)TEST_LIT(sizes[3]); + geometry->defines[BLOCK_COUNT_i] + = TEST_LIT(sizes[3]); } optarg = s; goto geometry_next; @@ -2165,16 +2166,16 @@ geometry_next: // comma-separated permutation if (*optarg == '{') { - lfs_testbd_powercycles_t *cycles = NULL; + lfs_emubd_powercycles_t *cycles = NULL; size_t cycle_count = 0; size_t cycle_capacity = 0; char *s = optarg + 1; while (true) { char *parsed = NULL; - *(lfs_testbd_powercycles_t*)mappend( + *(lfs_emubd_powercycles_t*)mappend( (void**)&cycles, - sizeof(lfs_testbd_powercycles_t), + sizeof(lfs_emubd_powercycles_t), &cycle_count, &cycle_capacity) = strtoumax(s, &parsed, 0); @@ -2202,7 +2203,7 @@ geometry_next: // leb16-encoded permutation if (*optarg == ':') { - lfs_testbd_powercycles_t *cycles = NULL; + lfs_emubd_powercycles_t *cycles = NULL; size_t cycle_count = 0; size_t cycle_capacity = 0; @@ -2214,9 +2215,9 @@ geometry_next: break; } - *(lfs_testbd_powercycles_t*)mappend( + *(lfs_emubd_powercycles_t*)mappend( (void**)&cycles, - sizeof(lfs_testbd_powercycles_t), + sizeof(lfs_emubd_powercycles_t), &cycle_count, &cycle_capacity) = x; s = parsed; @@ -2374,7 +2375,7 @@ getopt_done: ; for (; argc > optind; optind++) { test_define_t *defines = NULL; size_t define_count = 0; - lfs_testbd_powercycles_t *cycles = NULL; + lfs_emubd_powercycles_t *cycles = NULL; size_t cycle_count = 0; // parse name, can be suite or case @@ -2422,7 +2423,7 @@ getopt_done: ; (ncount-define_count)*sizeof(test_define_t)); define_count = ncount; } - defines[d] = (test_define_t)TEST_LIT(v); + defines[d] = TEST_LIT(v); } if (cycles_) { @@ -2430,9 +2431,9 @@ getopt_done: ; size_t cycle_capacity = 0; while (*cycles_ != '\0') { char *parsed = NULL; - *(lfs_testbd_powercycles_t*)mappend( + *(lfs_emubd_powercycles_t*)mappend( (void**)&cycles, - sizeof(lfs_testbd_powercycles_t), + sizeof(lfs_emubd_powercycles_t), &cycle_count, &cycle_capacity) = leb16_parse(cycles_, &parsed); diff --git a/runners/test_runner.h b/runners/test_runner.h index e5986acc..f9561509 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -11,11 +11,11 @@ void test_trace(const char *fmt, ...); __LINE__, \ __VA_ARGS__) #define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") -#define LFS_TESTBD_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") +#define LFS_EMUBD_TRACE(...) LFS_TRACE_(__VA_ARGS__, "") // note these are indirectly included in any generated files -#include "bd/lfs_testbd.h" +#include "bd/lfs_emubd.h" #include // give source a chance to define feature macros @@ -62,9 +62,10 @@ struct test_suite { // access generated test defines -//intmax_t test_predefine(size_t define); intmax_t test_define(size_t define); +#define TEST_DEFINE(i) test_define(i) + // a few preconfigured defines that control how tests run #define READ_SIZE_i 0 @@ -79,33 +80,33 @@ intmax_t test_define(size_t define); #define BADBLOCK_BEHAVIOR_i 9 #define POWERLOSS_BEHAVIOR_i 10 -#define READ_SIZE test_define(READ_SIZE_i) -#define PROG_SIZE test_define(PROG_SIZE_i) -#define BLOCK_SIZE test_define(BLOCK_SIZE_i) -#define BLOCK_COUNT test_define(BLOCK_COUNT_i) -#define CACHE_SIZE test_define(CACHE_SIZE_i) -#define LOOKAHEAD_SIZE test_define(LOOKAHEAD_SIZE_i) -#define BLOCK_CYCLES test_define(BLOCK_CYCLES_i) -#define ERASE_VALUE test_define(ERASE_VALUE_i) -#define ERASE_CYCLES test_define(ERASE_CYCLES_i) -#define BADBLOCK_BEHAVIOR test_define(BADBLOCK_BEHAVIOR_i) -#define POWERLOSS_BEHAVIOR test_define(POWERLOSS_BEHAVIOR_i) +#define READ_SIZE TEST_DEFINE(READ_SIZE_i) +#define PROG_SIZE TEST_DEFINE(PROG_SIZE_i) +#define BLOCK_SIZE TEST_DEFINE(BLOCK_SIZE_i) +#define BLOCK_COUNT TEST_DEFINE(BLOCK_COUNT_i) +#define CACHE_SIZE TEST_DEFINE(CACHE_SIZE_i) +#define LOOKAHEAD_SIZE TEST_DEFINE(LOOKAHEAD_SIZE_i) +#define BLOCK_CYCLES TEST_DEFINE(BLOCK_CYCLES_i) +#define ERASE_VALUE TEST_DEFINE(ERASE_VALUE_i) +#define ERASE_CYCLES TEST_DEFINE(ERASE_CYCLES_i) +#define BADBLOCK_BEHAVIOR TEST_DEFINE(BADBLOCK_BEHAVIOR_i) +#define POWERLOSS_BEHAVIOR TEST_DEFINE(POWERLOSS_BEHAVIOR_i) #define TEST_IMPLICIT_DEFINES \ - TEST_DEFINE(READ_SIZE, PROG_SIZE) \ - TEST_DEFINE(PROG_SIZE, BLOCK_SIZE) \ - TEST_DEFINE(BLOCK_SIZE, 0) \ - TEST_DEFINE(BLOCK_COUNT, (1024*1024)/BLOCK_SIZE) \ - TEST_DEFINE(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ - TEST_DEFINE(LOOKAHEAD_SIZE, 16) \ - TEST_DEFINE(BLOCK_CYCLES, -1) \ - TEST_DEFINE(ERASE_VALUE, 0xff) \ - TEST_DEFINE(ERASE_CYCLES, 0) \ - TEST_DEFINE(BADBLOCK_BEHAVIOR, LFS_TESTBD_BADBLOCK_PROGERROR) \ - TEST_DEFINE(POWERLOSS_BEHAVIOR, LFS_TESTBD_POWERLOSS_NOOP) + TEST_DEF(READ_SIZE, PROG_SIZE) \ + TEST_DEF(PROG_SIZE, BLOCK_SIZE) \ + TEST_DEF(BLOCK_SIZE, 0) \ + TEST_DEF(BLOCK_COUNT, (1024*1024)/BLOCK_SIZE) \ + TEST_DEF(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ + TEST_DEF(LOOKAHEAD_SIZE, 16) \ + TEST_DEF(BLOCK_CYCLES, -1) \ + TEST_DEF(ERASE_VALUE, 0xff) \ + TEST_DEF(ERASE_CYCLES, 0) \ + TEST_DEF(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR) \ + TEST_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP) -#define TEST_GEOMETRY_DEFINE_COUNT 4 #define TEST_IMPLICIT_DEFINE_COUNT 11 +#define TEST_GEOMETRY_DEFINE_COUNT 4 #endif diff --git a/scripts/bench.py b/scripts/bench.py new file mode 100755 index 00000000..93c18a2d --- /dev/null +++ b/scripts/bench.py @@ -0,0 +1,1355 @@ +#!/usr/bin/env python3 +# +# Script to compile and runs benches. +# +# Example: +# ./scripts/bench.py runners/bench_runner -b +# +# Copyright (c) 2022, The littlefs authors. +# SPDX-License-Identifier: BSD-3-Clause +# + +import collections as co +import csv +import errno +import glob +import itertools as it +import math as m +import os +import pty +import re +import shlex +import shutil +import signal +import subprocess as sp +import threading as th +import time +import toml + + +RUNNER_PATH = 'runners/bench_runner' +HEADER_PATH = 'runners/bench_runner.h' + + +def openio(path, mode='r', buffering=-1, nb=False): + if path == '-': + if mode == 'r': + return os.fdopen(os.dup(sys.stdin.fileno()), 'r', buffering) + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w', buffering) + elif nb and 'a' in mode: + return os.fdopen(os.open( + path, + os.O_WRONLY | os.O_CREAT | os.O_APPEND | os.O_NONBLOCK, + 0o666), + mode, + buffering) + else: + return open(path, mode, buffering) + +class BenchCase: + # create a BenchCase object from a config + def __init__(self, config, args={}): + self.name = config.pop('name') + self.path = config.pop('path') + self.suite = config.pop('suite') + self.lineno = config.pop('lineno', None) + self.if_ = config.pop('if', None) + if isinstance(self.if_, bool): + self.if_ = 'true' if self.if_ else 'false' + self.code = config.pop('code') + self.code_lineno = config.pop('code_lineno', None) + self.in_ = config.pop('in', + config.pop('suite_in', None)) + + # figure out defines and build possible permutations + self.defines = set() + self.permutations = [] + + # defines can be a dict or a list or dicts + suite_defines = config.pop('suite_defines', {}) + if not isinstance(suite_defines, list): + suite_defines = [suite_defines] + defines = config.pop('defines', {}) + if not isinstance(defines, list): + defines = [defines] + + def csplit(v): + # split commas but only outside of parens + parens = 0 + i_ = 0 + for i in range(len(v)): + if v[i] == ',' and parens == 0: + yield v[i_:i] + i_ = i+1 + elif v[i] in '([{': + parens += 1 + elif v[i] in '}])': + parens -= 1 + if v[i_:].strip(): + yield v[i_:] + + def parse_define(v): + # a define entry can be a list + if isinstance(v, list): + for v_ in v: + yield from parse_define(v_) + # or a string + elif isinstance(v, str): + # which can be comma-separated values, with optional + # range statements. This matches the runtime define parser in + # the runner itself. + for v_ in csplit(v): + m = re.search(r'\brange\b\s*\(' + '(?P[^,\s]*)' + '\s*(?:,\s*(?P[^,\s]*)' + '\s*(?:,\s*(?P[^,\s]*)\s*)?)?\)', + v_) + if m: + start = (int(m.group('start'), 0) + if m.group('start') else 0) + stop = (int(m.group('stop'), 0) + if m.group('stop') else None) + step = (int(m.group('step'), 0) + if m.group('step') else 1) + if m.lastindex <= 1: + start, stop = 0, start + for x in range(start, stop, step): + yield from parse_define('%s(%d)%s' % ( + v_[:m.start()], x, v_[m.end():])) + else: + yield v_ + # or a literal value + else: + yield v + + # build possible permutations + for suite_defines_ in suite_defines: + self.defines |= suite_defines_.keys() + for defines_ in defines: + self.defines |= defines_.keys() + self.permutations.extend(dict(perm) for perm in it.product(*( + [(k, v) for v in parse_define(vs)] + for k, vs in sorted((suite_defines_ | defines_).items())))) + + for k in config.keys(): + print('%swarning:%s in %s, found unused key %r' % ( + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + self.name, + k), + file=sys.stderr) + + +class BenchSuite: + # create a BenchSuite object from a toml file + def __init__(self, path, args={}): + self.path = path + self.name = os.path.basename(path) + if self.name.endswith('.toml'): + self.name = self.name[:-len('.toml')] + + # load toml file and parse bench cases + with open(self.path) as f: + # load benches + config = toml.load(f) + + # find line numbers + f.seek(0) + case_linenos = [] + code_linenos = [] + for i, line in enumerate(f): + match = re.match( + '(?P\[\s*cases\s*\.\s*(?P\w+)\s*\])' + '|' '(?Pcode\s*=)', + line) + if match and match.group('case'): + case_linenos.append((i+1, match.group('name'))) + elif match and match.group('code'): + code_linenos.append(i+2) + + # sort in case toml parsing did not retain order + case_linenos.sort() + + cases = config.pop('cases') + for (lineno, name), (nlineno, _) in it.zip_longest( + case_linenos, case_linenos[1:], + fillvalue=(float('inf'), None)): + code_lineno = min( + (l for l in code_linenos if l >= lineno and l < nlineno), + default=None) + cases[name]['lineno'] = lineno + cases[name]['code_lineno'] = code_lineno + + self.if_ = config.pop('if', None) + if isinstance(self.if_, bool): + self.if_ = 'true' if self.if_ else 'false' + + self.code = config.pop('code', None) + self.code_lineno = min( + (l for l in code_linenos + if not case_linenos or l < case_linenos[0][0]), + default=None) + + # a couple of these we just forward to all cases + defines = config.pop('defines', {}) + in_ = config.pop('in', None) + + self.cases = [] + for name, case in sorted(cases.items(), + key=lambda c: c[1].get('lineno')): + self.cases.append(BenchCase(config={ + 'name': name, + 'path': path + (':%d' % case['lineno'] + if 'lineno' in case else ''), + 'suite': self.name, + 'suite_defines': defines, + 'suite_in': in_, + **case}, + args=args)) + + # combine per-case defines + self.defines = set.union(*( + set(case.defines) for case in self.cases)) + + for k in config.keys(): + print('%swarning:%s in %s, found unused key %r' % ( + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + self.name, + k), + file=sys.stderr) + + + +def compile(bench_paths, **args): + # find .toml files + paths = [] + for path in bench_paths: + if os.path.isdir(path): + path = path + '/*.toml' + + for path in glob.glob(path): + paths.append(path) + + if not paths: + print('no bench suites found in %r?' % bench_paths) + sys.exit(-1) + + # load the suites + suites = [BenchSuite(path, args) for path in paths] + suites.sort(key=lambda s: s.name) + + # check for name conflicts, these will cause ambiguity problems later + # when running benches + seen = {} + for suite in suites: + if suite.name in seen: + print('%swarning:%s conflicting suite %r, %s and %s' % ( + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + suite.name, + suite.path, + seen[suite.name].path), + file=sys.stderr) + seen[suite.name] = suite + + for case in suite.cases: + # only allow conflicts if a case and its suite share a name + if case.name in seen and not ( + isinstance(seen[case.name], BenchSuite) + and seen[case.name].cases == [case]): + print('%swarning:%s conflicting case %r, %s and %s' % ( + '\x1b[01;33m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + case.name, + case.path, + seen[case.name].path), + file=sys.stderr) + seen[case.name] = case + + # we can only compile one bench suite at a time + if not args.get('source'): + if len(suites) > 1: + print('more than one bench suite for compilation? (%r)' % bench_paths) + sys.exit(-1) + + suite = suites[0] + + # write generated bench source + if 'output' in args: + with openio(args['output'], 'w') as f: + _write = f.write + def write(s): + f.lineno += s.count('\n') + _write(s) + def writeln(s=''): + f.lineno += s.count('\n') + 1 + _write(s) + _write('\n') + f.lineno = 1 + f.write = write + f.writeln = writeln + + f.writeln("// Generated by %s:" % sys.argv[0]) + f.writeln("//") + f.writeln("// %s" % ' '.join(sys.argv)) + f.writeln("//") + f.writeln() + + # include bench_runner.h in every generated file + f.writeln("#include \"%s\"" % args['include']) + f.writeln() + + # write out generated functions, this can end up in different + # files depending on the "in" attribute + # + # note it's up to the specific generated file to declare + # the bench defines + def write_case_functions(f, suite, case): + # create case define functions + if case.defines: + # deduplicate defines by value to try to reduce the + # number of functions we generate + define_cbs = {} + for i, defines in enumerate(case.permutations): + for k, v in sorted(defines.items()): + if v not in define_cbs: + name = ('__bench__%s__%s__%s__%d' + % (suite.name, case.name, k, i)) + define_cbs[v] = name + f.writeln('intmax_t %s(' + '__attribute__((unused)) ' + 'void *data) {' % name) + f.writeln(4*' '+'return %s;' % v) + f.writeln('}') + f.writeln() + f.writeln('const bench_define_t ' + '__bench__%s__%s__defines[][' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d] = {' + % (suite.name, case.name, len(suite.defines))) + for defines in case.permutations: + f.writeln(4*' '+'{') + for k, v in sorted(defines.items()): + f.writeln(8*' '+'[%-24s] = {%s, NULL},' % ( + k+'_i', define_cbs[v])) + f.writeln(4*' '+'},') + f.writeln('};') + f.writeln() + + # create case filter function + if suite.if_ is not None or case.if_ is not None: + f.writeln('bool __bench__%s__%s__filter(void) {' + % (suite.name, case.name)) + f.writeln(4*' '+'return %s;' + % ' && '.join('(%s)' % if_ + for if_ in [suite.if_, case.if_] + if if_ is not None)) + f.writeln('}') + f.writeln() + + # create case run function + f.writeln('void __bench__%s__%s__run(' + '__attribute__((unused)) struct lfs_config *cfg) {' + % (suite.name, case.name)) + f.writeln(4*' '+'// bench case %s' % case.name) + if case.code_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (case.code_lineno, suite.path)) + f.write(case.code) + if case.code_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (f.lineno+1, args['output'])) + f.writeln('}') + f.writeln() + + if not args.get('source'): + if suite.code is not None: + if suite.code_lineno is not None: + f.writeln('#line %d "%s"' + % (suite.code_lineno, suite.path)) + f.write(suite.code) + if suite.code_lineno is not None: + f.writeln('#line %d "%s"' + % (f.lineno+1, args['output'])) + f.writeln() + + if suite.defines: + for i, define in enumerate(sorted(suite.defines)): + f.writeln('#ifndef %s' % define) + f.writeln('#define %-24s ' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d' % (define+'_i', i)) + f.writeln('#define %-24s ' + 'BENCH_DEFINE(%s)' % (define, define+'_i')) + f.writeln('#endif') + f.writeln() + + # create case functions + for case in suite.cases: + if case.in_ is None: + write_case_functions(f, suite, case) + else: + if case.defines: + f.writeln('extern const bench_define_t ' + '__bench__%s__%s__defines[][' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d];' + % (suite.name, case.name, len(suite.defines))) + if suite.if_ is not None or case.if_ is not None: + f.writeln('extern bool __bench__%s__%s__filter(' + 'void);' + % (suite.name, case.name)) + f.writeln('extern void __bench__%s__%s__run(' + 'struct lfs_config *cfg);' + % (suite.name, case.name)) + f.writeln() + + # create suite struct + # + # note we place this in the custom bench_suites section with + # minimum alignment, otherwise GCC ups the alignment to + # 32-bytes for some reason + f.writeln('__attribute__((section("_bench_suites"), ' + 'aligned(1)))') + f.writeln('const struct bench_suite __bench__%s__suite = {' + % suite.name) + f.writeln(4*' '+'.name = "%s",' % suite.name) + f.writeln(4*' '+'.path = "%s",' % suite.path) + f.writeln(4*' '+'.flags = 0,') + if suite.defines: + # create suite define names + f.writeln(4*' '+'.define_names = (const char *const[' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d]){' % ( + len(suite.defines))) + for k in sorted(suite.defines): + f.writeln(8*' '+'[%-24s] = "%s",' % (k+'_i', k)) + f.writeln(4*' '+'},') + f.writeln(4*' '+'.define_count = ' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d,' % len(suite.defines)) + f.writeln(4*' '+'.cases = (const struct bench_case[]){') + for case in suite.cases: + # create case structs + f.writeln(8*' '+'{') + f.writeln(12*' '+'.name = "%s",' % case.name) + f.writeln(12*' '+'.path = "%s",' % case.path) + f.writeln(12*' '+'.flags = 0,') + f.writeln(12*' '+'.permutations = %d,' + % len(case.permutations)) + if case.defines: + f.writeln(12*' '+'.defines ' + '= (const bench_define_t*)__bench__%s__%s__defines,' + % (suite.name, case.name)) + if suite.if_ is not None or case.if_ is not None: + f.writeln(12*' '+'.filter = __bench__%s__%s__filter,' + % (suite.name, case.name)) + f.writeln(12*' '+'.run = __bench__%s__%s__run,' + % (suite.name, case.name)) + f.writeln(8*' '+'},') + f.writeln(4*' '+'},') + f.writeln(4*' '+'.case_count = %d,' % len(suite.cases)) + f.writeln('};') + f.writeln() + + else: + # copy source + f.writeln('#line 1 "%s"' % args['source']) + with open(args['source']) as sf: + shutil.copyfileobj(sf, f) + f.writeln() + + # write any internal benches + for suite in suites: + for case in suite.cases: + if (case.in_ is not None + and os.path.normpath(case.in_) + == os.path.normpath(args['source'])): + # write defines, but note we need to undef any + # new defines since we're in someone else's file + if suite.defines: + for i, define in enumerate( + sorted(suite.defines)): + f.writeln('#ifndef %s' % define) + f.writeln('#define %-24s ' + 'BENCH_IMPLICIT_DEFINE_COUNT+%d' % ( + define+'_i', i)) + f.writeln('#define %-24s ' + 'BENCH_DEFINE(%s)' % ( + define, define+'_i')) + f.writeln('#define ' + '__BENCH__%s__NEEDS_UNDEF' % ( + define)) + f.writeln('#endif') + f.writeln() + + write_case_functions(f, suite, case) + + if suite.defines: + for define in sorted(suite.defines): + f.writeln('#ifdef __BENCH__%s__NEEDS_UNDEF' + % define) + f.writeln('#undef __BENCH__%s__NEEDS_UNDEF' + % define) + f.writeln('#undef %s' % define) + f.writeln('#undef %s' % (define+'_i')) + f.writeln('#endif') + f.writeln() + +def find_runner(runner, **args): + cmd = runner.copy() + + # run under some external command? + if args.get('exec'): + cmd[:0] = args['exec'] + + # run under valgrind? + if args.get('valgrind'): + cmd[:0] = filter(None, [ + 'valgrind', + '--leak-check=full', + '--track-origins=yes', + '--error-exitcode=4', + '-q']) + + # other context + if args.get('geometry'): + cmd.append('-g%s' % args['geometry']) + if args.get('disk'): + cmd.append('-d%s' % args['disk']) + if args.get('trace'): + cmd.append('-t%s' % args['trace']) + if args.get('read_sleep'): + cmd.append('--read-sleep=%s' % args['read_sleep']) + if args.get('prog_sleep'): + cmd.append('--prog-sleep=%s' % args['prog_sleep']) + if args.get('erase_sleep'): + cmd.append('--erase-sleep=%s' % args['erase_sleep']) + + # defines? + if args.get('define'): + for define in args.get('define'): + cmd.append('-D%s' % define) + + return cmd + +def list_(runner, bench_ids=[], **args): + cmd = find_runner(runner, **args) + bench_ids + if args.get('summary'): cmd.append('--summary') + if args.get('list_suites'): cmd.append('--list-suites') + if args.get('list_cases'): cmd.append('--list-cases') + if args.get('list_suite_paths'): cmd.append('--list-suite-paths') + if args.get('list_case_paths'): cmd.append('--list-case-paths') + if args.get('list_defines'): cmd.append('--list-defines') + if args.get('list_permutation_defines'): + cmd.append('--list-permutation-defines') + if args.get('list_implicit_defines'): + cmd.append('--list-implicit-defines') + if args.get('list_geometries'): cmd.append('--list-geometries') + + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + return sp.call(cmd) + + +def find_perms(runner_, ids=[], **args): + case_suites = {} + expected_case_perms = co.defaultdict(lambda: 0) + expected_perms = 0 + total_perms = 0 + + # query cases from the runner + cmd = runner_ + ['--list-cases'] + ids + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + pattern = re.compile( + '^(?P[^\s]+)' + '\s+(?P[^\s]+)' + '\s+(?P\d+)/(?P\d+)') + # skip the first line + for line in it.islice(proc.stdout, 1, None): + m = pattern.match(line) + if m: + filtered = int(m.group('filtered')) + perms = int(m.group('perms')) + expected_case_perms[m.group('case')] += filtered + expected_perms += filtered + total_perms += perms + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + # get which suite each case belongs to via paths + cmd = runner_ + ['--list-case-paths'] + ids + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + pattern = re.compile( + '^(?P[^\s]+)' + '\s+(?P[^:]+):(?P\d+)') + # skip the first line + for line in it.islice(proc.stdout, 1, None): + m = pattern.match(line) + if m: + path = m.group('path') + # strip path/suffix here + suite = os.path.basename(path) + if suite.endswith('.toml'): + suite = suite[:-len('.toml')] + case_suites[m.group('case')] = suite + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + # figure out expected suite perms + expected_suite_perms = co.defaultdict(lambda: 0) + for case, suite in case_suites.items(): + expected_suite_perms[suite] += expected_case_perms[case] + + return ( + case_suites, + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) + +def find_path(runner_, id, **args): + path = None + # query from runner + cmd = runner_ + ['--list-case-paths', id] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + pattern = re.compile( + '^(?P[^\s]+)' + '\s+(?P[^:]+):(?P\d+)') + # skip the first line + for line in it.islice(proc.stdout, 1, None): + m = pattern.match(line) + if m and path is None: + path_ = m.group('path') + lineno = int(m.group('lineno')) + path = (path_, lineno) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return path + +def find_defines(runner_, id, **args): + # query permutation defines from runner + cmd = runner_ + ['--list-permutation-defines', id] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + defines = co.OrderedDict() + pattern = re.compile('^(?P\w+)=(?P.+)') + for line in proc.stdout: + m = pattern.match(line) + if m: + define = m.group('define') + value = m.group('value') + defines[define] = value + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return defines + + +# Thread-safe CSV writer +class BenchOutput: + def __init__(self, path, head=None, tail=None): + self.f = openio(path, 'w+', 1) + self.lock = th.Lock() + self.head = head or [] + self.tail = tail or [] + self.writer = csv.DictWriter(self.f, self.head + self.tail) + self.rows = [] + + def close(self): + self.f.close() + + def __enter__(self): + return self + + def __exit__(self, *_): + self.f.close() + + def writerow(self, row): + with self.lock: + self.rows.append(row) + if all(k in self.head or k in self.tail for k in row.keys()): + # can simply append + self.writer.writerow(row) + else: + # need to rewrite the file + self.head.extend(row.keys() - (self.head + self.tail)) + self.f.seek(0) + self.f.truncate() + self.writer = csv.DictWriter(self.f, self.head + self.tail) + self.writer.writeheader() + for row in self.rows: + self.writer.writerow(row) + +# A bench failure +class BenchFailure(Exception): + def __init__(self, id, returncode, stdout, assert_=None): + self.id = id + self.returncode = returncode + self.stdout = stdout + self.assert_ = assert_ + +def run_stage(name, runner_, ids, output_, **args): + # get expected suite/case/perm counts + (case_suites, + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) = find_perms(runner_, ids, **args) + + passed_suite_perms = co.defaultdict(lambda: 0) + passed_case_perms = co.defaultdict(lambda: 0) + passed_perms = 0 + read = 0 + prog = 0 + erased = 0 + failures = [] + killed = False + + pattern = re.compile('^(?:' + '(?Prunning|finished|skipped|powerloss)' + ' (?P(?P[^:]+)[^\s]*)' + '(?: (?P\d+))?' + '(?: (?P\d+))?' + '(?: (?P\d+))?' + '|' '(?P[^:]+):(?P\d+):(?Passert):' + ' *(?P.*)' + ')$') + locals = th.local() + children = set() + + def run_runner(runner_, ids=[]): + nonlocal passed_suite_perms + nonlocal passed_case_perms + nonlocal passed_perms + nonlocal read + nonlocal prog + nonlocal erased + nonlocal locals + + # run the benches! + cmd = runner_ + ids + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + + mpty, spty = pty.openpty() + proc = sp.Popen(cmd, stdout=spty, stderr=spty, close_fds=False) + os.close(spty) + children.add(proc) + mpty = os.fdopen(mpty, 'r', 1) + stdout = None + + last_id = None + last_stdout = [] + last_assert = None + try: + while True: + # parse a line for state changes + try: + line = mpty.readline() + except OSError as e: + if e.errno == errno.EIO: + break + raise + if not line: + break + last_stdout.append(line) + if args.get('stdout'): + try: + if not stdout: + stdout = openio(args['stdout'], 'a', 1, nb=True) + stdout.write(line) + except OSError as e: + if e.errno not in [ + errno.ENXIO, + errno.EPIPE, + errno.EAGAIN]: + raise + stdout = None + if args.get('verbose'): + sys.stdout.write(line) + + m = pattern.match(line) + if m: + op = m.group('op') or m.group('op_') + if op == 'running': + locals.seen_perms += 1 + last_id = m.group('id') + last_stdout = [] + last_assert = None + elif op == 'finished': + case = m.group('case') + suite = case_suites[case] + read_ = int(m.group('read')) + prog_ = int(m.group('prog')) + erased_ = int(m.group('erased')) + passed_suite_perms[suite] += 1 + passed_case_perms[case] += 1 + passed_perms += 1 + read += read_ + prog += prog_ + erased += erased_ + if output_: + # get defines and write to csv + defines = find_defines( + runner_, m.group('id'), **args) + output_.writerow({ + 'suite': suite, + 'case': case, + 'bench_read': read_, + 'bench_prog': prog_, + 'bench_erased': erased_, + **defines}) + elif op == 'skipped': + locals.seen_perms += 1 + elif op == 'assert': + last_assert = ( + m.group('path'), + int(m.group('lineno')), + m.group('message')) + # go ahead and kill the process, aborting takes a while + if args.get('keep_going'): + proc.kill() + except KeyboardInterrupt: + raise BenchFailure(last_id, 1, last_stdout) + finally: + children.remove(proc) + mpty.close() + + proc.wait() + if proc.returncode != 0: + raise BenchFailure( + last_id, + proc.returncode, + last_stdout, + last_assert) + + def run_job(runner_, ids=[], start=None, step=None): + nonlocal failures + nonlocal killed + nonlocal locals + + start = start or 0 + step = step or 1 + while start < total_perms: + job_runner = runner_.copy() + if args.get('isolate') or args.get('valgrind'): + job_runner.append('-s%s,%s,%s' % (start, start+step, step)) + else: + job_runner.append('-s%s,,%s' % (start, step)) + + try: + # run the benches + locals.seen_perms = 0 + run_runner(job_runner, ids) + assert locals.seen_perms > 0 + start += locals.seen_perms*step + + except BenchFailure as failure: + # keep track of failures + if output_: + case, _ = failure.id.split(':', 1) + suite = case_suites[case] + # get defines and write to csv + defines = find_defines(runner_, failure.id, **args) + output_.writerow({ + 'suite': suite, + 'case': case, + **defines}) + + # race condition for multiple failures? + if failures and not args.get('keep_going'): + break + + failures.append(failure) + + if args.get('keep_going') and not killed: + # resume after failed bench + assert locals.seen_perms > 0 + start += locals.seen_perms*step + continue + else: + # stop other benches + killed = True + for child in children.copy(): + child.kill() + break + + + # parallel jobs? + runners = [] + if 'jobs' in args: + for job in range(args['jobs']): + runners.append(th.Thread( + target=run_job, args=(runner_, ids, job, args['jobs']), + daemon=True)) + else: + runners.append(th.Thread( + target=run_job, args=(runner_, ids, None, None), + daemon=True)) + + def print_update(done): + if not args.get('verbose') and (args['color'] or done): + sys.stdout.write('%s%srunning %s%s:%s %s%s' % ( + '\r\x1b[K' if args['color'] else '', + '\x1b[?7l' if not done else '', + ('\x1b[34m' if not failures else '\x1b[31m') + if args['color'] else '', + name, + '\x1b[m' if args['color'] else '', + ', '.join(filter(None, [ + '%d/%d suites' % ( + sum(passed_suite_perms[k] == v + for k, v in expected_suite_perms.items()), + len(expected_suite_perms)) + if (not args.get('by_suites') + and not args.get('by_cases')) else None, + '%d/%d cases' % ( + sum(passed_case_perms[k] == v + for k, v in expected_case_perms.items()), + len(expected_case_perms)) + if not args.get('by_cases') else None, + '%d/%d perms' % (passed_perms, expected_perms), + '%s%d/%d failures%s' % ( + '\x1b[31m' if args['color'] else '', + len(failures), + expected_perms, + '\x1b[m' if args['color'] else '') + if failures else None])), + '\x1b[?7h' if not done else '\n')) + sys.stdout.flush() + + for r in runners: + r.start() + + try: + while any(r.is_alive() for r in runners): + time.sleep(0.01) + print_update(False) + except KeyboardInterrupt: + # this is handled by the runner threads, we just + # need to not abort here + killed = True + finally: + print_update(True) + + for r in runners: + r.join() + + return ( + expected_perms, + passed_perms, + read, + prog, + erased, + failures, + killed) + + +def run(runner, bench_ids=[], **args): + # query runner for benches + runner_ = find_runner(runner, **args) + print('using runner: %s' % ' '.join(shlex.quote(c) for c in runner_)) + (_, + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) = find_perms(runner_, bench_ids, **args) + print('found %d suites, %d cases, %d/%d permutations' % ( + len(expected_suite_perms), + len(expected_case_perms), + expected_perms, + total_perms)) + print() + + # truncate and open logs here so they aren't disconnected between benches + stdout = None + if args.get('stdout'): + stdout = openio(args['stdout'], 'w', 1) + trace = None + if args.get('trace'): + trace = openio(args['trace'], 'w', 1) + output = None + if args.get('output'): + output = BenchOutput(args['output'], + ['suite', 'case'], + ['bench_read', 'bench_prog', 'bench_erased']) + + # measure runtime + start = time.time() + + # spawn runners + expected = 0 + passed = 0 + read = 0 + prog = 0 + erased = 0 + failures = [] + for by in (expected_case_perms.keys() if args.get('by_cases') + else expected_suite_perms.keys() if args.get('by_suites') + else [None]): + # spawn jobs for stage + (expected_, + passed_, + read_, + prog_, + erased_, + failures_, + killed) = run_stage( + by or 'benches', + runner_, + [by] if by is not None else bench_ids, + output, + **args) + # collect passes/failures + expected += expected_ + passed += passed_ + read += read_ + prog += prog_ + erased += erased_ + failures.extend(failures_) + if (failures and not args.get('keep_going')) or killed: + break + + stop = time.time() + + if stdout: + stdout.close() + if trace: + trace.close() + if output: + output.close() + + # show summary + print() + print('%sdone:%s %s' % ( + ('\x1b[34m' if not failures else '\x1b[31m') + if args['color'] else '', + '\x1b[m' if args['color'] else '', + ', '.join(filter(None, [ + '%d read' % read, + '%d prog' % prog, + '%d erased' % erased, + 'in %.2fs' % (stop-start)])))) + print() + + # print each failure + for failure in failures: + assert failure.id is not None, '%s broken? %r' % ( + ' '.join(shlex.quote(c) for c in runner_), + failure) + + # get some extra info from runner + path, lineno = find_path(runner_, failure.id, **args) + defines = find_defines(runner_, failure.id, **args) + + # show summary of failure + print('%s%s:%d:%sfailure:%s %s%s failed' % ( + '\x1b[01m' if args['color'] else '', + path, lineno, + '\x1b[01;31m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + failure.id, + ' (%s)' % ', '.join('%s=%s' % (k,v) for k,v in defines.items()) + if defines else '')) + + if failure.stdout: + stdout = failure.stdout + if failure.assert_ is not None: + stdout = stdout[:-1] + for line in stdout[-args.get('context', 5):]: + sys.stdout.write(line) + + if failure.assert_ is not None: + path, lineno, message = failure.assert_ + print('%s%s:%d:%sassert:%s %s' % ( + '\x1b[01m' if args['color'] else '', + path, lineno, + '\x1b[01;31m' if args['color'] else '', + '\x1b[m' if args['color'] else '', + message)) + with open(path) as f: + line = next(it.islice(f, lineno-1, None)).strip('\n') + print(line) + print() + + # drop into gdb? + if failures and (args.get('gdb') + or args.get('gdb_case') + or args.get('gdb_main')): + failure = failures[0] + cmd = runner_ + [failure.id] + + if args.get('gdb_main'): + cmd[:0] = ['gdb', + '-ex', 'break main', + '-ex', 'run', + '--args'] + elif args.get('gdb_case'): + path, lineno = find_path(runner_, failure.id, **args) + cmd[:0] = ['gdb', + '-ex', 'break %s:%d' % (path, lineno), + '-ex', 'run', + '--args'] + elif failure.assert_ is not None: + cmd[:0] = ['gdb', + '-ex', 'run', + '-ex', 'frame function raise', + '-ex', 'up 2', + '--args'] + else: + cmd[:0] = ['gdb', + '-ex', 'run', + '--args'] + + # exec gdb interactively + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + os.execvp(cmd[0], cmd) + + return 1 if failures else 0 + + +def main(**args): + # figure out what color should be + if args.get('color') == 'auto': + args['color'] = sys.stdout.isatty() + elif args.get('color') == 'always': + args['color'] = True + else: + args['color'] = False + + if args.get('compile'): + return compile(**args) + elif (args.get('summary') + or args.get('list_suites') + or args.get('list_cases') + or args.get('list_suite_paths') + or args.get('list_case_paths') + or args.get('list_defines') + or args.get('list_permutation_defines') + or args.get('list_implicit_defines') + or args.get('list_geometries')): + return list_(**args) + else: + return run(**args) + + +if __name__ == "__main__": + import argparse + import sys + argparse.ArgumentParser._handle_conflict_ignore = lambda *_: None + argparse._ArgumentGroup._handle_conflict_ignore = lambda *_: None + parser = argparse.ArgumentParser( + description="Build and run benches.", + conflict_handler='ignore') + parser.add_argument( + '-v', '--verbose', + action='store_true', + help="Output commands that run behind the scenes.") + parser.add_argument( + '--color', + choices=['never', 'always', 'auto'], + default='auto', + help="When to use terminal colors. Defaults to 'auto'.") + + # bench flags + bench_parser = parser.add_argument_group('bench options') + bench_parser.add_argument( + 'runner', + nargs='?', + type=lambda x: x.split(), + help="Bench runner to use for benching. Defaults to %r." % RUNNER_PATH) + bench_parser.add_argument( + 'bench_ids', + nargs='*', + help="Description of benches to run.") + bench_parser.add_argument( + '-Y', '--summary', + action='store_true', + help="Show quick summary.") + bench_parser.add_argument( + '-l', '--list-suites', + action='store_true', + help="List bench suites.") + bench_parser.add_argument( + '-L', '--list-cases', + action='store_true', + help="List bench cases.") + bench_parser.add_argument( + '--list-suite-paths', + action='store_true', + help="List the path for each bench suite.") + bench_parser.add_argument( + '--list-case-paths', + action='store_true', + help="List the path and line number for each bench case.") + bench_parser.add_argument( + '--list-defines', + action='store_true', + help="List all defines in this bench-runner.") + bench_parser.add_argument( + '--list-permutation-defines', + action='store_true', + help="List explicit defines in this bench-runner.") + bench_parser.add_argument( + '--list-implicit-defines', + action='store_true', + help="List implicit defines in this bench-runner.") + bench_parser.add_argument( + '--list-geometries', + action='store_true', + help="List the available disk geometries.") + bench_parser.add_argument( + '-D', '--define', + action='append', + help="Override a bench define.") + bench_parser.add_argument( + '-g', '--geometry', + help="Comma-separated list of disk geometries to bench. " + "Defaults to d,e,E,n,N.") + bench_parser.add_argument( + '-d', '--disk', + help="Direct block device operations to this file.") + bench_parser.add_argument( + '-t', '--trace', + help="Direct trace output to this file.") + bench_parser.add_argument( + '-O', '--stdout', + help="Direct stdout to this file. Note stderr is already merged here.") + bench_parser.add_argument( + '-o', '--output', + help="CSV file to store results.") + bench_parser.add_argument( + '--read-sleep', + help="Artificial read delay in seconds.") + bench_parser.add_argument( + '--prog-sleep', + help="Artificial prog delay in seconds.") + bench_parser.add_argument( + '--erase-sleep', + help="Artificial erase delay in seconds.") + bench_parser.add_argument( + '-j', '--jobs', + nargs='?', + type=lambda x: int(x, 0), + const=len(os.sched_getaffinity(0)), + help="Number of parallel runners to run.") + bench_parser.add_argument( + '-k', '--keep-going', + action='store_true', + help="Don't stop on first error.") + bench_parser.add_argument( + '-i', '--isolate', + action='store_true', + help="Run each bench permutation in a separate process.") + bench_parser.add_argument( + '-b', '--by-suites', + action='store_true', + help="Step through benches by suite.") + bench_parser.add_argument( + '-B', '--by-cases', + action='store_true', + help="Step through benches by case.") + bench_parser.add_argument( + '--context', + type=lambda x: int(x, 0), + default=5, + help="Show this many lines of stdout on bench failure. " + "Defaults to 5.") + bench_parser.add_argument( + '--gdb', + action='store_true', + help="Drop into gdb on bench failure.") + bench_parser.add_argument( + '--gdb-case', + action='store_true', + help="Drop into gdb on bench failure but stop at the beginning " + "of the failing bench case.") + bench_parser.add_argument( + '--gdb-main', + action='store_true', + help="Drop into gdb on bench failure but stop at the beginning " + "of main.") + bench_parser.add_argument( + '--exec', + type=lambda e: e.split(), + help="Run under another executable.") + bench_parser.add_argument( + '--valgrind', + action='store_true', + help="Run under Valgrind to find memory errors. Implicitly sets " + "--isolate.") + + # compilation flags + comp_parser = parser.add_argument_group('compilation options') + comp_parser.add_argument( + 'bench_paths', + nargs='*', + help="Description of *.toml files to compile. May be a directory " + "or a list of paths.") + comp_parser.add_argument( + '-c', '--compile', + action='store_true', + help="Compile a bench suite or source file.") + comp_parser.add_argument( + '-s', '--source', + help="Source file to compile, possibly injecting internal benches.") + comp_parser.add_argument( + '--include', + default=HEADER_PATH, + help="Inject this header file into every compiled bench file. " + "Defaults to %r." % HEADER_PATH) + comp_parser.add_argument( + '-o', '--output', + help="Output file.") + + # runner + bench_ids overlaps bench_paths, so we need to do some munging here + args = parser.parse_intermixed_args() + args.bench_paths = [' '.join(args.runner or [])] + args.bench_ids + args.runner = args.runner or [RUNNER_PATH] + + sys.exit(main(**{k: v + for k, v in vars(args).items() + if v is not None})) diff --git a/scripts/summary.py b/scripts/summary.py index 680a7150..0855ffb2 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -28,7 +28,8 @@ MERGES = { 'add': ( ['code_size', 'data_size', 'stack_frame', 'struct_size', 'coverage_lines', 'coverage_branches', - 'test_passed'], + 'test_passed', + 'bench_read', 'bench_prog', 'bench_erased'], lambda xs: sum(xs[1:], start=xs[0]) ), 'mul': ( diff --git a/scripts/tailpipe.py b/scripts/tailpipe.py index c9e742ca..08213cf6 100755 --- a/scripts/tailpipe.py +++ b/scripts/tailpipe.py @@ -48,7 +48,7 @@ def main(path='-', *, lines=1, sleep=0.01, keep_open=False): if not keep_open: break # don't just flood open calls - time.sleep(sleep) + time.sleep(sleep or 0.1) done = True th.Thread(target=read, daemon=True).start() diff --git a/scripts/test.py b/scripts/test.py index 42bc308a..71688547 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -388,7 +388,7 @@ def compile(test_paths, **args): f.writeln('#define %-24s ' 'TEST_IMPLICIT_DEFINE_COUNT+%d' % (define+'_i', i)) f.writeln('#define %-24s ' - 'test_define(%s)' % (define, define+'_i')) + 'TEST_DEFINE(%s)' % (define, define+'_i')) f.writeln('#endif') f.writeln() @@ -486,7 +486,7 @@ def compile(test_paths, **args): 'TEST_IMPLICIT_DEFINE_COUNT+%d' % ( define+'_i', i)) f.writeln('#define %-24s ' - 'test_define(%s)' % ( + 'TEST_DEFINE(%s)' % ( define, define+'_i')) f.writeln('#define ' '__TEST__%s__NEEDS_UNDEF' % ( @@ -1018,7 +1018,9 @@ def run(runner, test_ids=[], **args): trace = openio(args['trace'], 'w', 1) output = None if args.get('output'): - output = TestOutput(args['output'], ['suite', 'case'], ['test_passed']) + output = TestOutput(args['output'], + ['suite', 'case'], + ['test_passed']) # measure runtime start = time.time() diff --git a/scripts/tracebd.py b/scripts/tracebd.py index ff0bbbea..a8bdf8a4 100755 --- a/scripts/tracebd.py +++ b/scripts/tracebd.py @@ -607,7 +607,7 @@ def main(path='-', *, if not keep_open: break # don't just flood open calls - time.sleep(sleep) + time.sleep(sleep or 0.1) except KeyboardInterrupt: pass else: @@ -627,7 +627,7 @@ def main(path='-', *, if not keep_open: break # don't just flood open calls - time.sleep(sleep) + time.sleep(sleep or 0.1) done = True th.Thread(target=parse, daemon=True).start() diff --git a/tests/test_alloc.toml b/tests/test_alloc.toml index 64b805fa..205efbb1 100644 --- a/tests/test_alloc.toml +++ b/tests/test_alloc.toml @@ -370,7 +370,7 @@ code = ''' [cases.test_alloc_bad_blocks] in = "lfs.c" defines.ERASE_CYCLES = 0xffffffff -defines.BADBLOCK_BEHAVIOR = 'LFS_TESTBD_BADBLOCK_READERROR' +defines.BADBLOCK_BEHAVIOR = 'LFS_EMUBD_BADBLOCK_READERROR' code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -409,7 +409,7 @@ code = ''' // but mark the head of our file as a "bad block", this is force our // scan to bail early - lfs_testbd_setwear(cfg, fileblock, 0xffffffff) => 0; + lfs_emubd_setwear(cfg, fileblock, 0xffffffff) => 0; lfs_file_open(&lfs, &file, "ghost", LFS_O_WRONLY | LFS_O_CREAT) => 0; strcpy((char*)buffer, "chomp"); size = strlen("chomp"); @@ -424,7 +424,7 @@ code = ''' // now reverse the "bad block" and try to write the file again until we // run out of space - lfs_testbd_setwear(cfg, fileblock, 0) => 0; + lfs_emubd_setwear(cfg, fileblock, 0) => 0; lfs_file_open(&lfs, &file, "ghost", LFS_O_WRONLY | LFS_O_CREAT) => 0; strcpy((char*)buffer, "chomp"); size = strlen("chomp"); diff --git a/tests/test_badblocks.toml b/tests/test_badblocks.toml index 012d8765..b50b3933 100644 --- a/tests/test_badblocks.toml +++ b/tests/test_badblocks.toml @@ -6,18 +6,18 @@ defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] defines.BADBLOCK_BEHAVIOR = [ - 'LFS_TESTBD_BADBLOCK_PROGERROR', - 'LFS_TESTBD_BADBLOCK_ERASEERROR', - 'LFS_TESTBD_BADBLOCK_READERROR', - 'LFS_TESTBD_BADBLOCK_PROGNOOP', - 'LFS_TESTBD_BADBLOCK_ERASENOOP', + 'LFS_EMUBD_BADBLOCK_PROGERROR', + 'LFS_EMUBD_BADBLOCK_ERASEERROR', + 'LFS_EMUBD_BADBLOCK_READERROR', + 'LFS_EMUBD_BADBLOCK_PROGNOOP', + 'LFS_EMUBD_BADBLOCK_ERASENOOP', ] defines.NAMEMULT = 64 defines.FILEMULT = 1 code = ''' for (lfs_block_t badblock = 2; badblock < BLOCK_COUNT; badblock++) { - lfs_testbd_setwear(cfg, badblock-1, 0) => 0; - lfs_testbd_setwear(cfg, badblock, 0xffffffff) => 0; + lfs_emubd_setwear(cfg, badblock-1, 0) => 0; + lfs_emubd_setwear(cfg, badblock, 0xffffffff) => 0; lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -86,17 +86,17 @@ defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] defines.BADBLOCK_BEHAVIOR = [ - 'LFS_TESTBD_BADBLOCK_PROGERROR', - 'LFS_TESTBD_BADBLOCK_ERASEERROR', - 'LFS_TESTBD_BADBLOCK_READERROR', - 'LFS_TESTBD_BADBLOCK_PROGNOOP', - 'LFS_TESTBD_BADBLOCK_ERASENOOP', + 'LFS_EMUBD_BADBLOCK_PROGERROR', + 'LFS_EMUBD_BADBLOCK_ERASEERROR', + 'LFS_EMUBD_BADBLOCK_READERROR', + 'LFS_EMUBD_BADBLOCK_PROGNOOP', + 'LFS_EMUBD_BADBLOCK_ERASENOOP', ] defines.NAMEMULT = 64 defines.FILEMULT = 1 code = ''' for (lfs_block_t i = 0; i < (BLOCK_COUNT-2)/2; i++) { - lfs_testbd_setwear(cfg, i+2, 0xffffffff) => 0; + lfs_emubd_setwear(cfg, i+2, 0xffffffff) => 0; } lfs_t lfs; @@ -165,17 +165,17 @@ defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] defines.BADBLOCK_BEHAVIOR = [ - 'LFS_TESTBD_BADBLOCK_PROGERROR', - 'LFS_TESTBD_BADBLOCK_ERASEERROR', - 'LFS_TESTBD_BADBLOCK_READERROR', - 'LFS_TESTBD_BADBLOCK_PROGNOOP', - 'LFS_TESTBD_BADBLOCK_ERASENOOP', + 'LFS_EMUBD_BADBLOCK_PROGERROR', + 'LFS_EMUBD_BADBLOCK_ERASEERROR', + 'LFS_EMUBD_BADBLOCK_READERROR', + 'LFS_EMUBD_BADBLOCK_PROGNOOP', + 'LFS_EMUBD_BADBLOCK_ERASENOOP', ] defines.NAMEMULT = 64 defines.FILEMULT = 1 code = ''' for (lfs_block_t i = 0; i < (BLOCK_COUNT-2)/2; i++) { - lfs_testbd_setwear(cfg, (2*i) + 2, 0xffffffff) => 0; + lfs_emubd_setwear(cfg, (2*i) + 2, 0xffffffff) => 0; } lfs_t lfs; @@ -244,15 +244,15 @@ code = ''' defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] defines.BADBLOCK_BEHAVIOR = [ - 'LFS_TESTBD_BADBLOCK_PROGERROR', - 'LFS_TESTBD_BADBLOCK_ERASEERROR', - 'LFS_TESTBD_BADBLOCK_READERROR', - 'LFS_TESTBD_BADBLOCK_PROGNOOP', - 'LFS_TESTBD_BADBLOCK_ERASENOOP', + 'LFS_EMUBD_BADBLOCK_PROGERROR', + 'LFS_EMUBD_BADBLOCK_ERASEERROR', + 'LFS_EMUBD_BADBLOCK_READERROR', + 'LFS_EMUBD_BADBLOCK_PROGNOOP', + 'LFS_EMUBD_BADBLOCK_ERASENOOP', ] code = ''' - lfs_testbd_setwear(cfg, 0, 0xffffffff) => 0; - lfs_testbd_setwear(cfg, 1, 0xffffffff) => 0; + lfs_emubd_setwear(cfg, 0, 0xffffffff) => 0; + lfs_emubd_setwear(cfg, 1, 0xffffffff) => 0; lfs_t lfs; lfs_format(&lfs, cfg) => LFS_ERR_NOSPC; diff --git a/tests/test_exhaustion.toml b/tests/test_exhaustion.toml index fdcef24c..6003af47 100644 --- a/tests/test_exhaustion.toml +++ b/tests/test_exhaustion.toml @@ -4,11 +4,11 @@ defines.ERASE_CYCLES = 10 defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' defines.BADBLOCK_BEHAVIOR = [ - 'LFS_TESTBD_BADBLOCK_PROGERROR', - 'LFS_TESTBD_BADBLOCK_ERASEERROR', - 'LFS_TESTBD_BADBLOCK_READERROR', - 'LFS_TESTBD_BADBLOCK_PROGNOOP', - 'LFS_TESTBD_BADBLOCK_ERASENOOP', + 'LFS_EMUBD_BADBLOCK_PROGERROR', + 'LFS_EMUBD_BADBLOCK_ERASEERROR', + 'LFS_EMUBD_BADBLOCK_READERROR', + 'LFS_EMUBD_BADBLOCK_PROGNOOP', + 'LFS_EMUBD_BADBLOCK_ERASENOOP', ] defines.FILES = 10 code = ''' @@ -97,11 +97,11 @@ defines.ERASE_CYCLES = 10 defines.BLOCK_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' defines.BADBLOCK_BEHAVIOR = [ - 'LFS_TESTBD_BADBLOCK_PROGERROR', - 'LFS_TESTBD_BADBLOCK_ERASEERROR', - 'LFS_TESTBD_BADBLOCK_READERROR', - 'LFS_TESTBD_BADBLOCK_PROGNOOP', - 'LFS_TESTBD_BADBLOCK_ERASENOOP', + 'LFS_EMUBD_BADBLOCK_PROGERROR', + 'LFS_EMUBD_BADBLOCK_ERASEERROR', + 'LFS_EMUBD_BADBLOCK_READERROR', + 'LFS_EMUBD_BADBLOCK_PROGNOOP', + 'LFS_EMUBD_BADBLOCK_ERASENOOP', ] defines.FILES = 10 code = ''' @@ -197,7 +197,7 @@ code = ''' for (int run = 0; run < 2; run++) { for (lfs_block_t b = 0; b < BLOCK_COUNT; b++) { - lfs_testbd_setwear(cfg, b, + lfs_emubd_setwear(cfg, b, (b < run_block_count[run]) ? 0 : ERASE_CYCLES) => 0; } @@ -297,7 +297,7 @@ code = ''' for (int run = 0; run < 2; run++) { for (lfs_block_t b = 0; b < BLOCK_COUNT; b++) { - lfs_testbd_setwear(cfg, b, + lfs_emubd_setwear(cfg, b, (b < run_block_count[run]) ? 0 : ERASE_CYCLES) => 0; } @@ -469,12 +469,12 @@ exhausted: LFS_WARN("completed %d cycles", cycle); // check the wear on our block device - lfs_testbd_wear_t minwear = -1; - lfs_testbd_wear_t totalwear = 0; - lfs_testbd_wear_t maxwear = 0; + lfs_emubd_wear_t minwear = -1; + lfs_emubd_wear_t totalwear = 0; + lfs_emubd_wear_t maxwear = 0; // skip 0 and 1 as superblock movement is intentionally avoided for (lfs_block_t b = 2; b < BLOCK_COUNT; b++) { - lfs_testbd_wear_t wear = lfs_testbd_getwear(cfg, b); + lfs_emubd_wear_t wear = lfs_emubd_getwear(cfg, b); printf("%08x: wear %d\n", b, wear); assert(wear >= 0); if (wear < minwear) { @@ -485,17 +485,17 @@ exhausted: } totalwear += wear; } - lfs_testbd_wear_t avgwear = totalwear / BLOCK_COUNT; + lfs_emubd_wear_t avgwear = totalwear / BLOCK_COUNT; LFS_WARN("max wear: %d cycles", maxwear); LFS_WARN("avg wear: %d cycles", totalwear / (int)BLOCK_COUNT); LFS_WARN("min wear: %d cycles", minwear); // find standard deviation^2 - lfs_testbd_wear_t dev2 = 0; + lfs_emubd_wear_t dev2 = 0; for (lfs_block_t b = 2; b < BLOCK_COUNT; b++) { - lfs_testbd_wear_t wear = lfs_testbd_getwear(cfg, b); + lfs_emubd_wear_t wear = lfs_emubd_getwear(cfg, b); assert(wear >= 0); - lfs_testbd_swear_t diff = wear - avgwear; + lfs_emubd_swear_t diff = wear - avgwear; dev2 += diff*diff; } dev2 /= totalwear; diff --git a/tests/test_move.toml b/tests/test_move.toml index 6c89766c..0537f486 100644 --- a/tests/test_move.toml +++ b/tests/test_move.toml @@ -1638,15 +1638,15 @@ code = ''' if (RELOCATIONS & 0x1) { lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/parent"); - lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; - lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; lfs_dir_close(&lfs, &dir) => 0; } if (RELOCATIONS & 0x2) { lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/parent/child"); - lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; - lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; lfs_dir_close(&lfs, &dir) => 0; } @@ -1784,22 +1784,22 @@ code = ''' if (RELOCATIONS & 0x1) { lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/parent"); - lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; - lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; lfs_dir_close(&lfs, &dir) => 0; } if (RELOCATIONS & 0x2) { lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/parent/sibling"); - lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; - lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; lfs_dir_close(&lfs, &dir) => 0; } if (RELOCATIONS & 0x4) { lfs_dir_t dir; lfs_dir_open(&lfs, &dir, "/parent/child"); - lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; - lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0; + lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0; lfs_dir_close(&lfs, &dir) => 0; } From 9a0e3be84ebfcabe45167c1fe7af25eb40a11abb Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 20 Sep 2022 13:35:45 -0500 Subject: [PATCH 42/81] Added a quick trie to avoid running redundant test/bench permutations Without this redundant permutations can easily happen with runtime overrides because the different define layers aren't aware of each other. This causes problems for collecting benchmark results. --- runners/bench_runner.c | 97 ++++++++++++++++++++++++++++---- runners/test_runner.c | 122 +++++++++++++++++++++++++++++++++-------- 2 files changed, 183 insertions(+), 36 deletions(-) diff --git a/runners/bench_runner.c b/runners/bench_runner.c index 39a38f38..073760f7 100644 --- a/runners/bench_runner.c +++ b/runners/bench_runner.c @@ -530,6 +530,68 @@ static void perm_printid( } } +// a quick trie for keeping track of permutations we've seen +typedef struct bench_seen { + struct bench_seen_branch *branches; + size_t branch_count; + size_t branch_capacity; +} bench_seen_t; + +struct bench_seen_branch { + intmax_t define; + struct bench_seen branch; +}; + +bool bench_seen_insert( + bench_seen_t *seen, + const struct bench_suite *suite, + const struct bench_case *case_) { + (void)case_; + bool was_seen = true; + + // use the currently set defines + for (size_t d = 0; + d < lfs_max( + suite->define_count, + BENCH_IMPLICIT_DEFINE_COUNT); + d++) { + // treat unpermuted defines the same as 0 + intmax_t define = bench_define_ispermutation(d) ? BENCH_DEFINE(d) : 0; + + // already seen? + struct bench_seen_branch *branch = NULL; + for (size_t i = 0; i < seen->branch_count; i++) { + if (seen->branches[i].define == define) { + branch = &seen->branches[i]; + break; + } + } + + // need to create a new node + if (!branch) { + was_seen = false; + branch = mappend( + (void**)&seen->branches, + sizeof(struct bench_seen_branch), + &seen->branch_count, + &seen->branch_capacity); + branch->define = define; + branch->branch = (bench_seen_t){NULL, 0, 0}; + } + + seen = &branch->branch; + } + + return was_seen; +} + +void bench_seen_cleanup(bench_seen_t *seen) { + for (size_t i = 0; i < seen->branch_count; i++) { + bench_seen_cleanup(&seen->branches[i].branch); + } + free(seen->branches); +} + // iterate through permutations in a bench case static void case_forperm( const struct bench_suite *suite, @@ -546,25 +608,36 @@ static void case_forperm( bench_define_flush(); cb(data, suite, case_); - } else { - for (size_t k = 0; k < case_->permutations; k++) { - // define permutation - bench_define_perm(suite, case_, k); + return; + } - for (size_t v = 0; v < bench_override_define_permutations; v++) { - // define override permutation - bench_define_override(v); + bench_seen_t seen = {NULL, 0, 0}; - for (size_t g = 0; g < bench_geometry_count; g++) { - // define geometry - bench_define_geometry(&bench_geometries[g]); - bench_define_flush(); + for (size_t k = 0; k < case_->permutations; k++) { + // define permutation + bench_define_perm(suite, case_, k); - cb(data, suite, case_); + for (size_t v = 0; v < bench_override_define_permutations; v++) { + // define override permutation + bench_define_override(v); + + for (size_t g = 0; g < bench_geometry_count; g++) { + // define geometry + bench_define_geometry(&bench_geometries[g]); + bench_define_flush(); + + // have we seen this permutation before? + bool was_seen = bench_seen_insert(&seen, suite, case_); + if (!(k == 0 && v == 0 && g == 0) && was_seen) { + continue; } + + cb(data, suite, case_); } } } + + bench_seen_cleanup(&seen); } diff --git a/runners/test_runner.c b/runners/test_runner.c index 8333e388..34c31e1e 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -511,6 +511,69 @@ static void perm_printid( } } + +// a quick trie for keeping track of permutations we've seen +typedef struct test_seen { + struct test_seen_branch *branches; + size_t branch_count; + size_t branch_capacity; +} test_seen_t; + +struct test_seen_branch { + intmax_t define; + struct test_seen branch; +}; + +bool test_seen_insert( + test_seen_t *seen, + const struct test_suite *suite, + const struct test_case *case_) { + (void)case_; + bool was_seen = true; + + // use the currently set defines + for (size_t d = 0; + d < lfs_max( + suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + // treat unpermuted defines the same as 0 + intmax_t define = test_define_ispermutation(d) ? TEST_DEFINE(d) : 0; + + // already seen? + struct test_seen_branch *branch = NULL; + for (size_t i = 0; i < seen->branch_count; i++) { + if (seen->branches[i].define == define) { + branch = &seen->branches[i]; + break; + } + } + + // need to create a new node + if (!branch) { + was_seen = false; + branch = mappend( + (void**)&seen->branches, + sizeof(struct test_seen_branch), + &seen->branch_count, + &seen->branch_capacity); + branch->define = define; + branch->branch = (test_seen_t){NULL, 0, 0}; + } + + seen = &branch->branch; + } + + return was_seen; +} + +void test_seen_cleanup(test_seen_t *seen) { + for (size_t i = 0; i < seen->branch_count; i++) { + test_seen_cleanup(&seen->branches[i].branch); + } + free(seen->branches); +} + static void run_powerloss_cycles( const lfs_emubd_powercycles_t *cycles, size_t cycle_count, @@ -551,40 +614,51 @@ static void case_forperm( cb(data, suite, case_, &test_powerlosses[p]); } } - } else { - for (size_t k = 0; k < case_->permutations; k++) { - // define permutation - test_define_perm(suite, case_, k); + return; + } - for (size_t v = 0; v < test_override_define_permutations; v++) { - // define override permutation - test_define_override(v); + test_seen_t seen = {NULL, 0, 0}; - for (size_t g = 0; g < test_geometry_count; g++) { - // define geometry - test_define_geometry(&test_geometries[g]); - test_define_flush(); + for (size_t k = 0; k < case_->permutations; k++) { + // define permutation + test_define_perm(suite, case_, k); - if (cycles) { - cb(data, suite, case_, &(test_powerloss_t){ - .run=run_powerloss_cycles, - .cycles=cycles, - .cycle_count=cycle_count}); - } else { - for (size_t p = 0; p < test_powerloss_count; p++) { - // skip non-reentrant tests when powerloss testing - if (test_powerlosses[p].short_name != '0' - && !(case_->flags & TEST_REENTRANT)) { - continue; - } + for (size_t v = 0; v < test_override_define_permutations; v++) { + // define override permutation + test_define_override(v); - cb(data, suite, case_, &test_powerlosses[p]); + for (size_t g = 0; g < test_geometry_count; g++) { + // define geometry + test_define_geometry(&test_geometries[g]); + test_define_flush(); + + // have we seen this permutation before? + bool was_seen = test_seen_insert(&seen, suite, case_); + if (!(k == 0 && v == 0 && g == 0) && was_seen) { + continue; + } + + if (cycles) { + cb(data, suite, case_, &(test_powerloss_t){ + .run=run_powerloss_cycles, + .cycles=cycles, + .cycle_count=cycle_count}); + } else { + for (size_t p = 0; p < test_powerloss_count; p++) { + // skip non-reentrant tests when powerloss testing + if (test_powerlosses[p].short_name != '0' + && !(case_->flags & TEST_REENTRANT)) { + continue; } + + cb(data, suite, case_, &test_powerlosses[p]); } } } } } + + test_seen_cleanup(&seen); } From 7591d9cf74efc20922146a95f7a9a6c0e4299576 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 22 Sep 2022 01:29:50 -0500 Subject: [PATCH 43/81] Added plot.py for in-terminal plotting --- scripts/code.py | 4 +- scripts/coverage.py | 4 +- scripts/data.py | 4 +- scripts/plot.py | 770 ++++++++++++++++++++++++++++++++++++++++++++ scripts/stack.py | 4 +- scripts/struct_.py | 4 +- scripts/summary.py | 8 +- scripts/tailpipe.py | 4 +- scripts/tracebd.py | 110 +++---- 9 files changed, 839 insertions(+), 73 deletions(-) create mode 100755 scripts/plot.py diff --git a/scripts/code.py b/scripts/code.py index 4adc0c94..083ab8af 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -364,7 +364,7 @@ def main(obj_paths, **args): else: results = [] with openio(args['use']) as f: - reader = csv.DictReader(f) + reader = csv.DictReader(f, restval='') for r in reader: try: results.append(CodeResult(**{ @@ -392,7 +392,7 @@ def main(obj_paths, **args): diff_results = [] try: with openio(args['diff']) as f: - reader = csv.DictReader(f) + reader = csv.DictReader(f, restval='') for r in reader: try: diff_results.append(CodeResult(**{ diff --git a/scripts/coverage.py b/scripts/coverage.py index 5f0e11a8..81bff111 100755 --- a/scripts/coverage.py +++ b/scripts/coverage.py @@ -610,7 +610,7 @@ def main(gcda_paths, **args): else: results = [] with openio(args['use']) as f: - reader = csv.DictReader(f) + reader = csv.DictReader(f, restval='') for r in reader: try: results.append(CoverageResult(**{ @@ -638,7 +638,7 @@ def main(gcda_paths, **args): diff_results = [] try: with openio(args['diff']) as f: - reader = csv.DictReader(f) + reader = csv.DictReader(f, restval='') for r in reader: try: diff_results.append(CoverageResult(**{ diff --git a/scripts/data.py b/scripts/data.py index d42f5319..e86bafdc 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -364,7 +364,7 @@ def main(obj_paths, **args): else: results = [] with openio(args['use']) as f: - reader = csv.DictReader(f) + reader = csv.DictReader(f, restval='') for r in reader: try: results.append(DataResult(**{ @@ -392,7 +392,7 @@ def main(obj_paths, **args): diff_results = [] try: with openio(args['diff']) as f: - reader = csv.DictReader(f) + reader = csv.DictReader(f, restval='') for r in reader: try: diff_results.append(DataResult(**{ diff --git a/scripts/plot.py b/scripts/plot.py new file mode 100755 index 00000000..b310cb0a --- /dev/null +++ b/scripts/plot.py @@ -0,0 +1,770 @@ +#!/usr/bin/env python3 +# +# Plot CSV files in terminal. +# +# Example: +# ./scripts/plot.py bench.csv -xSIZE -ybench_read -W80 -H17 +# +# Copyright (c) 2022, The littlefs authors. +# SPDX-License-Identifier: BSD-3-Clause +# + +import collections as co +import csv +import glob +import io +import itertools as it +import math as m +import os +import shutil +import time + +CSV_PATHS = ['*.csv'] +COLORS = [ + '1;34', # bold blue + '1;31', # bold red + '1;32', # bold green + '1;35', # bold purple + '1;33', # bold yellow + '1;36', # bold cyan + '34', # blue + '31', # red + '32', # green + '35', # purple + '33', # yellow + '36', # cyan +] + +CHARS_DOTS = " .':" +CHARS_BRAILLE = ( + '⠀⢀⡀⣀⠠⢠⡠⣠⠄⢄⡄⣄⠤⢤⡤⣤' '⠐⢐⡐⣐⠰⢰⡰⣰⠔⢔⡔⣔⠴⢴⡴⣴' + '⠂⢂⡂⣂⠢⢢⡢⣢⠆⢆⡆⣆⠦⢦⡦⣦' '⠒⢒⡒⣒⠲⢲⡲⣲⠖⢖⡖⣖⠶⢶⡶⣶' + '⠈⢈⡈⣈⠨⢨⡨⣨⠌⢌⡌⣌⠬⢬⡬⣬' '⠘⢘⡘⣘⠸⢸⡸⣸⠜⢜⡜⣜⠼⢼⡼⣼' + '⠊⢊⡊⣊⠪⢪⡪⣪⠎⢎⡎⣎⠮⢮⡮⣮' '⠚⢚⡚⣚⠺⢺⡺⣺⠞⢞⡞⣞⠾⢾⡾⣾' + '⠁⢁⡁⣁⠡⢡⡡⣡⠅⢅⡅⣅⠥⢥⡥⣥' '⠑⢑⡑⣑⠱⢱⡱⣱⠕⢕⡕⣕⠵⢵⡵⣵' + '⠃⢃⡃⣃⠣⢣⡣⣣⠇⢇⡇⣇⠧⢧⡧⣧' '⠓⢓⡓⣓⠳⢳⡳⣳⠗⢗⡗⣗⠷⢷⡷⣷' + '⠉⢉⡉⣉⠩⢩⡩⣩⠍⢍⡍⣍⠭⢭⡭⣭' '⠙⢙⡙⣙⠹⢹⡹⣹⠝⢝⡝⣝⠽⢽⡽⣽' + '⠋⢋⡋⣋⠫⢫⡫⣫⠏⢏⡏⣏⠯⢯⡯⣯' '⠛⢛⡛⣛⠻⢻⡻⣻⠟⢟⡟⣟⠿⢿⡿⣿') + +SI_PREFIXES = { + 18: 'E', + 15: 'P', + 12: 'T', + 9: 'G', + 6: 'M', + 3: 'K', + 0: '', + -3: 'm', + -6: 'u', + -9: 'n', + -12: 'p', + -15: 'f', + -18: 'a', +} + + +# format a number to a strict character width using SI prefixes +def si(x, w=4): + if x == 0: + return '0' + # figure out prefix and scale + p = 3*int(m.log(abs(x)*10, 10**3)) + p = min(18, max(-18, p)) + # format with enough digits + s = '%.*f' % (w, abs(x) / (10.0**p)) + s = s.lstrip('0') + # truncate but only digits that follow the dot + if '.' in s: + s = s[:max(s.find('.'), w-(2 if x < 0 else 1))] + s = s.rstrip('0') + s = s.rstrip('.') + return '%s%s%s' % ('-' if x < 0 else '', s, SI_PREFIXES[p]) + +def openio(path, mode='r'): + if path == '-': + if mode == 'r': + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) + + +# parse different data representations +def dat(x): + # allow the first part of an a/b fraction + if '/' in x: + x, _ = x.split('/', 1) + + # first try as int + try: + return int(x, 0) + except ValueError: + pass + + # then try as float + try: + x = float(x) + # just don't allow infinity or nan + if m.isinf(x) or m.isnan(x): + raise ValueError("invalid dat %r" % x) + except ValueError: + pass + + # else give up + raise ValueError("invalid dat %r" % x) + +# a hack log10 that preserves sign, and passes zero as zero +def slog10(x): + if x == 0: + return x + elif x > 0: + return m.log10(x) + else: + return -m.log10(-x) + + +class Plot: + def __init__(self, width, height, *, + xlim=None, + ylim=None, + xlog=False, + ylog=False, + **_): + self.width = width + self.height = height + self.xlim = xlim or (0, width) + self.ylim = ylim or (0, height) + self.xlog = xlog + self.ylog = ylog + self.grid = [('',False)]*(self.width*self.height) + + def scale(self, x, y): + # scale and clamp + try: + if self.xlog: + x = int(self.width * ( + (slog10(x)-slog10(self.xlim[0])) + / (slog10(self.xlim[1])-slog10(self.xlim[0])))) + else: + x = int(self.width * ( + (x-self.xlim[0]) + / (self.xlim[1]-self.xlim[0]))) + if self.ylog: + y = int(self.height * ( + (slog10(y)-slog10(self.ylim[0])) + / (slog10(self.ylim[1])-slog10(self.ylim[0])))) + else: + y = int(self.height * ( + (y-self.ylim[0]) + / (self.ylim[1]-self.ylim[0]))) + except ZeroDivisionError: + x = 0 + y = 0 + return x, y + + def point(self, x, y, *, + color=COLORS[0], + char=True): + # scale + x, y = self.scale(x, y) + + # ignore out of bounds points + if x >= 0 and x < self.width and y >= 0 and y < self.height: + self.grid[x + y*self.width] = (color, char) + + def line(self, x1, y1, x2, y2, *, + color=COLORS[0], + char=True): + # scale + x1, y1 = self.scale(x1, y1) + x2, y2 = self.scale(x2, y2) + + # incremental error line algorithm + ex = abs(x2 - x1) + ey = -abs(y2 - y1) + dx = +1 if x1 < x2 else -1 + dy = +1 if y1 < y2 else -1 + e = ex + ey + + while True: + if x1 >= 0 and x1 < self.width and y1 >= 0 and y1 < self.height: + self.grid[x1 + y1*self.width] = (color, char) + e2 = 2*e + + if x1 == x2 and y1 == y2: + break + + if e2 > ey: + e += ey + x1 += dx + + if x1 == x2 and y1 == y2: + break + + if e2 < ex: + e += ex + y1 += dy + + if x2 >= 0 and x2 < self.width and y2 >= 0 and y2 < self.height: + self.grid[x2 + y2*self.width] = (color, char) + + def plot(self, coords, *, + color=COLORS[0], + char=True, + line_char=True): + # draw lines + if line_char: + for (x1, y1), (x2, y2) in zip(coords, coords[1:]): + if y1 is not None and y2 is not None: + self.line(x1, y1, x2, y2, + color=color, + char=line_char) + + # draw points + if char and (not line_char or char is not True): + for x, y in coords: + if y is not None: + self.point(x, y, + color=color, + char=char) + + def draw(self, row, *, + dots=False, + braille=False, + color=False, + **_): + # scale if needed + if braille: + xscale, yscale = 2, 4 + elif dots: + xscale, yscale = 1, 2 + else: + xscale, yscale = 1, 1 + + y = self.height//yscale-1 - row + row_ = [] + for x in range(self.width//xscale): + best_f = '' + best_c = False + + # encode into a byte + b = 0 + for i in range(xscale*yscale): + f, c = self.grid[x*xscale+(xscale-1-(i%xscale)) + + (y*yscale+(i//xscale))*self.width] + if c: + b |= 1 << i + + if f: + best_f = f + if c and c is not True: + best_c = c + + # use byte to lookup character + if b: + if best_c: + c = best_c + elif braille: + c = CHARS_BRAILLE[b] + else: + c = CHARS_DOTS[b] + else: + c = ' ' + + # color? + if b and color and best_f: + c = '\x1b[%sm%s\x1b[m' % (best_f, c) + + # draw axis in blank spaces + if not b: + zx, zy = self.scale(0, 0) + if x == zx // xscale and y == zy // yscale: + c = '+' + elif x == zx // xscale and y == 0: + c = 'v' + elif x == zx // xscale and y == self.height//yscale-1: + c = '^' + elif y == zy // yscale and x == 0: + c = '<' + elif y == zy // yscale and x == self.width//xscale-1: + c = '>' + elif x == zx // xscale: + c = '|' + elif y == zy // yscale: + c = '-' + + row_.append(c) + + return ''.join(row_) + + +def collect(csv_paths, renames=[]): + # collect results from CSV files + paths = [] + for path in csv_paths: + if os.path.isdir(path): + path = path + '/*.csv' + + for path in glob.glob(path): + paths.append(path) + + results = [] + for path in paths: + try: + with openio(path) as f: + reader = csv.DictReader(f, restval='') + for r in reader: + results.append(r) + except FileNotFoundError: + pass + + if renames: + for r in results: + # make a copy so renames can overlap + r_ = {} + for new_k, old_k in renames: + if old_k in r: + r_[new_k] = r[old_k] + r.update(r_) + + return results + +def dataset(results, x=None, y=None, defines={}): + # organize by 'by', x, and y + dataset = {} + for i, r in enumerate(results): + # filter results by matching defines + if not all(k in r and r[k] in vs for k, vs in defines.items()): + continue + + # find xs + if x is not None: + if x not in r: + continue + try: + x_ = dat(r[x]) + except ValueError: + continue + else: + x_ = i + + # find ys + if y is not None: + if y not in r: + y_ = None + else: + try: + y_ = dat(r[y]) + except ValueError: + y_ = None + else: + y_ = None + + if y_ is not None: + dataset[x_] = y_ + dataset.get(x_, 0) + else: + dataset[x_] = y_ or dataset.get(x_, None) + + return dataset + +def datasets(results, by=None, x=None, y=None, defines={}): + # filter results by matching defines + results_ = [] + for r in results: + if all(k in r and r[k] in vs for k, vs in defines.items()): + results_.append(r) + results = results_ + + if by is not None: + # find all 'by' values + ks = set() + for r in results: + ks.add(tuple(r.get(k, '') for k in by)) + ks = sorted(ks) + + # collect all datasets + datasets = co.OrderedDict() + for ks_ in (ks if by is not None else [()]): + for x_ in (x if x is not None else [None]): + for y_ in (y if y is not None else [None]): + datasets[ks_ + (x_, y_)] = dataset( + results, + x_, + y_, + {by_: {k_} for by_, k_ in zip(by, ks_)} + if by is not None else {}) + + return datasets + + +def main(csv_paths, *, + by=None, + x=None, + y=None, + define=[], + xlim=None, + ylim=None, + width=None, + height=None, + color=False, + braille=False, + colors=None, + chars=None, + line_chars=None, + no_lines=False, + legend=None, + keep_open=False, + sleep=None, + **args): + # figure out what color should be + if color == 'auto': + color = sys.stdout.isatty() + elif color == 'always': + color = True + else: + color = False + + # allow shortened ranges + if xlim is not None and len(xlim) == 1: + xlim = (0, xlim[0]) + if ylim is not None and len(ylim) == 1: + ylim = (0, ylim[0]) + + # seperate out renames + renames = [k.split('=', 1) + for k in it.chain(by or [], x or [], y or []) + if '=' in k] + if by is not None: + by = [k.split('=', 1)[0] for k in by] + if x is not None: + x = [k.split('=', 1)[0] for k in x] + if y is not None: + y = [k.split('=', 1)[0] for k in y] + + def draw(f): + def writeln(s=''): + f.write(s) + f.write('\n') + f.writeln = writeln + + # first collect results from CSV files + results = collect(csv_paths, renames) + + # then extract the requested datasets + datasets_ = datasets(results, by, x, y, dict(define)) + + # what colors to use? + if colors is not None: + colors_ = colors + else: + colors_ = COLORS + + if chars is not None: + chars_ = chars + else: + chars_ = [True] + + if line_chars is not None: + line_chars_ = line_chars + elif not no_lines: + line_chars_ = [True] + else: + line_chars_ = [False] + + # build legend? + legend_width = 0 + if legend: + legend_ = [] + for i, k in enumerate(datasets_.keys()): + label = '%s%s' % ( + '%s ' % chars_[i % len(chars_)] + if chars is not None + else '%s ' % line_chars_[i % len(line_chars_)] + if line_chars is not None + else '', + ','.join(k_ for i, k_ in enumerate(k) + if k_ + if not (i == len(k)-2 and len(x) == 1) + if not (i == len(k)-1 and len(y) == 1))) + + if label: + legend_.append(label) + legend_width = max(legend_width, len(label)+1) + + # find xlim/ylim + if xlim is not None: + xlim_ = xlim + else: + xlim_ = ( + min(it.chain([0], (k + for r in datasets_.values() + for k, v in r.items() + if v is not None))), + max(it.chain([0], (k + for r in datasets_.values() + for k, v in r.items() + if v is not None)))) + + if ylim is not None: + ylim_ = ylim + else: + ylim_ = ( + min(it.chain([0], (v + for r in datasets_.values() + for _, v in r.items() + if v is not None))), + max(it.chain([0], (v + for r in datasets_.values() + for _, v in r.items() + if v is not None)))) + + # figure out our plot size + if width is not None: + width_ = width + else: + width_ = shutil.get_terminal_size((80, 8))[0] + # make space for units + width_ -= 5 + # make space for legend + if legend in {'left', 'right'} and legend_: + width_ -= legend_width + # limit a bit + width_ = max(2*4, width_) + + if height is not None: + height_ = height + else: + height_ = shutil.get_terminal_size((80, 8))[1] + # make space for shell prompt + if not keep_open: + height_ -= 1 + # make space for units + height_ -= 1 + # make space for legend + if legend in {'above', 'below'} and legend_: + legend_cols = min(len(legend_), max(1, width_//legend_width)) + height_ -= (len(legend_)+legend_cols-1) // legend_cols + # limit a bit + height_ = max(2, height_) + + # create a plot and draw our coordinates + plot = Plot( + # scale if we're printing with dots or braille + 2*width_ if line_chars is None and braille else width_, + 4*height_ if line_chars is None and braille + else 2*height_ if line_chars is None + else height_, + xlim=xlim_, + ylim=ylim_, + **args) + + for i, (k, dataset) in enumerate(datasets_.items()): + plot.plot( + sorted((x,y) for x,y in dataset.items()), + color=colors_[i % len(colors_)], + char=chars_[i % len(chars_)], + line_char=line_chars_[i % len(line_chars_)]) + + # draw legend=above? + if legend == 'above' and legend_: + for i in range(0, len(legend_), legend_cols): + f.writeln('%4s %*s%s' % ( + '', + max(width_ - sum(len(label)+1 + for label in legend_[i:i+legend_cols]), + 0) // 2, + '', + ' '.join('%s%s%s' % ( + '\x1b[%sm' % colors_[j % len(colors_)] if color else '', + legend_[j], + '\x1b[m' if color else '') + for j in range(i, min(i+legend_cols, len(legend_)))))) + for row in range(height_): + f.writeln('%s%4s %s%s' % ( + # draw legend=left? + ('%s%-*s %s' % ( + '\x1b[%sm' % colors_[row % len(colors_)] if color else '', + legend_width-1, + legend_[row] if row < len(legend_) else '', + '\x1b[m' if color else '')) + if legend == 'left' and legend_ else '', + # draw plot + si(ylim_[0], 4) if row == height_-1 + else si(ylim_[1], 4) if row == 0 + else '', + plot.draw(row, + braille=line_chars is None and braille, + dots=line_chars is None and not braille, + color=color, + **args), + # draw legend=right? + (' %s%s%s' % ( + '\x1b[%sm' % colors_[row % len(colors_)] if color else '', + legend_[row] if row < len(legend_) else '', + '\x1b[m' if color else '')) + if legend == 'right' and legend_ else '')) + f.writeln('%*s %-4s%*s%4s' % ( + 4 + (legend_width if legend == 'left' and legend_ else 0), + '', + si(xlim_[0], 4), + width_ - 2*4, + '', + si(xlim_[1], 4))) + # draw legend=below? + if legend == 'below' and legend_: + for i in range(0, len(legend_), legend_cols): + f.writeln('%4s %*s%s' % ( + '', + max(width_ - sum(len(label)+1 + for label in legend_[i:i+legend_cols]), + 0) // 2, + '', + ' '.join('%s%s%s' % ( + '\x1b[%sm' % colors_[j % len(colors_)] if color else '', + legend_[j], + '\x1b[m' if color else '') + for j in range(i, min(i+legend_cols, len(legend_)))))) + + + last_lines = 1 + def redraw(): + nonlocal last_lines + + canvas = io.StringIO() + draw(canvas) + canvas = canvas.getvalue().splitlines() + + # give ourself a canvas + while last_lines < len(canvas): + sys.stdout.write('\n') + last_lines += 1 + + for i, line in enumerate(canvas): + jump = len(canvas)-1-i + # move cursor, clear line, disable/reenable line wrapping + sys.stdout.write('\r') + if jump > 0: + sys.stdout.write('\x1b[%dA' % jump) + sys.stdout.write('\x1b[K') + sys.stdout.write('\x1b[?7l') + sys.stdout.write(line) + sys.stdout.write('\x1b[?7h') + if jump > 0: + sys.stdout.write('\x1b[%dB' % jump) + + sys.stdout.flush() + + if keep_open: + try: + while True: + redraw() + # don't just flood open calls + time.sleep(sleep or 0.1) + except KeyboardInterrupt: + pass + + redraw() + sys.stdout.write('\n') + else: + draw(sys.stdout) + + +if __name__ == "__main__": + import sys + import argparse + parser = argparse.ArgumentParser( + description="Plot CSV files in terminal.") + parser.add_argument( + 'csv_paths', + nargs='*', + default=CSV_PATHS, + help="Description of where to find *.csv files. May be a directory " + "or list of paths. Defaults to %r." % CSV_PATHS) + parser.add_argument( + '-b', '--by', + type=lambda x: [x.strip() for x in x.split(',')], + help="Fields to render as separate plots. All other fields will be " + "summed. Can rename fields with new_name=old_name.") + parser.add_argument( + '-x', + type=lambda x: [x.strip() for x in x.split(',')], + help="Fields to use for the x-axis. Can rename fields with " + "new_name=old_name.") + parser.add_argument( + '-y', + type=lambda x: [x.strip() for x in x.split(',')], + required=True, + help="Fields to use for the y-axis. Can rename fields with " + "new_name=old_name.") + parser.add_argument( + '-D', '--define', + type=lambda x: (lambda k, v: (k, set(v.split(','))))(*x.split('=', 1)), + action='append', + help="Only include rows where this field is this value (field=value). " + "May include comma-separated options.") + parser.add_argument( + '--color', + choices=['never', 'always', 'auto'], + default='auto', + help="When to use terminal colors. Defaults to 'auto'.") + parser.add_argument( + '--braille', + action='store_true', + help="Use unicode braille characters. Note that braille characters " + "sometimes suffer from inconsistent widths.") + parser.add_argument( + '--colors', + type=lambda x: x.split(','), + help="Colors to use.") + parser.add_argument( + '--chars', + help="Characters to use for points.") + parser.add_argument( + '--line-chars', + help="Characters to use for lines.") + parser.add_argument( + '-L', '--no-lines', + action='store_true', + help="Only draw the data points.") + parser.add_argument( + '-W', '--width', + type=lambda x: int(x, 0), + help="Width in columns. A width of 0 indicates no limit. Defaults " + "to terminal width or 80.") + parser.add_argument( + '-H', '--height', + type=lambda x: int(x, 0), + help="Height in rows. Defaults to terminal height or 8.") + parser.add_argument( + '-X', '--xlim', + type=lambda x: tuple(dat(x) if x else None for x in x.split(',')), + help="Range for the x-axis.") + parser.add_argument( + '-Y', '--ylim', + type=lambda x: tuple(dat(x) if x else None for x in x.split(',')), + help="Range for the y-axis.") + parser.add_argument( + '--xlog', + action='store_true', + help="Use a logarithmic x-axis.") + parser.add_argument( + '--ylog', + action='store_true', + help="Use a logarithmic y-axis.") + parser.add_argument( + '-l', '--legend', + choices=['above', 'below', 'left', 'right'], + help="Place a legend here.") + parser.add_argument( + '-k', '--keep-open', + action='store_true', + help="Continue to open and redraw the CSV files in a loop.") + parser.add_argument( + '-s', '--sleep', + type=float, + help="Time in seconds to sleep between redraws when running with -k. " + "Defaults to 0.01.") + sys.exit(main(**{k: v + for k, v in vars(parser.parse_intermixed_args()).items() + if v is not None})) diff --git a/scripts/stack.py b/scripts/stack.py index 36ef3dca..b53fecb7 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -508,7 +508,7 @@ def main(ci_paths, **args): else: results = [] with openio(args['use']) as f: - reader = csv.DictReader(f) + reader = csv.DictReader(f, restval='') for r in reader: try: results.append(StackResult(**{ @@ -538,7 +538,7 @@ def main(ci_paths, **args): diff_results = [] try: with openio(args['diff']) as f: - reader = csv.DictReader(f) + reader = csv.DictReader(f, restval='') for r in reader: try: diff_results.append(StackResult(**{ diff --git a/scripts/struct_.py b/scripts/struct_.py index 49994977..a024cad6 100755 --- a/scripts/struct_.py +++ b/scripts/struct_.py @@ -407,7 +407,7 @@ def main(obj_paths, **args): else: results = [] with openio(args['use']) as f: - reader = csv.DictReader(f) + reader = csv.DictReader(f, restval='') for r in reader: try: results.append(StructResult(**{ @@ -435,7 +435,7 @@ def main(obj_paths, **args): diff_results = [] try: with openio(args['diff']) as f: - reader = csv.DictReader(f) + reader = csv.DictReader(f, restval='') for r in reader: try: diff_results.append(StructResult(**{ diff --git a/scripts/summary.py b/scripts/summary.py index 0855ffb2..d73e882a 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -607,7 +607,7 @@ def main(csv_paths, *, fields=None, by=None, **args): for path in paths: try: with openio(path) as f: - reader = csv.DictReader(f) + reader = csv.DictReader(f, restval='') for r in reader: results.append(r) except FileNotFoundError: @@ -634,7 +634,7 @@ def main(csv_paths, *, fields=None, by=None, **args): diff_results = [] try: with openio(args['diff']) as f: - reader = csv.DictReader(f) + reader = csv.DictReader(f, restval='') for r in reader: diff_results.append(r) except FileNotFoundError: @@ -693,12 +693,12 @@ if __name__ == "__main__": '-f', '--fields', type=lambda x: [x.strip() for x in x.split(',')], help="Only show these fields. Can rename fields " - "with old_name=new_name.") + "with new_name=old_name.") parser.add_argument( '-b', '--by', type=lambda x: [x.strip() for x in x.split(',')], help="Group by these fields. Can rename fields " - "with old_name=new_name.") + "with new_name=old_name.") parser.add_argument( '--add', type=lambda x: [x.strip() for x in x.split(',')], diff --git a/scripts/tailpipe.py b/scripts/tailpipe.py index 08213cf6..ae477e22 100755 --- a/scripts/tailpipe.py +++ b/scripts/tailpipe.py @@ -104,12 +104,12 @@ if __name__ == "__main__": '-n', '--lines', type=lambda x: int(x, 0), - help="Number of lines to show, defaults to 1.") + help="Number of lines to show. Defaults to 1.") parser.add_argument( '-s', '--sleep', type=float, - help="Seconds to sleep between reads, defaults to 0.01.") + help="Seconds to sleep between reads. Defaults to 0.01.") parser.add_argument( '-k', '--keep-open', diff --git a/scripts/tracebd.py b/scripts/tracebd.py index a8bdf8a4..1905f5d8 100755 --- a/scripts/tracebd.py +++ b/scripts/tracebd.py @@ -11,6 +11,7 @@ import collections as co import functools as ft +import io import itertools as it import math as m import os @@ -424,7 +425,7 @@ def main(path='-', *, '\s*(?P\w+)\s*' '\)' '|' '(?Psync)\(' '\s*(?P\w+)\s*' '\)' ')') - def parse_line(line): + def parse(line): # string searching is actually much faster than # the regex here if 'trace' not in line or 'bd' not in line: @@ -508,7 +509,7 @@ def main(path='-', *, # print a pretty line of trace output history = [] - def push_line(): + def push(): # create copy to avoid corrupt output with lock: resmoosh() @@ -564,30 +565,43 @@ def main(path='-', *, history.append(line) del history[:-lines] - last_rows = 1 - def print_line(): - nonlocal last_rows - if not lines: - return + def draw(f): + def writeln(s=''): + f.write(s) + f.write('\n') + f.writeln = writeln + + for line in it.chain.from_iterable(history): + f.writeln(line) + + last_lines = 1 + def redraw(): + nonlocal last_lines + + canvas = io.StringIO() + draw(canvas) + canvas = canvas.getvalue().splitlines() # give ourself a canvas - while last_rows < len(history)*height: + while last_lines < len(canvas): sys.stdout.write('\n') - last_rows += 1 + last_lines += 1 - for i, row in enumerate(it.chain.from_iterable(history)): - jump = len(history)*height-1-i + for i, line in enumerate(canvas): + jump = len(canvas)-1-i # move cursor, clear line, disable/reenable line wrapping sys.stdout.write('\r') if jump > 0: sys.stdout.write('\x1b[%dA' % jump) sys.stdout.write('\x1b[K') sys.stdout.write('\x1b[?7l') - sys.stdout.write(row) + sys.stdout.write(line) sys.stdout.write('\x1b[?7h') if jump > 0: sys.stdout.write('\x1b[%dB' % jump) + sys.stdout.flush() + if sleep is None or (coalesce and not lines): # read/parse coalesce number of operations @@ -596,11 +610,11 @@ def main(path='-', *, with openio(path) as f: changes = 0 for line in f: - change = parse_line(line) + change = parse(line) changes += change if change and changes % (coalesce or 1) == 0: - push_line() - print_line() + push() + redraw() # sleep between coalesced lines? if sleep is not None: time.sleep(sleep) @@ -612,17 +626,17 @@ def main(path='-', *, pass else: # read/parse in a background thread - def parse(): + def background_parse(): nonlocal done while True: with openio(path) as f: changes = 0 for line in f: - change = parse_line(line) + change = parse(line) changes += change if change and changes % (coalesce or 1) == 0: if coalesce: - push_line() + push() event.set() if not keep_open: break @@ -630,7 +644,7 @@ def main(path='-', *, time.sleep(sleep or 0.1) done = True - th.Thread(target=parse, daemon=True).start() + th.Thread(target=background_parse, daemon=True).start() try: while not done: @@ -638,8 +652,8 @@ def main(path='-', *, event.wait() event.clear() if not coalesce: - push_line() - print_line() + push() + redraw() except KeyboardInterrupt: pass @@ -658,23 +672,19 @@ if __name__ == "__main__": nargs='?', help="Path to read from.") parser.add_argument( - '-r', - '--read', + '-r', '--read', action='store_true', help="Render reads.") parser.add_argument( - '-p', - '--prog', + '-p', '--prog', action='store_true', help="Render progs.") parser.add_argument( - '-e', - '--erase', + '-e', '--erase', action='store_true', help="Render erases.") parser.add_argument( - '-w', - '--wear', + '-w', '--wear', action='store_true', help="Render wear.") parser.add_argument( @@ -692,18 +702,15 @@ if __name__ == "__main__": default='auto', help="When to use terminal colors. Defaults to 'auto'.") parser.add_argument( - '-b', - '--block', + '-b', '--block', type=lambda x: tuple(int(x,0) if x else None for x in x.split(',',1)), help="Show a specific block or range of blocks.") parser.add_argument( - '-i', - '--off', + '-i', '--off', type=lambda x: tuple(int(x,0) if x else None for x in x.split(',',1)), help="Show a specific offset or range of offsets.") parser.add_argument( - '-B', - '--block-size', + '-B', '--block-size', type=lambda x: int(x, 0), help="Assume a specific block size.") parser.add_argument( @@ -711,60 +718,49 @@ if __name__ == "__main__": type=lambda x: int(x, 0), help="Assume a specific block count.") parser.add_argument( - '-C', - '--block-cycles', + '-C', '--block-cycles', type=lambda x: int(x, 0), help="Assumed maximum number of erase cycles when measuring wear.") parser.add_argument( - '-R', - '--reset', + '-R', '--reset', action='store_true', help="Reset wear on block device initialization.") parser.add_argument( - '-W', - '--width', + '-W', '--width', type=lambda x: int(x, 0), help="Width in columns. A width of 0 indicates no limit. Defaults " "to terminal width or 80.") parser.add_argument( - '-H', - '--height', + '-H', '--height', type=lambda x: int(x, 0), help="Height in rows. Defaults to 1.") parser.add_argument( - '-x', - '--scale', + '-x', '--scale', type=float, help="Number of characters per block, ignores --width if set.") parser.add_argument( - '-n', - '--lines', + '-n', '--lines', type=lambda x: int(x, 0), help="Number of lines to show.") parser.add_argument( - '-c', - '--coalesce', + '-c', '--coalesce', type=lambda x: int(x, 0), help="Number of operations to coalesce together.") parser.add_argument( - '-s', - '--sleep', + '-s', '--sleep', type=float, help="Time in seconds to sleep between reads, while coalescing " "operations.") parser.add_argument( - '-I', - '--hilbert', + '-I', '--hilbert', action='store_true', help="Render as a space-filling Hilbert curve.") parser.add_argument( - '-Z', - '--lebesgue', + '-Z', '--lebesgue', action='store_true', help="Render as a space-filling Z-curve.") parser.add_argument( - '-k', - '--keep-open', + '-k', '--keep-open', action='store_true', help="Reopen the pipe on EOF, useful when multiple " "processes are writing.") From fb58148df298c8a373fe0ce6a691bb4e08cb6305 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 23 Sep 2022 00:27:09 -0500 Subject: [PATCH 44/81] Consistent handling of by/field arguments for plot.py and summary.py Now both scripts also fallback to guessing what fields to use based on what fields can be converted to integers. This is more falible, and doesn't work for tests/benchmarks, but in those cases explicit fields can be used (which is what would be needed without guessing anyways). --- Makefile | 3 +- scripts/plot.py | 57 +++++--- scripts/summary.py | 355 ++++++++++++++++++++++----------------------- 3 files changed, 209 insertions(+), 206 deletions(-) diff --git a/Makefile b/Makefile index 3eaa92d6..84aeb18f 100644 --- a/Makefile +++ b/Makefile @@ -170,10 +170,11 @@ coverage: $(GCDA) .PHONY: summary sizes summary sizes: $(BUILDDIR)lfs.csv $(strip ./scripts/summary.py -Y $^ \ - -f code=code_size,$\ + -fcode=code_size,$\ data=data_size,$\ stack=stack_limit,$\ struct=struct_size \ + --max=stack \ $(SUMMARYFLAGS)) diff --git a/scripts/plot.py b/scripts/plot.py index b310cb0a..6eeb69ad 100755 --- a/scripts/plot.py +++ b/scripts/plot.py @@ -330,12 +330,13 @@ def collect(csv_paths, renames=[]): return results -def dataset(results, x=None, y=None, defines={}): +def dataset(results, x=None, y=None, define=[]): # organize by 'by', x, and y dataset = {} - for i, r in enumerate(results): + i = 0 + for r in results: # filter results by matching defines - if not all(k in r and r[k] in vs for k, vs in defines.items()): + if not all(k in r and r[k] in vs for k, vs in define): continue # find xs @@ -348,6 +349,7 @@ def dataset(results, x=None, y=None, defines={}): continue else: x_ = i + i += 1 # find ys if y is not None: @@ -368,14 +370,29 @@ def dataset(results, x=None, y=None, defines={}): return dataset -def datasets(results, by=None, x=None, y=None, defines={}): +def datasets(results, by=None, x=None, y=None, define=[]): # filter results by matching defines results_ = [] for r in results: - if all(k in r and r[k] in vs for k, vs in defines.items()): + if all(k in r and r[k] in vs for k, vs in define): results_.append(r) results = results_ + # if y not specified, try to guess from data + if y is None: + y = co.OrderedDict() + for r in results: + for k, v in r.items(): + if by is not None and k in by: + continue + if y.get(k, True): + try: + dat(v) + y[k] = True + except ValueError: + y[k] = False + y = list(k for k,v in y.items() if v) + if by is not None: # find all 'by' values ks = set() @@ -387,13 +404,17 @@ def datasets(results, by=None, x=None, y=None, defines={}): datasets = co.OrderedDict() for ks_ in (ks if by is not None else [()]): for x_ in (x if x is not None else [None]): - for y_ in (y if y is not None else [None]): - datasets[ks_ + (x_, y_)] = dataset( + for y_ in y: + # hide x/y if there is only one field + k_x = x_ if len(x or []) > 1 else '' + k_y = y_ if len(y or []) > 1 else '' + + datasets[ks_ + (k_x, k_y)] = dataset( results, x_, y_, - {by_: {k_} for by_, k_ in zip(by, ks_)} - if by is not None else {}) + [(by_, k_) for by_, k_ in zip(by, ks_)] + if by is not None else []) return datasets @@ -431,7 +452,7 @@ def main(csv_paths, *, if ylim is not None and len(ylim) == 1: ylim = (0, ylim[0]) - # seperate out renames + # separate out renames renames = [k.split('=', 1) for k in it.chain(by or [], x or [], y or []) if '=' in k] @@ -452,7 +473,7 @@ def main(csv_paths, *, results = collect(csv_paths, renames) # then extract the requested datasets - datasets_ = datasets(results, by, x, y, dict(define)) + datasets_ = datasets(results, by, x, y, define) # what colors to use? if colors is not None: @@ -483,10 +504,7 @@ def main(csv_paths, *, else '%s ' % line_chars_[i % len(line_chars_)] if line_chars is not None else '', - ','.join(k_ for i, k_ in enumerate(k) - if k_ - if not (i == len(k)-2 and len(x) == 1) - if not (i == len(k)-1 and len(y) == 1))) + ','.join(k_ for k_ in k if k_)) if label: legend_.append(label) @@ -685,7 +703,7 @@ if __name__ == "__main__": '-b', '--by', type=lambda x: [x.strip() for x in x.split(',')], help="Fields to render as separate plots. All other fields will be " - "summed. Can rename fields with new_name=old_name.") + "summed as needed. Can rename fields with new_name=old_name.") parser.add_argument( '-x', type=lambda x: [x.strip() for x in x.split(',')], @@ -694,15 +712,14 @@ if __name__ == "__main__": parser.add_argument( '-y', type=lambda x: [x.strip() for x in x.split(',')], - required=True, help="Fields to use for the y-axis. Can rename fields with " "new_name=old_name.") parser.add_argument( '-D', '--define', - type=lambda x: (lambda k, v: (k, set(v.split(','))))(*x.split('=', 1)), + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), action='append', - help="Only include rows where this field is this value (field=value). " - "May include comma-separated options.") + help="Only include rows where this field is this value. May include " + "comma-separated options.") parser.add_argument( '--color', choices=['never', 'always', 'auto'], diff --git a/scripts/summary.py b/scripts/summary.py index d73e882a..36b556f5 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -16,6 +16,7 @@ import collections as co import csv import functools as ft import glob +import itertools as it import math as m import os import re @@ -23,31 +24,13 @@ import re CSV_PATHS = ['*.csv'] -# Defaults are common fields generated by other littlefs scripts -MERGES = { - 'add': ( - ['code_size', 'data_size', 'stack_frame', 'struct_size', - 'coverage_lines', 'coverage_branches', - 'test_passed', - 'bench_read', 'bench_prog', 'bench_erased'], - lambda xs: sum(xs[1:], start=xs[0]) - ), - 'mul': ( - [], - lambda xs: m.prod(xs[1:], start=xs[0]) - ), - 'min': ( - [], - min - ), - 'max': ( - ['stack_limit', 'coverage_hits'], - max - ), - 'avg': ( - [], - lambda xs: sum(xs[1:], start=xs[0]) / len(xs) - ), +# supported merge operations +OPS = { + 'add': lambda xs: sum(xs[1:], start=xs[0]), + 'mul': lambda xs: m.prod(xs[1:], start=xs[0]), + 'min': min, + 'max': max, + 'avg': lambda xs: sum(xs[1:], start=xs[0]) / len(xs), } @@ -273,112 +256,142 @@ class FracField(co.namedtuple('FracField', 'a,b')): def __truediv__(self, n): return FracField(self.a / n, self.b / n) +# available types +TYPES = [IntField, FloatField, FracField] + def homogenize(results, *, + by=None, fields=None, - merges=None, - renames=None, + renames=[], + define={}, types=None, **_): + results = results.copy() + # rename fields? - if renames is not None: + if renames: + for r in results: + # make a copy so renames can overlap + r_ = {} + for new_k, old_k in renames: + if old_k in r: + r_[new_k] = r[old_k] + r.update(r_) + + # filter by matching defines + if define: results_ = [] for r in results: - results_.append({renames.get(k, k): v for k, v in r.items()}) + if all(k in r and r[k] in vs for k, vs in define): + results_.append(r) results = results_ - # find all fields - if not fields: + # if fields not specified, try to guess from data + if fields is None: fields = co.OrderedDict() for r in results: - # also remove None fields, these can get introduced by - # csv.DictReader when header and rows mismatch - fields.update((k, v) for k, v in r.items() if k is not None) - fields = list(fields.keys()) + for k, v in r.items(): + if by is not None and k in by: + continue + types_ = [] + for type in fields.get(k, TYPES): + try: + type(v) + types_.append(type) + except ValueError: + pass + fields[k] = types_ + fields = list(k for k,v in fields.items() if v) + + # infer 'by' fields? + if by is None: + by = co.OrderedDict() + for r in results: + # also ignore None keys, these are introduced by csv.DictReader + # when header + row mismatch + by.update((k, True) for k in r.keys() + if k is not None + and k not in fields + and not any(k == old_k for _, old_k in renames)) + by = list(by.keys()) # go ahead and clean up none values, these can have a few forms results_ = [] for r in results: results_.append({ - k: r[k] for k in fields - if r.get(k) is not None and not( + k: r[k] for k in it.chain(by, fields) + if r.get(k) is not None and not ( isinstance(r[k], str) and re.match('^\s*[+-]?\s*$', r[k]))}) + results = results_ # find best type for all fields - def try_(x, type): - try: - type(x) - return True - except ValueError: - return False - if types is None: + def is_type(x, type): + try: + type(x) + return True + except ValueError: + return False + types = {} for k in fields: - if merges is not None and merges.get(k): - for type in [IntField, FloatField, FracField]: - if all(k not in r or try_(r[k], type) for r in results_): - types[k] = type - break - else: - print("no type matches field %r?" % k) - sys.exit(-1) + for type in TYPES: + if all(k not in r or is_type(r[k], type) for r in results_): + types[k] = type + break + else: + print("no type matches field %r?" % k) + sys.exit(-1) # homogenize types - for k in fields: - if k in types: - for r in results_: - if k in r: - r[k] = types[k](r[k]) + for r in results: + for k in fields: + if k in r: + r[k] = types[k](r[k]) - return fields, types, results_ + return by, fields, types, results def fold(results, *, - fields=None, - merges=None, - by=None, + by=[], + fields=[], + ops={}, **_): folding = co.OrderedDict() - if by is None: - by = [k for k in fields if k not in merges] - for r in results: - name = tuple(r.get(k) for k in by) + name = tuple(r.get(k, '') for k in by) if name not in folding: - folding[name] = {k: [] for k in fields if k in merges} + folding[name] = {k: [] for k in fields} for k in fields: - # drop all fields fields without a type - if k in merges and k in r: + if k in r: folding[name][k].append(r[k]) # merge fields, we need the count at this point for averages folded = [] - types = {} for name, r in folding.items(): r_ = {} for k, vs in r.items(): if vs: - _, merge = MERGES[merges[k]] - r_[k] = merge(vs) + # sum fields by default + op = OPS[ops.get(k, 'add')] + r_[k] = op(vs) - # drop all rows without any fields - # and drop all empty keys + # drop any rows without fields and any empty keys if r_: folded.append(dict( - {k: n for k, n in zip(by, name) if n}, + {k: v for k, v in zip(by, name) if v}, **r_)) - fields_ = by + [k for k in fields if k in merges] - return fields_, folded + return folded def table(results, diff_results=None, *, + by=None, fields=None, types=None, - merges=None, - by=None, + ops=None, sort=None, reverse_sort=None, summary=False, @@ -387,29 +400,18 @@ def table(results, diff_results=None, *, **_): all_, all = all, __builtins__.all - # fold - if by is not None: - fields, results = fold(results, fields=fields, merges=merges, by=by) - if diff_results is not None: - _, diff_results = fold(diff_results, - fields=fields, merges=merges, by=by) - - table = { - tuple(r.get(k,'') for k in fields if k not in merges): r - for r in results} - diff_table = { - tuple(r.get(k,'') for k in fields if k not in merges): r - for r in diff_results or []} + table = {tuple(r.get(k,'') for k in by): r for r in results} + diff_table = {tuple(r.get(k,'') for k in by): r for r in diff_results or []} # sort, note that python's sort is stable names = list(table.keys() | diff_table.keys()) names.sort() if diff_results is not None: - names.sort(key=lambda n: [ + names.sort(key=lambda n: tuple( -types[k].ratio( table.get(n,{}).get(k), diff_table.get(n,{}).get(k)) - for k in fields if k in merges]) + for k in fields)) if sort: names.sort(key=lambda n: tuple( (table[n][k],) if k in table.get(n,{}) else () @@ -423,7 +425,7 @@ def table(results, diff_results=None, *, # print header print('%-36s' % ('%s%s' % ( - ','.join(k for k in fields if k not in merges), + ','.join(k for k in by), ' (%d added, %d removed)' % ( sum(1 for n in table if n not in diff_table), sum(1 for n in diff_table if n not in table)) @@ -433,19 +435,19 @@ def table(results, diff_results=None, *, if diff_results is None: print(' %s' % ( ' '.join(k.rjust(len(types[k].none)) - for k in fields if k in merges))) + for k in fields))) elif percent: print(' %s' % ( ' '.join(k.rjust(len(types[k].diff_none)) - for k in fields if k in merges))) + for k in fields))) else: print(' %s %s %s' % ( ' '.join(('o'+k).rjust(len(types[k].diff_none)) - for k in fields if k in merges), + for k in fields), ' '.join(('n'+k).rjust(len(types[k].diff_none)) - for k in fields if k in merges), + for k in fields), ' '.join(('d'+k).rjust(len(types[k].diff_none)) - for k in fields if k in merges))) + for k in fields))) # print entries if not summary: @@ -454,7 +456,7 @@ def table(results, diff_results=None, *, if diff_results is not None: diff_r = diff_table.get(name, {}) ratios = [types[k].ratio(r.get(k), diff_r.get(k)) - for k in fields if k in merges] + for k in fields] if not any(ratios) and not all_: continue @@ -463,12 +465,12 @@ def table(results, diff_results=None, *, print(' %s' % ( ' '.join(r[k].table() if k in r else types[k].none - for k in fields if k in merges))) + for k in fields))) elif percent: print(' %s%s' % ( ' '.join(r[k].diff_table() if k in r else types[k].diff_none - for k in fields if k in merges), + for k in fields), ' (%s)' % ', '.join( '+∞%' if t == float('+inf') else '-∞%' if t == float('-inf') @@ -478,13 +480,13 @@ def table(results, diff_results=None, *, print(' %s %s %s%s' % ( ' '.join(diff_r[k].diff_table() if k in diff_r else types[k].diff_none - for k in fields if k in merges), + for k in fields), ' '.join(r[k].diff_table() if k in r else types[k].diff_none - for k in fields if k in merges), + for k in fields), ' '.join(types[k].diff_diff(r.get(k), diff_r.get(k)) if k in r or k in diff_r else types[k].diff_none - for k in fields if k in merges), + for k in fields), ' (%s)' % ', '.join( '+∞%' if t == float('+inf') else '-∞%' if t == float('-inf') @@ -494,26 +496,25 @@ def table(results, diff_results=None, *, if any(ratios) else '')) # print total - _, total = fold(results, fields=fields, merges=merges, by=[]) + total = fold(results, by=[], fields=fields, ops=ops) r = total[0] if total else {} if diff_results is not None: - _, diff_total = fold(diff_results, - fields=fields, merges=merges, by=[]) + diff_total = fold(diff_results, by=[], fields=fields, ops=ops) diff_r = diff_total[0] if diff_total else {} ratios = [types[k].ratio(r.get(k), diff_r.get(k)) - for k in fields if k in merges] + for k in fields] print('%-36s' % 'TOTAL', end='') if diff_results is None: print(' %s' % ( ' '.join(r[k].table() if k in r else types[k].none - for k in fields if k in merges))) + for k in fields))) elif percent: print(' %s%s' % ( ' '.join(r[k].diff_table() if k in r else types[k].diff_none - for k in fields if k in merges), + for k in fields), ' (%s)' % ', '.join( '+∞%' if t == float('+inf') else '-∞%' if t == float('-inf') @@ -523,13 +524,13 @@ def table(results, diff_results=None, *, print(' %s %s %s%s' % ( ' '.join(diff_r[k].diff_table() if k in diff_r else types[k].diff_none - for k in fields if k in merges), + for k in fields), ' '.join(r[k].diff_table() if k in r else types[k].diff_none - for k in fields if k in merges), + for k in fields), ' '.join(types[k].diff_diff(r.get(k), diff_r.get(k)) if k in r or k in diff_r else types[k].diff_none - for k in fields if k in merges), + for k in fields), ' (%s)' % ', '.join( '+∞%' if t == float('+inf') else '-∞%' if t == float('-inf') @@ -539,56 +540,35 @@ def table(results, diff_results=None, *, if any(ratios) else '')) -def main(csv_paths, *, fields=None, by=None, **args): - # figure out what fields to use - renames = {} - +def main(csv_paths, *, + by=None, + fields=None, + define=[], + **args): + # separate out renames + renames = [k.split('=', 1) + for k in it.chain(by or [], fields or []) + if '=' in k] + if by is not None: + by = [k.split('=', 1)[0] for k in by] if fields is not None: - fields_ = [] - for name in fields: - if '=' in name: - a, b = name.split('=', 1) - renames[b] = a - name = a - fields_.append(name) - fields = fields_ + fields = [k.split('=', 1)[0] for k in fields] - if by is not None: - by_ = [] - for name in by: - if '=' in name: - a, b = name.split('=', 1) - renames[b] = a - name = a - by_.append(name) - by = by_ - - # include 'by' fields in fields, it doesn't make sense to not - if fields is not None and by is not None: - fields[:0] = [k for k in by if k not in fields] - - # use preconfigured merge operations unless any merge operation is - # explictly specified - merge_args = (args - if any(args.get(m) for m in MERGES.keys()) - else {m: k for m, (k, _) in MERGES.items()}) - merges = {} - for m in MERGES.keys(): - for k in merge_args.get(m, []): - if k in merges: - print("conflicting merge type for field %r?" % k) + # figure out merge operations + ops = {} + for m in OPS.keys(): + for k in args.get(m, []): + if k in ops: + print("conflicting op for field %r?" % k) sys.exit(-1) - merges[k] = m - # allow renames to apply to merges - for m in MERGES.keys(): - for k in merge_args.get(m, []): - if renames.get(k, k) not in merges: - merges[renames.get(k, k)] = m - # ignore merges that conflict with 'by' fields - if by is not None: - for k in by: - if k in merges: - del merges[k] + ops[k] = m + # rename ops? + if renames: + ops_ = {} + for new_k, old_k in renames: + if old_k in ops: + ops_[new_k] = ops[old_k] + ops.update(ops_) # find CSV files paths = [] @@ -614,17 +594,17 @@ def main(csv_paths, *, fields=None, by=None, **args): pass # homogenize - fields, types, results = homogenize(results, - fields=fields, merges=merges, renames=renames) + by, fields, types, results = homogenize(results, + by=by, fields=fields, renames=renames, define=define) # fold to remove duplicates - fields, results = fold(results, - fields=fields, merges=merges) + results = fold(results, + by=by, fields=fields, ops=ops) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, fields) + writer = csv.DictWriter(f, by + fields) writer.writeheader() for r in results: writer.writerow(r) @@ -641,22 +621,22 @@ def main(csv_paths, *, fields=None, by=None, **args): pass # homogenize - _, _, diff_results = homogenize(diff_results, - fields=fields, merges=merges, renames=renames, types=types) + _, _, _, diff_results = homogenize(diff_results, + by=by, fields=fields, renames=renames, define=define, types=types) # fold to remove duplicates - _, diff_results = fold(diff_results, - fields=fields, merges=merges) + diff_results = fold(diff_results, + by=by, fields=fields, ops=ops) # print table if not args.get('quiet'): table( results, diff_results if args.get('diff') else None, - fields=fields, - types=types, - merges=merges, by=by, + fields=fields, + ops=ops, + types=types, **args) @@ -689,36 +669,41 @@ if __name__ == "__main__": '-p', '--percent', action='store_true', help="Only show percentage change, not a full diff.") - parser.add_argument( - '-f', '--fields', - type=lambda x: [x.strip() for x in x.split(',')], - help="Only show these fields. Can rename fields " - "with new_name=old_name.") parser.add_argument( '-b', '--by', type=lambda x: [x.strip() for x in x.split(',')], - help="Group by these fields. Can rename fields " - "with new_name=old_name.") + help="Group by these fields. All other fields will be merged as " + "needed. Can rename fields with new_name=old_name.") + parser.add_argument( + '-f', '--fields', + type=lambda x: [x.strip() for x in x.split(',')], + help="Use these fields. Can rename fields with new_name=old_name.") + parser.add_argument( + '-D', '--define', + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), + action='append', + help="Only include rows where this field is this value. May include " + "comma-separated options.") parser.add_argument( '--add', type=lambda x: [x.strip() for x in x.split(',')], - help="Add these fields when merging.") + help="Add these fields (the default).") parser.add_argument( '--mul', type=lambda x: [x.strip() for x in x.split(',')], - help="Multiply these fields when merging.") + help="Multiply these fields.") parser.add_argument( '--min', type=lambda x: [x.strip() for x in x.split(',')], - help="Take the minimum of these fields when merging.") + help="Take the minimum of these fields.") parser.add_argument( '--max', type=lambda x: [x.strip() for x in x.split(',')], - help="Take the maximum of these fields when merging.") + help="Take the maximum of these fields.") parser.add_argument( '--avg', type=lambda x: [x.strip() for x in x.split(',')], - help="Average these fields when merging.") + help="Average these fields.") parser.add_argument( '-s', '--sort', type=lambda x: [x.strip() for x in x.split(',')], From 42d889e14174c60829aa750d5f70bec23a528755 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 24 Sep 2022 20:30:55 -0500 Subject: [PATCH 45/81] Reworked/simplified tracebd.py a bit Instead of trying to align to block-boundaries tracebd.py now just aliases to whatever dimensions are provided. Also reworked how scripts handle default sizing. Now using reasonable defaults with 0 being a placeholder for automatic sizing. The addition of -z/--cat makes it possible to pipe directly to stdout. Also added support for dots/braille output which can capture more detail, though care needs to be taken to not rely on accurate coloring. --- scripts/plot.py | 127 ++++-- scripts/tailpipe.py | 143 ++++--- scripts/tracebd.py | 944 ++++++++++++++++++++++++++------------------ 3 files changed, 731 insertions(+), 483 deletions(-) diff --git a/scripts/plot.py b/scripts/plot.py index 6eeb69ad..ad59adca 100755 --- a/scripts/plot.py +++ b/scripts/plot.py @@ -89,6 +89,61 @@ def openio(path, mode='r'): else: return open(path, mode) +class LinesIO: + def __init__(self, maxlen=None): + self.maxlen = maxlen + self.lines = co.deque(maxlen=maxlen) + self.tail = io.StringIO() + + # trigger automatic sizing + if maxlen == 0: + self.resize(0) + + def write(self, s): + # note using split here ensures the trailing string has no newline + lines = s.split('\n') + + if len(lines) > 1 and self.tail.getvalue(): + self.tail.write(lines[0]) + lines[0] = self.tail.getvalue() + self.tail = io.StringIO() + + self.lines.extend(lines[:-1]) + + if lines[-1]: + self.tail.write(lines[-1]) + + def resize(self, maxlen): + self.maxlen = maxlen + if maxlen == 0: + maxlen = shutil.get_terminal_size((80, 5))[1] + if maxlen != self.lines.maxlen: + self.lines = co.deque(self.lines, maxlen=maxlen) + + last_lines = 1 + def draw(self): + # did terminal size change? + if self.maxlen == 0: + self.resize(0) + + # first thing first, give ourself a canvas + while LinesIO.last_lines < len(self.lines): + sys.stdout.write('\n') + LinesIO.last_lines += 1 + + for j, line in enumerate(self.lines): + # move cursor, clear line, disable/reenable line wrapping + sys.stdout.write('\r') + if len(self.lines)-1-j > 0: + sys.stdout.write('\x1b[%dA' % (len(self.lines)-1-j)) + sys.stdout.write('\x1b[K') + sys.stdout.write('\x1b[?7l') + sys.stdout.write(line) + sys.stdout.write('\x1b[?7h') + if len(self.lines)-1-j > 0: + sys.stdout.write('\x1b[%dB' % (len(self.lines)-1-j)) + sys.stdout.flush() + # parse different data representations def dat(x): @@ -114,6 +169,7 @@ def dat(x): # else give up raise ValueError("invalid dat %r" % x) + # a hack log10 that preserves sign, and passes zero as zero def slog10(x): if x == 0: @@ -123,7 +179,6 @@ def slog10(x): else: return -m.log10(-x) - class Plot: def __init__(self, width, height, *, xlim=None, @@ -427,7 +482,8 @@ def main(csv_paths, *, xlim=None, ylim=None, width=None, - height=None, + height=17, + cat=False, color=False, braille=False, colors=None, @@ -538,10 +594,12 @@ def main(csv_paths, *, if v is not None)))) # figure out our plot size - if width is not None: + if width is None: + width_ = min(80, shutil.get_terminal_size((80, 17))[0]) + elif width: width_ = width else: - width_ = shutil.get_terminal_size((80, 8))[0] + width_ = shutil.get_terminal_size((80, 17))[0] # make space for units width_ -= 5 # make space for legend @@ -550,10 +608,10 @@ def main(csv_paths, *, # limit a bit width_ = max(2*4, width_) - if height is not None: + if height: height_ = height else: - height_ = shutil.get_terminal_size((80, 8))[1] + height_ = shutil.get_terminal_size((80, 17))[1] # make space for shell prompt if not keep_open: height_ -= 1 @@ -644,45 +702,26 @@ def main(csv_paths, *, '\x1b[m' if color else '') for j in range(i, min(i+legend_cols, len(legend_)))))) - - last_lines = 1 - def redraw(): - nonlocal last_lines - - canvas = io.StringIO() - draw(canvas) - canvas = canvas.getvalue().splitlines() - - # give ourself a canvas - while last_lines < len(canvas): - sys.stdout.write('\n') - last_lines += 1 - - for i, line in enumerate(canvas): - jump = len(canvas)-1-i - # move cursor, clear line, disable/reenable line wrapping - sys.stdout.write('\r') - if jump > 0: - sys.stdout.write('\x1b[%dA' % jump) - sys.stdout.write('\x1b[K') - sys.stdout.write('\x1b[?7l') - sys.stdout.write(line) - sys.stdout.write('\x1b[?7h') - if jump > 0: - sys.stdout.write('\x1b[%dB' % jump) - - sys.stdout.flush() - if keep_open: try: while True: - redraw() + if cat: + draw(sys.stdout) + else: + ring = LinesIO() + draw(ring) + ring.draw() # don't just flood open calls time.sleep(sleep or 0.1) except KeyboardInterrupt: pass - redraw() + if cat: + draw(sys.stdout) + else: + ring = LinesIO() + draw(ring) + ring.draw() sys.stdout.write('\n') else: draw(sys.stdout) @@ -726,9 +765,9 @@ if __name__ == "__main__": default='auto', help="When to use terminal colors. Defaults to 'auto'.") parser.add_argument( - '--braille', + '-⣿', '--braille', action='store_true', - help="Use unicode braille characters. Note that braille characters " + help="Use 2x4 unicode braille characters. Note that braille characters " "sometimes suffer from inconsistent widths.") parser.add_argument( '--colors', @@ -747,12 +786,16 @@ if __name__ == "__main__": parser.add_argument( '-W', '--width', type=lambda x: int(x, 0), - help="Width in columns. A width of 0 indicates no limit. Defaults " - "to terminal width or 80.") + help="Width in columns. 0 uses the terminal width. Defaults to " + "min(terminal, 80).") parser.add_argument( '-H', '--height', type=lambda x: int(x, 0), - help="Height in rows. Defaults to terminal height or 8.") + help="Height in rows. 0 uses the terminal height. Defaults to 17.") + parser.add_argument( + '-z', '--cat', + action='store_true', + help="Pipe directly to stdout.") parser.add_argument( '-X', '--xlim', type=lambda x: tuple(dat(x) if x else None for x in x.split(',')), diff --git a/scripts/tailpipe.py b/scripts/tailpipe.py index ae477e22..3db7612f 100755 --- a/scripts/tailpipe.py +++ b/scripts/tailpipe.py @@ -9,9 +9,11 @@ # SPDX-License-Identifier: BSD-3-Clause # +import collections as co +import io import os +import shutil import sys -import threading as th import time @@ -24,71 +26,89 @@ def openio(path, mode='r'): else: return open(path, mode) -def main(path='-', *, lines=1, sleep=0.01, keep_open=False): - ring = [None] * lines - i = 0 - count = 0 - lock = th.Lock() - event = th.Event() - done = False +class LinesIO: + def __init__(self, maxlen=None): + self.maxlen = maxlen + self.lines = co.deque(maxlen=maxlen) + self.tail = io.StringIO() - # do the actual reading in a background thread - def read(): - nonlocal i - nonlocal count - nonlocal done + # trigger automatic sizing + if maxlen == 0: + self.resize(0) + + def write(self, s): + # note using split here ensures the trailing string has no newline + lines = s.split('\n') + + if len(lines) > 1 and self.tail.getvalue(): + self.tail.write(lines[0]) + lines[0] = self.tail.getvalue() + self.tail = io.StringIO() + + self.lines.extend(lines[:-1]) + + if lines[-1]: + self.tail.write(lines[-1]) + + def resize(self, maxlen): + self.maxlen = maxlen + if maxlen == 0: + maxlen = shutil.get_terminal_size((80, 5))[1] + if maxlen != self.lines.maxlen: + self.lines = co.deque(self.lines, maxlen=maxlen) + + last_lines = 1 + def draw(self): + # did terminal size change? + if self.maxlen == 0: + self.resize(0) + + # first thing first, give ourself a canvas + while LinesIO.last_lines < len(self.lines): + sys.stdout.write('\n') + LinesIO.last_lines += 1 + + for j, line in enumerate(self.lines): + # move cursor, clear line, disable/reenable line wrapping + sys.stdout.write('\r') + if len(self.lines)-1-j > 0: + sys.stdout.write('\x1b[%dA' % (len(self.lines)-1-j)) + sys.stdout.write('\x1b[K') + sys.stdout.write('\x1b[?7l') + sys.stdout.write(line) + sys.stdout.write('\x1b[?7h') + if len(self.lines)-1-j > 0: + sys.stdout.write('\x1b[%dB' % (len(self.lines)-1-j)) + sys.stdout.flush() + + +def main(path='-', *, lines=5, cat=False, sleep=0.01, keep_open=False): + if cat: + ring = sys.stdout + else: + ring = LinesIO(lines) + + ptime = time.time() + try: while True: with openio(path) as f: for line in f: - with lock: - ring[i] = line - i = (i + 1) % lines - count = min(lines, count + 1) - event.set() + ring.write(line) + + # need to redraw? + if not cat and time.time()-ptime >= sleep: + ring.draw() + ptime = time.time() + if not keep_open: break # don't just flood open calls time.sleep(sleep or 0.1) - done = True - - th.Thread(target=read, daemon=True).start() - - try: - last_count = 1 - while not done: - time.sleep(sleep) - event.wait() - event.clear() - - # create a copy to avoid corrupt output - with lock: - ring_ = ring.copy() - i_ = i - count_ = count - - # first thing first, give ourself a canvas - while last_count < count_: - sys.stdout.write('\n') - last_count += 1 - - for j in range(count_): - # move cursor, clear line, disable/reenable line wrapping - sys.stdout.write('\r') - if count_-1-j > 0: - sys.stdout.write('\x1b[%dA' % (count_-1-j)) - sys.stdout.write('\x1b[K') - sys.stdout.write('\x1b[?7l') - sys.stdout.write(ring_[(i_-count_+j) % lines][:-1]) - sys.stdout.write('\x1b[?7h') - if count_-1-j > 0: - sys.stdout.write('\x1b[%dB' % (count_-1-j)) - - sys.stdout.flush() - except KeyboardInterrupt: pass - sys.stdout.write('\n') + if not cat: + sys.stdout.write('\n') if __name__ == "__main__": @@ -104,15 +124,18 @@ if __name__ == "__main__": '-n', '--lines', type=lambda x: int(x, 0), - help="Number of lines to show. Defaults to 1.") + help="Show this many lines of history. 0 uses the terminal height. " + "Defaults to 5.") parser.add_argument( - '-s', - '--sleep', + '-z', '--cat', + action='store_true', + help="Pipe directly to stdout.") + parser.add_argument( + '-s', '--sleep', type=float, help="Seconds to sleep between reads. Defaults to 0.01.") parser.add_argument( - '-k', - '--keep-open', + '-k', '--keep-open', action='store_true', help="Reopen the pipe on EOF, useful when multiple " "processes are writing.") diff --git a/scripts/tracebd.py b/scripts/tracebd.py index 1905f5d8..84b0b29a 100755 --- a/scripts/tracebd.py +++ b/scripts/tracebd.py @@ -17,10 +17,30 @@ import math as m import os import re import shutil -import threading as th import time + +CHARS = 'rpe.' +COLORS = ['42', '45', '44', ''] + +WEAR_CHARS = '0123456789' +WEAR_CHARS_SUBSCRIPTS = '.₁₂₃₄₅₆789' +WEAR_COLORS = ['', '', '', '', '', '', '', '35', '35', '1;31'] + +CHARS_DOTS = " .':" +COLORS_DOTS = ['32', '35', '34', ''] +CHARS_BRAILLE = ( + '⠀⢀⡀⣀⠠⢠⡠⣠⠄⢄⡄⣄⠤⢤⡤⣤' '⠐⢐⡐⣐⠰⢰⡰⣰⠔⢔⡔⣔⠴⢴⡴⣴' + '⠂⢂⡂⣂⠢⢢⡢⣢⠆⢆⡆⣆⠦⢦⡦⣦' '⠒⢒⡒⣒⠲⢲⡲⣲⠖⢖⡖⣖⠶⢶⡶⣶' + '⠈⢈⡈⣈⠨⢨⡨⣨⠌⢌⡌⣌⠬⢬⡬⣬' '⠘⢘⡘⣘⠸⢸⡸⣸⠜⢜⡜⣜⠼⢼⡼⣼' + '⠊⢊⡊⣊⠪⢪⡪⣪⠎⢎⡎⣎⠮⢮⡮⣮' '⠚⢚⡚⣚⠺⢺⡺⣺⠞⢞⡞⣞⠾⢾⡾⣾' + '⠁⢁⡁⣁⠡⢡⡡⣡⠅⢅⡅⣅⠥⢥⡥⣥' '⠑⢑⡑⣑⠱⢱⡱⣱⠕⢕⡕⣕⠵⢵⡵⣵' + '⠃⢃⡃⣃⠣⢣⡣⣣⠇⢇⡇⣇⠧⢧⡧⣧' '⠓⢓⡓⣓⠳⢳⡳⣳⠗⢗⡗⣗⠷⢷⡷⣷' + '⠉⢉⡉⣉⠩⢩⡩⣩⠍⢍⡍⣍⠭⢭⡭⣭' '⠙⢙⡙⣙⠹⢹⡹⣹⠝⢝⡝⣝⠽⢽⡽⣽' + '⠋⢋⡋⣋⠫⢫⡫⣫⠏⢏⡏⣏⠯⢯⡯⣯' '⠛⢛⡛⣛⠻⢻⡻⣻⠟⢟⡟⣟⠿⢿⡿⣿') + + def openio(path, mode='r'): if path == '-': if mode == 'r': @@ -30,6 +50,63 @@ def openio(path, mode='r'): else: return open(path, mode) +class LinesIO: + def __init__(self, maxlen=None): + self.maxlen = maxlen + self.lines = co.deque(maxlen=maxlen) + self.tail = io.StringIO() + + # trigger automatic sizing + if maxlen == 0: + self.resize(0) + + def write(self, s): + # note using split here ensures the trailing string has no newline + lines = s.split('\n') + + if len(lines) > 1 and self.tail.getvalue(): + self.tail.write(lines[0]) + lines[0] = self.tail.getvalue() + self.tail = io.StringIO() + + self.lines.extend(lines[:-1]) + + if lines[-1]: + self.tail.write(lines[-1]) + + def resize(self, maxlen): + self.maxlen = maxlen + if maxlen == 0: + maxlen = shutil.get_terminal_size((80, 5))[1] + if maxlen != self.lines.maxlen: + self.lines = co.deque(self.lines, maxlen=maxlen) + + last_lines = 1 + def draw(self): + # did terminal size change? + if self.maxlen == 0: + self.resize(0) + + # first thing first, give ourself a canvas + while LinesIO.last_lines < len(self.lines): + sys.stdout.write('\n') + LinesIO.last_lines += 1 + + for j, line in enumerate(self.lines): + # move cursor, clear line, disable/reenable line wrapping + sys.stdout.write('\r') + if len(self.lines)-1-j > 0: + sys.stdout.write('\x1b[%dA' % (len(self.lines)-1-j)) + sys.stdout.write('\x1b[K') + sys.stdout.write('\x1b[?7l') + sys.stdout.write(line) + sys.stdout.write('\x1b[?7h') + if len(self.lines)-1-j > 0: + sys.stdout.write('\x1b[%dB' % (len(self.lines)-1-j)) + sys.stdout.flush() + + + # space filling Hilbert-curve # # note we memoize the last curve since this is a bit expensive @@ -113,209 +190,362 @@ def lebesgue_curve(width, height): return curve -class Block: - def __init__(self, wear=0, readed=False, proged=False, erased=False): - self._ = ((wear << 3) +class Block(int): + __slots__ = () + def __new__(cls, state=0, *, + wear=0, + readed=False, + proged=False, + erased=False): + return super().__new__(cls, + state + | (wear << 3) | (1 if readed else 0) | (2 if proged else 0) - | (4 if erased else False)) + | (4 if erased else 0)) @property def wear(self): - return self._ >> 3 + return self >> 3 @property def readed(self): - return (self._ & 1) != 0 + return (self & 1) != 0 @property def proged(self): - return (self._ & 2) != 0 + return (self & 2) != 0 @property def erased(self): - return (self._ & 4) != 0 + return (self & 4) != 0 def read(self): - self._ |= 1 + return Block(int(self) | 1) def prog(self): - self._ |= 2 + return Block(int(self) | 2) def erase(self): - self._ = (self._ | 4) + 8 + return Block((int(self) | 4) + 8) def clear(self): - self._ &= ~7 + return Block(int(self) & ~7) - def reset(self): - self._ = 0 - - def copy(self): - return Block(self.wear, self.readed, self.proged, self.erased) - - def __add__(self, other): + def __or__(self, other): return Block( - max(self.wear, other.wear), - self.readed | other.readed, - self.proged | other.proged, - self.erased | other.erased) + (int(self) | int(other)) & 7, + wear=max(self.wear, other.wear)) - def draw(self, *, - subscripts=False, - chars=None, + def worn(self, max_wear, *, + block_cycles=None, wear_chars=None, - color=True, + **_): + if wear_chars is None: + wear_chars = WEAR_CHARS + + if block_cycles: + return self.wear / block_cycles + else: + return self.wear / max(max_wear, len(wear_chars)) + + def draw(self, max_wear, char=None, *, read=True, prog=True, erase=True, wear=False, - max_wear=None, block_cycles=None, + color=True, + subscripts=False, + dots=False, + braille=False, + chars=None, + wear_chars=None, + colors=None, + wear_colors=None, **_): - if not chars: chars = '.rpe' - c = chars[0] - f = [] + # fallback to default chars/colors + if chars is None: + chars = CHARS + if len(chars) < len(CHARS): + chars = chars + CHARS[len(chars):] + + if colors is None: + if braille or dots: + colors = COLORS_DOTS + else: + colors = COLORS + if len(colors) < len(COLORS): + colors = colors + COLORS[len(colors):] + + if wear_chars is None: + if subscripts: + wear_chars = WEAR_CHARS_SUBSCRIPTS + else: + wear_chars = WEAR_CHARS + + if wear_colors is None: + wear_colors = WEAR_COLORS + + # compute char/color + c = chars[3] + f = [colors[3]] if wear: - if not wear_chars and subscripts: wear_chars = '.₁₂₃₄₅₆789' - elif not wear_chars: wear_chars = '0123456789' + w = min( + self.worn( + max_wear, + block_cycles=block_cycles, + wear_chars=wear_chars), + 1) - if block_cycles: - w = self.wear / block_cycles - else: - w = self.wear / max(max_wear, len(wear_chars)-1) + c = wear_chars[int(w * (len(wear_chars)-1))] + f.append(wear_colors[int(w * (len(wear_colors)-1))]) - c = wear_chars[min( - int(w*(len(wear_chars)-1)), - len(wear_chars)-1)] - if color: - if w*9 >= 9: f.append('\x1b[1;31m') - elif w*9 >= 7: f.append('\x1b[35m') + if erase and self.erased: + c = chars[2] + f.append(colors[2]) + elif prog and self.proged: + c = chars[1] + f.append(colors[1]) + elif read and self.readed: + c = chars[0] + f.append(colors[0]) - if erase and self.erased: c = chars[3] - elif prog and self.proged: c = chars[2] - elif read and self.readed: c = chars[1] + # override char? + if char: + c = char - if color: - if erase and self.erased: f.append('\x1b[44m') - elif prog and self.proged: f.append('\x1b[45m') - elif read and self.readed: f.append('\x1b[42m') + # apply colors + if f and color: + c = '%s%s\x1b[m' % ( + ''.join('\x1b[%sm' % f_ for f_ in f), + c) + + return c - if color: - return '%s%c\x1b[m' % (''.join(f), c) - else: - return c class Bd: - def __init__(self, *, blocks=None, size=1, count=1, width=80): - if blocks is not None: + def __init__(self, *, + size=1, + count=1, + width=None, + height=1, + blocks=None): + if width is None: + width = count + + if blocks is None: + self.blocks = [Block() for _ in range(width*height)] + else: self.blocks = blocks - self.size = size - self.count = count - self.width = width - else: - self.blocks = [] - self.size = None - self.count = None - self.width = None - self.smoosh(size=size, count=count, width=width) - - def get(self, block=slice(None), off=slice(None)): - if not isinstance(block, slice): - block = slice(block, block+1) - if not isinstance(off, slice): - off = slice(off, off+1) - - if (not self.blocks - or not self.width - or not self.size - or not self.count): - return - - if self.count >= self.width: - scale = (self.count+self.width-1) // self.width - for i in range( - (block.start if block.start is not None else 0)//scale, - (min(block.stop if block.stop is not None else self.count, - self.count)+scale-1)//scale): - yield self.blocks[i] - else: - scale = self.width // self.count - for i in range( - block.start if block.start is not None else 0, - min(block.stop if block.stop is not None else self.count, - self.count)): - for j in range( - ((off.start if off.start is not None else 0) - *scale)//self.size, - (min(off.stop if off.stop is not None else self.size, - self.size)*scale+self.size-1)//self.size): - yield self.blocks[i*scale+j] - - def __getitem__(self, block=slice(None), off=slice(None)): - if isinstance(block, tuple): - block, off = block - if not isinstance(block, slice): - block = slice(block, block+1) - if not isinstance(off, slice): - off = slice(off, off+1) - - # needs resize? - if ((block.stop is not None and block.stop > self.count) - or (off.stop is not None and off.stop > self.size)): - self.smoosh( - count=max(block.stop or self.count, self.count), - size=max(off.stop or self.size, self.size)) - - return self.get(block, off) - - def smoosh(self, *, size=None, count=None, width=None): - size = size or self.size - count = count or self.count - width = width or self.width - - if count >= width: - scale = (count+width-1) // width - self.blocks = [ - sum(self.get(slice(i,i+scale)), start=Block()) - for i in range(0, count, scale)] - else: - scale = width // count - self.blocks = [ - sum(self.get(i, slice(j*(size//width),(j+1)*(size//width))), - start=Block()) - for i in range(0, count) - for j in range(scale)] - self.size = size self.count = count self.width = width + self.height = height - def read(self, block=slice(None), off=slice(None)): - for c in self[block, off]: - c.read() + def _op(self, f, block=None, off=None, size=None): + if block is None: + range_ = range(len(self.blocks)) + else: + if off is None: + off, size = 0, self.size + elif size is None: + off, size = 0, off - def prog(self, block=slice(None), off=slice(None)): - for c in self[block, off]: - c.prog() + # update our geometry? this will do nothing if we haven't changed + self.resize( + size=max(self.size, off+size), + count=max(self.count, block+1)) - def erase(self, block=slice(None), off=slice(None)): - for c in self[block, off]: - c.erase() + # map to our block space + start = (block*self.size + off) / (self.size*self.count) + stop = (block*self.size + off+size) / (self.size*self.count) - def clear(self, block=slice(None), off=slice(None)): - for c in self[block, off]: - c.clear() + range_ = range( + m.floor(start*len(self.blocks)), + m.ceil(stop*len(self.blocks))) - def reset(self, block=slice(None), off=slice(None)): - for c in self[block, off]: - c.reset() + # apply the op + for i in range_: + self.blocks[i] = f(self.blocks[i]) + + def read(self, block=None, off=None, size=None): + self._op(Block.read, block, off, size) + + def prog(self, block=None, off=None, size=None): + self._op(Block.prog, block, off, size) + + def erase(self, block=None, off=None, size=None): + self._op(Block.erase, block, off, size) + + def clear(self, block=None, off=None, size=None): + self._op(Block.clear, block, off, size) def copy(self): return Bd( - blocks=[b.copy() for b in self.blocks], - size=self.size, count=self.count, width=self.width) + blocks=self.blocks.copy(), + size=self.size, + count=self.count, + width=self.width, + height=self.height) + + def resize(self, *, + size=None, + count=None, + width=None, + height=None): + size = size if size is not None else self.size + count = count if count is not None else self.count + width = width if width is not None else self.width + height = height if height is not None else self.height + + if (size == self.size + and count == self.count + and width == self.width + and height == self.height): + return + + # transform our blocks + blocks = [] + for x in range(width*height): + # map from new bd space + start = m.floor(x * (size*count)/(width*height)) + stop = m.ceil((x+1) * (size*count)/(width*height)) + start_block = start // size + start_off = start % size + stop_block = stop // size + stop_off = stop % size + # map to old bd space + start = start_block*self.size + start_off + stop = stop_block*self.size + stop_off + start = m.floor(start * len(self.blocks)/(self.size*self.count)) + stop = m.ceil(stop * len(self.blocks)/(self.size*self.count)) + + # aggregate state + blocks.append(ft.reduce( + Block.__or__, + self.blocks[start:stop], + Block())) + + self.size = size + self.count = count + self.width = width + self.height = height + self.blocks = blocks + + def draw(self, row, *, + read=False, + prog=False, + erase=False, + wear=False, + hilbert=False, + lebesgue=False, + dots=False, + braille=False, + **args): + # find max wear? + max_wear = None + if wear: + max_wear = max(b.wear for b in self.blocks) + + # fold via a curve? + if hilbert: + grid = [None]*(self.width*self.height) + for (x,y), b in zip( + hilbert_curve(self.width, self.height), + self.blocks): + grid[x + y*self.width] = b + elif lebesgue: + grid = [None]*(self.width*self.height) + for (x,y), b in zip( + lebesgue_curve(self.width, self.height), + self.blocks): + grid[x + y*self.width] = b + else: + grid = self.blocks + + # need to wait for more trace output before rendering + # + # this is sort of a hack that knows the output is going to a terminal + if (braille and self.height < 4) or (dots and self.height < 2): + needed_height = 4 if braille else 2 + + self.history = getattr(self, 'history', []) + self.history.append(grid) + + if len(self.history)*self.height < needed_height: + # skip for now + return None + + grid = list(it.chain.from_iterable( + # did we resize? + it.islice(it.chain(h, it.repeat(Block())), + self.width*self.height) + for h in self.history)) + self.history = [] + + line = [] + if braille: + # encode into a byte + for x in range(0, self.width, 2): + byte_b = 0 + best_b = Block() + for i in range(2*4): + b = grid[x+(2-1-(i%2)) + ((row*4)+(4-1-(i//2)))*self.width] + best_b |= b + if ((read and b.readed) + or (prog and b.proged) + or (erase and b.erased) + or (not read and not prog and not erase + and wear and b.worn(max_wear, **args) >= 0.7)): + byte_b |= 1 << i + + line.append(best_b.draw( + max_wear, + CHARS_BRAILLE[byte_b], + braille=True, + read=read, + prog=prog, + erase=erase, + wear=wear, + **args)) + elif dots: + # encode into a byte + for x in range(self.width): + byte_b = 0 + best_b = Block() + for i in range(2): + b = grid[x + ((row*2)+(2-1-i))*self.width] + best_b |= b + if ((read and b.readed) + or (prog and b.proged) + or (erase and b.erased) + or (not read and not prog and not erase + and wear and b.worn(max_wear, **args) >= 0.7)): + byte_b |= 1 << i + + line.append(best_b.draw( + max_wear, + CHARS_DOTS[byte_b], + dots=True, + read=read, + prog=prog, + erase=erase, + wear=wear, + **args)) + else: + for x in range(self.width): + line.append(grid[x + row*self.width].draw( + max_wear, + read=read, + prog=prog, + erase=erase, + wear=wear, + **args)) + + return ''.join(line) + def main(path='-', *, @@ -323,28 +553,25 @@ def main(path='-', *, prog=False, erase=False, wear=False, - color='auto', block=(None,None), off=(None,None), block_size=None, block_count=None, block_cycles=None, reset=False, + color='auto', + dots=False, + braille=False, width=None, - height=1, - scale=None, + height=None, lines=None, - coalesce=None, - sleep=None, + cat=False, hilbert=False, lebesgue=False, + coalesce=None, + sleep=None, keep_open=False, **args): - # exclusive wear or read/prog/erase by default - if not read and not prog and not erase and not wear: - read = True - prog = True - erase = True # figure out what color should be if color == 'auto': color = sys.stdout.isatty() @@ -353,6 +580,30 @@ def main(path='-', *, else: color = False + # exclusive wear or read/prog/erase by default + if not read and not prog and not erase and not wear: + read = True + prog = True + erase = True + + # assume a reasonable lines/height if not specified + # + # note that we let height = None if neither hilbert or lebesgue + # are specified, this is a bit special as the default may be less + # than one character in height. + if height is None and (hilbert or lebesgue): + if lines is not None: + height = lines + else: + height = 5 + + if lines is None: + if height is not None: + lines = height + else: + lines = 5 + + # allow ranges for blocks/offs block_start = block[0] block_stop = block[1] if len(block) > 1 else block[0]+1 off_start = off[0] @@ -367,38 +618,48 @@ def main(path='-', *, if off_stop is None and block_size is not None: off_stop = block_size - bd = Bd( - size=(block_size if block_size is not None - else off_stop-off_start if off_stop is not None - else 1), - count=(block_count if block_count is not None - else block_stop-block_start if block_stop is not None - else 1), - width=(width or 80)*height) - lock = th.Lock() - event = th.Event() - done = False + # create a block device representation + bd = Bd() - # adjust width? - def resmoosh(): + def resize(*, size=None, count=None): + nonlocal bd + + # size may be overriden by cli args + if block_size is not None: + size = block_size + elif off_stop is not None: + size = off_stop-off_start + + if block_count is not None: + count = block_count + elif block_stop is not None: + count = block_stop-block_start + + # figure out best width/height if width is None: - w = shutil.get_terminal_size((80, 0))[0] * height - elif width == 0: - w = max(int(bd.count*(scale or 1)), 1) + width_ = min(80, shutil.get_terminal_size((80, 5))[0]) + elif width: + width_ = width else: - w = width * height + width_ = shutil.get_terminal_size((80, 5))[0] - if scale and int(bd.count*scale) > w: - c = int(w/scale) - elif scale and int(bd.count*scale) < w: - w = max(int(bd.count*(scale or 1)), 1) - c = bd.count + if height is None: + height_ = 0 + elif height: + height_ = height else: - c = bd.count + height_ = shutil.get_terminal_size((80, 5))[1] - if w != bd.width or c != bd.count: - bd.smoosh(width=w, count=c) - resmoosh() + bd.resize( + size=size, + count=count, + # scale if we're printing with dots or braille + width=2*width_ if braille else width_, + height=max(1, + 4*height_ if braille + else 2*height_ if dots + else height_)) + resize() # parse a line of trace output pattern = re.compile( @@ -426,8 +687,11 @@ def main(path='-', *, '|' '(?Psync)\(' '\s*(?P\w+)\s*' '\)' ')') def parse(line): - # string searching is actually much faster than - # the regex here + nonlocal bd + + # string searching is much faster than the regex here, and this + # actually has a big impact given how much trace output comes + # through here if 'trace' not in line or 'bd' not in line: return False m = pattern.search(line) @@ -439,21 +703,13 @@ def main(path='-', *, size = int(m.group('block_size'), 0) count = int(m.group('block_count'), 0) - if off_stop is not None: - size = off_stop-off_start - if block_stop is not None: - count = block_stop-block_start - - with lock: - if reset: - bd.reset() - - # ignore the new values if block_stop/off_stop is explicit - bd.smoosh( - size=(size if off_stop is None - else off_stop-off_start), - count=(count if block_stop is None - else block_stop-block_start)) + resize(size=size, count=count) + if reset: + bd = Bd( + size=bd.size, + count=bd.count, + width=bd.width, + height=bd.height) return True elif m.group('read') and read: @@ -470,8 +726,7 @@ def main(path='-', *, size = min(size, off_stop-off) off -= off_start - with lock: - bd.read(block, slice(off,off+size)) + bd.read(block, off, size) return True elif m.group('prog') and prog: @@ -488,8 +743,7 @@ def main(path='-', *, size = min(size, off_stop-off) off -= off_start - with lock: - bd.prog(block, slice(off,off+size)) + bd.prog(block, off, size) return True elif m.group('erase') and (erase or wear): @@ -499,165 +753,75 @@ def main(path='-', *, return False block -= block_start - with lock: - bd.erase(block) + bd.erase(block) return True else: return False - - # print a pretty line of trace output - history = [] - def push(): - # create copy to avoid corrupt output - with lock: - resmoosh() - bd_ = bd.copy() - bd.clear() - - max_wear = None - if wear: - max_wear = max(b.wear for b in bd_.blocks) - - def draw(b): - return b.draw( - read=read, - prog=prog, - erase=erase, - wear=wear, - color=color, - max_wear=max_wear, - block_cycles=block_cycles, - **args) - - # fold via a curve? - if height > 1: - w = (len(bd.blocks)+height-1) // height - if hilbert: - grid = {} - for (x,y),b in zip(hilbert_curve(w, height), bd_.blocks): - grid[(x,y)] = draw(b) - line = [ - ''.join(grid.get((x,y), ' ') for x in range(w)) - for y in range(height)] - elif lebesgue: - grid = {} - for (x,y),b in zip(lebesgue_curve(w, height), bd_.blocks): - grid[(x,y)] = draw(b) - line = [ - ''.join(grid.get((x,y), ' ') for x in range(w)) - for y in range(height)] - else: - line = [ - ''.join(draw(b) for b in bd_.blocks[y*w:y*w+w]) - for y in range(height)] - else: - line = [''.join(draw(b) for b in bd_.blocks)] - - if not lines: - # just go ahead and print here - for row in line: - sys.stdout.write(row) - sys.stdout.write('\n') - sys.stdout.flush() - else: - history.append(line) - del history[:-lines] - + # print trace output def draw(f): def writeln(s=''): f.write(s) f.write('\n') f.writeln = writeln - for line in it.chain.from_iterable(history): - f.writeln(line) + # don't forget we've scaled this for braille/dots! + for row in range( + m.ceil(bd.height/4) if braille + else m.ceil(bd.height/2) if dots + else bd.height): + line = bd.draw(row, + read=read, + prog=prog, + erase=erase, + wear=wear, + block_cycles=block_cycles, + color=color, + dots=dots, + braille=braille, + hilbert=hilbert, + lebesgue=lebesgue, + **args) + if line: + f.writeln(line) - last_lines = 1 - def redraw(): - nonlocal last_lines - - canvas = io.StringIO() - draw(canvas) - canvas = canvas.getvalue().splitlines() - - # give ourself a canvas - while last_lines < len(canvas): - sys.stdout.write('\n') - last_lines += 1 - - for i, line in enumerate(canvas): - jump = len(canvas)-1-i - # move cursor, clear line, disable/reenable line wrapping - sys.stdout.write('\r') - if jump > 0: - sys.stdout.write('\x1b[%dA' % jump) - sys.stdout.write('\x1b[K') - sys.stdout.write('\x1b[?7l') - sys.stdout.write(line) - sys.stdout.write('\x1b[?7h') - if jump > 0: - sys.stdout.write('\x1b[%dB' % jump) - - sys.stdout.flush() + bd.clear() + resize() - if sleep is None or (coalesce and not lines): - # read/parse coalesce number of operations - try: - while True: - with openio(path) as f: - changes = 0 - for line in f: - change = parse(line) - changes += change - if change and changes % (coalesce or 1) == 0: - push() - redraw() - # sleep between coalesced lines? - if sleep is not None: - time.sleep(sleep) - if not keep_open: - break - # don't just flood open calls - time.sleep(sleep or 0.1) - except KeyboardInterrupt: - pass + # read/parse/coalesce operations + if cat: + ring = sys.stdout else: - # read/parse in a background thread - def background_parse(): - nonlocal done - while True: - with openio(path) as f: - changes = 0 - for line in f: - change = parse(line) - changes += change - if change and changes % (coalesce or 1) == 0: - if coalesce: - push() - event.set() - if not keep_open: - break - # don't just flood open calls - time.sleep(sleep or 0.1) - done = True + ring = LinesIO(lines) - th.Thread(target=background_parse, daemon=True).start() + ptime = time.time() + try: + while True: + with openio(path) as f: + changed = 0 + for line in f: + changed += parse(line) - try: - while not done: - time.sleep(sleep) - event.wait() - event.clear() - if not coalesce: - push() - redraw() - except KeyboardInterrupt: - pass + # need to redraw? + if (changed + and (not coalesce or changed >= coalesce) + and (not sleep or time.time()-ptime >= sleep)): + draw(ring) + if not cat: + ring.draw() + changed = 0 + ptime = time.time() - if lines: + if not keep_open: + break + # don't just flood open calls + time.sleep(sleep or 0.1) + except KeyboardInterrupt: + pass + + if not cat: sys.stdout.write('\n') @@ -687,20 +851,6 @@ if __name__ == "__main__": '-w', '--wear', action='store_true', help="Render wear.") - parser.add_argument( - '--subscripts', - help="Use unicode subscripts for showing wear.") - parser.add_argument( - '--chars', - help="Characters to use for noop, read, prog, erase operations.") - parser.add_argument( - '--wear-chars', - help="Characters to use to show wear.") - parser.add_argument( - '--color', - choices=['never', 'always', 'auto'], - default='auto', - help="When to use terminal colors. Defaults to 'auto'.") parser.add_argument( '-b', '--block', type=lambda x: tuple(int(x,0) if x else None for x in x.split(',',1)), @@ -725,23 +875,64 @@ if __name__ == "__main__": '-R', '--reset', action='store_true', help="Reset wear on block device initialization.") + parser.add_argument( + '--color', + choices=['never', 'always', 'auto'], + default='auto', + help="When to use terminal colors. Defaults to 'auto'.") + parser.add_argument( + '--subscripts', + action='store_true', + help="Use unicode subscripts for showing wear.") + parser.add_argument( + '-:', '--dots', + action='store_true', + help="Use 1x2 ascii dot characters.") + parser.add_argument( + '-⣿', '--braille', + action='store_true', + help="Use 2x4 unicode braille characters. Note that braille characters " + "sometimes suffer from inconsistent widths.") + parser.add_argument( + '--chars', + help="Characters to use for read, prog, erase, noop operations.") + parser.add_argument( + '--wear-chars', + help="Characters to use for showing wear.") + parser.add_argument( + '--colors', + type=lambda x: x.split(','), + help="Colors to use for read, prog, erase, noop operations.") + parser.add_argument( + '--wear-colors', + type=lambda x: x.split(','), + help="Colors to use for showing wear.") parser.add_argument( '-W', '--width', type=lambda x: int(x, 0), - help="Width in columns. A width of 0 indicates no limit. Defaults " - "to terminal width or 80.") + help="Width in columns. 0 uses the terminal width. Defaults to " + "min(terminal, 80).") parser.add_argument( '-H', '--height', type=lambda x: int(x, 0), - help="Height in rows. Defaults to 1.") - parser.add_argument( - '-x', '--scale', - type=float, - help="Number of characters per block, ignores --width if set.") + help="Height in rows. 0 uses the terminal height. Defaults to 1.") parser.add_argument( '-n', '--lines', type=lambda x: int(x, 0), - help="Number of lines to show.") + help="Show this many lines of history. 0 uses the terminal height. " + "Defaults to 5.") + parser.add_argument( + '-z', '--cat', + action='store_true', + help="Pipe directly to stdout.") + parser.add_argument( + '-U', '--hilbert', + action='store_true', + help="Render as a space-filling Hilbert curve.") + parser.add_argument( + '-Z', '--lebesgue', + action='store_true', + help="Render as a space-filling Z-curve.") parser.add_argument( '-c', '--coalesce', type=lambda x: int(x, 0), @@ -749,16 +940,7 @@ if __name__ == "__main__": parser.add_argument( '-s', '--sleep', type=float, - help="Time in seconds to sleep between reads, while coalescing " - "operations.") - parser.add_argument( - '-I', '--hilbert', - action='store_true', - help="Render as a space-filling Hilbert curve.") - parser.add_argument( - '-Z', '--lebesgue', - action='store_true', - help="Render as a space-filling Z-curve.") + help="Time in seconds to sleep between reads, coalescing operations.") parser.add_argument( '-k', '--keep-open', action='store_true', From 9507e6243c7987739b8f275ee9ea30df27604e88 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 26 Sep 2022 19:19:40 -0500 Subject: [PATCH 46/81] Several tweaks to script flags - Changed multi-field flags to action=append instead of comma-separated. - Dropped short-names for geometries/powerlosses - Renamed -Pexponential -> -Plog - Allowed omitting the 0 for -W0/-H0/-n0 and made -j0 consistent - Better handling of --xlim/--ylim --- Makefile | 8 ++-- runners/bench_runner.c | 44 +++++++++---------- runners/test_runner.c | 95 +++++++++++++++++++----------------------- scripts/bench.py | 15 ++++--- scripts/plot.py | 56 ++++++++++++++----------- scripts/summary.py | 20 ++++----- scripts/tailpipe.py | 5 ++- scripts/test.py | 22 +++++----- scripts/tracebd.py | 18 ++++++-- 9 files changed, 146 insertions(+), 137 deletions(-) diff --git a/Makefile b/Makefile index 84aeb18f..e848e9c4 100644 --- a/Makefile +++ b/Makefile @@ -170,10 +170,10 @@ coverage: $(GCDA) .PHONY: summary sizes summary sizes: $(BUILDDIR)lfs.csv $(strip ./scripts/summary.py -Y $^ \ - -fcode=code_size,$\ - data=data_size,$\ - stack=stack_limit,$\ - struct=struct_size \ + -fcode=code_size \ + -fdata=data_size \ + -fstack=stack_limit \ + -fstruct=struct_size \ --max=stack \ $(SUMMARYFLAGS)) diff --git a/runners/bench_runner.c b/runners/bench_runner.c index 073760f7..f7e33479 100644 --- a/runners/bench_runner.c +++ b/runners/bench_runner.c @@ -90,9 +90,7 @@ static uintmax_t leb16_parse(const char *s, char **tail) { // bench_runner types typedef struct bench_geometry { - char short_name; - const char *long_name; - + const char *name; bench_define_t defines[BENCH_GEOMETRY_DEFINE_COUNT]; } bench_geometry_t; @@ -1057,7 +1055,7 @@ static void list_implicit_defines(void) { // make sure to include builtin geometries here extern const bench_geometry_t builtin_geometries[]; - for (size_t g = 0; builtin_geometries[g].long_name; g++) { + for (size_t g = 0; builtin_geometries[g].name; g++) { bench_define_geometry(&builtin_geometries[g]); bench_define_flush(); @@ -1089,12 +1087,12 @@ static void list_implicit_defines(void) { // geometries to bench const bench_geometry_t builtin_geometries[] = { - {'d', "default", {{NULL}, BENCH_CONST(16), BENCH_CONST(512), {NULL}}}, - {'e', "eeprom", {{NULL}, BENCH_CONST(1), BENCH_CONST(512), {NULL}}}, - {'E', "emmc", {{NULL}, {NULL}, BENCH_CONST(512), {NULL}}}, - {'n', "nor", {{NULL}, BENCH_CONST(1), BENCH_CONST(4096), {NULL}}}, - {'N', "nand", {{NULL}, BENCH_CONST(4096), BENCH_CONST(32768), {NULL}}}, - {0, NULL, {{NULL}, {NULL}, {NULL}, {NULL}}}, + {"default", {{NULL}, BENCH_CONST(16), BENCH_CONST(512), {NULL}}}, + {"eeprom", {{NULL}, BENCH_CONST(1), BENCH_CONST(512), {NULL}}}, + {"emmc", {{NULL}, {NULL}, BENCH_CONST(512), {NULL}}}, + {"nor", {{NULL}, BENCH_CONST(1), BENCH_CONST(4096), {NULL}}}, + {"nand", {{NULL}, BENCH_CONST(4096), BENCH_CONST(32768), {NULL}}}, + {NULL, {{NULL}, {NULL}, {NULL}, {NULL}}}, }; const bench_geometry_t *bench_geometries = builtin_geometries; @@ -1107,12 +1105,11 @@ static void list_geometries(void) { printf("%-24s %7s %7s %7s %7s %11s\n", "geometry", "read", "prog", "erase", "count", "size"); - for (size_t g = 0; builtin_geometries[g].long_name; g++) { + for (size_t g = 0; builtin_geometries[g].name; g++) { bench_define_geometry(&builtin_geometries[g]); bench_define_flush(); - printf("%c,%-22s %7ju %7ju %7ju %7ju %11ju\n", - builtin_geometries[g].short_name, - builtin_geometries[g].long_name, + printf("%-24s %7ju %7ju %7ju %7ju %11ju\n", + builtin_geometries[g].name, READ_SIZE, PROG_SIZE, BLOCK_SIZE, @@ -1253,7 +1250,7 @@ enum opt_flags { OPT_LIST_IMPLICIT_DEFINES = 5, OPT_LIST_GEOMETRIES = 6, OPT_DEFINE = 'D', - OPT_GEOMETRY = 'g', + OPT_GEOMETRY = 'G', OPT_STEP = 's', OPT_DISK = 'd', OPT_TRACE = 't', @@ -1262,7 +1259,7 @@ enum opt_flags { OPT_ERASE_SLEEP = 9, }; -const char *short_opts = "hYlLD:g:s:d:t:"; +const char *short_opts = "hYlLD:G:s:d:t:"; const struct option long_opts[] = { {"help", no_argument, NULL, OPT_HELP}, @@ -1300,7 +1297,7 @@ const char *const help_text[] = { "List implicit defines in this bench-runner.", "List the available disk geometries.", "Override a bench define.", - "Comma-separated list of disk geometries to bench. Defaults to d,e,E,n,N.", + "Comma-separated list of disk geometries to bench.", "Comma-separated range of bench permutations to run (start,stop,step).", "Redirect block device operations to this file.", "Redirect trace output to this file.", @@ -1555,14 +1552,11 @@ invalid_define: // named disk geometry size_t len = strcspn(optarg, " ,"); - for (size_t i = 0; builtin_geometries[i].long_name; i++) { - if ((len == 1 - && *optarg == builtin_geometries[i].short_name) - || (len == strlen( - builtin_geometries[i].long_name) - && memcmp(optarg, - builtin_geometries[i].long_name, - len) == 0)) { + for (size_t i = 0; builtin_geometries[i].name; i++) { + if (len == strlen(builtin_geometries[i].name) + && memcmp(optarg, + builtin_geometries[i].name, + len) == 0) { *geometry = builtin_geometries[i]; optarg += len; goto geometry_next; diff --git a/runners/test_runner.c b/runners/test_runner.c index 34c31e1e..f0e1ff6e 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -90,16 +90,12 @@ static uintmax_t leb16_parse(const char *s, char **tail) { // test_runner types typedef struct test_geometry { - char short_name; - const char *long_name; - + const char *name; test_define_t defines[TEST_GEOMETRY_DEFINE_COUNT]; } test_geometry_t; typedef struct test_powerloss { - char short_name; - const char *long_name; - + const char *name; void (*run)( const lfs_emubd_powercycles_t *cycles, size_t cycle_count, @@ -574,6 +570,11 @@ void test_seen_cleanup(test_seen_t *seen) { free(seen->branches); } +static void run_powerloss_none( + const lfs_emubd_powercycles_t *cycles, + size_t cycle_count, + const struct test_suite *suite, + const struct test_case *case_); static void run_powerloss_cycles( const lfs_emubd_powercycles_t *cycles, size_t cycle_count, @@ -606,7 +607,7 @@ static void case_forperm( } else { for (size_t p = 0; p < test_powerloss_count; p++) { // skip non-reentrant tests when powerloss testing - if (test_powerlosses[p].short_name != '0' + if (test_powerlosses[p].run != run_powerloss_none && !(case_->flags & TEST_REENTRANT)) { continue; } @@ -646,7 +647,7 @@ static void case_forperm( } else { for (size_t p = 0; p < test_powerloss_count; p++) { // skip non-reentrant tests when powerloss testing - if (test_powerlosses[p].short_name != '0' + if (test_powerlosses[p].run != run_powerloss_none && !(case_->flags & TEST_REENTRANT)) { continue; } @@ -1094,7 +1095,7 @@ static void list_implicit_defines(void) { // make sure to include builtin geometries here extern const test_geometry_t builtin_geometries[]; - for (size_t g = 0; builtin_geometries[g].long_name; g++) { + for (size_t g = 0; builtin_geometries[g].name; g++) { test_define_geometry(&builtin_geometries[g]); test_define_flush(); @@ -1126,12 +1127,12 @@ static void list_implicit_defines(void) { // geometries to test const test_geometry_t builtin_geometries[] = { - {'d', "default", {{NULL}, TEST_CONST(16), TEST_CONST(512), {NULL}}}, - {'e', "eeprom", {{NULL}, TEST_CONST(1), TEST_CONST(512), {NULL}}}, - {'E', "emmc", {{NULL}, {NULL}, TEST_CONST(512), {NULL}}}, - {'n', "nor", {{NULL}, TEST_CONST(1), TEST_CONST(4096), {NULL}}}, - {'N', "nand", {{NULL}, TEST_CONST(4096), TEST_CONST(32768), {NULL}}}, - {0, NULL, {{NULL}, {NULL}, {NULL}, {NULL}}}, + {"default", {{NULL}, TEST_CONST(16), TEST_CONST(512), {NULL}}}, + {"eeprom", {{NULL}, TEST_CONST(1), TEST_CONST(512), {NULL}}}, + {"emmc", {{NULL}, {NULL}, TEST_CONST(512), {NULL}}}, + {"nor", {{NULL}, TEST_CONST(1), TEST_CONST(4096), {NULL}}}, + {"nand", {{NULL}, TEST_CONST(4096), TEST_CONST(32768), {NULL}}}, + {NULL, {{NULL}, {NULL}, {NULL}, {NULL}}}, }; const test_geometry_t *test_geometries = builtin_geometries; @@ -1144,12 +1145,11 @@ static void list_geometries(void) { printf("%-24s %7s %7s %7s %7s %11s\n", "geometry", "read", "prog", "erase", "count", "size"); - for (size_t g = 0; builtin_geometries[g].long_name; g++) { + for (size_t g = 0; builtin_geometries[g].name; g++) { test_define_geometry(&builtin_geometries[g]); test_define_flush(); - printf("%c,%-22s %7ju %7ju %7ju %7ju %11ju\n", - builtin_geometries[g].short_name, - builtin_geometries[g].long_name, + printf("%-24s %7ju %7ju %7ju %7ju %11ju\n", + builtin_geometries[g].name, READ_SIZE, PROG_SIZE, BLOCK_SIZE, @@ -1314,7 +1314,7 @@ static void run_powerloss_linear( } } -static void run_powerloss_exponential( +static void run_powerloss_log( const lfs_emubd_powercycles_t *cycles, size_t cycle_count, const struct test_suite *suite, @@ -1646,11 +1646,11 @@ static void run_powerloss_exhaustive( const test_powerloss_t builtin_powerlosses[] = { - {'0', "none", run_powerloss_none, NULL, 0}, - {'e', "exponential", run_powerloss_exponential, NULL, 0}, - {'l', "linear", run_powerloss_linear, NULL, 0}, - {'x', "exhaustive", run_powerloss_exhaustive, NULL, SIZE_MAX}, - {0, NULL, NULL, NULL, 0}, + {"none", run_powerloss_none, NULL, 0}, + {"log", run_powerloss_log, NULL, 0}, + {"linear", run_powerloss_linear, NULL, 0}, + {"exhaustive", run_powerloss_exhaustive, NULL, SIZE_MAX}, + {NULL, NULL, NULL, 0}, }; const char *const builtin_powerlosses_help[] = { @@ -1664,17 +1664,16 @@ const char *const builtin_powerlosses_help[] = { }; const test_powerloss_t *test_powerlosses = (const test_powerloss_t[]){ - {'0', "none", run_powerloss_none, NULL, 0}, + {"none", run_powerloss_none, NULL, 0}, }; size_t test_powerloss_count = 1; static void list_powerlosses(void) { printf("%-24s %s\n", "scenario", "description"); size_t i = 0; - for (; builtin_powerlosses[i].long_name; i++) { - printf("%c,%-22s %s\n", - builtin_powerlosses[i].short_name, - builtin_powerlosses[i].long_name, + for (; builtin_powerlosses[i].name; i++) { + printf("%-24s %s\n", + builtin_powerlosses[i].name, builtin_powerlosses_help[i]); } @@ -1765,8 +1764,8 @@ enum opt_flags { OPT_LIST_GEOMETRIES = 6, OPT_LIST_POWERLOSSES = 7, OPT_DEFINE = 'D', - OPT_GEOMETRY = 'g', - OPT_POWERLOSS = 'p', + OPT_GEOMETRY = 'G', + OPT_POWERLOSS = 'P', OPT_STEP = 's', OPT_DISK = 'd', OPT_TRACE = 't', @@ -1775,7 +1774,7 @@ enum opt_flags { OPT_ERASE_SLEEP = 10, }; -const char *short_opts = "hYlLD:g:p:s:d:t:"; +const char *short_opts = "hYlLD:G:P:s:d:t:"; const struct option long_opts[] = { {"help", no_argument, NULL, OPT_HELP}, @@ -1816,8 +1815,8 @@ const char *const help_text[] = { "List the available disk geometries.", "List the available power-loss scenarios.", "Override a test define.", - "Comma-separated list of disk geometries to test. Defaults to d,e,E,n,N.", - "Comma-separated list of power-loss scenarios to test. Defaults to 0,l.", + "Comma-separated list of disk geometries to test.", + "Comma-separated list of power-loss scenarios to test.", "Comma-separated range of test permutations to run (start,stop,step).", "Redirect block device operations to this file.", "Redirect trace output to this file.", @@ -2076,14 +2075,11 @@ invalid_define: // named disk geometry size_t len = strcspn(optarg, " ,"); - for (size_t i = 0; builtin_geometries[i].long_name; i++) { - if ((len == 1 - && *optarg == builtin_geometries[i].short_name) - || (len == strlen( - builtin_geometries[i].long_name) - && memcmp(optarg, - builtin_geometries[i].long_name, - len) == 0)) { + for (size_t i = 0; builtin_geometries[i].name; i++) { + if (len == strlen(builtin_geometries[i].name) + && memcmp(optarg, + builtin_geometries[i].name, + len) == 0) { *geometry = builtin_geometries[i]; optarg += len; goto geometry_next; @@ -2224,14 +2220,11 @@ geometry_next: // named power-loss scenario size_t len = strcspn(optarg, " ,"); - for (size_t i = 0; builtin_powerlosses[i].long_name; i++) { - if ((len == 1 - && *optarg == builtin_powerlosses[i].short_name) - || (len == strlen( - builtin_powerlosses[i].long_name) - && memcmp(optarg, - builtin_powerlosses[i].long_name, - len) == 0)) { + for (size_t i = 0; builtin_powerlosses[i].name; i++) { + if (len == strlen(builtin_powerlosses[i].name) + && memcmp(optarg, + builtin_powerlosses[i].name, + len) == 0) { *powerloss = builtin_powerlosses[i]; optarg += len; goto powerloss_next; diff --git a/scripts/bench.py b/scripts/bench.py index 93c18a2d..a59d80d3 100755 --- a/scripts/bench.py +++ b/scripts/bench.py @@ -511,7 +511,7 @@ def find_runner(runner, **args): # other context if args.get('geometry'): - cmd.append('-g%s' % args['geometry']) + cmd.append('-G%s' % args['geometry']) if args.get('disk'): cmd.append('-d%s' % args['disk']) if args.get('trace'): @@ -1003,6 +1003,10 @@ def run(runner, bench_ids=[], **args): total_perms)) print() + # automatic job detection? + if args.get('jobs') == 0: + args['jobs'] = len(os.sched_getaffinity(0)) + # truncate and open logs here so they aren't disconnected between benches stdout = None if args.get('stdout'): @@ -1246,9 +1250,8 @@ if __name__ == "__main__": action='append', help="Override a bench define.") bench_parser.add_argument( - '-g', '--geometry', - help="Comma-separated list of disk geometries to bench. " - "Defaults to d,e,E,n,N.") + '-G', '--geometry', + help="Comma-separated list of disk geometries to bench.") bench_parser.add_argument( '-d', '--disk', help="Direct block device operations to this file.") @@ -1274,8 +1277,8 @@ if __name__ == "__main__": '-j', '--jobs', nargs='?', type=lambda x: int(x, 0), - const=len(os.sched_getaffinity(0)), - help="Number of parallel runners to run.") + const=0, + help="Number of parallel runners to run. 0 runs one runner per core.") bench_parser.add_argument( '-k', '--keep-going', action='store_true', diff --git a/scripts/plot.py b/scripts/plot.py index ad59adca..9aef2aee 100755 --- a/scripts/plot.py +++ b/scripts/plot.py @@ -479,8 +479,8 @@ def main(csv_paths, *, x=None, y=None, define=[], - xlim=None, - ylim=None, + xlim=(None,None), + ylim=(None,None), width=None, height=17, cat=False, @@ -489,7 +489,7 @@ def main(csv_paths, *, colors=None, chars=None, line_chars=None, - no_lines=False, + points=False, legend=None, keep_open=False, sleep=None, @@ -503,9 +503,9 @@ def main(csv_paths, *, color = False # allow shortened ranges - if xlim is not None and len(xlim) == 1: + if len(xlim) == 1: xlim = (0, xlim[0]) - if ylim is not None and len(ylim) == 1: + if len(ylim) == 1: ylim = (0, ylim[0]) # separate out renames @@ -544,7 +544,7 @@ def main(csv_paths, *, if line_chars is not None: line_chars_ = line_chars - elif not no_lines: + elif not points: line_chars_ = [True] else: line_chars_ = [False] @@ -567,28 +567,26 @@ def main(csv_paths, *, legend_width = max(legend_width, len(label)+1) # find xlim/ylim - if xlim is not None: - xlim_ = xlim - else: - xlim_ = ( - min(it.chain([0], (k + xlim_ = ( + xlim[0] if xlim[0] is not None + else min(it.chain([0], (k for r in datasets_.values() for k, v in r.items() if v is not None))), - max(it.chain([0], (k + xlim[1] if xlim[1] is not None + else max(it.chain([0], (k for r in datasets_.values() for k, v in r.items() if v is not None)))) - if ylim is not None: - ylim_ = ylim - else: - ylim_ = ( - min(it.chain([0], (v + ylim_ = ( + ylim[0] if ylim[0] is not None + else min(it.chain([0], (v for r in datasets_.values() for _, v in r.items() if v is not None))), - max(it.chain([0], (v + ylim[1] if ylim[1] is not None + else max(it.chain([0], (v for r in datasets_.values() for _, v in r.items() if v is not None)))) @@ -740,17 +738,17 @@ if __name__ == "__main__": "or list of paths. Defaults to %r." % CSV_PATHS) parser.add_argument( '-b', '--by', - type=lambda x: [x.strip() for x in x.split(',')], + action='append', help="Fields to render as separate plots. All other fields will be " "summed as needed. Can rename fields with new_name=old_name.") parser.add_argument( '-x', - type=lambda x: [x.strip() for x in x.split(',')], + action='append', help="Fields to use for the x-axis. Can rename fields with " "new_name=old_name.") parser.add_argument( '-y', - type=lambda x: [x.strip() for x in x.split(',')], + action='append', help="Fields to use for the y-axis. Can rename fields with " "new_name=old_name.") parser.add_argument( @@ -771,7 +769,7 @@ if __name__ == "__main__": "sometimes suffer from inconsistent widths.") parser.add_argument( '--colors', - type=lambda x: x.split(','), + type=lambda x: [x.strip() for x in x.split(',')], help="Colors to use.") parser.add_argument( '--chars', @@ -780,17 +778,21 @@ if __name__ == "__main__": '--line-chars', help="Characters to use for lines.") parser.add_argument( - '-L', '--no-lines', + '-.', '--points', action='store_true', help="Only draw the data points.") parser.add_argument( '-W', '--width', + nargs='?', type=lambda x: int(x, 0), + const=0, help="Width in columns. 0 uses the terminal width. Defaults to " "min(terminal, 80).") parser.add_argument( '-H', '--height', + nargs='?', type=lambda x: int(x, 0), + const=0, help="Height in rows. 0 uses the terminal height. Defaults to 17.") parser.add_argument( '-z', '--cat', @@ -798,11 +800,15 @@ if __name__ == "__main__": help="Pipe directly to stdout.") parser.add_argument( '-X', '--xlim', - type=lambda x: tuple(dat(x) if x else None for x in x.split(',')), + type=lambda x: tuple( + dat(x) if x.strip() else None + for x in x.split(',')), help="Range for the x-axis.") parser.add_argument( '-Y', '--ylim', - type=lambda x: tuple(dat(x) if x else None for x in x.split(',')), + type=lambda x: tuple( + dat(x) if x.strip() else None + for x in x.split(',')), help="Range for the y-axis.") parser.add_argument( '--xlog', diff --git a/scripts/summary.py b/scripts/summary.py index 36b556f5..60bf948e 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -671,46 +671,46 @@ if __name__ == "__main__": help="Only show percentage change, not a full diff.") parser.add_argument( '-b', '--by', - type=lambda x: [x.strip() for x in x.split(',')], + action='append', help="Group by these fields. All other fields will be merged as " "needed. Can rename fields with new_name=old_name.") parser.add_argument( '-f', '--fields', - type=lambda x: [x.strip() for x in x.split(',')], + action='append', help="Use these fields. Can rename fields with new_name=old_name.") parser.add_argument( '-D', '--define', - type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), action='append', + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), help="Only include rows where this field is this value. May include " "comma-separated options.") parser.add_argument( '--add', - type=lambda x: [x.strip() for x in x.split(',')], + action='append', help="Add these fields (the default).") parser.add_argument( '--mul', - type=lambda x: [x.strip() for x in x.split(',')], + action='append', help="Multiply these fields.") parser.add_argument( '--min', - type=lambda x: [x.strip() for x in x.split(',')], + action='append', help="Take the minimum of these fields.") parser.add_argument( '--max', - type=lambda x: [x.strip() for x in x.split(',')], + action='append', help="Take the maximum of these fields.") parser.add_argument( '--avg', - type=lambda x: [x.strip() for x in x.split(',')], + action='append', help="Average these fields.") parser.add_argument( '-s', '--sort', - type=lambda x: [x.strip() for x in x.split(',')], + action='append', help="Sort by these fields.") parser.add_argument( '-S', '--reverse-sort', - type=lambda x: [x.strip() for x in x.split(',')], + action='append', help="Sort by these fields, but backwards.") parser.add_argument( '-Y', '--summary', diff --git a/scripts/tailpipe.py b/scripts/tailpipe.py index 3db7612f..7e8c4543 100755 --- a/scripts/tailpipe.py +++ b/scripts/tailpipe.py @@ -121,9 +121,10 @@ if __name__ == "__main__": nargs='?', help="Path to read from.") parser.add_argument( - '-n', - '--lines', + '-n', '--lines', + nargs='?', type=lambda x: int(x, 0), + const=0, help="Show this many lines of history. 0 uses the terminal height. " "Defaults to 5.") parser.add_argument( diff --git a/scripts/test.py b/scripts/test.py index 71688547..eaab3e38 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -525,9 +525,9 @@ def find_runner(runner, **args): # other context if args.get('geometry'): - cmd.append('-g%s' % args['geometry']) + cmd.append('-G%s' % args['geometry']) if args.get('powerloss'): - cmd.append('-p%s' % args['powerloss']) + cmd.append('-P%s' % args['powerloss']) if args.get('disk'): cmd.append('-d%s' % args['disk']) if args.get('trace'): @@ -1009,6 +1009,10 @@ def run(runner, test_ids=[], **args): total_perms)) print() + # automatic job detection? + if args.get('jobs') == 0: + args['jobs'] = len(os.sched_getaffinity(0)) + # truncate and open logs here so they aren't disconnected between tests stdout = None if args.get('stdout'): @@ -1251,13 +1255,11 @@ if __name__ == "__main__": action='append', help="Override a test define.") test_parser.add_argument( - '-g', '--geometry', - help="Comma-separated list of disk geometries to test. " - "Defaults to d,e,E,n,N.") + '-G', '--geometry', + help="Comma-separated list of disk geometries to test.") test_parser.add_argument( - '-p', '--powerloss', - help="Comma-separated list of power-loss scenarios to test. " - "Defaults to 0,l.") + '-P', '--powerloss', + help="Comma-separated list of power-loss scenarios to test.") test_parser.add_argument( '-d', '--disk', help="Direct block device operations to this file.") @@ -1283,8 +1285,8 @@ if __name__ == "__main__": '-j', '--jobs', nargs='?', type=lambda x: int(x, 0), - const=len(os.sched_getaffinity(0)), - help="Number of parallel runners to run.") + const=0, + help="Number of parallel runners to run. 0 runs one runner per core.") test_parser.add_argument( '-k', '--keep-going', action='store_true', diff --git a/scripts/tracebd.py b/scripts/tracebd.py index 84b0b29a..d69c1329 100755 --- a/scripts/tracebd.py +++ b/scripts/tracebd.py @@ -853,11 +853,15 @@ if __name__ == "__main__": help="Render wear.") parser.add_argument( '-b', '--block', - type=lambda x: tuple(int(x,0) if x else None for x in x.split(',',1)), + type=lambda x: tuple( + int(x, 0) if x.strip() else None + for x in x.split(',')), help="Show a specific block or range of blocks.") parser.add_argument( '-i', '--off', - type=lambda x: tuple(int(x,0) if x else None for x in x.split(',',1)), + type=lambda x: tuple( + int(x, 0) if x.strip() else None + for x in x.split(',')), help="Show a specific offset or range of offsets.") parser.add_argument( '-B', '--block-size', @@ -901,24 +905,30 @@ if __name__ == "__main__": help="Characters to use for showing wear.") parser.add_argument( '--colors', - type=lambda x: x.split(','), + type=lambda x: [x.strip() for x in x.split(',')], help="Colors to use for read, prog, erase, noop operations.") parser.add_argument( '--wear-colors', - type=lambda x: x.split(','), + type=lambda x: [x.strip() for x in x.split(',')], help="Colors to use for showing wear.") parser.add_argument( '-W', '--width', + nargs='?', type=lambda x: int(x, 0), + const=0, help="Width in columns. 0 uses the terminal width. Defaults to " "min(terminal, 80).") parser.add_argument( '-H', '--height', + nargs='?', type=lambda x: int(x, 0), + const=0, help="Height in rows. 0 uses the terminal height. Defaults to 1.") parser.add_argument( '-n', '--lines', + nargs='?', type=lambda x: int(x, 0), + const=0, help="Show this many lines of history. 0 uses the terminal height. " "Defaults to 5.") parser.add_argument( From a2fb7089ddcf49ed328280492aef867d04586116 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 27 Sep 2022 13:30:43 -0500 Subject: [PATCH 47/81] Added stddev/gmean/gstddev to summary.py --- scripts/code.py | 56 ++++++------- scripts/coverage.py | 75 ++++++++--------- scripts/data.py | 57 +++++++------ scripts/stack.py | 56 ++++++------- scripts/struct_.py | 56 ++++++------- scripts/summary.py | 196 +++++++++++++++++++++++++++----------------- 6 files changed, 266 insertions(+), 230 deletions(-) diff --git a/scripts/code.py b/scripts/code.py index 083ab8af..eeaf8a0c 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -31,7 +31,7 @@ TYPE = 'tTrRdD' # integer fields class IntField(co.namedtuple('IntField', 'x')): __slots__ = () - def __new__(cls, x): + def __new__(cls, x=0): if isinstance(x, IntField): return x if isinstance(x, str): @@ -40,13 +40,22 @@ class IntField(co.namedtuple('IntField', 'x')): except ValueError: # also accept +-∞ and +-inf if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): - x = float('inf') + x = m.inf elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): - x = float('-inf') + x = -m.inf else: raise + assert isinstance(x, int) or m.isinf(x), x return super().__new__(cls, x) + def __str__(self): + if self.x == m.inf: + return '∞' + elif self.x == -m.inf: + return '-∞' + else: + return str(self.x) + def __int__(self): assert not m.isinf(self.x) return self.x @@ -54,14 +63,6 @@ class IntField(co.namedtuple('IntField', 'x')): def __float__(self): return float(self.x) - def __str__(self): - if self.x == float('inf'): - return '∞' - elif self.x == float('-inf'): - return '-∞' - else: - return str(self.x) - none = '%7s' % '-' def table(self): return '%7s' % (self,) @@ -73,9 +74,9 @@ class IntField(co.namedtuple('IntField', 'x')): new = self.x if self else 0 old = other.x if other else 0 diff = new - old - if diff == float('+inf'): + if diff == +m.inf: return '%7s' % '+∞' - elif diff == float('-inf'): + elif diff == -m.inf: return '%7s' % '-∞' else: return '%+7d' % diff @@ -86,9 +87,9 @@ class IntField(co.namedtuple('IntField', 'x')): if m.isinf(new) and m.isinf(old): return 0.0 elif m.isinf(new): - return float('+inf') + return +m.inf elif m.isinf(old): - return float('-inf') + return -m.inf elif not old and not new: return 0.0 elif not old: @@ -99,6 +100,9 @@ class IntField(co.namedtuple('IntField', 'x')): def __add__(self, other): return IntField(self.x + other.x) + def __sub__(self, other): + return IntField(self.x - other.x) + def __mul__(self, other): return IntField(self.x * other.x) @@ -114,12 +118,6 @@ class IntField(co.namedtuple('IntField', 'x')): def __ge__(self, other): return not self.__lt__(other) - def __truediv__(self, n): - if m.isinf(self.x): - return self - else: - return IntField(round(self.x / n)) - # code size results class CodeResult(co.namedtuple('CodeResult', 'file,function,code_size')): __slots__ = () @@ -285,8 +283,8 @@ def table(results, diff_results=None, *, r.code_size.diff_table() if r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)))) else: print(' %s %s %s%s' % ( @@ -299,8 +297,8 @@ def table(results, diff_results=None, *, diff_r.code_size if diff_r else None) if r or diff_r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)) if ratio else '')) @@ -324,8 +322,8 @@ def table(results, diff_results=None, *, r.code_size.diff_table() if r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)))) else: print(' %s %s %s%s' % ( @@ -338,8 +336,8 @@ def table(results, diff_results=None, *, diff_r.code_size if diff_r else None) if r or diff_r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)) if ratio else '')) diff --git a/scripts/coverage.py b/scripts/coverage.py index 81bff111..f24744e8 100755 --- a/scripts/coverage.py +++ b/scripts/coverage.py @@ -32,7 +32,7 @@ GCOV_TOOL = ['gcov'] # integer fields class IntField(co.namedtuple('IntField', 'x')): __slots__ = () - def __new__(cls, x): + def __new__(cls, x=0): if isinstance(x, IntField): return x if isinstance(x, str): @@ -41,13 +41,22 @@ class IntField(co.namedtuple('IntField', 'x')): except ValueError: # also accept +-∞ and +-inf if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): - x = float('inf') + x = m.inf elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): - x = float('-inf') + x = -m.inf else: raise + assert isinstance(x, int) or m.isinf(x), x return super().__new__(cls, x) + def __str__(self): + if self.x == m.inf: + return '∞' + elif self.x == -m.inf: + return '-∞' + else: + return str(self.x) + def __int__(self): assert not m.isinf(self.x) return self.x @@ -55,14 +64,6 @@ class IntField(co.namedtuple('IntField', 'x')): def __float__(self): return float(self.x) - def __str__(self): - if self.x == float('inf'): - return '∞' - elif self.x == float('-inf'): - return '-∞' - else: - return str(self.x) - none = '%7s' % '-' def table(self): return '%7s' % (self,) @@ -74,9 +75,9 @@ class IntField(co.namedtuple('IntField', 'x')): new = self.x if self else 0 old = other.x if other else 0 diff = new - old - if diff == float('+inf'): + if diff == +m.inf: return '%7s' % '+∞' - elif diff == float('-inf'): + elif diff == -m.inf: return '%7s' % '-∞' else: return '%+7d' % diff @@ -87,9 +88,9 @@ class IntField(co.namedtuple('IntField', 'x')): if m.isinf(new) and m.isinf(old): return 0.0 elif m.isinf(new): - return float('+inf') + return +m.inf elif m.isinf(old): - return float('-inf') + return -m.inf elif not old and not new: return 0.0 elif not old: @@ -100,6 +101,9 @@ class IntField(co.namedtuple('IntField', 'x')): def __add__(self, other): return IntField(self.x + other.x) + def __sub__(self, other): + return IntField(self.x - other.x) + def __mul__(self, other): return IntField(self.x * other.x) @@ -115,16 +119,10 @@ class IntField(co.namedtuple('IntField', 'x')): def __ge__(self, other): return not self.__lt__(other) - def __truediv__(self, n): - if m.isinf(self.x): - return self - else: - return IntField(round(self.x / n)) - # fractional fields, a/b class FracField(co.namedtuple('FracField', 'a,b')): __slots__ = () - def __new__(cls, a, b=None): + def __new__(cls, a=0, b=None): if isinstance(a, FracField) and b is None: return a if isinstance(a, str) and b is None: @@ -136,6 +134,9 @@ class FracField(co.namedtuple('FracField', 'a,b')): def __str__(self): return '%s/%s' % (self.a, self.b) + def __float__(self): + return float(self.a) + none = '%11s %7s' % ('-', '-') def table(self): if not self.b.x: @@ -144,8 +145,8 @@ class FracField(co.namedtuple('FracField', 'a,b')): t = self.a.x/self.b.x return '%11s %7s' % ( self, - '∞%' if t == float('+inf') - else '-∞%' if t == float('-inf') + '∞%' if t == +m.inf + else '-∞%' if t == -m.inf else '%.1f%%' % (100*t)) diff_none = '%11s' % '-' @@ -172,12 +173,15 @@ class FracField(co.namedtuple('FracField', 'a,b')): def __add__(self, other): return FracField(self.a + other.a, self.b + other.b) + def __sub__(self, other): + return FracField(self.a - other.a, self.b - other.b) + def __mul__(self, other): return FracField(self.a * other.a, self.b + other.b) def __lt__(self, other): - self_r = self.a.x/self.b.x if self.b.x else float('-inf') - other_r = other.a.x/other.b.x if other.b.x else float('-inf') + self_r = self.a.x/self.b.x if self.b.x else -m.inf + other_r = other.a.x/other.b.x if other.b.x else -m.inf return self_r < other_r def __gt__(self, other): @@ -189,9 +193,6 @@ class FracField(co.namedtuple('FracField', 'a,b')): def __ge__(self, other): return not self.__lt__(other) - def __truediv__(self, n): - return FracField(self.a / n, self.b / n) - # coverage results class CoverageResult(co.namedtuple('CoverageResult', 'file,function,line,' @@ -416,8 +417,8 @@ def table(results, diff_results=None, *, r.coverage_branches.diff_table() if r else FracField.diff_none, ' (%s)' % ', '.join( - '+∞%' if t == float('+inf') - else '-∞%' if t == float('-inf') + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf else '%+.1f%%' % (100*t) for t in [line_ratio, branch_ratio]))) else: @@ -439,8 +440,8 @@ def table(results, diff_results=None, *, diff_r.coverage_branches if diff_r else None) if r or diff_r else FracField.diff_none, ' (%s)' % ', '.join( - '+∞%' if t == float('+inf') - else '-∞%' if t == float('-inf') + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf else '%+.1f%%' % (100*t) for t in [line_ratio, branch_ratio] if t) @@ -473,8 +474,8 @@ def table(results, diff_results=None, *, r.coverage_branches.diff_table() if r else FracField.diff_none, ' (%s)' % ', '.join( - '+∞%' if t == float('+inf') - else '-∞%' if t == float('-inf') + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf else '%+.1f%%' % (100*t) for t in [line_ratio, branch_ratio]))) else: @@ -496,8 +497,8 @@ def table(results, diff_results=None, *, diff_r.coverage_branches if diff_r else None) if r or diff_r else FracField.diff_none, ' (%s)' % ', '.join( - '+∞%' if t == float('+inf') - else '-∞%' if t == float('-inf') + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf else '%+.1f%%' % (100*t) for t in [line_ratio, branch_ratio] if t) diff --git a/scripts/data.py b/scripts/data.py index e86bafdc..690ac025 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -28,10 +28,11 @@ NM_TOOL = ['nm'] TYPE = 'dDbB' + # integer fields class IntField(co.namedtuple('IntField', 'x')): __slots__ = () - def __new__(cls, x): + def __new__(cls, x=0): if isinstance(x, IntField): return x if isinstance(x, str): @@ -40,13 +41,22 @@ class IntField(co.namedtuple('IntField', 'x')): except ValueError: # also accept +-∞ and +-inf if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): - x = float('inf') + x = m.inf elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): - x = float('-inf') + x = -m.inf else: raise + assert isinstance(x, int) or m.isinf(x), x return super().__new__(cls, x) + def __str__(self): + if self.x == m.inf: + return '∞' + elif self.x == -m.inf: + return '-∞' + else: + return str(self.x) + def __int__(self): assert not m.isinf(self.x) return self.x @@ -54,14 +64,6 @@ class IntField(co.namedtuple('IntField', 'x')): def __float__(self): return float(self.x) - def __str__(self): - if self.x == float('inf'): - return '∞' - elif self.x == float('-inf'): - return '-∞' - else: - return str(self.x) - none = '%7s' % '-' def table(self): return '%7s' % (self,) @@ -73,9 +75,9 @@ class IntField(co.namedtuple('IntField', 'x')): new = self.x if self else 0 old = other.x if other else 0 diff = new - old - if diff == float('+inf'): + if diff == +m.inf: return '%7s' % '+∞' - elif diff == float('-inf'): + elif diff == -m.inf: return '%7s' % '-∞' else: return '%+7d' % diff @@ -86,9 +88,9 @@ class IntField(co.namedtuple('IntField', 'x')): if m.isinf(new) and m.isinf(old): return 0.0 elif m.isinf(new): - return float('+inf') + return +m.inf elif m.isinf(old): - return float('-inf') + return -m.inf elif not old and not new: return 0.0 elif not old: @@ -99,6 +101,9 @@ class IntField(co.namedtuple('IntField', 'x')): def __add__(self, other): return IntField(self.x + other.x) + def __sub__(self, other): + return IntField(self.x - other.x) + def __mul__(self, other): return IntField(self.x * other.x) @@ -114,12 +119,6 @@ class IntField(co.namedtuple('IntField', 'x')): def __ge__(self, other): return not self.__lt__(other) - def __truediv__(self, n): - if m.isinf(self.x): - return self - else: - return IntField(round(self.x / n)) - # data size results class DataResult(co.namedtuple('DataResult', 'file,function,data_size')): __slots__ = () @@ -285,8 +284,8 @@ def table(results, diff_results=None, *, r.data_size.diff_table() if r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)))) else: print(' %s %s %s%s' % ( @@ -299,8 +298,8 @@ def table(results, diff_results=None, *, diff_r.data_size if diff_r else None) if r or diff_r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)) if ratio else '')) @@ -324,8 +323,8 @@ def table(results, diff_results=None, *, r.data_size.diff_table() if r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)))) else: print(' %s %s %s%s' % ( @@ -338,8 +337,8 @@ def table(results, diff_results=None, *, diff_r.data_size if diff_r else None) if r or diff_r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)) if ratio else '')) diff --git a/scripts/stack.py b/scripts/stack.py index b53fecb7..a3bbd486 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -25,7 +25,7 @@ CI_PATHS = ['*.ci'] # integer fields class IntField(co.namedtuple('IntField', 'x')): __slots__ = () - def __new__(cls, x): + def __new__(cls, x=0): if isinstance(x, IntField): return x if isinstance(x, str): @@ -34,13 +34,22 @@ class IntField(co.namedtuple('IntField', 'x')): except ValueError: # also accept +-∞ and +-inf if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): - x = float('inf') + x = m.inf elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): - x = float('-inf') + x = -m.inf else: raise + assert isinstance(x, int) or m.isinf(x), x return super().__new__(cls, x) + def __str__(self): + if self.x == m.inf: + return '∞' + elif self.x == -m.inf: + return '-∞' + else: + return str(self.x) + def __int__(self): assert not m.isinf(self.x) return self.x @@ -48,14 +57,6 @@ class IntField(co.namedtuple('IntField', 'x')): def __float__(self): return float(self.x) - def __str__(self): - if self.x == float('inf'): - return '∞' - elif self.x == float('-inf'): - return '-∞' - else: - return str(self.x) - none = '%7s' % '-' def table(self): return '%7s' % (self,) @@ -67,9 +68,9 @@ class IntField(co.namedtuple('IntField', 'x')): new = self.x if self else 0 old = other.x if other else 0 diff = new - old - if diff == float('+inf'): + if diff == +m.inf: return '%7s' % '+∞' - elif diff == float('-inf'): + elif diff == -m.inf: return '%7s' % '-∞' else: return '%+7d' % diff @@ -80,9 +81,9 @@ class IntField(co.namedtuple('IntField', 'x')): if m.isinf(new) and m.isinf(old): return 0.0 elif m.isinf(new): - return float('+inf') + return +m.inf elif m.isinf(old): - return float('-inf') + return -m.inf elif not old and not new: return 0.0 elif not old: @@ -93,6 +94,9 @@ class IntField(co.namedtuple('IntField', 'x')): def __add__(self, other): return IntField(self.x + other.x) + def __sub__(self, other): + return IntField(self.x - other.x) + def __mul__(self, other): return IntField(self.x * other.x) @@ -108,12 +112,6 @@ class IntField(co.namedtuple('IntField', 'x')): def __ge__(self, other): return not self.__lt__(other) - def __truediv__(self, n): - if m.isinf(self.x): - return self - else: - return IntField(round(self.x / n)) - # size results class StackResult(co.namedtuple('StackResult', 'file,function,stack_frame,stack_limit')): @@ -394,8 +392,8 @@ def table(results, calls, diff_results=None, *, r.stack_limit.diff_table() if r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)))) else: print(' %s %s %s %s %s %s%s' % ( @@ -416,8 +414,8 @@ def table(results, calls, diff_results=None, *, diff_r.stack_limit if diff_r else None) if r or diff_r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)) if ratio else '')) @@ -460,8 +458,8 @@ def table(results, calls, diff_results=None, *, r.stack_limit.diff_table() if r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)))) else: print(' %s %s %s %s %s %s%s' % ( @@ -482,8 +480,8 @@ def table(results, calls, diff_results=None, *, diff_r.stack_limit if diff_r else None) if r or diff_r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)) if ratio else '')) diff --git a/scripts/struct_.py b/scripts/struct_.py index a024cad6..9351bb33 100755 --- a/scripts/struct_.py +++ b/scripts/struct_.py @@ -27,7 +27,7 @@ OBJDUMP_TOOL = ['objdump'] # integer fields class IntField(co.namedtuple('IntField', 'x')): __slots__ = () - def __new__(cls, x): + def __new__(cls, x=0): if isinstance(x, IntField): return x if isinstance(x, str): @@ -36,13 +36,22 @@ class IntField(co.namedtuple('IntField', 'x')): except ValueError: # also accept +-∞ and +-inf if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): - x = float('inf') + x = m.inf elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): - x = float('-inf') + x = -m.inf else: raise + assert isinstance(x, int) or m.isinf(x), x return super().__new__(cls, x) + def __str__(self): + if self.x == m.inf: + return '∞' + elif self.x == -m.inf: + return '-∞' + else: + return str(self.x) + def __int__(self): assert not m.isinf(self.x) return self.x @@ -50,14 +59,6 @@ class IntField(co.namedtuple('IntField', 'x')): def __float__(self): return float(self.x) - def __str__(self): - if self.x == float('inf'): - return '∞' - elif self.x == float('-inf'): - return '-∞' - else: - return str(self.x) - none = '%7s' % '-' def table(self): return '%7s' % (self,) @@ -69,9 +70,9 @@ class IntField(co.namedtuple('IntField', 'x')): new = self.x if self else 0 old = other.x if other else 0 diff = new - old - if diff == float('+inf'): + if diff == +m.inf: return '%7s' % '+∞' - elif diff == float('-inf'): + elif diff == -m.inf: return '%7s' % '-∞' else: return '%+7d' % diff @@ -82,9 +83,9 @@ class IntField(co.namedtuple('IntField', 'x')): if m.isinf(new) and m.isinf(old): return 0.0 elif m.isinf(new): - return float('+inf') + return +m.inf elif m.isinf(old): - return float('-inf') + return -m.inf elif not old and not new: return 0.0 elif not old: @@ -95,6 +96,9 @@ class IntField(co.namedtuple('IntField', 'x')): def __add__(self, other): return IntField(self.x + other.x) + def __sub__(self, other): + return IntField(self.x - other.x) + def __mul__(self, other): return IntField(self.x * other.x) @@ -110,12 +114,6 @@ class IntField(co.namedtuple('IntField', 'x')): def __ge__(self, other): return not self.__lt__(other) - def __truediv__(self, n): - if m.isinf(self.x): - return self - else: - return IntField(round(self.x / n)) - # struct size results class StructResult(co.namedtuple('StructResult', 'file,struct,struct_size')): __slots__ = () @@ -328,8 +326,8 @@ def table(results, diff_results=None, *, r.struct_size.diff_table() if r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)))) else: print(' %s %s %s%s' % ( @@ -342,8 +340,8 @@ def table(results, diff_results=None, *, diff_r.struct_size if diff_r else None) if r or diff_r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)) if ratio else '')) @@ -367,8 +365,8 @@ def table(results, diff_results=None, *, r.struct_size.diff_table() if r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)))) else: print(' %s %s %s%s' % ( @@ -381,8 +379,8 @@ def table(results, diff_results=None, *, diff_r.struct_size if diff_r else None) if r or diff_r else IntField.diff_none, ' (%s)' % ( - '+∞%' if ratio == float('+inf') - else '-∞%' if ratio == float('-inf') + '+∞%' if ratio == +m.inf + else '-∞%' if ratio == -m.inf else '%+.1f%%' % (100*ratio)) if ratio else '')) diff --git a/scripts/summary.py b/scripts/summary.py index 60bf948e..d257a3fa 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -25,12 +25,26 @@ import re CSV_PATHS = ['*.csv'] # supported merge operations +# +# this is a terrible way to express these +# OPS = { - 'add': lambda xs: sum(xs[1:], start=xs[0]), - 'mul': lambda xs: m.prod(xs[1:], start=xs[0]), - 'min': min, - 'max': max, - 'avg': lambda xs: sum(xs[1:], start=xs[0]) / len(xs), + 'sum': lambda xs: sum(xs[1:], start=xs[0]), + 'prod': lambda xs: m.prod(xs[1:], start=xs[0]), + 'min': min, + 'max': max, + 'mean': lambda xs: FloatField(sum(float(x) for x in xs) / len(xs)), + 'stddev': lambda xs: ( + lambda mean: FloatField( + m.sqrt(sum((float(x) - mean)**2 for x in xs) / len(xs))) + )(sum(float(x) for x in xs) / len(xs)), + 'gmean': lambda xs: FloatField(m.prod(float(x) for x in xs)**(1/len(xs))), + 'gstddev': lambda xs: ( + lambda gmean: FloatField( + m.exp(m.sqrt(sum(m.log(float(x)/gmean)**2 for x in xs) / len(xs))) + if gmean else m.inf) + )(m.prod(float(x) for x in xs)**(1/len(xs))) + } @@ -47,7 +61,7 @@ def openio(path, mode='r'): # integer fields class IntField(co.namedtuple('IntField', 'x')): __slots__ = () - def __new__(cls, x): + def __new__(cls, x=0): if isinstance(x, IntField): return x if isinstance(x, str): @@ -56,13 +70,22 @@ class IntField(co.namedtuple('IntField', 'x')): except ValueError: # also accept +-∞ and +-inf if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): - x = float('inf') + x = m.inf elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): - x = float('-inf') + x = -m.inf else: raise + assert isinstance(x, int) or m.isinf(x), x return super().__new__(cls, x) + def __str__(self): + if self.x == m.inf: + return '∞' + elif self.x == -m.inf: + return '-∞' + else: + return str(self.x) + def __int__(self): assert not m.isinf(self.x) return self.x @@ -70,14 +93,6 @@ class IntField(co.namedtuple('IntField', 'x')): def __float__(self): return float(self.x) - def __str__(self): - if self.x == float('inf'): - return '∞' - elif self.x == float('-inf'): - return '-∞' - else: - return str(self.x) - none = '%7s' % '-' def table(self): return '%7s' % (self,) @@ -89,9 +104,9 @@ class IntField(co.namedtuple('IntField', 'x')): new = self.x if self else 0 old = other.x if other else 0 diff = new - old - if diff == float('+inf'): + if diff == +m.inf: return '%7s' % '+∞' - elif diff == float('-inf'): + elif diff == -m.inf: return '%7s' % '-∞' else: return '%+7d' % diff @@ -102,9 +117,9 @@ class IntField(co.namedtuple('IntField', 'x')): if m.isinf(new) and m.isinf(old): return 0.0 elif m.isinf(new): - return float('+inf') + return +m.inf elif m.isinf(old): - return float('-inf') + return -m.inf elif not old and not new: return 0.0 elif not old: @@ -115,6 +130,9 @@ class IntField(co.namedtuple('IntField', 'x')): def __add__(self, other): return IntField(self.x + other.x) + def __sub__(self, other): + return IntField(self.x - other.x) + def __mul__(self, other): return IntField(self.x * other.x) @@ -130,16 +148,10 @@ class IntField(co.namedtuple('IntField', 'x')): def __ge__(self, other): return not self.__lt__(other) - def __truediv__(self, n): - if m.isinf(self.x): - return self - else: - return IntField(round(self.x / n)) - # float fields class FloatField(co.namedtuple('FloatField', 'x')): __slots__ = () - def __new__(cls, x): + def __new__(cls, x=0.0): if isinstance(x, FloatField): return x if isinstance(x, str): @@ -148,24 +160,25 @@ class FloatField(co.namedtuple('FloatField', 'x')): except ValueError: # also accept +-∞ and +-inf if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): - x = float('inf') + x = m.inf elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): - x = float('-inf') + x = -m.inf else: raise + assert isinstance(x, float), x return super().__new__(cls, x) - def __float__(self): - return float(self.x) - def __str__(self): - if self.x == float('inf'): + if self.x == m.inf: return '∞' - elif self.x == float('-inf'): + elif self.x == -m.inf: return '-∞' else: return '%.1f' % self.x + def __float__(self): + return float(self.x) + none = IntField.none table = IntField.table diff_none = IntField.diff_none @@ -173,22 +186,17 @@ class FloatField(co.namedtuple('FloatField', 'x')): diff_diff = IntField.diff_diff ratio = IntField.ratio __add__ = IntField.__add__ + __sub__ = IntField.__sub__ __mul__ = IntField.__mul__ __lt__ = IntField.__lt__ __gt__ = IntField.__gt__ __le__ = IntField.__le__ __ge__ = IntField.__ge__ - def __truediv__(self, n): - if m.isinf(self.x): - return self - else: - return FloatField(self.x / n) - # fractional fields, a/b class FracField(co.namedtuple('FracField', 'a,b')): __slots__ = () - def __new__(cls, a, b=None): + def __new__(cls, a=0, b=None): if isinstance(a, FracField) and b is None: return a if isinstance(a, str) and b is None: @@ -200,6 +208,9 @@ class FracField(co.namedtuple('FracField', 'a,b')): def __str__(self): return '%s/%s' % (self.a, self.b) + def __float__(self): + return float(self.a) + none = '%11s %7s' % ('-', '-') def table(self): if not self.b.x: @@ -208,8 +219,8 @@ class FracField(co.namedtuple('FracField', 'a,b')): t = self.a.x/self.b.x return '%11s %7s' % ( self, - '∞%' if t == float('+inf') - else '-∞%' if t == float('-inf') + '∞%' if t == +m.inf + else '-∞%' if t == -m.inf else '%.1f%%' % (100*t)) diff_none = '%11s' % '-' @@ -236,12 +247,15 @@ class FracField(co.namedtuple('FracField', 'a,b')): def __add__(self, other): return FracField(self.a + other.a, self.b + other.b) + def __sub__(self, other): + return FracField(self.a - other.a, self.b - other.b) + def __mul__(self, other): return FracField(self.a * other.a, self.b + other.b) def __lt__(self, other): - self_r = self.a.x/self.b.x if self.b.x else float('-inf') - other_r = other.a.x/other.b.x if other.b.x else float('-inf') + self_r = self.a.x/self.b.x if self.b.x else -m.inf + other_r = other.a.x/other.b.x if other.b.x else -m.inf return self_r < other_r def __gt__(self, other): @@ -253,9 +267,6 @@ class FracField(co.namedtuple('FracField', 'a,b')): def __ge__(self, other): return not self.__lt__(other) - def __truediv__(self, n): - return FracField(self.a / n, self.b / n) - # available types TYPES = [IntField, FloatField, FracField] @@ -314,7 +325,7 @@ def homogenize(results, *, if k is not None and k not in fields and not any(k == old_k for _, old_k in renames)) - by = list(by.keys()) + by = list(by.keys()) # go ahead and clean up none values, these can have a few forms results_ = [] @@ -357,6 +368,7 @@ def homogenize(results, *, def fold(results, *, by=[], fields=[], + types=None, ops={}, **_): folding = co.OrderedDict() @@ -375,7 +387,7 @@ def fold(results, *, for k, vs in r.items(): if vs: # sum fields by default - op = OPS[ops.get(k, 'add')] + op = OPS[ops.get(k, 'sum')] r_[k] = op(vs) # drop any rows without fields and any empty keys @@ -384,14 +396,24 @@ def fold(results, *, {k: v for k, v in zip(by, name) if v}, **r_)) - return folded + # what is the type of merged fields? + if types is not None: + types_ = {} + for k in fields: + op = OPS[ops.get(k, 'sum')] + types_[k] = op([types[k]()]).__class__ + + if types is None: + return folded + else: + return types_, folded -def table(results, diff_results=None, *, - by=None, - fields=None, - types=None, - ops=None, +def table(results, total, diff_results=None, diff_total=None, *, + by=[], + fields=[], + types={}, + ops={}, sort=None, reverse_sort=None, summary=False, @@ -472,8 +494,8 @@ def table(results, diff_results=None, *, if k in r else types[k].diff_none for k in fields), ' (%s)' % ', '.join( - '+∞%' if t == float('+inf') - else '-∞%' if t == float('-inf') + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf else '%+.1f%%' % (100*t) for t in ratios))) else: @@ -488,19 +510,17 @@ def table(results, diff_results=None, *, if k in r or k in diff_r else types[k].diff_none for k in fields), ' (%s)' % ', '.join( - '+∞%' if t == float('+inf') - else '-∞%' if t == float('-inf') + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf else '%+.1f%%' % (100*t) for t in ratios if t) if any(ratios) else '')) # print total - total = fold(results, by=[], fields=fields, ops=ops) - r = total[0] if total else {} - if diff_results is not None: - diff_total = fold(diff_results, by=[], fields=fields, ops=ops) - diff_r = diff_total[0] if diff_total else {} + r = total + if diff_total is not None: + diff_r = diff_total ratios = [types[k].ratio(r.get(k), diff_r.get(k)) for k in fields] @@ -516,8 +536,8 @@ def table(results, diff_results=None, *, if k in r else types[k].diff_none for k in fields), ' (%s)' % ', '.join( - '+∞%' if t == float('+inf') - else '-∞%' if t == float('-inf') + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf else '%+.1f%%' % (100*t) for t in ratios))) else: @@ -532,8 +552,8 @@ def table(results, diff_results=None, *, if k in r or k in diff_r else types[k].diff_none for k in fields), ' (%s)' % ', '.join( - '+∞%' if t == float('+inf') - else '-∞%' if t == float('-inf') + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf else '%+.1f%%' % (100*t) for t in ratios if t) @@ -597,9 +617,13 @@ def main(csv_paths, *, by, fields, types, results = homogenize(results, by=by, fields=fields, renames=renames, define=define) + # fold for total, note we do this with the raw data to avoid + # issues with lossy operations + total = fold(results, fields=fields, ops=ops) + total = total[0] if total else {} + # fold to remove duplicates - results = fold(results, - by=by, fields=fields, ops=ops) + types_, results = fold(results, by=by, fields=fields, types=types, ops=ops) # write results to CSV if args.get('output'): @@ -624,19 +648,25 @@ def main(csv_paths, *, _, _, _, diff_results = homogenize(diff_results, by=by, fields=fields, renames=renames, define=define, types=types) + # fold for total, note we do this with the raw data to avoid + # issues with lossy operations + diff_total = fold(diff_results, fields=fields, ops=ops) + diff_total = diff_total[0] if diff_total else {} + # fold to remove duplicates - diff_results = fold(diff_results, - by=by, fields=fields, ops=ops) + diff_results = fold(diff_results, by=by, fields=fields, ops=ops) # print table if not args.get('quiet'): table( results, + total, diff_results if args.get('diff') else None, + diff_total if args.get('diff') else None, by=by, fields=fields, + types=types_, ops=ops, - types=types, **args) @@ -685,11 +715,11 @@ if __name__ == "__main__": help="Only include rows where this field is this value. May include " "comma-separated options.") parser.add_argument( - '--add', + '--sum', action='append', help="Add these fields (the default).") parser.add_argument( - '--mul', + '--prod', action='append', help="Multiply these fields.") parser.add_argument( @@ -701,9 +731,21 @@ if __name__ == "__main__": action='append', help="Take the maximum of these fields.") parser.add_argument( - '--avg', + '--mean', action='append', help="Average these fields.") + parser.add_argument( + '--stddev', + action='append', + help="Find the standard deviation of these fields.") + parser.add_argument( + '--gmean', + action='append', + help="Find the geometric mean of these fields.") + parser.add_argument( + '--gstddev', + action='append', + help="Find the geometric standard deviation of these fields.") parser.add_argument( '-s', '--sort', action='append', From 274222b5186dad50c13d9f409636359627bf8435 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 27 Sep 2022 15:25:53 -0500 Subject: [PATCH 48/81] Added some automatic sizing for field-names in scripts/runners --- runners/bench_runner.c | 84 ++++++++++++++++++++---- runners/test_runner.c | 105 ++++++++++++++++++++++++----- scripts/code.py | 26 +++++--- scripts/coverage.py | 34 +++++----- scripts/data.py | 26 +++++--- scripts/stack.py | 27 +++++--- scripts/struct_.py | 26 +++++--- scripts/summary.py | 146 +++++++++++++++++++++++++---------------- 8 files changed, 331 insertions(+), 143 deletions(-) diff --git a/runners/bench_runner.c b/runners/bench_runner.c index f7e33479..7a6e2fa5 100644 --- a/runners/bench_runner.c +++ b/runners/bench_runner.c @@ -665,7 +665,7 @@ void perm_count( // operations we can do static void summary(void) { - printf("%-36s %7s %7s %7s %11s\n", + printf("%-23s %7s %7s %7s %11s\n", "", "flags", "suites", "cases", "perms"); size_t suites = 0; size_t cases = 0; @@ -707,7 +707,7 @@ static void summary(void) { sprintf(flag_buf, "%s%s", (flags & BENCH_REENTRANT) ? "r" : "", (!flags) ? "-" : ""); - printf("%-36s %7s %7zu %7zu %11s\n", + printf("%-23s %7s %7zu %7zu %11s\n", "TOTAL", flag_buf, suites, @@ -716,8 +716,18 @@ static void summary(void) { } static void list_suites(void) { - printf("%-36s %7s %7s %11s\n", "suite", "flags", "cases", "perms"); + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + size_t len = strlen(bench_suites[i].name); + if (len > name_width) { + name_width = len; + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + printf("%-*s %7s %7s %11s\n", + name_width, "suite", "flags", "cases", "perms"); for (size_t t = 0; t < bench_id_count; t++) { for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { bench_define_suite(&bench_suites[i]); @@ -756,7 +766,8 @@ static void list_suites(void) { sprintf(flag_buf, "%s%s", (bench_suites[i].flags & BENCH_REENTRANT) ? "r" : "", (!bench_suites[i].flags) ? "-" : ""); - printf("%-36s %7s %7zu %11s\n", + printf("%-*s %7s %7zu %11s\n", + name_width, bench_suites[i].name, flag_buf, cases, @@ -766,8 +777,19 @@ static void list_suites(void) { } static void list_cases(void) { - printf("%-36s %7s %11s\n", "case", "flags", "perms"); + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + size_t len = strlen(bench_suites[i].cases[j].name); + if (len > name_width) { + name_width = len; + } + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + printf("%-*s %7s %11s\n", name_width, "case", "flags", "perms"); for (size_t t = 0; t < bench_id_count; t++) { for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { bench_define_suite(&bench_suites[i]); @@ -799,7 +821,8 @@ static void list_cases(void) { ? "r" : "", (!bench_suites[i].cases[j].flags) ? "-" : ""); - printf("%-36s %7s %11s\n", + printf("%-*s %7s %11s\n", + name_width, bench_suites[i].cases[j].name, flag_buf, perm_buf); @@ -809,8 +832,17 @@ static void list_cases(void) { } static void list_suite_paths(void) { - printf("%-36s %s\n", "suite", "path"); + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + size_t len = strlen(bench_suites[i].name); + if (len > name_width) { + name_width = len; + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + printf("%-*s %s\n", name_width, "suite", "path"); for (size_t t = 0; t < bench_id_count; t++) { for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { size_t cases = 0; @@ -823,6 +855,8 @@ static void list_suite_paths(void) { || strcmp(bench_ids[t].name, bench_suites[i].cases[j].name) == 0)) { continue; + + cases += 1; } } @@ -831,7 +865,8 @@ static void list_suite_paths(void) { continue; } - printf("%-36s %s\n", + printf("%-*s %s\n", + name_width, bench_suites[i].name, bench_suites[i].path); } @@ -839,8 +874,19 @@ static void list_suite_paths(void) { } static void list_case_paths(void) { - printf("%-36s %s\n", "case", "path"); + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { + for (size_t j = 0; j < bench_suites[i].case_count; j++) { + size_t len = strlen(bench_suites[i].cases[j].name); + if (len > name_width) { + name_width = len; + } + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + printf("%-*s %s\n", name_width, "case", "path"); for (size_t t = 0; t < bench_id_count; t++) { for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) { for (size_t j = 0; j < bench_suites[i].case_count; j++) { @@ -853,7 +899,8 @@ static void list_case_paths(void) { continue; } - printf("%-36s %s\n", + printf("%-*s %s\n", + name_width, bench_suites[i].cases[j].name, bench_suites[i].cases[j].path); } @@ -1099,16 +1146,27 @@ const bench_geometry_t *bench_geometries = builtin_geometries; size_t bench_geometry_count = 5; static void list_geometries(void) { + // at least size so that names fit + unsigned name_width = 23; + for (size_t g = 0; builtin_geometries[g].name; g++) { + size_t len = strlen(builtin_geometries[g].name); + if (len > name_width) { + name_width = len; + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + // yes we do need to define a suite, this does a bit of bookeeping // such as setting up the define cache bench_define_suite(&(const struct bench_suite){0}); - printf("%-24s %7s %7s %7s %7s %11s\n", - "geometry", "read", "prog", "erase", "count", "size"); + printf("%-*s %7s %7s %7s %7s %11s\n", + name_width, "geometry", "read", "prog", "erase", "count", "size"); for (size_t g = 0; builtin_geometries[g].name; g++) { bench_define_geometry(&builtin_geometries[g]); bench_define_flush(); - printf("%-24s %7ju %7ju %7ju %7ju %11ju\n", + printf("%-*s %7ju %7ju %7ju %7ju %11ju\n", + name_width, builtin_geometries[g].name, READ_SIZE, PROG_SIZE, diff --git a/runners/test_runner.c b/runners/test_runner.c index f0e1ff6e..7b754474 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -691,7 +691,7 @@ void perm_count( // operations we can do static void summary(void) { - printf("%-36s %7s %7s %7s %11s\n", + printf("%-23s %7s %7s %7s %11s\n", "", "flags", "suites", "cases", "perms"); size_t suites = 0; size_t cases = 0; @@ -735,7 +735,7 @@ static void summary(void) { sprintf(flag_buf, "%s%s", (flags & TEST_REENTRANT) ? "r" : "", (!flags) ? "-" : ""); - printf("%-36s %7s %7zu %7zu %11s\n", + printf("%-23s %7s %7zu %7zu %11s\n", "TOTAL", flag_buf, suites, @@ -744,8 +744,18 @@ static void summary(void) { } static void list_suites(void) { - printf("%-36s %7s %7s %11s\n", "suite", "flags", "cases", "perms"); + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + size_t len = strlen(test_suites[i].name); + if (len > name_width) { + name_width = len; + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + printf("%-*s %7s %7s %11s\n", + name_width, "suite", "flags", "cases", "perms"); for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { test_define_suite(&test_suites[i]); @@ -786,7 +796,8 @@ static void list_suites(void) { sprintf(flag_buf, "%s%s", (test_suites[i].flags & TEST_REENTRANT) ? "r" : "", (!test_suites[i].flags) ? "-" : ""); - printf("%-36s %7s %7zu %11s\n", + printf("%-*s %7s %7zu %11s\n", + name_width, test_suites[i].name, flag_buf, cases, @@ -796,8 +807,19 @@ static void list_suites(void) { } static void list_cases(void) { - printf("%-36s %7s %11s\n", "case", "flags", "perms"); + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + for (size_t j = 0; j < test_suites[i].case_count; j++) { + size_t len = strlen(test_suites[i].cases[j].name); + if (len > name_width) { + name_width = len; + } + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + printf("%-*s %7s %11s\n", name_width, "case", "flags", "perms"); for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { test_define_suite(&test_suites[i]); @@ -831,7 +853,8 @@ static void list_cases(void) { ? "r" : "", (!test_suites[i].cases[j].flags) ? "-" : ""); - printf("%-36s %7s %11s\n", + printf("%-*s %7s %11s\n", + name_width, test_suites[i].cases[j].name, flag_buf, perm_buf); @@ -841,8 +864,17 @@ static void list_cases(void) { } static void list_suite_paths(void) { - printf("%-36s %s\n", "suite", "path"); + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + size_t len = strlen(test_suites[i].name); + if (len > name_width) { + name_width = len; + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + printf("%-*s %s\n", name_width, "suite", "path"); for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { size_t cases = 0; @@ -856,6 +888,8 @@ static void list_suite_paths(void) { test_suites[i].cases[j].name) == 0)) { continue; } + + cases += 1; } // no tests found? @@ -863,7 +897,8 @@ static void list_suite_paths(void) { continue; } - printf("%-36s %s\n", + printf("%-*s %s\n", + name_width, test_suites[i].name, test_suites[i].path); } @@ -871,8 +906,19 @@ static void list_suite_paths(void) { } static void list_case_paths(void) { - printf("%-36s %s\n", "case", "path"); + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + for (size_t j = 0; j < test_suites[i].case_count; j++) { + size_t len = strlen(test_suites[i].cases[j].name); + if (len > name_width) { + name_width = len; + } + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + printf("%-*s %s\n", name_width, "case", "path"); for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { for (size_t j = 0; j < test_suites[i].case_count; j++) { @@ -885,7 +931,8 @@ static void list_case_paths(void) { continue; } - printf("%-36s %s\n", + printf("%-*s %s\n", + name_width, test_suites[i].cases[j].name, test_suites[i].cases[j].path); } @@ -1139,16 +1186,27 @@ const test_geometry_t *test_geometries = builtin_geometries; size_t test_geometry_count = 5; static void list_geometries(void) { + // at least size so that names fit + unsigned name_width = 23; + for (size_t g = 0; builtin_geometries[g].name; g++) { + size_t len = strlen(builtin_geometries[g].name); + if (len > name_width) { + name_width = len; + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + // yes we do need to define a suite, this does a bit of bookeeping // such as setting up the define cache test_define_suite(&(const struct test_suite){0}); - printf("%-24s %7s %7s %7s %7s %11s\n", - "geometry", "read", "prog", "erase", "count", "size"); + printf("%-*s %7s %7s %7s %7s %11s\n", + name_width, "geometry", "read", "prog", "erase", "count", "size"); for (size_t g = 0; builtin_geometries[g].name; g++) { test_define_geometry(&builtin_geometries[g]); test_define_flush(); - printf("%-24s %7ju %7ju %7ju %7ju %11ju\n", + printf("%-*s %7ju %7ju %7ju %7ju %11ju\n", + name_width, builtin_geometries[g].name, READ_SIZE, PROG_SIZE, @@ -1669,18 +1727,29 @@ const test_powerloss_t *test_powerlosses = (const test_powerloss_t[]){ size_t test_powerloss_count = 1; static void list_powerlosses(void) { - printf("%-24s %s\n", "scenario", "description"); + // at least size so that names fit + unsigned name_width = 23; + for (size_t i = 0; builtin_powerlosses[i].name; i++) { + size_t len = strlen(builtin_powerlosses[i].name); + if (len > name_width) { + name_width = len; + } + } + name_width = 4*((name_width+1+4-1)/4)-1; + + printf("%-*s %s\n", name_width, "scenario", "description"); size_t i = 0; for (; builtin_powerlosses[i].name; i++) { - printf("%-24s %s\n", + printf("%-*s %s\n", + name_width, builtin_powerlosses[i].name, builtin_powerlosses_help[i]); } // a couple more options with special parsing - printf("%-24s %s\n", "1,2,3", builtin_powerlosses_help[i+0]); - printf("%-24s %s\n", "{1,2,3}", builtin_powerlosses_help[i+1]); - printf("%-24s %s\n", ":1248g1", builtin_powerlosses_help[i+2]); + printf("%-*s %s\n", name_width, "1,2,3", builtin_powerlosses_help[i+0]); + printf("%-*s %s\n", name_width, "{1,2,3}", builtin_powerlosses_help[i+1]); + printf("%-*s %s\n", name_width, ":1248g1", builtin_powerlosses_help[i+2]); } diff --git a/scripts/code.py b/scripts/code.py index eeaf8a0c..df16ece6 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -243,14 +243,20 @@ def table(results, diff_results=None, *, reverse=False) # print header - print('%-36s' % ('%s%s' % ( - 'file' if by_file else 'function', - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - if not summary else ''), - end='') + if not summary: + title = '%s%s' % ( + 'file' if by_file else 'function', + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + name_width = max(it.chain([23, len(title)], (len(n) for n in names))) + else: + title = '' + name_width = 23 + name_width = 4*((name_width+1+4-1)//4)-1 + + print('%-*s ' % (name_width, title), end='') if diff_results is None: print(' %s' % ('size'.rjust(len(IntField.none)))) elif percent: @@ -273,7 +279,7 @@ def table(results, diff_results=None, *, if not ratio and not all_: continue - print('%-36s' % name, end='') + print('%-*s ' % (name_width, name), end='') if diff_results is None: print(' %s' % ( r.code_size.table() @@ -312,7 +318,7 @@ def table(results, diff_results=None, *, r.code_size if r else None, diff_r.code_size if diff_r else None) - print('%-36s' % 'TOTAL', end='') + print('%-*s ' % (name_width, 'TOTAL'), end='') if diff_results is None: print(' %s' % ( r.code_size.table() diff --git a/scripts/coverage.py b/scripts/coverage.py index f24744e8..4e4541ac 100755 --- a/scripts/coverage.py +++ b/scripts/coverage.py @@ -320,12 +320,12 @@ def table(results, diff_results=None, *, else ['function']) table = { - (r.file, r.line) if by_line + '%s:%s' % (r.file, r.line) if by_line else r.file if by_file else r.function: r for r in results} diff_table = { - (r.file, r.line) if by_line + '%s:%s' % (r.file, r.line) if by_line else r.file if by_file else r.function: r for r in diff_results or []} @@ -359,16 +359,20 @@ def table(results, diff_results=None, *, reverse=False) # print header - print('%-36s' % ('%s%s' % ( - 'line' if by_line - else 'file' if by_file - else 'function', - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - if not summary else ''), - end='') + if not summary: + title = '%s%s' % ( + 'line' if by_line else 'file' if by_file else 'function', + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + name_width = max(it.chain([23, len(title)], (len(n) for n in names))) + else: + title = '' + name_width = 23 + name_width = 4*((name_width+1+4-1)//4)-1 + + print('%-*s ' % (name_width, title), end='') if diff_results is None: print(' %s %s' % ( 'hits/line'.rjust(len(FracField.none)), @@ -401,9 +405,7 @@ def table(results, diff_results=None, *, if not line_ratio and not branch_ratio and not all_: continue - print('%-36s' % ( - ':'.join('%s' % n for n in name) - if by_line else name), end='') + print('%-*s ' % (name_width, name), end='') if diff_results is None: print(' %s %s' % ( r.coverage_lines.table() @@ -460,7 +462,7 @@ def table(results, diff_results=None, *, r.coverage_branches if r else None, diff_r.coverage_branches if diff_r else None) - print('%-36s' % 'TOTAL', end='') + print('%-*s ' % (name_width, 'TOTAL'), end='') if diff_results is None: print(' %s %s' % ( r.coverage_lines.table() diff --git a/scripts/data.py b/scripts/data.py index 690ac025..60707567 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -244,14 +244,20 @@ def table(results, diff_results=None, *, reverse=False) # print header - print('%-36s' % ('%s%s' % ( - 'file' if by_file else 'function', - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - if not summary else ''), - end='') + if not summary: + title = '%s%s' % ( + 'file' if by_file else 'function', + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + name_width = max(it.chain([23, len(title)], (len(n) for n in names))) + else: + title = '' + name_width = 23 + name_width = 4*((name_width+1+4-1)//4)-1 + + print('%-*s ' % (name_width, title), end='') if diff_results is None: print(' %s' % ('size'.rjust(len(IntField.none)))) elif percent: @@ -274,7 +280,7 @@ def table(results, diff_results=None, *, if not ratio and not all_: continue - print('%-36s' % name, end='') + print('%-*s ' % (name_width, name), end='') if diff_results is None: print(' %s' % ( r.data_size.table() @@ -313,7 +319,7 @@ def table(results, diff_results=None, *, r.data_size if r else None, diff_r.data_size if diff_r else None) - print('%-36s' % 'TOTAL', end='') + print('%-*s ' % (name_width, 'TOTAL'), end='') if diff_results is None: print(' %s' % ( r.data_size.table() diff --git a/scripts/stack.py b/scripts/stack.py index a3bbd486..ed5f1fce 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -329,20 +329,27 @@ def table(results, calls, diff_results=None, *, names.sort(key=lambda n: (table[n].stack_frame,) if n in table else (), reverse=False) - # adjust the name width based on the expected call depth, note that we - # can't always find the depth due to recursion - width = 36 + (4*depth if not m.isinf(depth) else 0) - # print header - if not tree: - print('%-*s' % (width, '%s%s' % ( + if not summary: + title = '%s%s' % ( 'file' if by_file else 'function', ' (%d added, %d removed)' % ( sum(1 for n in table if n not in diff_table), sum(1 for n in diff_table if n not in table)) if diff_results is not None and not percent else '') - if not summary else ''), - end='') + name_width = max(it.chain([23, len(title)], (len(n) for n in names))) + else: + title = '' + name_width = 23 + name_width = 4*((name_width+1+4-1)//4)-1 + + # adjust the name width based on the expected call depth, note that we + # can't always find the depth due to recursion + if not m.isinf(depth): + name_width += 4*depth + + if not tree: + print('%-*s ' % (name_width, title), end='') if diff_results is None: print(' %s %s' % ( 'frame'.rjust(len(IntField.none)), @@ -376,7 +383,7 @@ def table(results, calls, diff_results=None, *, continue is_last = (i == len(names_)-1) - print('%-*s' % (width, prefixes[0+is_last] + name), end='') + print('%-*s ' % (name_width, prefixes[0+is_last]+name), end='') if tree: print() elif diff_results is None: @@ -444,7 +451,7 @@ def table(results, calls, diff_results=None, *, r.stack_limit if r else None, diff_r.stack_limit if diff_r else None) - print('%-*s' % (width, 'TOTAL'), end='') + print('%-*s ' % (name_width, 'TOTAL'), end='') if diff_results is None: print(' %s %s' % ( r.stack_frame.table() diff --git a/scripts/struct_.py b/scripts/struct_.py index 9351bb33..bdb98f73 100755 --- a/scripts/struct_.py +++ b/scripts/struct_.py @@ -286,14 +286,20 @@ def table(results, diff_results=None, *, reverse=False) # print header - print('%-36s' % ('%s%s' % ( - 'file' if by_file else 'struct', - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - if not summary else ''), - end='') + if not summary: + title = '%s%s' % ( + 'file' if by_file else 'struct', + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + name_width = max(it.chain([23, len(title)], (len(n) for n in names))) + else: + title = '' + name_width = 23 + name_width = 4*((name_width+1+4-1)//4)-1 + + print('%-*s ' % (name_width, title), end='') if diff_results is None: print(' %s' % ('size'.rjust(len(IntField.none)))) elif percent: @@ -316,7 +322,7 @@ def table(results, diff_results=None, *, if not ratio and not all_: continue - print('%-36s' % name, end='') + print('%-*s ' % (name_width, name), end='') if diff_results is None: print(' %s' % ( r.struct_size.table() @@ -355,7 +361,7 @@ def table(results, diff_results=None, *, r.struct_size if r else None, diff_r.struct_size if diff_r else None) - print('%-36s' % 'TOTAL', end='') + print('%-*s ' % (name_width, 'TOTAL'), end='') if diff_results is None: print(' %s' % ( r.struct_size.table() diff --git a/scripts/summary.py b/scripts/summary.py index d257a3fa..98c1a538 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -422,8 +422,12 @@ def table(results, total, diff_results=None, diff_total=None, *, **_): all_, all = all, __builtins__.all - table = {tuple(r.get(k,'') for k in by): r for r in results} - diff_table = {tuple(r.get(k,'') for k in by): r for r in diff_results or []} + table = { + ','.join(r.get(k,'') for k in by): r + for r in results} + diff_table = { + ','.join(r.get(k,'') for k in by): r + for r in diff_results or []} # sort, note that python's sort is stable names = list(table.keys() | diff_table.keys()) @@ -446,30 +450,40 @@ def table(results, total, diff_results=None, diff_total=None, *, reverse=False) # print header - print('%-36s' % ('%s%s' % ( - ','.join(k for k in by), - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - if not summary else ''), - end='') - if diff_results is None: - print(' %s' % ( - ' '.join(k.rjust(len(types[k].none)) - for k in fields))) - elif percent: - print(' %s' % ( - ' '.join(k.rjust(len(types[k].diff_none)) - for k in fields))) + if not summary: + title = '%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + name_width = max(it.chain([23, len(title)], (len(n) for n in names))) else: + title = '' + name_width = 23 + name_width = 4*((name_width+1+4-1)//4)-1 + + print('%-*s ' % (name_width, title), end='') + if diff_results is None: + widths = [ + 4*((max(len(types[k].none), len(k))+1+4-1)//4)-1 + for k in fields] + print(' %s' % ( + ' '.join(k.rjust(w) for w, k in zip(widths, fields)))) + elif percent: + widths = [ + 4*((max(len(types[k].diff_none), len(k))+1+4-1)//4)-1 + for k in fields] + print(' %s' % ( + ' '.join(k.rjust(w) for w, k in zip(widths, fields)))) + else: + widths = [ + 4*((max(len(types[k].diff_none), 1+len(k))+1+4-1)//4)-1 + for k in fields] print(' %s %s %s' % ( - ' '.join(('o'+k).rjust(len(types[k].diff_none)) - for k in fields), - ' '.join(('n'+k).rjust(len(types[k].diff_none)) - for k in fields), - ' '.join(('d'+k).rjust(len(types[k].diff_none)) - for k in fields))) + ' '.join(('o'+k).rjust(w) for w, k in zip(widths, fields)), + ' '.join(('n'+k).rjust(w) for w, k in zip(widths, fields)), + ' '.join(('d'+k).rjust(w) for w, k in zip(widths, fields)))) # print entries if not summary: @@ -482,17 +496,21 @@ def table(results, total, diff_results=None, diff_total=None, *, if not any(ratios) and not all_: continue - print('%-36s' % ','.join(name), end='') + print('%-*s ' % (name_width, name), end='') if diff_results is None: print(' %s' % ( - ' '.join(r[k].table() - if k in r else types[k].none - for k in fields))) + ' '.join( + (r[k].table() + if k in r + else types[k].none).rjust(w) + for w, k in zip(widths, fields)))) elif percent: print(' %s%s' % ( - ' '.join(r[k].diff_table() - if k in r else types[k].diff_none - for k in fields), + ' '.join( + (r[k].diff_table().rjust(w) + if k in r + else types[k].diff_none).rjust(w) + for w, k in zip(widths, fields)), ' (%s)' % ', '.join( '+∞%' if t == +m.inf else '-∞%' if t == -m.inf @@ -500,15 +518,21 @@ def table(results, total, diff_results=None, diff_total=None, *, for t in ratios))) else: print(' %s %s %s%s' % ( - ' '.join(diff_r[k].diff_table() - if k in diff_r else types[k].diff_none - for k in fields), - ' '.join(r[k].diff_table() - if k in r else types[k].diff_none - for k in fields), - ' '.join(types[k].diff_diff(r.get(k), diff_r.get(k)) - if k in r or k in diff_r else types[k].diff_none - for k in fields), + ' '.join( + (diff_r[k].diff_table() + if k in diff_r + else types[k].diff_none).rjust(w) + for w, k in zip(widths, fields)), + ' '.join( + (r[k].diff_table() + if k in r + else types[k].diff_none).rjust(w) + for w, k in zip(widths, fields)), + ' '.join( + (types[k].diff_diff(r.get(k), diff_r.get(k)) + if k in r or k in diff_r + else types[k].diff_none).rjust(w) + for w, k in zip(widths, fields)), ' (%s)' % ', '.join( '+∞%' if t == +m.inf else '-∞%' if t == -m.inf @@ -524,17 +548,21 @@ def table(results, total, diff_results=None, diff_total=None, *, ratios = [types[k].ratio(r.get(k), diff_r.get(k)) for k in fields] - print('%-36s' % 'TOTAL', end='') + print('%-*s ' % (name_width, 'TOTAL'), end='') if diff_results is None: print(' %s' % ( - ' '.join(r[k].table() - if k in r else types[k].none - for k in fields))) + ' '.join( + (r[k].table() + if k in r + else types[k].none).rjust(w) + for w, k in zip(widths, fields)))) elif percent: print(' %s%s' % ( - ' '.join(r[k].diff_table() - if k in r else types[k].diff_none - for k in fields), + ' '.join( + (r[k].diff_table().rjust(w) + if k in r + else types[k].diff_none).rjust(w) + for w, k in zip(widths, fields)), ' (%s)' % ', '.join( '+∞%' if t == +m.inf else '-∞%' if t == -m.inf @@ -542,15 +570,21 @@ def table(results, total, diff_results=None, diff_total=None, *, for t in ratios))) else: print(' %s %s %s%s' % ( - ' '.join(diff_r[k].diff_table() - if k in diff_r else types[k].diff_none - for k in fields), - ' '.join(r[k].diff_table() - if k in r else types[k].diff_none - for k in fields), - ' '.join(types[k].diff_diff(r.get(k), diff_r.get(k)) - if k in r or k in diff_r else types[k].diff_none - for k in fields), + ' '.join( + (diff_r[k].diff_table() + if k in diff_r + else types[k].diff_none).rjust(w) + for w, k in zip(widths, fields)), + ' '.join( + (r[k].diff_table() + if k in r + else types[k].diff_none).rjust(w) + for w, k in zip(widths, fields)), + ' '.join( + (types[k].diff_diff(r.get(k), diff_r.get(k)) + if k in r or k in diff_r + else types[k].diff_none).rjust(w) + for w, k in zip(widths, fields)), ' (%s)' % ', '.join( '+∞%' if t == +m.inf else '-∞%' if t == -m.inf From 296c5afea7b4c449bb0c74d2b9fe6db6dcab216d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 27 Sep 2022 15:35:43 -0500 Subject: [PATCH 49/81] Renamed bench_read/prog/erased -> bench_readed/proged/erased Yes this isn't really correct english anymore, but these names avoid the read/read ambiguity. --- bd/lfs_emubd.c | 40 ++++++++++++++++++------------------ bd/lfs_emubd.h | 12 +++++------ runners/bench_runner.c | 44 ++++++++++++++++++++-------------------- scripts/bench.py | 46 +++++++++++++++++++++--------------------- 4 files changed, 71 insertions(+), 71 deletions(-) diff --git a/bd/lfs_emubd.c b/bd/lfs_emubd.c index 8a9da7a7..7372c978 100644 --- a/bd/lfs_emubd.c +++ b/bd/lfs_emubd.c @@ -125,8 +125,8 @@ int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, memset(bd->blocks, 0, cfg->block_count * sizeof(lfs_emubd_block_t*)); // setup testing things - bd->read = 0; - bd->prog = 0; + bd->readed = 0; + bd->proged = 0; bd->erased = 0; bd->power_cycles = bd->cfg->power_cycles; bd->disk = NULL; @@ -249,7 +249,7 @@ int lfs_emubd_read(const struct lfs_config *cfg, lfs_block_t block, } // track reads - bd->read += size; + bd->readed += size; if (bd->cfg->read_sleep) { int err = nanosleep(&(struct timespec){ .tv_sec=bd->cfg->read_sleep/1000000000, @@ -331,7 +331,7 @@ int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, } // track progs - bd->prog += size; + bd->proged += size; if (bd->cfg->prog_sleep) { int err = nanosleep(&(struct timespec){ .tv_sec=bd->cfg->prog_sleep/1000000000, @@ -454,18 +454,18 @@ int lfs_emubd_sync(const struct lfs_config *cfg) { /// Additional extended API for driving test features /// -lfs_emubd_sio_t lfs_emubd_getread(const struct lfs_config *cfg) { - LFS_EMUBD_TRACE("lfs_emubd_getread(%p)", (void*)cfg); +lfs_emubd_sio_t lfs_emubd_getreaded(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_getreaded(%p)", (void*)cfg); lfs_emubd_t *bd = cfg->context; - LFS_EMUBD_TRACE("lfs_emubd_getread -> %"PRIu64, bd->read); - return bd->read; + LFS_EMUBD_TRACE("lfs_emubd_getreaded -> %"PRIu64, bd->readed); + return bd->readed; } -lfs_emubd_sio_t lfs_emubd_getprog(const struct lfs_config *cfg) { - LFS_EMUBD_TRACE("lfs_emubd_getprog(%p)", (void*)cfg); +lfs_emubd_sio_t lfs_emubd_getproged(const struct lfs_config *cfg) { + LFS_EMUBD_TRACE("lfs_emubd_getproged(%p)", (void*)cfg); lfs_emubd_t *bd = cfg->context; - LFS_EMUBD_TRACE("lfs_emubd_getprog -> %"PRIu64, bd->prog); - return bd->prog; + LFS_EMUBD_TRACE("lfs_emubd_getproged -> %"PRIu64, bd->proged); + return bd->proged; } lfs_emubd_sio_t lfs_emubd_geterased(const struct lfs_config *cfg) { @@ -475,19 +475,19 @@ lfs_emubd_sio_t lfs_emubd_geterased(const struct lfs_config *cfg) { return bd->erased; } -int lfs_emubd_setread(const struct lfs_config *cfg, lfs_emubd_io_t read) { - LFS_EMUBD_TRACE("lfs_emubd_setread(%p, %"PRIu64")", (void*)cfg, read); +int lfs_emubd_setreaded(const struct lfs_config *cfg, lfs_emubd_io_t readed) { + LFS_EMUBD_TRACE("lfs_emubd_setreaded(%p, %"PRIu64")", (void*)cfg, readed); lfs_emubd_t *bd = cfg->context; - bd->read = read; - LFS_EMUBD_TRACE("lfs_emubd_setread -> %d", 0); + bd->readed = readed; + LFS_EMUBD_TRACE("lfs_emubd_setreaded -> %d", 0); return 0; } -int lfs_emubd_setprog(const struct lfs_config *cfg, lfs_emubd_io_t prog) { - LFS_EMUBD_TRACE("lfs_emubd_setprog(%p, %"PRIu64")", (void*)cfg, prog); +int lfs_emubd_setproged(const struct lfs_config *cfg, lfs_emubd_io_t proged) { + LFS_EMUBD_TRACE("lfs_emubd_setproged(%p, %"PRIu64")", (void*)cfg, proged); lfs_emubd_t *bd = cfg->context; - bd->prog = prog; - LFS_EMUBD_TRACE("lfs_emubd_setprog -> %d", 0); + bd->proged = proged; + LFS_EMUBD_TRACE("lfs_emubd_setproged -> %d", 0); return 0; } diff --git a/bd/lfs_emubd.h b/bd/lfs_emubd.h index 8aff161f..0fbac1f9 100644 --- a/bd/lfs_emubd.h +++ b/bd/lfs_emubd.h @@ -136,8 +136,8 @@ typedef struct lfs_emubd { lfs_emubd_block_t **blocks; // some other test state - lfs_emubd_io_t read; - lfs_emubd_io_t prog; + lfs_emubd_io_t readed; + lfs_emubd_io_t proged; lfs_emubd_io_t erased; lfs_emubd_powercycles_t power_cycles; lfs_emubd_disk_t *disk; @@ -182,19 +182,19 @@ int lfs_emubd_sync(const struct lfs_config *cfg); /// Additional extended API for driving test features /// // Get total amount of bytes read -lfs_emubd_sio_t lfs_emubd_getread(const struct lfs_config *cfg); +lfs_emubd_sio_t lfs_emubd_getreaded(const struct lfs_config *cfg); // Get total amount of bytes programmed -lfs_emubd_sio_t lfs_emubd_getprog(const struct lfs_config *cfg); +lfs_emubd_sio_t lfs_emubd_getproged(const struct lfs_config *cfg); // Get total amount of bytes erased lfs_emubd_sio_t lfs_emubd_geterased(const struct lfs_config *cfg); // Manually set amount of bytes read -int lfs_emubd_setread(const struct lfs_config *cfg, lfs_emubd_io_t read); +int lfs_emubd_setreaded(const struct lfs_config *cfg, lfs_emubd_io_t readed); // Manually set amount of bytes programmed -int lfs_emubd_setprog(const struct lfs_config *cfg, lfs_emubd_io_t prog); +int lfs_emubd_setproged(const struct lfs_config *cfg, lfs_emubd_io_t proged); // Manually set amount of bytes erased int lfs_emubd_seterased(const struct lfs_config *cfg, lfs_emubd_io_t erased); diff --git a/runners/bench_runner.c b/runners/bench_runner.c index 7a6e2fa5..d58ece82 100644 --- a/runners/bench_runner.c +++ b/runners/bench_runner.c @@ -464,47 +464,47 @@ void bench_trace(const char *fmt, ...) { // bench recording state static struct lfs_config *bench_cfg = NULL; -static lfs_emubd_io_t bench_last_read = 0; -static lfs_emubd_io_t bench_last_prog = 0; +static lfs_emubd_io_t bench_last_readed = 0; +static lfs_emubd_io_t bench_last_proged = 0; static lfs_emubd_io_t bench_last_erased = 0; -lfs_emubd_io_t bench_read = 0; -lfs_emubd_io_t bench_prog = 0; +lfs_emubd_io_t bench_readed = 0; +lfs_emubd_io_t bench_proged = 0; lfs_emubd_io_t bench_erased = 0; void bench_reset(void) { - bench_read = 0; - bench_prog = 0; + bench_readed = 0; + bench_proged = 0; bench_erased = 0; - bench_last_read = 0; - bench_last_prog = 0; + bench_last_readed = 0; + bench_last_proged = 0; bench_last_erased = 0; } void bench_start(void) { assert(bench_cfg); - lfs_emubd_sio_t read = lfs_emubd_getread(bench_cfg); - assert(read >= 0); - lfs_emubd_sio_t prog = lfs_emubd_getprog(bench_cfg); - assert(prog >= 0); + lfs_emubd_sio_t readed = lfs_emubd_getreaded(bench_cfg); + assert(readed >= 0); + lfs_emubd_sio_t proged = lfs_emubd_getproged(bench_cfg); + assert(proged >= 0); lfs_emubd_sio_t erased = lfs_emubd_geterased(bench_cfg); assert(erased >= 0); - bench_last_read = read; - bench_last_prog = prog; + bench_last_readed = readed; + bench_last_proged = proged; bench_last_erased = erased; } void bench_stop(void) { assert(bench_cfg); - lfs_emubd_sio_t read = lfs_emubd_getread(bench_cfg); - assert(read >= 0); - lfs_emubd_sio_t prog = lfs_emubd_getprog(bench_cfg); - assert(prog >= 0); + lfs_emubd_sio_t readed = lfs_emubd_getreaded(bench_cfg); + assert(readed >= 0); + lfs_emubd_sio_t proged = lfs_emubd_getproged(bench_cfg); + assert(proged >= 0); lfs_emubd_sio_t erased = lfs_emubd_geterased(bench_cfg); assert(erased >= 0); - bench_read += read - bench_last_read; - bench_prog += prog - bench_last_prog; + bench_readed += readed - bench_last_readed; + bench_proged += proged - bench_last_proged; bench_erased += erased - bench_last_erased; } @@ -1250,8 +1250,8 @@ void perm_run( printf("finished "); perm_printid(suite, case_); printf(" %"PRIu64" %"PRIu64" %"PRIu64, - bench_read, - bench_prog, + bench_readed, + bench_proged, bench_erased); printf("\n"); diff --git a/scripts/bench.py b/scripts/bench.py index a59d80d3..e401d7cc 100755 --- a/scripts/bench.py +++ b/scripts/bench.py @@ -741,8 +741,8 @@ def run_stage(name, runner_, ids, output_, **args): passed_suite_perms = co.defaultdict(lambda: 0) passed_case_perms = co.defaultdict(lambda: 0) passed_perms = 0 - read = 0 - prog = 0 + readed = 0 + proged = 0 erased = 0 failures = [] killed = False @@ -750,8 +750,8 @@ def run_stage(name, runner_, ids, output_, **args): pattern = re.compile('^(?:' '(?Prunning|finished|skipped|powerloss)' ' (?P(?P[^:]+)[^\s]*)' - '(?: (?P\d+))?' - '(?: (?P\d+))?' + '(?: (?P\d+))?' + '(?: (?P\d+))?' '(?: (?P\d+))?' '|' '(?P[^:]+):(?P\d+):(?Passert):' ' *(?P.*)' @@ -763,8 +763,8 @@ def run_stage(name, runner_, ids, output_, **args): nonlocal passed_suite_perms nonlocal passed_case_perms nonlocal passed_perms - nonlocal read - nonlocal prog + nonlocal readed + nonlocal proged nonlocal erased nonlocal locals @@ -821,14 +821,14 @@ def run_stage(name, runner_, ids, output_, **args): elif op == 'finished': case = m.group('case') suite = case_suites[case] - read_ = int(m.group('read')) - prog_ = int(m.group('prog')) + readed_ = int(m.group('readed')) + proged_ = int(m.group('proged')) erased_ = int(m.group('erased')) passed_suite_perms[suite] += 1 passed_case_perms[case] += 1 passed_perms += 1 - read += read_ - prog += prog_ + readed += readed_ + proged += proged_ erased += erased_ if output_: # get defines and write to csv @@ -837,8 +837,8 @@ def run_stage(name, runner_, ids, output_, **args): output_.writerow({ 'suite': suite, 'case': case, - 'bench_read': read_, - 'bench_prog': prog_, + 'bench_readed': readed_, + 'bench_proged': proged_, 'bench_erased': erased_, **defines}) elif op == 'skipped': @@ -980,8 +980,8 @@ def run_stage(name, runner_, ids, output_, **args): return ( expected_perms, passed_perms, - read, - prog, + readed, + proged, erased, failures, killed) @@ -1018,7 +1018,7 @@ def run(runner, bench_ids=[], **args): if args.get('output'): output = BenchOutput(args['output'], ['suite', 'case'], - ['bench_read', 'bench_prog', 'bench_erased']) + ['bench_readed', 'bench_proged', 'bench_erased']) # measure runtime start = time.time() @@ -1026,8 +1026,8 @@ def run(runner, bench_ids=[], **args): # spawn runners expected = 0 passed = 0 - read = 0 - prog = 0 + readed = 0 + proged = 0 erased = 0 failures = [] for by in (expected_case_perms.keys() if args.get('by_cases') @@ -1036,8 +1036,8 @@ def run(runner, bench_ids=[], **args): # spawn jobs for stage (expected_, passed_, - read_, - prog_, + readed_, + proged_, erased_, failures_, killed) = run_stage( @@ -1049,8 +1049,8 @@ def run(runner, bench_ids=[], **args): # collect passes/failures expected += expected_ passed += passed_ - read += read_ - prog += prog_ + readed += readed_ + proged += proged_ erased += erased_ failures.extend(failures_) if (failures and not args.get('keep_going')) or killed: @@ -1072,8 +1072,8 @@ def run(runner, bench_ids=[], **args): if args['color'] else '', '\x1b[m' if args['color'] else '', ', '.join(filter(None, [ - '%d read' % read, - '%d prog' % prog, + '%d readed' % readed, + '%d proged' % proged, '%d erased' % erased, 'in %.2fs' % (stop-start)])))) print() From ca669938125370bbc93c4189739c55220e099b1c Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 2 Oct 2022 03:07:30 -0500 Subject: [PATCH 50/81] Tweaked scripts to share more code, added coverage calls/hits The main change is requiring field names for -b/-f/-s/-S, this is a bit more powerful, and supports hidden extra fields, but can require a bit more typing in some cases. --- Makefile | 10 +- scripts/code.py | 421 ++++++++++++++--------- scripts/coverage.py | 634 ++++++++++++++++++---------------- scripts/data.py | 424 ++++++++++++++--------- scripts/plot.py | 36 +- scripts/stack.py | 668 ++++++++++++++++++++---------------- scripts/struct_.py | 425 ++++++++++++++--------- scripts/summary.py | 808 ++++++++++++++++++++++++-------------------- 8 files changed, 1972 insertions(+), 1454 deletions(-) diff --git a/Makefile b/Makefile index e848e9c4..1c6a4584 100644 --- a/Makefile +++ b/Makefile @@ -149,23 +149,23 @@ bench-list: bench-runner .PHONY: code code: $(OBJ) - ./scripts/code.py $^ -S $(CODEFLAGS) + ./scripts/code.py $^ -Ssize $(CODEFLAGS) .PHONY: data data: $(OBJ) - ./scripts/data.py $^ -S $(DATAFLAGS) + ./scripts/data.py $^ -Ssize $(DATAFLAGS) .PHONY: stack stack: $(CI) - ./scripts/stack.py $^ -S $(STACKFLAGS) + ./scripts/stack.py $^ -Slimit -Sframe $(STACKFLAGS) .PHONY: struct struct: $(OBJ) - ./scripts/struct_.py $^ -S $(STRUCTFLAGS) + ./scripts/struct_.py $^ -Ssize $(STRUCTFLAGS) .PHONY: coverage coverage: $(GCDA) - ./scripts/coverage.py $^ -s $(COVERAGEFLAGS) + ./scripts/coverage.py $^ -slines -sbranches $(COVERAGEFLAGS) .PHONY: summary sizes summary sizes: $(BUILDDIR)lfs.csv diff --git a/scripts/code.py b/scripts/code.py index df16ece6..6b373fcf 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -29,10 +29,10 @@ TYPE = 'tTrRdD' # integer fields -class IntField(co.namedtuple('IntField', 'x')): +class Int(co.namedtuple('Int', 'x')): __slots__ = () def __new__(cls, x=0): - if isinstance(x, IntField): + if isinstance(x, Int): return x if isinstance(x, str): try: @@ -98,35 +98,30 @@ class IntField(co.namedtuple('IntField', 'x')): return (new-old) / old def __add__(self, other): - return IntField(self.x + other.x) + return self.__class__(self.x + other.x) def __sub__(self, other): - return IntField(self.x - other.x) + return self.__class__(self.x - other.x) def __mul__(self, other): - return IntField(self.x * other.x) - - def __lt__(self, other): - return self.x < other.x - - def __gt__(self, other): - return self.__class__.__lt__(other, self) - - def __le__(self, other): - return not self.__gt__(other) - - def __ge__(self, other): - return not self.__lt__(other) + return self.__class__(self.x * other.x) # code size results -class CodeResult(co.namedtuple('CodeResult', 'file,function,code_size')): +class CodeResult(co.namedtuple('CodeResult', [ + 'file', 'function', + 'size'])): + _by = ['file', 'function'] + _fields = ['size'] + _types = {'size': Int} + __slots__ = () - def __new__(cls, file, function, code_size): - return super().__new__(cls, file, function, IntField(code_size)) + def __new__(cls, file='', function='', size=0): + return super().__new__(cls, file, function, + Int(size)) def __add__(self, other): return CodeResult(self.file, self.function, - self.code_size + other.code_size) + self.size + other.size) def openio(path, mode='r'): @@ -188,9 +183,27 @@ def collect(paths, *, return results -def fold(results, *, - by=['file', 'function'], +def fold(Result, results, *, + by=None, + defines=None, **_): + if by is None: + by = Result._by + + for k in it.chain(by or [], (k for k, _ in defines or [])): + if k not in Result._by and k not in Result._fields: + print("error: could not find field %r?" % k) + sys.exit(-1) + + # filter by matching defines + if defines is not None: + results_ = [] + for r in results: + if all(getattr(r, k) in vs for k, vs in defines): + results_.append(r) + results = results_ + + # organize results into conflicts folding = co.OrderedDict() for r in results: name = tuple(getattr(r, k) for k in by) @@ -198,157 +211,220 @@ def fold(results, *, folding[name] = [] folding[name].append(r) + # merge conflicts folded = [] - for rs in folding.values(): + for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) return folded - -def table(results, diff_results=None, *, - by_file=False, - size_sort=False, - reverse_size_sort=False, +def table(Result, results, diff_results=None, *, + by=None, + fields=None, + sort=None, summary=False, all=False, percent=False, **_): all_, all = all, __builtins__.all - # fold - results = fold(results, by=['file' if by_file else 'function']) - if diff_results is not None: - diff_results = fold(diff_results, - by=['file' if by_file else 'function']) + if by is None: + by = Result._by + if fields is None: + fields = Result._fields + types = Result._types + # fold again + results = fold(Result, results, by=by) + if diff_results is not None: + diff_results = fold(Result, diff_results, by=by) + + # organize by name table = { - r.file if by_file else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in results} diff_table = { - r.file if by_file else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in diff_results or []} - - # sort, note that python's sort is stable names = list(table.keys() | diff_table.keys()) + + # sort again, now with diff info, note that python's sort is stable names.sort() if diff_results is not None: - names.sort(key=lambda n: -IntField.ratio( - table[n].code_size if n in table else None, - diff_table[n].code_size if n in diff_table else None)) - if size_sort: - names.sort(key=lambda n: (table[n].code_size,) if n in table else (), + names.sort(key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), reverse=True) - elif reverse_size_sort: - names.sort(key=lambda n: (table[n].code_size,) if n in table else (), - reverse=False) + if sort: + for k, reverse in reversed(sort): + names.sort(key=lambda n: (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None else (), + reverse=reverse ^ (not k or k in Result._fields)) - # print header - if not summary: - title = '%s%s' % ( - 'file' if by_file else 'function', - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - name_width = max(it.chain([23, len(title)], (len(n) for n in names))) - else: - title = '' - name_width = 23 - name_width = 4*((name_width+1+4-1)//4)-1 - print('%-*s ' % (name_width, title), end='') + # build up our lines + lines = [] + + # header + line = [] + line.append('%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else '') if diff_results is None: - print(' %s' % ('size'.rjust(len(IntField.none)))) + for k in fields: + line.append(k) elif percent: - print(' %s' % ('size'.rjust(len(IntField.diff_none)))) + for k in fields: + line.append(k) else: - print(' %s %s %s' % ( - 'old'.rjust(len(IntField.diff_none)), - 'new'.rjust(len(IntField.diff_none)), - 'diff'.rjust(len(IntField.diff_none)))) + for k in fields: + line.append('o'+k) + for k in fields: + line.append('n'+k) + for k in fields: + line.append('d'+k) + line.append('') + lines.append(line) - # print entries + # entries if not summary: for name in names: r = table.get(name) if diff_results is not None: diff_r = diff_table.get(name) - ratio = IntField.ratio( - r.code_size if r else None, - diff_r.code_size if diff_r else None) - if not ratio and not all_: + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + if not any(ratios) and not all_: continue - print('%-*s ' % (name_width, name), end='') + line = [] + line.append(name) if diff_results is None: - print(' %s' % ( - r.code_size.table() - if r else IntField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - r.code_size.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - diff_r.code_size.diff_table() - if diff_r else IntField.diff_none, - r.code_size.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.code_size if r else None, - diff_r.code_size if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) - # print total - total = fold(results, by=[]) - r = total[0] if total else None + # total + r = next(iter(fold(Result, results, by=[])), None) if diff_results is not None: - diff_total = fold(diff_results, by=[]) - diff_r = diff_total[0] if diff_total else None - ratio = IntField.ratio( - r.code_size if r else None, - diff_r.code_size if diff_r else None) + diff_r = next(iter(fold(Result, diff_results, by=[])), None) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] - print('%-*s ' % (name_width, 'TOTAL'), end='') + line = [] + line.append('TOTAL') if diff_results is None: - print(' %s' % ( - r.code_size.table() - if r else IntField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - r.code_size.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - diff_r.code_size.diff_table() - if diff_r else IntField.diff_none, - r.code_size.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.code_size if r else None, - diff_r.code_size if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) + + # find the best widths, note that column 0 contains the names and column -1 + # the ratios, so those are handled a bit differently + widths = [ + ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1 + for w, i in zip( + it.chain([23], it.repeat(7)), + range(len(lines[0])-1))] + + # print our table + for line in lines: + print('%-*s %s%s' % ( + widths[0], line[0], + ' '.join('%*s' % (w, x) + for w, x in zip(widths[1:], line[1:-1])), + line[-1])) -def main(obj_paths, **args): +def main(obj_paths, *, + by=None, + fields=None, + defines=None, + sort=None, + **args): # find sizes if not args.get('use', None): # find .o files @@ -361,7 +437,7 @@ def main(obj_paths, **args): paths.append(path) if not paths: - print('no .obj files found in %r?' % obj_paths) + print("error: no .obj files found in %r?" % obj_paths) sys.exit(-1) results = collect(paths, **args) @@ -371,25 +447,35 @@ def main(obj_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - results.append(CodeResult(**{ - k: v for k, v in r.items() - if k in CodeResult._fields})) + results.append(CodeResult( + **{k: r[k] for k in CodeResult._by + if k in r and r[k].strip()}, + **{k: r['code_'+k] for k in CodeResult._fields + if 'code_'+k in r and r['code_'+k].strip()})) except TypeError: pass - # fold to remove duplicates - results = fold(results) + # fold + results = fold(CodeResult, results, by=by, defines=defines) - # sort because why not + # sort, note that python's sort is stable results.sort() + if sort: + for k, reverse in reversed(sort): + results.sort(key=lambda r: (getattr(r, k),) + if getattr(r, k) is not None else (), + reverse=reverse ^ (not k or k in CodeResult._fields)) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, CodeResult._fields) + writer = csv.DictWriter(f, CodeResult._by + + ['code_'+k for k in CodeResult._fields]) writer.writeheader() for r in results: - writer.writerow(r._asdict()) + writer.writerow( + {k: getattr(r, k) for k in CodeResult._by} + | {'code_'+k: getattr(r, k) for k in CodeResult._fields}) # find previous results? if args.get('diff'): @@ -399,22 +485,26 @@ def main(obj_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - diff_results.append(CodeResult(**{ - k: v for k, v in r.items() - if k in CodeResult._fields})) + diff_results.append(CodeResult( + **{k: r[k] for k in CodeResult._by + if k in r and r[k].strip()}, + **{k: r['code_'+k] for k in CodeResult._fields + if 'code_'+k in r and r['code_'+k].strip()})) except TypeError: pass except FileNotFoundError: pass - # fold to remove duplicates - diff_results = fold(diff_results) + # fold + diff_results = fold(CodeResult, diff_results, by=by, defines=defines) # print table if not args.get('quiet'): - table( - results, + table(CodeResult, results, diff_results if args.get('diff') else None, + by=by if by is not None else ['function'], + fields=fields, + sort=sort, **args) @@ -455,22 +545,39 @@ if __name__ == "__main__": action='store_true', help="Only show percentage change, not a full diff.") parser.add_argument( - '-b', '--by-file', - action='store_true', - help="Group by file. Note this does not include padding " - "so sizes may differ from other tools.") + '-b', '--by', + action='append', + choices=CodeResult._by, + help="Group by this field.") parser.add_argument( - '-s', '--size-sort', - action='store_true', - help="Sort by size.") + '-f', '--field', + dest='fields', + action='append', + choices=CodeResult._fields, + help="Show this field.") parser.add_argument( - '-S', '--reverse-size-sort', - action='store_true', - help="Sort by size, but backwards.") + '-D', '--define', + dest='defines', + action='append', + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), + help="Only include results where this field is this value.") + class AppendSort(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.sort is None: + namespace.sort = [] + namespace.sort.append((value, True if option == '-S' else False)) + parser.add_argument( + '-s', '--sort', + action=AppendSort, + help="Sort by this fields.") + parser.add_argument( + '-S', '--reverse-sort', + action=AppendSort, + help="Sort by this fields, but backwards.") parser.add_argument( '-Y', '--summary', action='store_true', - help="Only show the total size.") + help="Only show the total.") parser.add_argument( '-A', '--everything', action='store_true', diff --git a/scripts/coverage.py b/scripts/coverage.py index 4e4541ac..7d36a47e 100755 --- a/scripts/coverage.py +++ b/scripts/coverage.py @@ -30,10 +30,10 @@ GCOV_TOOL = ['gcov'] # integer fields -class IntField(co.namedtuple('IntField', 'x')): +class Int(co.namedtuple('Int', 'x')): __slots__ = () def __new__(cls, x=0): - if isinstance(x, IntField): + if isinstance(x, Int): return x if isinstance(x, str): try: @@ -99,37 +99,25 @@ class IntField(co.namedtuple('IntField', 'x')): return (new-old) / old def __add__(self, other): - return IntField(self.x + other.x) + return self.__class__(self.x + other.x) def __sub__(self, other): - return IntField(self.x - other.x) + return self.__class__(self.x - other.x) def __mul__(self, other): - return IntField(self.x * other.x) - - def __lt__(self, other): - return self.x < other.x - - def __gt__(self, other): - return self.__class__.__lt__(other, self) - - def __le__(self, other): - return not self.__gt__(other) - - def __ge__(self, other): - return not self.__lt__(other) + return self.__class__(self.x * other.x) # fractional fields, a/b -class FracField(co.namedtuple('FracField', 'a,b')): +class Frac(co.namedtuple('Frac', 'a,b')): __slots__ = () def __new__(cls, a=0, b=None): - if isinstance(a, FracField) and b is None: + if isinstance(a, Frac) and b is None: return a if isinstance(a, str) and b is None: a, b = a.split('/', 1) if b is None: b = a - return super().__new__(cls, IntField(a), IntField(b)) + return super().__new__(cls, Int(a), Int(b)) def __str__(self): return '%s/%s' % (self.a, self.b) @@ -139,10 +127,7 @@ class FracField(co.namedtuple('FracField', 'a,b')): none = '%11s %7s' % ('-', '-') def table(self): - if not self.b.x: - return self.none - - t = self.a.x/self.b.x + t = self.a.x/self.b.x if self.b.x else 1.0 return '%11s %7s' % ( self, '∞%' if t == +m.inf @@ -151,38 +136,35 @@ class FracField(co.namedtuple('FracField', 'a,b')): diff_none = '%11s' % '-' def diff_table(self): - if not self.b.x: - return self.diff_none - return '%11s' % (self,) def diff_diff(self, other): - new_a, new_b = self if self else (IntField(0), IntField(0)) - old_a, old_b = other if other else (IntField(0), IntField(0)) + new_a, new_b = self if self else (Int(0), Int(0)) + old_a, old_b = other if other else (Int(0), Int(0)) return '%11s' % ('%s/%s' % ( new_a.diff_diff(old_a).strip(), new_b.diff_diff(old_b).strip())) def ratio(self, other): - new_a, new_b = self if self else (IntField(0), IntField(0)) - old_a, old_b = other if other else (IntField(0), IntField(0)) + new_a, new_b = self if self else (Int(0), Int(0)) + old_a, old_b = other if other else (Int(0), Int(0)) new = new_a.x/new_b.x if new_b.x else 1.0 old = old_a.x/old_b.x if old_b.x else 1.0 return new - old def __add__(self, other): - return FracField(self.a + other.a, self.b + other.b) + return self.__class__(self.a + other.a, self.b + other.b) def __sub__(self, other): - return FracField(self.a - other.a, self.b - other.b) + return self.__class__(self.a - other.a, self.b - other.b) def __mul__(self, other): - return FracField(self.a * other.a, self.b + other.b) + return self.__class__(self.a * other.a, self.b + other.b) def __lt__(self, other): - self_r = self.a.x/self.b.x if self.b.x else -m.inf - other_r = other.a.x/other.b.x if other.b.x else -m.inf - return self_r < other_r + self_t = self.a.x/self.b.x if self.b.x else 1.0 + other_t = other.a.x/other.b.x if other.b.x else 1.0 + return (self_t, self.a.x) < (other_t, other.a.x) def __gt__(self, other): return self.__class__.__lt__(other, self) @@ -194,22 +176,28 @@ class FracField(co.namedtuple('FracField', 'a,b')): return not self.__lt__(other) # coverage results -class CoverageResult(co.namedtuple('CoverageResult', - 'file,function,line,' - 'coverage_hits,coverage_lines,coverage_branches')): +class CoverageResult(co.namedtuple('CoverageResult', [ + 'file', 'function', 'line', + 'calls', 'hits', 'funcs', 'lines', 'branches'])): + _by = ['file', 'function', 'line'] + _fields = ['calls', 'hits', 'funcs', 'lines', 'branches'] + _types = { + 'calls': Int, 'hits': Int, + 'funcs': Frac, 'lines': Frac, 'branches': Frac} + __slots__ = () - def __new__(cls, file, function, line, - coverage_hits, coverage_lines, coverage_branches): - return super().__new__(cls, file, function, int(IntField(line)), - IntField(coverage_hits), - FracField(coverage_lines), - FracField(coverage_branches)) + def __new__(cls, file='', function='', line=0, + calls=0, hits=0, funcs=0, lines=0, branches=0): + return super().__new__(cls, file, function, int(Int(line)), + Int(calls), Int(hits), Frac(funcs), Frac(lines), Frac(branches)) def __add__(self, other): return CoverageResult(self.file, self.function, self.line, - max(self.coverage_hits, other.coverage_hits), - self.coverage_lines + other.coverage_lines, - self.coverage_branches + other.coverage_branches) + max(self.calls, other.calls), + max(self.hits, other.hits), + self.funcs + other.funcs, + self.lines + other.lines, + self.branches + other.branches) def openio(path, mode='r'): @@ -257,20 +245,37 @@ def collect(paths, *, if file['file'] != src_path: continue - for line in file['lines']: - func = line.get('function_name', '(inlined)') + for func in file['functions']: + func_name = func.get('name', '(inlined)') # discard internal function (this includes injected test cases) if not everything: - if func.startswith('__'): + if func_name.startswith('__'): continue + # go ahead and add functions, later folding will merge this if + # there are other hits on this line results.append(CoverageResult( - src_path, func, line['line_number'], - line['count'], - FracField( - 1 if line['count'] > 0 else 0, - 1), - FracField( + src_path, func_name, func['start_line'], + func['execution_count'], 0, + Frac(1 if func['execution_count'] > 0 else 0, 1), + 0, + 0)) + + for line in file['lines']: + func_name = line.get('function_name', '(inlined)') + # discard internal function (this includes injected test cases) + if not everything: + if func_name.startswith('__'): + continue + + # go ahead and add lines, later folding will merge this if + # there are other hits on this line + results.append(CoverageResult( + src_path, func_name, line['line_number'], + 0, line['count'], + 0, + Frac(1 if line['count'] > 0 else 0, 1), + Frac( sum(1 if branch['count'] > 0 else 0 for branch in line['branches']), len(line['branches'])))) @@ -278,9 +283,27 @@ def collect(paths, *, return results -def fold(results, *, - by=['file', 'function', 'line'], +def fold(Result, results, *, + by=None, + defines=None, **_): + if by is None: + by = Result._by + + for k in it.chain(by or [], (k for k, _ in defines or [])): + if k not in Result._by and k not in Result._fields: + print("error: could not find field %r?" % k) + sys.exit(-1) + + # filter by matching defines + if defines is not None: + results_ = [] + for r in results: + if all(getattr(r, k) in vs for k, vs in defines): + results_.append(r) + results = results_ + + # organize results into conflicts folding = co.OrderedDict() for r in results: name = tuple(getattr(r, k) for k in by) @@ -288,231 +311,224 @@ def fold(results, *, folding[name] = [] folding[name].append(r) + # merge conflicts folded = [] - for rs in folding.values(): + for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) return folded - -def table(results, diff_results=None, *, - by_file=False, - by_line=False, - line_sort=False, - reverse_line_sort=False, - branch_sort=False, - reverse_branch_sort=False, +def table(Result, results, diff_results=None, *, + by=None, + fields=None, + sort=None, summary=False, all=False, percent=False, **_): all_, all = all, __builtins__.all - # fold - results = fold(results, - by=['file', 'line'] if by_line - else ['file'] if by_file - else ['function']) - if diff_results is not None: - diff_results = fold(diff_results, - by=['file', 'line'] if by_line - else ['file'] if by_file - else ['function']) + if by is None: + by = Result._by + if fields is None: + fields = Result._fields + types = Result._types + # fold again + results = fold(Result, results, by=by) + if diff_results is not None: + diff_results = fold(Result, diff_results, by=by) + + # organize by name table = { - '%s:%s' % (r.file, r.line) if by_line - else r.file if by_file - else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in results} diff_table = { - '%s:%s' % (r.file, r.line) if by_line - else r.file if by_file - else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in diff_results or []} - - # sort, note that python's sort is stable names = list(table.keys() | diff_table.keys()) + + # sort again, now with diff info, note that python's sort is stable names.sort() if diff_results is not None: - names.sort(key=lambda n: ( - -FracField.ratio( - table[n].coverage_lines if n in table else None, - diff_table[n].coverage_lines if n in diff_table else None), - -FracField.ratio( - table[n].coverage_branches if n in table else None, - diff_table[n].coverage_branches if n in diff_table else None))) - if line_sort: - names.sort(key=lambda n: (table[n].coverage_lines,) - if n in table else (), + names.sort(key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), reverse=True) - elif reverse_line_sort: - names.sort(key=lambda n: (table[n].coverage_lines,) - if n in table else (), - reverse=False) - elif branch_sort: - names.sort(key=lambda n: (table[n].coverage_branches,) - if n in table else (), - reverse=True) - elif reverse_branch_sort: - names.sort(key=lambda n: (table[n].coverage_branches,) - if n in table else (), - reverse=False) + if sort: + for k, reverse in reversed(sort): + names.sort(key=lambda n: (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None else (), + reverse=reverse ^ (not k or k in Result._fields)) - # print header - if not summary: - title = '%s%s' % ( - 'line' if by_line else 'file' if by_file else 'function', - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - name_width = max(it.chain([23, len(title)], (len(n) for n in names))) - else: - title = '' - name_width = 23 - name_width = 4*((name_width+1+4-1)//4)-1 - print('%-*s ' % (name_width, title), end='') + # build up our lines + lines = [] + + # header + line = [] + line.append('%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else '') if diff_results is None: - print(' %s %s' % ( - 'hits/line'.rjust(len(FracField.none)), - 'hits/branch'.rjust(len(FracField.none)))) + for k in fields: + line.append(k) elif percent: - print(' %s %s' % ( - 'hits/line'.rjust(len(FracField.diff_none)), - 'hits/branch'.rjust(len(FracField.diff_none)))) + for k in fields: + line.append(k) else: - print(' %s %s %s %s %s %s' % ( - 'oh/line'.rjust(len(FracField.diff_none)), - 'oh/branch'.rjust(len(FracField.diff_none)), - 'nh/line'.rjust(len(FracField.diff_none)), - 'nh/branch'.rjust(len(FracField.diff_none)), - 'dh/line'.rjust(len(FracField.diff_none)), - 'dh/branch'.rjust(len(FracField.diff_none)))) + for k in fields: + line.append('o'+k) + for k in fields: + line.append('n'+k) + for k in fields: + line.append('d'+k) + line.append('') + lines.append(line) - # print entries + # entries if not summary: for name in names: r = table.get(name) if diff_results is not None: diff_r = diff_table.get(name) - line_ratio = FracField.ratio( - r.coverage_lines if r else None, - diff_r.coverage_lines if diff_r else None) - branch_ratio = FracField.ratio( - r.coverage_branches if r else None, - diff_r.coverage_branches if diff_r else None) - if not line_ratio and not branch_ratio and not all_: + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + if not any(ratios) and not all_: continue - print('%-*s ' % (name_width, name), end='') + line = [] + line.append(name) if diff_results is None: - print(' %s %s' % ( - r.coverage_lines.table() - if r else FracField.none, - r.coverage_branches.table() - if r else FracField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s %s%s' % ( - r.coverage_lines.diff_table() - if r else FracField.diff_none, - r.coverage_branches.diff_table() - if r else FracField.diff_none, - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in [line_ratio, branch_ratio]))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s %s %s %s%s' % ( - diff_r.coverage_lines.diff_table() - if diff_r else FracField.diff_none, - diff_r.coverage_branches.diff_table() - if diff_r else FracField.diff_none, - r.coverage_lines.diff_table() - if r else FracField.diff_none, - r.coverage_branches.diff_table() - if r else FracField.diff_none, - FracField.diff_diff( - r.coverage_lines if r else None, - diff_r.coverage_lines if diff_r else None) - if r or diff_r else FracField.diff_none, - FracField.diff_diff( - r.coverage_branches if r else None, - diff_r.coverage_branches if diff_r else None) - if r or diff_r else FracField.diff_none, - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in [line_ratio, branch_ratio] - if t) - if line_ratio or branch_ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) - # print total - total = fold(results, by=[]) - r = total[0] if total else None + # total + r = next(iter(fold(Result, results, by=[])), None) if diff_results is not None: - diff_total = fold(diff_results, by=[]) - diff_r = diff_total[0] if diff_total else None - line_ratio = FracField.ratio( - r.coverage_lines if r else None, - diff_r.coverage_lines if diff_r else None) - branch_ratio = FracField.ratio( - r.coverage_branches if r else None, - diff_r.coverage_branches if diff_r else None) + diff_r = next(iter(fold(Result, diff_results, by=[])), None) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] - print('%-*s ' % (name_width, 'TOTAL'), end='') + line = [] + line.append('TOTAL') if diff_results is None: - print(' %s %s' % ( - r.coverage_lines.table() - if r else FracField.none, - r.coverage_branches.table() - if r else FracField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s %s%s' % ( - r.coverage_lines.diff_table() - if r else FracField.diff_none, - r.coverage_branches.diff_table() - if r else FracField.diff_none, - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in [line_ratio, branch_ratio]))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s %s %s %s%s' % ( - diff_r.coverage_lines.diff_table() - if diff_r else FracField.diff_none, - diff_r.coverage_branches.diff_table() - if diff_r else FracField.diff_none, - r.coverage_lines.diff_table() - if r else FracField.diff_none, - r.coverage_branches.diff_table() - if r else FracField.diff_none, - FracField.diff_diff( - r.coverage_lines if r else None, - diff_r.coverage_lines if diff_r else None) - if r or diff_r else FracField.diff_none, - FracField.diff_diff( - r.coverage_branches if r else None, - diff_r.coverage_branches if diff_r else None) - if r or diff_r else FracField.diff_none, - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in [line_ratio, branch_ratio] - if t) - if line_ratio or branch_ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) + + # find the best widths, note that column 0 contains the names and column -1 + # the ratios, so those are handled a bit differently + widths = [ + ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1 + for w, i in zip( + it.chain([23], it.repeat(7)), + range(len(lines[0])-1))] + + # print our table + for line in lines: + print('%-*s %s%s' % ( + widths[0], line[0], + ' '.join('%*s' % (w, x) + for w, x in zip(widths[1:], line[1:-1])), + line[-1])) -def annotate(paths, results, *, +def annotate(Result, results, paths, *, annotate=False, lines=False, branches=False, build_dir=None, **args): + # if neither branches/lines specified, color both + if annotate and not lines and not branches: + lines, branches = True, True + for path in paths: # map to source file src_path = re.sub('\.t\.a\.gcda$', '.c', path) @@ -521,7 +537,7 @@ def annotate(paths, results, *, src_path) # flatten to line info - results = fold(results, by=['file', 'line']) + results = fold(Result, results, by=['file', 'line']) table = {r.line: r for r in results if r.file == src_path} # calculate spans to show @@ -529,10 +545,8 @@ def annotate(paths, results, *, spans = [] last = None for line, r in sorted(table.items()): - if ((lines and int(r.coverage_hits) == 0) - or (branches - and r.coverage_branches.a - < r.coverage_branches.b)): + if ((lines and int(r.hits) == 0) + or (branches and r.branches.a < r.branches.b)): if last is not None and line - last.stop <= args['context']: last = range( last.start, @@ -568,24 +582,29 @@ def annotate(paths, results, *, if i+1 in table: r = table[i+1] - line = '%-*s // %s hits, %s branches' % ( + line = '%-*s // %s hits%s' % ( args['width'], line, - r.coverage_hits, - r.coverage_branches) + r.hits, + ', %s branches' % (r.branches,) + if int(r.branches.b) else '') if args['color']: - if lines and int(r.coverage_hits) == 0: + if lines and int(r.hits) == 0: line = '\x1b[1;31m%s\x1b[m' % line - elif (branches - and r.coverage_branches.a - < r.coverage_branches.b): + elif branches and r.branches.a < r.branches.b: line = '\x1b[35m%s\x1b[m' % line print(line) -def main(gcda_paths, **args): +def main(gcda_paths, *, + by=None, + fields=None, + defines=None, + sort=None, + hits=False, + **args): # figure out what color should be if args.get('color') == 'auto': args['color'] = sys.stdout.isatty() @@ -606,7 +625,7 @@ def main(gcda_paths, **args): paths.append(path) if not paths: - print('no .gcda files found in %r?' % gcda_paths) + print("error: no .gcda files found in %r?" % gcda_paths) sys.exit(-1) results = collect(paths, **args) @@ -616,25 +635,38 @@ def main(gcda_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - results.append(CoverageResult(**{ - k: v for k, v in r.items() - if k in CoverageResult._fields})) + results.append(CoverageResult( + **{k: r[k] for k in CoverageResult._by + if k in r and r[k].strip()}, + **{k: r['coverage_'+k] + for k in CoverageResult._fields + if 'coverage_'+k in r + and r['coverage_'+k].strip()})) except TypeError: pass - # fold to remove duplicates - results = fold(results) + # fold + results = fold(CoverageResult, results, by=by, defines=defines) - # sort because why not + # sort, note that python's sort is stable results.sort() + if sort: + for k, reverse in reversed(sort): + results.sort(key=lambda r: (getattr(r, k),) + if getattr(r, k) is not None else (), + reverse=reverse ^ (not k or k in CoverageResult._fields)) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, CoverageResult._fields) + writer = csv.DictWriter(f, CoverageResult._by + + ['coverage_'+k for k in CoverageResult._fields]) writer.writeheader() for r in results: - writer.writerow(r._asdict()) + writer.writerow( + {k: getattr(r, k) for k in CoverageResult._by} + | {'coverage_'+k: getattr(r, k) + for k in CoverageResult._fields}) # find previous results? if args.get('diff'): @@ -644,31 +676,39 @@ def main(gcda_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - diff_results.append(CoverageResult(**{ - k: v for k, v in r.items() - if k in CoverageResult._fields})) + diff_results.append(CoverageResult( + **{k: r[k] for k in CoverageResult._by + if k in r and r[k].strip()}, + **{k: r['coverage_'+k] + for k in CoverageResult._fields + if 'coverage_'+k in r + and r['coverage_'+k].strip()})) except TypeError: pass except FileNotFoundError: pass - # fold to remove duplicates - diff_results = fold(diff_results) + # fold + diff_results = fold(CoverageResult, diff_results, + by=by, defines=defines) + # print table if not args.get('quiet'): if (args.get('annotate') or args.get('lines') or args.get('branches')): # annotate sources - annotate( - paths, - results, + annotate(CoverageResult, results, paths, **args) else: # print table - table( - results, + table(CoverageResult, results, diff_results if args.get('diff') else None, + by=by if by is not None else ['function'], + fields=fields if fields is not None + else ['lines', 'branches'] if not hits + else ['calls', 'hits'], + sort=sort, **args) # catch lack of coverage @@ -717,33 +757,47 @@ if __name__ == "__main__": action='store_true', help="Only show percentage change, not a full diff.") parser.add_argument( - '-b', '--by-file', - action='store_true', - help="Group by file.") + '-b', '--by', + action='append', + choices=CoverageResult._by, + help="Group by this field.") parser.add_argument( - '--by-line', - action='store_true', - help="Group by line.") + '-f', '--field', + dest='fields', + action='append', + choices=CoverageResult._fields, + help="Show this field.") parser.add_argument( - '-s', '--line-sort', - action='store_true', - help="Sort by line coverage.") + '-D', '--define', + dest='defines', + action='append', + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), + help="Only include results where this field is this value.") + class AppendSort(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.sort is None: + namespace.sort = [] + namespace.sort.append((value, True if option == '-S' else False)) parser.add_argument( - '-S', '--reverse-line-sort', - action='store_true', - help="Sort by line coverage, but backwards.") + '-s', '--sort', + action=AppendSort, + help="Sort by this field.") parser.add_argument( - '--branch-sort', - action='store_true', - help="Sort by branch coverage.") - parser.add_argument( - '--reverse-branch-sort', - action='store_true', - help="Sort by branch coverage, but backwards.") + '-S', '--reverse-sort', + action=AppendSort, + help="Sort by this field, but backwards.") parser.add_argument( '-Y', '--summary', action='store_true', - help="Only show the total size.") + help="Only show the total.") + parser.add_argument( + '-A', '--everything', + action='store_true', + help="Include builtin and libc specific symbols.") + parser.add_argument( + '-H', '--hits', + action='store_true', + help="Show total hits instead of coverage.") parser.add_argument( '-l', '--annotate', action='store_true', @@ -779,10 +833,6 @@ if __name__ == "__main__": '-E', '--error-on-branches', action='store_true', help="Error if any branches are not covered.") - parser.add_argument( - '-A', '--everything', - action='store_true', - help="Include builtin and libc specific symbols.") parser.add_argument( '--gcov-tool', default=GCOV_TOOL, diff --git a/scripts/data.py b/scripts/data.py index 60707567..05ef8681 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Script to find data size at the function level. Basically just a bit wrapper +# Script to find data size at the function level. Basically just a big wrapper # around nm with some extra conveniences for comparing builds. Heavily inspired # by Linux's Bloat-O-Meter. # @@ -28,12 +28,11 @@ NM_TOOL = ['nm'] TYPE = 'dDbB' - # integer fields -class IntField(co.namedtuple('IntField', 'x')): +class Int(co.namedtuple('Int', 'x')): __slots__ = () def __new__(cls, x=0): - if isinstance(x, IntField): + if isinstance(x, Int): return x if isinstance(x, str): try: @@ -99,35 +98,30 @@ class IntField(co.namedtuple('IntField', 'x')): return (new-old) / old def __add__(self, other): - return IntField(self.x + other.x) + return self.__class__(self.x + other.x) def __sub__(self, other): - return IntField(self.x - other.x) + return self.__class__(self.x - other.x) def __mul__(self, other): - return IntField(self.x * other.x) - - def __lt__(self, other): - return self.x < other.x - - def __gt__(self, other): - return self.__class__.__lt__(other, self) - - def __le__(self, other): - return not self.__gt__(other) - - def __ge__(self, other): - return not self.__lt__(other) + return self.__class__(self.x * other.x) # data size results -class DataResult(co.namedtuple('DataResult', 'file,function,data_size')): +class DataResult(co.namedtuple('DataResult', [ + 'file', 'function', + 'size'])): + _by = ['file', 'function'] + _fields = ['size'] + _types = {'size': Int} + __slots__ = () - def __new__(cls, file, function, data_size): - return super().__new__(cls, file, function, IntField(data_size)) + def __new__(cls, file='', function='', size=0): + return super().__new__(cls, file, function, + Int(size)) def __add__(self, other): return DataResult(self.file, self.function, - self.data_size + other.data_size) + self.size + other.size) def openio(path, mode='r'): @@ -189,9 +183,27 @@ def collect(paths, *, return results -def fold(results, *, - by=['file', 'function'], +def fold(Result, results, *, + by=None, + defines=None, **_): + if by is None: + by = Result._by + + for k in it.chain(by or [], (k for k, _ in defines or [])): + if k not in Result._by and k not in Result._fields: + print("error: could not find field %r?" % k) + sys.exit(-1) + + # filter by matching defines + if defines is not None: + results_ = [] + for r in results: + if all(getattr(r, k) in vs for k, vs in defines): + results_.append(r) + results = results_ + + # organize results into conflicts folding = co.OrderedDict() for r in results: name = tuple(getattr(r, k) for k in by) @@ -199,157 +211,220 @@ def fold(results, *, folding[name] = [] folding[name].append(r) + # merge conflicts folded = [] - for rs in folding.values(): + for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) return folded - -def table(results, diff_results=None, *, - by_file=False, - size_sort=False, - reverse_size_sort=False, +def table(Result, results, diff_results=None, *, + by=None, + fields=None, + sort=None, summary=False, all=False, percent=False, **_): all_, all = all, __builtins__.all - # fold - results = fold(results, by=['file' if by_file else 'function']) - if diff_results is not None: - diff_results = fold(diff_results, - by=['file' if by_file else 'function']) + if by is None: + by = Result._by + if fields is None: + fields = Result._fields + types = Result._types + # fold again + results = fold(Result, results, by=by) + if diff_results is not None: + diff_results = fold(Result, diff_results, by=by) + + # organize by name table = { - r.file if by_file else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in results} diff_table = { - r.file if by_file else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in diff_results or []} - - # sort, note that python's sort is stable names = list(table.keys() | diff_table.keys()) + + # sort again, now with diff info, note that python's sort is stable names.sort() if diff_results is not None: - names.sort(key=lambda n: -IntField.ratio( - table[n].data_size if n in table else None, - diff_table[n].data_size if n in diff_table else None)) - if size_sort: - names.sort(key=lambda n: (table[n].data_size,) if n in table else (), + names.sort(key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), reverse=True) - elif reverse_size_sort: - names.sort(key=lambda n: (table[n].data_size,) if n in table else (), - reverse=False) + if sort: + for k, reverse in reversed(sort): + names.sort(key=lambda n: (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None else (), + reverse=reverse ^ (not k or k in Result._fields)) - # print header - if not summary: - title = '%s%s' % ( - 'file' if by_file else 'function', - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - name_width = max(it.chain([23, len(title)], (len(n) for n in names))) - else: - title = '' - name_width = 23 - name_width = 4*((name_width+1+4-1)//4)-1 - print('%-*s ' % (name_width, title), end='') + # build up our lines + lines = [] + + # header + line = [] + line.append('%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else '') if diff_results is None: - print(' %s' % ('size'.rjust(len(IntField.none)))) + for k in fields: + line.append(k) elif percent: - print(' %s' % ('size'.rjust(len(IntField.diff_none)))) + for k in fields: + line.append(k) else: - print(' %s %s %s' % ( - 'old'.rjust(len(IntField.diff_none)), - 'new'.rjust(len(IntField.diff_none)), - 'diff'.rjust(len(IntField.diff_none)))) + for k in fields: + line.append('o'+k) + for k in fields: + line.append('n'+k) + for k in fields: + line.append('d'+k) + line.append('') + lines.append(line) - # print entries + # entries if not summary: for name in names: r = table.get(name) if diff_results is not None: diff_r = diff_table.get(name) - ratio = IntField.ratio( - r.data_size if r else None, - diff_r.data_size if diff_r else None) - if not ratio and not all_: + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + if not any(ratios) and not all_: continue - print('%-*s ' % (name_width, name), end='') + line = [] + line.append(name) if diff_results is None: - print(' %s' % ( - r.data_size.table() - if r else IntField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - r.data_size.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - diff_r.data_size.diff_table() - if diff_r else IntField.diff_none, - r.data_size.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.data_size if r else None, - diff_r.data_size if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) - # print total - total = fold(results, by=[]) - r = total[0] if total else None + # total + r = next(iter(fold(Result, results, by=[])), None) if diff_results is not None: - diff_total = fold(diff_results, by=[]) - diff_r = diff_total[0] if diff_total else None - ratio = IntField.ratio( - r.data_size if r else None, - diff_r.data_size if diff_r else None) + diff_r = next(iter(fold(Result, diff_results, by=[])), None) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] - print('%-*s ' % (name_width, 'TOTAL'), end='') + line = [] + line.append('TOTAL') if diff_results is None: - print(' %s' % ( - r.data_size.table() - if r else IntField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - r.data_size.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - diff_r.data_size.diff_table() - if diff_r else IntField.diff_none, - r.data_size.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.data_size if r else None, - diff_r.data_size if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) + + # find the best widths, note that column 0 contains the names and column -1 + # the ratios, so those are handled a bit differently + widths = [ + ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1 + for w, i in zip( + it.chain([23], it.repeat(7)), + range(len(lines[0])-1))] + + # print our table + for line in lines: + print('%-*s %s%s' % ( + widths[0], line[0], + ' '.join('%*s' % (w, x) + for w, x in zip(widths[1:], line[1:-1])), + line[-1])) -def main(obj_paths, **args): +def main(obj_paths, *, + by=None, + fields=None, + defines=None, + sort=None, + **args): # find sizes if not args.get('use', None): # find .o files @@ -362,7 +437,7 @@ def main(obj_paths, **args): paths.append(path) if not paths: - print('no .obj files found in %r?' % obj_paths) + print("error: no .obj files found in %r?" % obj_paths) sys.exit(-1) results = collect(paths, **args) @@ -372,25 +447,35 @@ def main(obj_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - results.append(DataResult(**{ - k: v for k, v in r.items() - if k in DataResult._fields})) + results.append(DataResult( + **{k: r[k] for k in DataResult._by + if k in r and r[k].strip()}, + **{k: r['data_'+k] for k in DataResult._fields + if 'data_'+k in r and r['data_'+k].strip()})) except TypeError: pass - # fold to remove duplicates - results = fold(results) + # fold + results = fold(DataResult, results, by=by, defines=defines) - # sort because why not + # sort, note that python's sort is stable results.sort() + if sort: + for k, reverse in reversed(sort): + results.sort(key=lambda r: (getattr(r, k),) + if getattr(r, k) is not None else (), + reverse=reverse ^ (not k or k in DataResult._fields)) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, DataResult._fields) + writer = csv.DictWriter(f, DataResult._by + + ['data_'+k for k in DataResult._fields]) writer.writeheader() for r in results: - writer.writerow(r._asdict()) + writer.writerow( + {k: getattr(r, k) for k in DataResult._by} + | {'data_'+k: getattr(r, k) for k in DataResult._fields}) # find previous results? if args.get('diff'): @@ -400,22 +485,26 @@ def main(obj_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - diff_results.append(DataResult(**{ - k: v for k, v in r.items() - if k in DataResult._fields})) + diff_results.append(DataResult( + **{k: r[k] for k in DataResult._by + if k in r and r[k].strip()}, + **{k: r['data_'+k] for k in DataResult._fields + if 'data_'+k in r and r['data_'+k].strip()})) except TypeError: pass except FileNotFoundError: pass - # fold to remove duplicates - diff_results = fold(diff_results) + # fold + diff_results = fold(DataResult, diff_results, by=by, defines=defines) # print table if not args.get('quiet'): - table( - results, + table(DataResult, results, diff_results if args.get('diff') else None, + by=by if by is not None else ['function'], + fields=fields, + sort=sort, **args) @@ -456,22 +545,39 @@ if __name__ == "__main__": action='store_true', help="Only show percentage change, not a full diff.") parser.add_argument( - '-b', '--by-file', - action='store_true', - help="Group by file. Note this does not include padding " - "so sizes may differ from other tools.") + '-b', '--by', + action='append', + choices=DataResult._by, + help="Group by this field.") parser.add_argument( - '-s', '--size-sort', - action='store_true', - help="Sort by size.") + '-f', '--field', + dest='fields', + action='append', + choices=DataResult._fields, + help="Show this field.") parser.add_argument( - '-S', '--reverse-size-sort', - action='store_true', - help="Sort by size, but backwards.") + '-D', '--define', + dest='defines', + action='append', + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), + help="Only include results where this field is this value.") + class AppendSort(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.sort is None: + namespace.sort = [] + namespace.sort.append((value, True if option == '-S' else False)) + parser.add_argument( + '-s', '--sort', + action=AppendSort, + help="Sort by this fields.") + parser.add_argument( + '-S', '--reverse-sort', + action=AppendSort, + help="Sort by this fields, but backwards.") parser.add_argument( '-Y', '--summary', action='store_true', - help="Only show the total size.") + help="Only show the total.") parser.add_argument( '-A', '--everything', action='store_true', diff --git a/scripts/plot.py b/scripts/plot.py index 9aef2aee..15ec8462 100755 --- a/scripts/plot.py +++ b/scripts/plot.py @@ -438,9 +438,7 @@ def datasets(results, by=None, x=None, y=None, define=[]): y = co.OrderedDict() for r in results: for k, v in r.items(): - if by is not None and k in by: - continue - if y.get(k, True): + if (by is None or k not in by) and v.strip(): try: dat(v) y[k] = True @@ -462,7 +460,7 @@ def datasets(results, by=None, x=None, y=None, define=[]): for y_ in y: # hide x/y if there is only one field k_x = x_ if len(x or []) > 1 else '' - k_y = y_ if len(y or []) > 1 else '' + k_y = y_ if len(y or []) > 1 or (not ks_ and not k_x) else '' datasets[ks_ + (k_x, k_y)] = dataset( results, @@ -509,15 +507,15 @@ def main(csv_paths, *, ylim = (0, ylim[0]) # separate out renames - renames = [k.split('=', 1) - for k in it.chain(by or [], x or [], y or []) - if '=' in k] + renames = list(it.chain.from_iterable( + ((k, v) for v in vs) + for k, vs in it.chain(by or [], x or [], y or []))) if by is not None: - by = [k.split('=', 1)[0] for k in by] + by = [k for k, _ in by] if x is not None: - x = [k.split('=', 1)[0] for k in x] + x = [k for k, _ in x] if y is not None: - y = [k.split('=', 1)[0] for k in y] + y = [k for k, _ in y] def draw(f): def writeln(s=''): @@ -739,23 +737,31 @@ if __name__ == "__main__": parser.add_argument( '-b', '--by', action='append', - help="Fields to render as separate plots. All other fields will be " - "summed as needed. Can rename fields with new_name=old_name.") + type=lambda x: ( + lambda k,v=None: (k, v.split(',') if v is not None else ()) + )(*x.split('=', 1)), + help="Group by this field. Can rename fields with new_name=old_name.") parser.add_argument( '-x', action='append', - help="Fields to use for the x-axis. Can rename fields with " + type=lambda x: ( + lambda k,v=None: (k, v.split(',') if v is not None else ()) + )(*x.split('=', 1)), + help="Field to use for the x-axis. Can rename fields with " "new_name=old_name.") parser.add_argument( '-y', action='append', - help="Fields to use for the y-axis. Can rename fields with " + type=lambda x: ( + lambda k,v=None: (k, v.split(',') if v is not None else ()) + )(*x.split('=', 1)), + help="Field to use for the y-axis. Can rename fields with " "new_name=old_name.") parser.add_argument( '-D', '--define', type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), action='append', - help="Only include rows where this field is this value. May include " + help="Only include results where this field is this value. May include " "comma-separated options.") parser.add_argument( '--color', diff --git a/scripts/stack.py b/scripts/stack.py index ed5f1fce..6cb20ffa 100755 --- a/scripts/stack.py +++ b/scripts/stack.py @@ -23,10 +23,10 @@ CI_PATHS = ['*.ci'] # integer fields -class IntField(co.namedtuple('IntField', 'x')): +class Int(co.namedtuple('Int', 'x')): __slots__ = () def __new__(cls, x=0): - if isinstance(x, IntField): + if isinstance(x, Int): return x if isinstance(x, str): try: @@ -92,38 +92,33 @@ class IntField(co.namedtuple('IntField', 'x')): return (new-old) / old def __add__(self, other): - return IntField(self.x + other.x) + return self.__class__(self.x + other.x) def __sub__(self, other): - return IntField(self.x - other.x) + return self.__class__(self.x - other.x) def __mul__(self, other): - return IntField(self.x * other.x) - - def __lt__(self, other): - return self.x < other.x - - def __gt__(self, other): - return self.__class__.__lt__(other, self) - - def __le__(self, other): - return not self.__gt__(other) - - def __ge__(self, other): - return not self.__lt__(other) + return self.__class__(self.x * other.x) # size results -class StackResult(co.namedtuple('StackResult', - 'file,function,stack_frame,stack_limit')): +class StackResult(co.namedtuple('StackResult', [ + 'file', 'function', 'frame', 'limit', 'calls'])): + _by = ['file', 'function'] + _fields = ['frame', 'limit'] + _types = {'frame': Int, 'limit': Int} + __slots__ = () - def __new__(cls, file, function, stack_frame, stack_limit): + def __new__(cls, file='', function='', + frame=0, limit=0, calls=set()): return super().__new__(cls, file, function, - IntField(stack_frame), IntField(stack_limit)) + Int(frame), Int(limit), + calls) def __add__(self, other): return StackResult(self.file, self.function, - self.stack_frame + other.stack_frame, - max(self.stack_limit, other.stack_limit)) + self.frame + other.frame, + max(self.limit, other.limit), + self.calls | other.calls) def openio(path, mode='r'): @@ -135,7 +130,6 @@ def openio(path, mode='r'): else: return open(path, mode) - def collect(paths, *, everything=False, **args): @@ -147,10 +141,10 @@ def collect(paths, *, node = [] while True: rest = rest.lstrip() - m = k_pattern.match(rest) - if not m: + m_ = k_pattern.match(rest) + if not m_: return (node, rest) - k, rest = m.group(1), rest[m.end(0):] + k, rest = m_.group(1), rest[m_.end(0):] rest = rest.lstrip() if rest.startswith('{'): @@ -159,9 +153,9 @@ def collect(paths, *, rest = rest[1:] node.append((k, v)) else: - m = v_pattern.match(rest) - assert m, "unexpected %r" % rest[0:1] - v, rest = m.group(1) or m.group(2), rest[m.end(0):] + m_ = v_pattern.match(rest) + assert m_, "unexpected %r" % rest[0:1] + v, rest = m_.group(1) or m_.group(2), rest[m_.end(0):] node.append((k, v)) node, rest = parse_vcg(rest) @@ -181,13 +175,13 @@ def collect(paths, *, for k, info in graph: if k == 'node': info = dict(info) - m = f_pattern.match(info['label']) - if m: - function, file, size, type = m.groups() + m_ = f_pattern.match(info['label']) + if m_: + function, file, size, type = m_.groups() if (not args.get('quiet') and 'static' not in type and 'bounded' not in type): - print('warning: found non-static stack for %s (%s)' + print("warning: found non-static stack for %s (%s)" % (function, type, size)) _, _, _, targets = callgraph[info['title']] callgraph[info['title']] = ( @@ -217,7 +211,7 @@ def collect(paths, *, for target in targets: if target in seen: # found a cycle - return float('inf') + return m.inf limit_ = find_limit(target, seen | {target}) limit = max(limit, limit_) @@ -233,19 +227,35 @@ def collect(paths, *, # build results results = [] - calls = {} for source, (s_file, s_function, frame, targets) in callgraph.items(): limit = find_limit(source) - cs = find_calls(targets) - results.append(StackResult(s_file, s_function, frame, limit)) - calls[(s_file, s_function)] = cs + calls = find_calls(targets) + results.append(StackResult(s_file, s_function, frame, limit, calls)) - return results, calls + return results -def fold(results, *, - by=['file', 'function'], +def fold(Result, results, *, + by=None, + defines=None, **_): + if by is None: + by = Result._by + + for k in it.chain(by or [], (k for k, _ in defines or [])): + if k not in Result._by and k not in Result._fields: + print("error: could not find field %r?" % k) + sys.exit(-1) + + # filter by matching defines + if defines is not None: + results_ = [] + for r in results: + if all(getattr(r, k) in vs for k, vs in defines): + results_.append(r) + results = results_ + + # organize results into conflicts folding = co.OrderedDict() for r in results: name = tuple(getattr(r, k) for k in by) @@ -253,36 +263,17 @@ def fold(results, *, folding[name] = [] folding[name].append(r) + # merge conflicts folded = [] - for rs in folding.values(): + for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) return folded -def fold_calls(calls, *, - by=['file', 'function'], - **_): - def by_(name): - file, function = name - return (((file,) if 'file' in by else ()) - + ((function,) if 'function' in by else ())) - - folded = {} - for name, cs in calls.items(): - name = by_(name) - if name not in folded: - folded[name] = set() - folded[name] |= {by_(c) for c in cs} - - return folded - - -def table(results, calls, diff_results=None, *, - by_file=False, - limit_sort=False, - reverse_limit_sort=False, - frame_sort=False, - reverse_frame_sort=False, +def table(Result, results, diff_results=None, *, + by=None, + fields=None, + sort=None, summary=False, all=False, percent=False, @@ -291,209 +282,268 @@ def table(results, calls, diff_results=None, *, **_): all_, all = all, __builtins__.all - # tree doesn't really make sense with depth=0, assume depth=inf - if depth is None: - depth = float('inf') if tree else 0 + if by is None: + by = Result._by + if fields is None: + fields = Result._fields + types = Result._types - # fold - results = fold(results, by=['file' if by_file else 'function']) - calls = fold_calls(calls, by=['file' if by_file else 'function']) + # fold again + results = fold(Result, results, by=by) if diff_results is not None: - diff_results = fold(diff_results, - by=['file' if by_file else 'function']) + diff_results = fold(Result, diff_results, by=by) + # organize by name table = { - r.file if by_file else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in results} diff_table = { - r.file if by_file else r.function: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in diff_results or []} - - # sort, note that python's sort is stable names = list(table.keys() | diff_table.keys()) + + # sort again, now with diff info, note that python's sort is stable names.sort() if diff_results is not None: - names.sort(key=lambda n: -IntField.ratio( - table[n].stack_frame if n in table else None, - diff_table[n].stack_frame if n in diff_table else None)) - if limit_sort: - names.sort(key=lambda n: (table[n].stack_limit,) if n in table else (), + names.sort(key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), reverse=True) - elif reverse_limit_sort: - names.sort(key=lambda n: (table[n].stack_limit,) if n in table else (), - reverse=False) - elif frame_sort: - names.sort(key=lambda n: (table[n].stack_frame,) if n in table else (), - reverse=True) - elif reverse_frame_sort: - names.sort(key=lambda n: (table[n].stack_frame,) if n in table else (), - reverse=False) + if sort: + for k, reverse in reversed(sort): + names.sort(key=lambda n: (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None else (), + reverse=reverse ^ (not k or k in Result._fields)) - # print header - if not summary: - title = '%s%s' % ( - 'file' if by_file else 'function', - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - name_width = max(it.chain([23, len(title)], (len(n) for n in names))) + + # build up our lines + lines = [] + + # header + line = [] + line.append('%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else '') + if diff_results is None: + for k in fields: + line.append(k) + elif percent: + for k in fields: + line.append(k) else: - title = '' - name_width = 23 - name_width = 4*((name_width+1+4-1)//4)-1 + for k in fields: + line.append('o'+k) + for k in fields: + line.append('n'+k) + for k in fields: + line.append('d'+k) + line.append('') + lines.append(line) - # adjust the name width based on the expected call depth, note that we - # can't always find the depth due to recursion - if not m.isinf(depth): - name_width += 4*depth - - if not tree: - print('%-*s ' % (name_width, title), end='') - if diff_results is None: - print(' %s %s' % ( - 'frame'.rjust(len(IntField.none)), - 'limit'.rjust(len(IntField.none)))) - elif percent: - print(' %s %s' % ( - 'frame'.rjust(len(IntField.diff_none)), - 'limit'.rjust(len(IntField.diff_none)))) - else: - print(' %s %s %s %s %s %s' % ( - 'oframe'.rjust(len(IntField.diff_none)), - 'olimit'.rjust(len(IntField.diff_none)), - 'nframe'.rjust(len(IntField.diff_none)), - 'nlimit'.rjust(len(IntField.diff_none)), - 'dframe'.rjust(len(IntField.diff_none)), - 'dlimit'.rjust(len(IntField.diff_none)))) - - # print entries + # entries if not summary: - # print the tree recursively - def table_calls(names_, depth, - prefixes=('', '', '', '')): - for i, name in enumerate(names_): - r = table.get(name) - if diff_results is not None: - diff_r = diff_table.get(name) - ratio = IntField.ratio( - r.stack_limit if r else None, - diff_r.stack_limit if diff_r else None) - if not ratio and not all_: - continue + for name in names: + r = table.get(name) + if diff_results is not None: + diff_r = diff_table.get(name) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + if not any(ratios) and not all_: + continue + line = [] + line.append(name) + if diff_results is None: + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) + elif percent: + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + else: + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) + + # total + r = next(iter(fold(Result, results, by=[])), None) + if diff_results is not None: + diff_r = next(iter(fold(Result, diff_results, by=[])), None) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + + line = [] + line.append('TOTAL') + if diff_results is None: + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) + elif percent: + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + else: + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) + + # find the best widths, note that column 0 contains the names and column -1 + # the ratios, so those are handled a bit differently + widths = [ + ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1 + for w, i in zip( + it.chain([23], it.repeat(7)), + range(len(lines[0])-1))] + + # adjust the name width based on the expected call depth, though + # note this doesn't really work with unbounded recursion + if not summary: + # it doesn't really make sense to not have a depth with tree, + # so assume depth=inf if tree by default + if depth is None: + depth = m.inf if tree else 0 + elif depth == 0: + depth = m.inf + + if not m.isinf(depth): + widths[0] += 4*depth + + # print our table with optional call info + # + # note we try to adjust our name width based on expected call depth, but + # this doesn't work if there's unbounded recursion + if not tree: + print('%-*s %s%s' % ( + widths[0], lines[0][0], + ' '.join('%*s' % (w, x) + for w, x, in zip(widths[1:], lines[0][1:-1])), + lines[0][-1])) + + # print the tree recursively + if not summary: + line_table = {n: l for n, l in zip(names, lines[1:-1])} + + def recurse(names_, depth_, prefixes=('', '', '', '')): + for i, name in enumerate(names_): + if name not in line_table: + continue + line = line_table[name] is_last = (i == len(names_)-1) - print('%-*s ' % (name_width, prefixes[0+is_last]+name), end='') - if tree: - print() - elif diff_results is None: - print(' %s %s' % ( - r.stack_frame.table() - if r else IntField.none, - r.stack_limit.table() - if r else IntField.none)) - elif percent: - print(' %s %s%s' % ( - r.stack_frame.diff_table() - if r else IntField.diff_none, - r.stack_limit.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) - else: - print(' %s %s %s %s %s %s%s' % ( - diff_r.stack_frame.diff_table() - if diff_r else IntField.diff_none, - diff_r.stack_limit.diff_table() - if diff_r else IntField.diff_none, - r.stack_frame.diff_table() - if r else IntField.diff_none, - r.stack_limit.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.stack_frame if r else None, - diff_r.stack_frame if diff_r else None) - if r or diff_r else IntField.diff_none, - IntField.diff_diff( - r.stack_limit if r else None, - diff_r.stack_limit if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + + print('%s%-*s ' % ( + prefixes[0+is_last], + widths[0] - ( + len(prefixes[0+is_last]) + if not m.isinf(depth) else 0), + line[0]), + end='') + if not tree: + print(' %s%s' % ( + ' '.join('%*s' % (w, x) + for w, x, in zip(widths[1:], line[1:-1])), + line[-1]), + end='') + print() # recurse? - if depth > 0: - cs = calls.get((name,), set()) - table_calls( - [n for n in names if (n,) in cs], - depth-1, - ( prefixes[2+is_last] + "|-> ", - prefixes[2+is_last] + "'-> ", - prefixes[2+is_last] + "| ", - prefixes[2+is_last] + " ")) + if name in table and depth_ > 0: + calls = { + ','.join(str(getattr(Result(*c), k) or '') for k in by) + for c in table[name].calls} + recurse( + # note we're maintaining sort order + [n for n in names if n in calls], + depth_-1, + (prefixes[2+is_last] + "|-> ", + prefixes[2+is_last] + "'-> ", + prefixes[2+is_last] + "| ", + prefixes[2+is_last] + " ")) + recurse(names, depth) - table_calls(names, depth) - - # print total if not tree: - total = fold(results, by=[]) - r = total[0] if total else None - if diff_results is not None: - diff_total = fold(diff_results, by=[]) - diff_r = diff_total[0] if diff_total else None - ratio = IntField.ratio( - r.stack_limit if r else None, - diff_r.stack_limit if diff_r else None) - - print('%-*s ' % (name_width, 'TOTAL'), end='') - if diff_results is None: - print(' %s %s' % ( - r.stack_frame.table() - if r else IntField.none, - r.stack_limit.table() - if r else IntField.none)) - elif percent: - print(' %s %s%s' % ( - r.stack_frame.diff_table() - if r else IntField.diff_none, - r.stack_limit.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) - else: - print(' %s %s %s %s %s %s%s' % ( - diff_r.stack_frame.diff_table() - if diff_r else IntField.diff_none, - diff_r.stack_limit.diff_table() - if diff_r else IntField.diff_none, - r.stack_frame.diff_table() - if r else IntField.diff_none, - r.stack_limit.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.stack_frame if r else None, - diff_r.stack_frame if diff_r else None) - if r or diff_r else IntField.diff_none, - IntField.diff_diff( - r.stack_limit if r else None, - diff_r.stack_limit if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + print('%-*s %s%s' % ( + widths[0], lines[-1][0], + ' '.join('%*s' % (w, x) + for w, x, in zip(widths[1:], lines[-1][1:-1])), + lines[-1][-1])) -def main(ci_paths, **args): +def main(ci_paths, + by=None, + fields=None, + defines=None, + sort=None, + **args): # find sizes if not args.get('use', None): # find .ci files @@ -506,37 +556,45 @@ def main(ci_paths, **args): paths.append(path) if not paths: - print('no .ci files found in %r?' % ci_paths) + print("error: no .ci files found in %r?" % ci_paths) sys.exit(-1) - results, calls = collect(paths, **args) + results = collect(paths, **args) else: results = [] with openio(args['use']) as f: reader = csv.DictReader(f, restval='') for r in reader: try: - results.append(StackResult(**{ - k: v for k, v in r.items() - if k in StackResult._fields})) + results.append(StackResult( + **{k: r[k] for k in StackResult._by + if k in r and r[k].strip()}, + **{k: r['stack_'+k] for k in StackResult._fields + if 'stack_'+k in r and r['stack_'+k].strip()})) except TypeError: pass - calls = {} + # fold + results = fold(StackResult, results, by=by, defines=defines) - # fold to remove duplicates - results = fold(results) - - # sort because why not + # sort, note that python's sort is stable results.sort() + if sort: + for k, reverse in reversed(sort): + results.sort(key=lambda r: (getattr(r, k),) + if getattr(r, k) is not None else (), + reverse=reverse ^ (not k or k in StackResult._fields)) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, StackResult._fields) + writer = csv.DictWriter(f, StackResult._by + + ['stack_'+k for k in StackResult._fields]) writer.writeheader() for r in results: - writer.writerow(r._asdict()) + writer.writerow( + {k: getattr(r, k) for k in StackResult._by} + | {'stack_'+k: getattr(r, k) for k in StackResult._fields}) # find previous results? if args.get('diff'): @@ -546,28 +604,31 @@ def main(ci_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - diff_results.append(StackResult(**{ - k: v for k, v in r.items() - if k in StackResult._fields})) + diff_results.append(StackResult( + **{k: r[k] for k in StackResult._by + if k in r and r[k].strip()}, + **{k: r['stack_'+k] for k in StackResult._fields + if 'stack_'+k in r and r['stack_'+k].strip()})) except TypeError: - pass + raise except FileNotFoundError: pass - # fold to remove duplicates - diff_results = fold(diff_results) + # fold + diff_results = fold(StackResult, diff_results, by=by, defines=defines) # print table if not args.get('quiet'): - table( - results, - calls, + table(StackResult, results, diff_results if args.get('diff') else None, + by=by if by is not None else ['function'], + fields=fields, + sort=sort, **args) # error on recursion if args.get('error_on_recursion') and any( - m.isinf(float(r.stack_limit)) for r in results): + m.isinf(float(r.limit)) for r in results): sys.exit(2) @@ -608,47 +669,58 @@ if __name__ == "__main__": action='store_true', help="Only show percentage change, not a full diff.") parser.add_argument( - '-t', '--tree', - action='store_true', - help="Only show the function call tree.") + '-b', '--by', + action='append', + choices=StackResult._by, + help="Group by this field.") parser.add_argument( - '-b', '--by-file', - action='store_true', - help="Group by file.") + '-f', '--field', + dest='fields', + action='append', + choices=StackResult._fields, + help="Show this field.") parser.add_argument( - '-s', '--limit-sort', - action='store_true', - help="Sort by stack limit.") + '-D', '--define', + dest='defines', + action='append', + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), + help="Only include results where this field is this value.") + class AppendSort(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.sort is None: + namespace.sort = [] + namespace.sort.append((value, True if option == '-S' else False)) parser.add_argument( - '-S', '--reverse-limit-sort', - action='store_true', - help="Sort by stack limit, but backwards.") + '-s', '--sort', + action=AppendSort, + help="Sort by this fields.") parser.add_argument( - '--frame-sort', - action='store_true', - help="Sort by stack frame.") - parser.add_argument( - '--reverse-frame-sort', - action='store_true', - help="Sort by stack frame, but backwards.") + '-S', '--reverse-sort', + action=AppendSort, + help="Sort by this fields, but backwards.") parser.add_argument( '-Y', '--summary', action='store_true', - help="Only show the total size.") - parser.add_argument( - '-L', '--depth', - nargs='?', - type=lambda x: int(x, 0), - const=float('inf'), - help="Depth of function calls to show.") - parser.add_argument( - '-e', '--error-on-recursion', - action='store_true', - help="Error if any functions are recursive.") + help="Only show the total.") parser.add_argument( '-A', '--everything', action='store_true', help="Include builtin and libc specific symbols.") + parser.add_argument( + '--tree', + action='store_true', + help="Only show the function call tree.") + parser.add_argument( + '-L', '--depth', + nargs='?', + type=lambda x: int(x, 0), + const=0, + help="Depth of function calls to show. 0 show all calls but may not " + "terminate!") + parser.add_argument( + '-e', '--error-on-recursion', + action='store_true', + help="Error if any functions are recursive.") parser.add_argument( '--build-dir', help="Specify the relative build directory. Used to map object files " diff --git a/scripts/struct_.py b/scripts/struct_.py index bdb98f73..ed6584cb 100755 --- a/scripts/struct_.py +++ b/scripts/struct_.py @@ -24,11 +24,12 @@ OBJ_PATHS = ['*.o'] OBJDUMP_TOOL = ['objdump'] + # integer fields -class IntField(co.namedtuple('IntField', 'x')): +class Int(co.namedtuple('Int', 'x')): __slots__ = () def __new__(cls, x=0): - if isinstance(x, IntField): + if isinstance(x, Int): return x if isinstance(x, str): try: @@ -94,35 +95,28 @@ class IntField(co.namedtuple('IntField', 'x')): return (new-old) / old def __add__(self, other): - return IntField(self.x + other.x) + return self.__class__(self.x + other.x) def __sub__(self, other): - return IntField(self.x - other.x) + return self.__class__(self.x - other.x) def __mul__(self, other): - return IntField(self.x * other.x) - - def __lt__(self, other): - return self.x < other.x - - def __gt__(self, other): - return self.__class__.__lt__(other, self) - - def __le__(self, other): - return not self.__gt__(other) - - def __ge__(self, other): - return not self.__lt__(other) + return self.__class__(self.x * other.x) # struct size results -class StructResult(co.namedtuple('StructResult', 'file,struct,struct_size')): +class StructResult(co.namedtuple('StructResult', ['file', 'struct', 'size'])): + _by = ['file', 'struct'] + _fields = ['size'] + _types = {'size': Int} + __slots__ = () - def __new__(cls, file, struct, struct_size): - return super().__new__(cls, file, struct, IntField(struct_size)) + def __new__(cls, file='', struct='', size=0): + return super().__new__(cls, file, struct, + Int(size)) def __add__(self, other): return StructResult(self.file, self.struct, - self.struct_size + other.struct_size) + self.size + other.size) def openio(path, mode='r'): @@ -231,9 +225,27 @@ def collect(paths, *, return results -def fold(results, *, - by=['file', 'struct'], +def fold(Result, results, *, + by=None, + defines=None, **_): + if by is None: + by = Result._by + + for k in it.chain(by or [], (k for k, _ in defines or [])): + if k not in Result._by and k not in Result._fields: + print("error: could not find field %r?" % k) + sys.exit(-1) + + # filter by matching defines + if defines is not None: + results_ = [] + for r in results: + if all(getattr(r, k) in vs for k, vs in defines): + results_.append(r) + results = results_ + + # organize results into conflicts folding = co.OrderedDict() for r in results: name = tuple(getattr(r, k) for k in by) @@ -241,157 +253,220 @@ def fold(results, *, folding[name] = [] folding[name].append(r) + # merge conflicts folded = [] - for rs in folding.values(): + for name, rs in folding.items(): folded.append(sum(rs[1:], start=rs[0])) return folded - -def table(results, diff_results=None, *, - by_file=False, - size_sort=False, - reverse_size_sort=False, +def table(Result, results, diff_results=None, *, + by=None, + fields=None, + sort=None, summary=False, all=False, percent=False, **_): all_, all = all, __builtins__.all - # fold - results = fold(results, by=['file' if by_file else 'struct']) - if diff_results is not None: - diff_results = fold(diff_results, - by=['file' if by_file else 'struct']) + if by is None: + by = Result._by + if fields is None: + fields = Result._fields + types = Result._types + # fold again + results = fold(Result, results, by=by) + if diff_results is not None: + diff_results = fold(Result, diff_results, by=by) + + # organize by name table = { - r.file if by_file else r.struct: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in results} diff_table = { - r.file if by_file else r.struct: r + ','.join(str(getattr(r, k) or '') for k in by): r for r in diff_results or []} - - # sort, note that python's sort is stable names = list(table.keys() | diff_table.keys()) + + # sort again, now with diff info, note that python's sort is stable names.sort() if diff_results is not None: - names.sort(key=lambda n: -IntField.ratio( - table[n].struct_size if n in table else None, - diff_table[n].struct_size if n in diff_table else None)) - if size_sort: - names.sort(key=lambda n: (table[n].struct_size,) if n in table else (), + names.sort(key=lambda n: tuple( + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), reverse=True) - elif reverse_size_sort: - names.sort(key=lambda n: (table[n].struct_size,) if n in table else (), - reverse=False) + if sort: + for k, reverse in reversed(sort): + names.sort(key=lambda n: (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None else (), + reverse=reverse ^ (not k or k in Result._fields)) - # print header - if not summary: - title = '%s%s' % ( - 'file' if by_file else 'struct', - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - name_width = max(it.chain([23, len(title)], (len(n) for n in names))) - else: - title = '' - name_width = 23 - name_width = 4*((name_width+1+4-1)//4)-1 - print('%-*s ' % (name_width, title), end='') + # build up our lines + lines = [] + + # header + line = [] + line.append('%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else '') if diff_results is None: - print(' %s' % ('size'.rjust(len(IntField.none)))) + for k in fields: + line.append(k) elif percent: - print(' %s' % ('size'.rjust(len(IntField.diff_none)))) + for k in fields: + line.append(k) else: - print(' %s %s %s' % ( - 'old'.rjust(len(IntField.diff_none)), - 'new'.rjust(len(IntField.diff_none)), - 'diff'.rjust(len(IntField.diff_none)))) + for k in fields: + line.append('o'+k) + for k in fields: + line.append('n'+k) + for k in fields: + line.append('d'+k) + line.append('') + lines.append(line) - # print entries + # entries if not summary: for name in names: r = table.get(name) if diff_results is not None: diff_r = diff_table.get(name) - ratio = IntField.ratio( - r.struct_size if r else None, - diff_r.struct_size if diff_r else None) - if not ratio and not all_: + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] + if not any(ratios) and not all_: continue - print('%-*s ' % (name_width, name), end='') + line = [] + line.append(name) if diff_results is None: - print(' %s' % ( - r.struct_size.table() - if r else IntField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - r.struct_size.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - diff_r.struct_size.diff_table() - if diff_r else IntField.diff_none, - r.struct_size.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.struct_size if r else None, - diff_r.struct_size if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) - # print total - total = fold(results, by=[]) - r = total[0] if total else None + # total + r = next(iter(fold(Result, results, by=[])), None) if diff_results is not None: - diff_total = fold(diff_results, by=[]) - diff_r = diff_total[0] if diff_total else None - ratio = IntField.ratio( - r.struct_size if r else None, - diff_r.struct_size if diff_r else None) + diff_r = next(iter(fold(Result, diff_results, by=[])), None) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) + for k in fields] - print('%-*s ' % (name_width, 'TOTAL'), end='') + line = [] + line.append('TOTAL') if diff_results is None: - print(' %s' % ( - r.struct_size.table() - if r else IntField.none)) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - r.struct_size.diff_table() - if r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - diff_r.struct_size.diff_table() - if diff_r else IntField.diff_none, - r.struct_size.diff_table() - if r else IntField.diff_none, - IntField.diff_diff( - r.struct_size if r else None, - diff_r.struct_size if diff_r else None) - if r or diff_r else IntField.diff_none, - ' (%s)' % ( - '+∞%' if ratio == +m.inf - else '-∞%' if ratio == -m.inf - else '%+.1f%%' % (100*ratio)) - if ratio else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) + + # find the best widths, note that column 0 contains the names and column -1 + # the ratios, so those are handled a bit differently + widths = [ + ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1 + for w, i in zip( + it.chain([23], it.repeat(7)), + range(len(lines[0])-1))] + + # print our table + for line in lines: + print('%-*s %s%s' % ( + widths[0], line[0], + ' '.join('%*s' % (w, x) + for w, x in zip(widths[1:], line[1:-1])), + line[-1])) -def main(obj_paths, **args): +def main(obj_paths, *, + by=None, + fields=None, + defines=None, + sort=None, + **args): # find sizes if not args.get('use', None): # find .o files @@ -404,7 +479,7 @@ def main(obj_paths, **args): paths.append(path) if not paths: - print('no .obj files found in %r?' % obj_paths) + print("error: no .obj files found in %r?" % obj_paths) sys.exit(-1) results = collect(paths, **args) @@ -414,25 +489,38 @@ def main(obj_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - results.append(StructResult(**{ - k: v for k, v in r.items() - if k in StructResult._fields})) + results.append(StructResult( + **{k: r[k] for k in StructResult._by + if k in r and r[k].strip()}, + **{k: r['struct_'+k] + for k in StructResult._fields + if 'struct_'+k in r + and r['struct_'+k].strip()})) except TypeError: pass - # fold to remove duplicates - results = fold(results) + # fold + results = fold(StructResult, results, by=by, defines=defines) - # sort because why not + # sort, note that python's sort is stable results.sort() + if sort: + for k, reverse in reversed(sort): + results.sort(key=lambda r: (getattr(r, k),) + if getattr(r, k) is not None else (), + reverse=reverse ^ (not k or k in StructResult._fields)) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, StructResult._fields) + writer = csv.DictWriter(f, StructResult._by + + ['struct_'+k for k in StructResult._fields]) writer.writeheader() for r in results: - writer.writerow(r._asdict()) + writer.writerow( + {k: getattr(r, k) for k in StructResult._by} + | {'struct_'+k: getattr(r, k) + for k in StructResult._fields}) # find previous results? if args.get('diff'): @@ -442,22 +530,28 @@ def main(obj_paths, **args): reader = csv.DictReader(f, restval='') for r in reader: try: - diff_results.append(StructResult(**{ - k: v for k, v in r.items() - if k in StructResult._fields})) + diff_results.append(StructResult( + **{k: r[k] for k in StructResult._by + if k in r and r[k].strip()}, + **{k: r['struct_'+k] + for k in StructResult._fields + if 'struct_'+k in r + and r['struct_'+k].strip()})) except TypeError: pass except FileNotFoundError: pass - # fold to remove duplicates - diff_results = fold(diff_results) + # fold + diff_results = fold(StructResult, diff_results, by=by, defines=defines) # print table if not args.get('quiet'): - table( - results, + table(StructResult, results, diff_results if args.get('diff') else None, + by=by if by is not None else ['struct'], + fields=fields, + sort=sort, **args) @@ -498,22 +592,39 @@ if __name__ == "__main__": action='store_true', help="Only show percentage change, not a full diff.") parser.add_argument( - '-b', '--by-file', - action='store_true', - help="Group by file. Note this does not include padding " - "so sizes may differ from other tools.") + '-b', '--by', + action='append', + choices=StructResult._by, + help="Group by this field.") parser.add_argument( - '-s', '--size-sort', - action='store_true', - help="Sort by size.") + '-f', '--field', + dest='fields', + action='append', + choices=StructResult._fields, + help="Show this field.") parser.add_argument( - '-S', '--reverse-size-sort', - action='store_true', - help="Sort by size, but backwards.") + '-D', '--define', + dest='defines', + action='append', + type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), + help="Only include results where this field is this value.") + class AppendSort(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.sort is None: + namespace.sort = [] + namespace.sort.append((value, True if option == '-S' else False)) + parser.add_argument( + '-s', '--sort', + action=AppendSort, + help="Sort by this field.") + parser.add_argument( + '-S', '--reverse-sort', + action=AppendSort, + help="Sort by this field, but backwards.") parser.add_argument( '-Y', '--summary', action='store_true', - help="Only show the total size.") + help="Only show the total.") parser.add_argument( '-A', '--everything', action='store_true', diff --git a/scripts/summary.py b/scripts/summary.py index 98c1a538..9a13703f 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -33,36 +33,25 @@ OPS = { 'prod': lambda xs: m.prod(xs[1:], start=xs[0]), 'min': min, 'max': max, - 'mean': lambda xs: FloatField(sum(float(x) for x in xs) / len(xs)), + 'mean': lambda xs: Float(sum(float(x) for x in xs) / len(xs)), 'stddev': lambda xs: ( - lambda mean: FloatField( + lambda mean: Float( m.sqrt(sum((float(x) - mean)**2 for x in xs) / len(xs))) )(sum(float(x) for x in xs) / len(xs)), - 'gmean': lambda xs: FloatField(m.prod(float(x) for x in xs)**(1/len(xs))), + 'gmean': lambda xs: Float(m.prod(float(x) for x in xs)**(1/len(xs))), 'gstddev': lambda xs: ( - lambda gmean: FloatField( + lambda gmean: Float( m.exp(m.sqrt(sum(m.log(float(x)/gmean)**2 for x in xs) / len(xs))) if gmean else m.inf) - )(m.prod(float(x) for x in xs)**(1/len(xs))) - + )(m.prod(float(x) for x in xs)**(1/len(xs))), } -def openio(path, mode='r'): - if path == '-': - if mode == 'r': - return os.fdopen(os.dup(sys.stdin.fileno()), 'r') - else: - return os.fdopen(os.dup(sys.stdout.fileno()), 'w') - else: - return open(path, mode) - - # integer fields -class IntField(co.namedtuple('IntField', 'x')): +class Int(co.namedtuple('Int', 'x')): __slots__ = () def __new__(cls, x=0): - if isinstance(x, IntField): + if isinstance(x, Int): return x if isinstance(x, str): try: @@ -128,31 +117,19 @@ class IntField(co.namedtuple('IntField', 'x')): return (new-old) / old def __add__(self, other): - return IntField(self.x + other.x) + return self.__class__(self.x + other.x) def __sub__(self, other): - return IntField(self.x - other.x) + return self.__class__(self.x - other.x) def __mul__(self, other): - return IntField(self.x * other.x) - - def __lt__(self, other): - return self.x < other.x - - def __gt__(self, other): - return self.__class__.__lt__(other, self) - - def __le__(self, other): - return not self.__gt__(other) - - def __ge__(self, other): - return not self.__lt__(other) + return self.__class__(self.x * other.x) # float fields -class FloatField(co.namedtuple('FloatField', 'x')): +class Float(co.namedtuple('Float', 'x')): __slots__ = () def __new__(cls, x=0.0): - if isinstance(x, FloatField): + if isinstance(x, Float): return x if isinstance(x, str): try: @@ -179,31 +156,27 @@ class FloatField(co.namedtuple('FloatField', 'x')): def __float__(self): return float(self.x) - none = IntField.none - table = IntField.table - diff_none = IntField.diff_none - diff_table = IntField.diff_table - diff_diff = IntField.diff_diff - ratio = IntField.ratio - __add__ = IntField.__add__ - __sub__ = IntField.__sub__ - __mul__ = IntField.__mul__ - __lt__ = IntField.__lt__ - __gt__ = IntField.__gt__ - __le__ = IntField.__le__ - __ge__ = IntField.__ge__ + none = Int.none + table = Int.table + diff_none = Int.diff_none + diff_table = Int.diff_table + diff_diff = Int.diff_diff + ratio = Int.ratio + __add__ = Int.__add__ + __sub__ = Int.__sub__ + __mul__ = Int.__mul__ # fractional fields, a/b -class FracField(co.namedtuple('FracField', 'a,b')): +class Frac(co.namedtuple('Frac', 'a,b')): __slots__ = () def __new__(cls, a=0, b=None): - if isinstance(a, FracField) and b is None: + if isinstance(a, Frac) and b is None: return a if isinstance(a, str) and b is None: a, b = a.split('/', 1) if b is None: b = a - return super().__new__(cls, IntField(a), IntField(b)) + return super().__new__(cls, Int(a), Int(b)) def __str__(self): return '%s/%s' % (self.a, self.b) @@ -213,10 +186,7 @@ class FracField(co.namedtuple('FracField', 'a,b')): none = '%11s %7s' % ('-', '-') def table(self): - if not self.b.x: - return self.none - - t = self.a.x/self.b.x + t = self.a.x/self.b.x if self.b.x else 1.0 return '%11s %7s' % ( self, '∞%' if t == +m.inf @@ -225,38 +195,35 @@ class FracField(co.namedtuple('FracField', 'a,b')): diff_none = '%11s' % '-' def diff_table(self): - if not self.b.x: - return self.diff_none - return '%11s' % (self,) def diff_diff(self, other): - new_a, new_b = self if self else (IntField(0), IntField(0)) - old_a, old_b = other if other else (IntField(0), IntField(0)) + new_a, new_b = self if self else (Int(0), Int(0)) + old_a, old_b = other if other else (Int(0), Int(0)) return '%11s' % ('%s/%s' % ( new_a.diff_diff(old_a).strip(), new_b.diff_diff(old_b).strip())) def ratio(self, other): - new_a, new_b = self if self else (IntField(0), IntField(0)) - old_a, old_b = other if other else (IntField(0), IntField(0)) + new_a, new_b = self if self else (Int(0), Int(0)) + old_a, old_b = other if other else (Int(0), Int(0)) new = new_a.x/new_b.x if new_b.x else 1.0 old = old_a.x/old_b.x if old_b.x else 1.0 return new - old def __add__(self, other): - return FracField(self.a + other.a, self.b + other.b) + return self.__class__(self.a + other.a, self.b + other.b) def __sub__(self, other): - return FracField(self.a - other.a, self.b - other.b) + return self.__class__(self.a - other.a, self.b - other.b) def __mul__(self, other): - return FracField(self.a * other.a, self.b + other.b) + return self.__class__(self.a * other.a, self.b + other.b) def __lt__(self, other): - self_r = self.a.x/self.b.x if self.b.x else -m.inf - other_r = other.a.x/other.b.x if other.b.x else -m.inf - return self_r < other_r + self_t = self.a.x/self.b.x if self.b.x else 1.0 + other_t = other.a.x/other.b.x if other.b.x else 1.0 + return (self_t, self.a.x) < (other_t, other.a.x) def __gt__(self, other): return self.__class__.__lt__(other, self) @@ -268,54 +235,41 @@ class FracField(co.namedtuple('FracField', 'a,b')): return not self.__lt__(other) # available types -TYPES = [IntField, FloatField, FracField] +TYPES = co.OrderedDict([ + ('int', Int), + ('float', Float), + ('frac', Frac) +]) -def homogenize(results, *, +def infer(results, *, by=None, fields=None, + types={}, + ops={}, renames=[], - define={}, - types=None, **_): - results = results.copy() - - # rename fields? - if renames: - for r in results: - # make a copy so renames can overlap - r_ = {} - for new_k, old_k in renames: - if old_k in r: - r_[new_k] = r[old_k] - r.update(r_) - - # filter by matching defines - if define: - results_ = [] - for r in results: - if all(k in r and r[k] in vs for k, vs in define): - results_.append(r) - results = results_ - # if fields not specified, try to guess from data if fields is None: fields = co.OrderedDict() for r in results: for k, v in r.items(): - if by is not None and k in by: - continue - types_ = [] - for type in fields.get(k, TYPES): - try: - type(v) - types_.append(type) - except ValueError: - pass - fields[k] = types_ - fields = list(k for k,v in fields.items() if v) + if (by is None or k not in by) and v.strip(): + types_ = [] + for t in fields.get(k, TYPES.values()): + try: + t(v) + types_.append(t) + except ValueError: + pass + fields[k] = types_ + fields = list(k for k, v in fields.items() if v) - # infer 'by' fields? + # deduplicate fields + fields = list(co.OrderedDict.fromkeys(fields).keys()) + + # if by not specified, guess it's anything not in fields and not a + # source of a rename if by is None: by = co.OrderedDict() for r in results: @@ -327,295 +281,355 @@ def homogenize(results, *, and not any(k == old_k for _, old_k in renames)) by = list(by.keys()) - # go ahead and clean up none values, these can have a few forms - results_ = [] - for r in results: - results_.append({ - k: r[k] for k in it.chain(by, fields) - if r.get(k) is not None and not ( - isinstance(r[k], str) - and re.match('^\s*[+-]?\s*$', r[k]))}) - results = results_ + # deduplicate fields + by = list(co.OrderedDict.fromkeys(by).keys()) # find best type for all fields - if types is None: - def is_type(x, type): - try: - type(x) - return True - except ValueError: - return False - - types = {} - for k in fields: - for type in TYPES: - if all(k not in r or is_type(r[k], type) for r in results_): - types[k] = type + types_ = {} + for k in fields: + if k in types: + types_[k] = types[k] + else: + for t in TYPES.values(): + for r in results: + if k in r and r[k].strip(): + try: + t(r[k]) + except ValueError: + break + else: + types_[k] = t break else: - print("no type matches field %r?" % k) + print("error: no type matches field %r?" % k) sys.exit(-1) + types = types_ - # homogenize types - for r in results: - for k in fields: - if k in r: - r[k] = types[k](r[k]) - - return by, fields, types, results + # does folding change the type? + types_ = {} + for k, t in types.items(): + types_[k] = ops.get(k, OPS['sum'])([t()]).__class__ -def fold(results, *, - by=[], - fields=[], - types=None, - ops={}, + # create result class + def __new__(cls, **r): + return cls.__mro__[1].__new__(cls, + **{k: r.get(k) for k in by}, + **{k: r[k] if k in r and isinstance(r[k], list) + else [types[k](r[k])] if k in r + else [] + for k in fields}) + + def __add__(self, other): + return self.__class__( + **{k: getattr(self, k) for k in by}, + **{k: object.__getattribute__(self, k) + + object.__getattribute__(other, k) + for k in fields}) + + def __getattribute__(self, k): + if k in fields: + if object.__getattribute__(self, k): + return ops.get(k, OPS['sum'])(object.__getattribute__(self, k)) + else: + return None + return object.__getattribute__(self, k) + + return type('Result', (co.namedtuple('Result', by + fields),), { + '__slots__': (), + '__new__': __new__, + '__add__': __add__, + '__getattribute__': __getattribute__, + '_by': by, + '_fields': fields, + '_types': types_, + }) + + +def fold(Result, results, *, + by=None, + defines=None, **_): + if by is None: + by = Result._by + + for k in it.chain(by or [], (k for k, _ in defines or [])): + if k not in Result._by and k not in Result._fields: + print("error: could not find field %r?" % k) + sys.exit(-1) + + # filter by matching defines + if defines is not None: + results_ = [] + for r in results: + if all(getattr(r, k) in vs for k, vs in defines): + results_.append(r) + results = results_ + + # organize results into conflicts folding = co.OrderedDict() for r in results: - name = tuple(r.get(k, '') for k in by) + name = tuple(getattr(r, k) for k in by) if name not in folding: - folding[name] = {k: [] for k in fields} - for k in fields: - if k in r: - folding[name][k].append(r[k]) + folding[name] = [] + folding[name].append(r) - # merge fields, we need the count at this point for averages + # merge conflicts folded = [] - for name, r in folding.items(): - r_ = {} - for k, vs in r.items(): - if vs: - # sum fields by default - op = OPS[ops.get(k, 'sum')] - r_[k] = op(vs) + for name, rs in folding.items(): + folded.append(sum(rs[1:], start=rs[0])) - # drop any rows without fields and any empty keys - if r_: - folded.append(dict( - {k: v for k, v in zip(by, name) if v}, - **r_)) + return folded - # what is the type of merged fields? - if types is not None: - types_ = {} - for k in fields: - op = OPS[ops.get(k, 'sum')] - types_[k] = op([types[k]()]).__class__ - - if types is None: - return folded - else: - return types_, folded - - -def table(results, total, diff_results=None, diff_total=None, *, - by=[], - fields=[], - types={}, - ops={}, +def table(Result, results, diff_results=None, *, + by=None, + fields=None, sort=None, - reverse_sort=None, summary=False, all=False, percent=False, **_): all_, all = all, __builtins__.all + if by is None: + by = Result._by + if fields is None: + fields = Result._fields + types = Result._types + + # fold again + results = fold(Result, results, by=by) + if diff_results is not None: + diff_results = fold(Result, diff_results, by=by) + + # organize by name table = { - ','.join(r.get(k,'') for k in by): r + ','.join(str(getattr(r, k) or '') for k in by): r for r in results} diff_table = { - ','.join(r.get(k,'') for k in by): r + ','.join(str(getattr(r, k) or '') for k in by): r for r in diff_results or []} - - # sort, note that python's sort is stable names = list(table.keys() | diff_table.keys()) + + # sort again, now with diff info, note that python's sort is stable names.sort() if diff_results is not None: names.sort(key=lambda n: tuple( - -types[k].ratio( - table.get(n,{}).get(k), - diff_table.get(n,{}).get(k)) - for k in fields)) - if sort: - names.sort(key=lambda n: tuple( - (table[n][k],) if k in table.get(n,{}) else () - for k in sort), + types[k].ratio( + getattr(table.get(n), k, None), + getattr(diff_table.get(n), k, None)) + for k in fields), reverse=True) - elif reverse_sort: - names.sort(key=lambda n: tuple( - (table[n][k],) if k in table.get(n,{}) else () - for k in reverse_sort), - reverse=False) + if sort: + for k, reverse in reversed(sort): + names.sort(key=lambda n: (getattr(table[n], k),) + if getattr(table.get(n), k, None) is not None else (), + reverse=reverse ^ (not k or k in Result._fields)) - # print header - if not summary: - title = '%s%s' % ( - ','.join(by), - ' (%d added, %d removed)' % ( - sum(1 for n in table if n not in diff_table), - sum(1 for n in diff_table if n not in table)) - if diff_results is not None and not percent else '') - name_width = max(it.chain([23, len(title)], (len(n) for n in names))) - else: - title = '' - name_width = 23 - name_width = 4*((name_width+1+4-1)//4)-1 - print('%-*s ' % (name_width, title), end='') + # build up our lines + lines = [] + + # header + line = [] + line.append('%s%s' % ( + ','.join(by), + ' (%d added, %d removed)' % ( + sum(1 for n in table if n not in diff_table), + sum(1 for n in diff_table if n not in table)) + if diff_results is not None and not percent else '') + if not summary else '') if diff_results is None: - widths = [ - 4*((max(len(types[k].none), len(k))+1+4-1)//4)-1 - for k in fields] - print(' %s' % ( - ' '.join(k.rjust(w) for w, k in zip(widths, fields)))) + for k in fields: + line.append(k) elif percent: - widths = [ - 4*((max(len(types[k].diff_none), len(k))+1+4-1)//4)-1 - for k in fields] - print(' %s' % ( - ' '.join(k.rjust(w) for w, k in zip(widths, fields)))) + for k in fields: + line.append(k) else: - widths = [ - 4*((max(len(types[k].diff_none), 1+len(k))+1+4-1)//4)-1 - for k in fields] - print(' %s %s %s' % ( - ' '.join(('o'+k).rjust(w) for w, k in zip(widths, fields)), - ' '.join(('n'+k).rjust(w) for w, k in zip(widths, fields)), - ' '.join(('d'+k).rjust(w) for w, k in zip(widths, fields)))) + for k in fields: + line.append('o'+k) + for k in fields: + line.append('n'+k) + for k in fields: + line.append('d'+k) + line.append('') + lines.append(line) - # print entries + # entries if not summary: for name in names: - r = table.get(name, {}) + r = table.get(name) if diff_results is not None: - diff_r = diff_table.get(name, {}) - ratios = [types[k].ratio(r.get(k), diff_r.get(k)) + diff_r = diff_table.get(name) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) for k in fields] if not any(ratios) and not all_: continue - print('%-*s ' % (name_width, name), end='') + line = [] + line.append(name) if diff_results is None: - print(' %s' % ( - ' '.join( - (r[k].table() - if k in r - else types[k].none).rjust(w) - for w, k in zip(widths, fields)))) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - ' '.join( - (r[k].diff_table().rjust(w) - if k in r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in ratios))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - ' '.join( - (diff_r[k].diff_table() - if k in diff_r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' '.join( - (r[k].diff_table() - if k in r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' '.join( - (types[k].diff_diff(r.get(k), diff_r.get(k)) - if k in r or k in diff_r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in ratios - if t) - if any(ratios) else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) - # print total - r = total - if diff_total is not None: - diff_r = diff_total - ratios = [types[k].ratio(r.get(k), diff_r.get(k)) + # total + r = next(iter(fold(Result, results, by=[])), None) + if diff_results is not None: + diff_r = next(iter(fold(Result, diff_results, by=[])), None) + ratios = [ + types[k].ratio( + getattr(r, k, None), + getattr(diff_r, k, None)) for k in fields] - print('%-*s ' % (name_width, 'TOTAL'), end='') + line = [] + line.append('TOTAL') if diff_results is None: - print(' %s' % ( - ' '.join( - (r[k].table() - if k in r - else types[k].none).rjust(w) - for w, k in zip(widths, fields)))) + for k in fields: + line.append(getattr(r, k).table() + if getattr(r, k, None) is not None + else types[k].none) elif percent: - print(' %s%s' % ( - ' '.join( - (r[k].diff_table().rjust(w) - if k in r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in ratios))) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) else: - print(' %s %s %s%s' % ( - ' '.join( - (diff_r[k].diff_table() - if k in diff_r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' '.join( - (r[k].diff_table() - if k in r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' '.join( - (types[k].diff_diff(r.get(k), diff_r.get(k)) - if k in r or k in diff_r - else types[k].diff_none).rjust(w) - for w, k in zip(widths, fields)), - ' (%s)' % ', '.join( - '+∞%' if t == +m.inf - else '-∞%' if t == -m.inf - else '%+.1f%%' % (100*t) - for t in ratios - if t) - if any(ratios) else '')) + for k in fields: + line.append(getattr(diff_r, k).diff_table() + if getattr(diff_r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(getattr(r, k).diff_table() + if getattr(r, k, None) is not None + else types[k].diff_none) + for k in fields: + line.append(types[k].diff_diff( + getattr(r, k, None), + getattr(diff_r, k, None))) + if diff_results is None: + line.append('') + elif percent: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios)) + else: + line.append(' (%s)' % ', '.join( + '+∞%' if t == +m.inf + else '-∞%' if t == -m.inf + else '%+.1f%%' % (100*t) + for t in ratios + if t) + if any(ratios) else '') + lines.append(line) + # find the best widths, note that column 0 contains the names and column -1 + # the ratios, so those are handled a bit differently + widths = [ + ((max(it.chain([w], (len(l[i]) for l in lines)))+1+4-1)//4)*4-1 + for w, i in zip( + it.chain([23], it.repeat(7)), + range(len(lines[0])-1))] + + # print our table + for line in lines: + print('%-*s %s%s' % ( + widths[0], line[0], + ' '.join('%*s' % (w, x) + for w, x in zip(widths[1:], line[1:-1])), + line[-1])) + + +def openio(path, mode='r'): + if path == '-': + if mode == 'r': + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) def main(csv_paths, *, by=None, fields=None, - define=[], + defines=None, + sort=None, **args): # separate out renames - renames = [k.split('=', 1) - for k in it.chain(by or [], fields or []) - if '=' in k] + renames = list(it.chain.from_iterable( + ((k, v) for v in vs) + for k, vs in it.chain(by or [], fields or []))) if by is not None: - by = [k.split('=', 1)[0] for k in by] + by = [k for k, _ in by] if fields is not None: - fields = [k.split('=', 1)[0] for k in fields] + fields = [k for k, _ in fields] + + # figure out types + types = {} + for t in TYPES.keys(): + for k in args.get(t, []): + if k in types: + print("error: conflicting type for field %r?" % k) + sys.exit(-1) + types[k] = TYPES[t] + # rename types? + if renames: + types_ = {} + for new_k, old_k in renames: + if old_k in types: + types_[new_k] = types[old_k] + types.update(types_) # figure out merge operations ops = {} - for m in OPS.keys(): - for k in args.get(m, []): + for o in OPS.keys(): + for k in args.get(o, []): if k in ops: - print("conflicting op for field %r?" % k) + print("error: conflicting op for field %r?" % k) sys.exit(-1) - ops[k] = m + ops[k] = OPS[o] # rename ops? if renames: ops_ = {} @@ -634,7 +648,7 @@ def main(csv_paths, *, paths.append(path) if not paths: - print('no .csv files found in %r?' % csv_paths) + print("error: no .csv files found in %r?" % csv_paths) sys.exit(-1) results = [] @@ -643,29 +657,56 @@ def main(csv_paths, *, with openio(path) as f: reader = csv.DictReader(f, restval='') for r in reader: + # rename fields? + if renames: + # make a copy so renames can overlap + r_ = {} + for new_k, old_k in renames: + if old_k in r: + r_[new_k] = r[old_k] + r.update(r_) + results.append(r) except FileNotFoundError: pass # homogenize - by, fields, types, results = homogenize(results, - by=by, fields=fields, renames=renames, define=define) + Result = infer(results, + by=by, + fields=fields, + types=types, + ops=ops, + renames=renames) + results_ = [] + for r in results: + try: + results_.append(Result(**{ + k: r[k] for k in Result._by + Result._fields + if k in r and r[k].strip()})) + except TypeError: + pass + results = results_ - # fold for total, note we do this with the raw data to avoid - # issues with lossy operations - total = fold(results, fields=fields, ops=ops) - total = total[0] if total else {} + # fold + results = fold(Result, results, by=by, defines=defines) - # fold to remove duplicates - types_, results = fold(results, by=by, fields=fields, types=types, ops=ops) + # sort, note that python's sort is stable + results.sort() + if sort: + for k, reverse in reversed(sort): + results.sort(key=lambda r: (getattr(r, k),) + if getattr(r, k) is not None else (), + reverse=reverse ^ (not k or k in Result._fields)) # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, by + fields) + writer = csv.DictWriter(f, Result._by + Result._fields) writer.writeheader() for r in results: - writer.writerow(r) + # note we need to go through getattr to resolve lazy fields + writer.writerow({ + k: getattr(r, k) for k in Result._by + Result._fields}) # find previous results? if args.get('diff'): @@ -674,33 +715,34 @@ def main(csv_paths, *, with openio(args['diff']) as f: reader = csv.DictReader(f, restval='') for r in reader: - diff_results.append(r) + # rename fields? + if renames: + # make a copy so renames can overlap + r_ = {} + for new_k, old_k in renames: + if old_k in r: + r_[new_k] = r[old_k] + r.update(r_) + + try: + diff_results.append(Result(**{ + k: r[k] for k in Result._by + Result._fields + if k in r and r[k].strip()})) + except TypeError: + pass except FileNotFoundError: pass - # homogenize - _, _, _, diff_results = homogenize(diff_results, - by=by, fields=fields, renames=renames, define=define, types=types) - - # fold for total, note we do this with the raw data to avoid - # issues with lossy operations - diff_total = fold(diff_results, fields=fields, ops=ops) - diff_total = diff_total[0] if diff_total else {} - - # fold to remove duplicates - diff_results = fold(diff_results, by=by, fields=fields, ops=ops) + # fold + diff_results = fold(Result, diff_results, by=by, defines=defines) # print table if not args.get('quiet'): - table( - results, - total, + table(Result, results, diff_results if args.get('diff') else None, - diff_total if args.get('diff') else None, by=by, fields=fields, - types=types_, - ops=ops, + sort=sort, **args) @@ -736,18 +778,54 @@ if __name__ == "__main__": parser.add_argument( '-b', '--by', action='append', - help="Group by these fields. All other fields will be merged as " - "needed. Can rename fields with new_name=old_name.") + type=lambda x: ( + lambda k,v=None: (k, v.split(',') if v is not None else ()) + )(*x.split('=', 1)), + help="Group by this field. Can rename fields with new_name=old_name.") parser.add_argument( - '-f', '--fields', + '-f', '--field', + dest='fields', action='append', - help="Use these fields. Can rename fields with new_name=old_name.") + type=lambda x: ( + lambda k,v=None: (k, v.split(',') if v is not None else ()) + )(*x.split('=', 1)), + help="Show this field. Can rename fields with new_name=old_name.") parser.add_argument( '-D', '--define', + dest='defines', action='append', type=lambda x: (lambda k,v: (k, set(v.split(','))))(*x.split('=', 1)), - help="Only include rows where this field is this value. May include " + help="Only include results where this field is this value. May include " "comma-separated options.") + class AppendSort(argparse.Action): + def __call__(self, parser, namespace, value, option): + if namespace.sort is None: + namespace.sort = [] + namespace.sort.append((value, True if option == '-S' else False)) + parser.add_argument( + '-s', '--sort', + action=AppendSort, + help="Sort by this fields.") + parser.add_argument( + '-S', '--reverse-sort', + action=AppendSort, + help="Sort by this fields, but backwards.") + parser.add_argument( + '-Y', '--summary', + action='store_true', + help="Only show the total.") + parser.add_argument( + '--int', + action='append', + help="Treat these fields as ints.") + parser.add_argument( + '--float', + action='append', + help="Treat these fields as floats.") + parser.add_argument( + '--frac', + action='append', + help="Treat these fields as fractions.") parser.add_argument( '--sum', action='append', @@ -780,18 +858,6 @@ if __name__ == "__main__": '--gstddev', action='append', help="Find the geometric standard deviation of these fields.") - parser.add_argument( - '-s', '--sort', - action='append', - help="Sort by these fields.") - parser.add_argument( - '-S', '--reverse-sort', - action='append', - help="Sort by these fields, but backwards.") - parser.add_argument( - '-Y', '--summary', - action='store_true', - help="Only show the totals.") sys.exit(main(**{k: v for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) From 490e1c461645e5905f2d1d8f20f8f14ae7efed8f Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 2 Oct 2022 18:35:46 -0500 Subject: [PATCH 51/81] Added perf.py a wrapper around Linux's perf tool for perf sampling This provides 2 things: 1. perf integration with the bench/test runners - This is a bit tricky with perf as it doesn't have its own way to combine perf measurements across multiple processes. perf.py works around this by writing everything to a zip file, using flock to synchronize. As a plus, free compression! 2. Parsing and presentation of perf results in a format consistent with the other CSV-based tools. This actually ran into a surprising number of issues: - We need to process raw events to get the information we want, this ends up being a lot of data (~16MiB at 100Hz uncompressed), so we paralellize the parsing of each decompressed perf file. - perf reports raw addresses post-ASLR. It does provide sym+off which is very useful, but to find the source of static functions we need to reverse the ASLR by finding the delta the produces the best symbol<->addr matches. - This isn't related to perf, but decoding dwarf line-numbers is really complicated. You basically need to write a tiny VM. This also turns on perf measurement by default for the bench-runner, but at a low frequency (100 Hz). This can be decreased or removed in the future if it causes any slowdown. --- .gitignore | 1 + Makefile | 189 ++++-- scripts/bench.py | 78 ++- scripts/code.py | 200 +++++- scripts/coverage.py | 105 ++-- scripts/data.py | 200 +++++- scripts/perf.py | 1263 ++++++++++++++++++++++++++++++++++++++ scripts/plot.py | 3 +- scripts/prettyasserts.py | 3 +- scripts/stack.py | 99 ++- scripts/struct_.py | 159 +++-- scripts/summary.py | 3 +- scripts/tailpipe.py | 3 +- scripts/test.py | 78 ++- scripts/tracebd.py | 3 +- 15 files changed, 2104 insertions(+), 283 deletions(-) create mode 100755 scripts/perf.py diff --git a/.gitignore b/.gitignore index 8f1b90e6..640a47c3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ *.a.c *.gcno *.gcda +*.perf # Testing things blocks/ diff --git a/Makefile b/Makefile index 1c6a4584..16957fd1 100644 --- a/Makefile +++ b/Makefile @@ -19,44 +19,52 @@ TARGET ?= $(BUILDDIR)lfs.a endif -CC ?= gcc -AR ?= ar -SIZE ?= size -CTAGS ?= ctags -NM ?= nm -OBJDUMP ?= objdump -LCOV ?= lcov +CC ?= gcc +AR ?= ar +SIZE ?= size +CTAGS ?= ctags +NM ?= nm +OBJDUMP ?= objdump +VALGRIND ?= valgrind +GDB ?= gdb +PERF ?= perf -SRC ?= $(filter-out $(wildcard *.*.c),$(wildcard *.c)) -OBJ := $(SRC:%.c=$(BUILDDIR)%.o) -DEP := $(SRC:%.c=$(BUILDDIR)%.d) -ASM := $(SRC:%.c=$(BUILDDIR)%.s) -CI := $(SRC:%.c=$(BUILDDIR)%.ci) +SRC ?= $(filter-out $(wildcard *.*.c),$(wildcard *.c)) +OBJ := $(SRC:%.c=$(BUILDDIR)%.o) +DEP := $(SRC:%.c=$(BUILDDIR)%.d) +ASM := $(SRC:%.c=$(BUILDDIR)%.s) +CI := $(SRC:%.c=$(BUILDDIR)%.ci) GCDA := $(SRC:%.c=$(BUILDDIR)%.t.a.gcda) TESTS ?= $(wildcard tests/*.toml) TEST_SRC ?= $(SRC) \ $(filter-out $(wildcard bd/*.*.c),$(wildcard bd/*.c)) \ runners/test_runner.c -TEST_TC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) $(TEST_SRC:%.c=$(BUILDDIR)%.t.c) -TEST_TAC := $(TEST_TC:%.t.c=%.t.a.c) -TEST_OBJ := $(TEST_TAC:%.t.a.c=%.t.a.o) -TEST_DEP := $(TEST_TAC:%.t.a.c=%.t.a.d) -TEST_CI := $(TEST_TAC:%.t.a.c=%.t.a.ci) +TEST_RUNNER ?= $(BUILDDIR)runners/test_runner +TEST_TC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) \ + $(TEST_SRC:%.c=$(BUILDDIR)%.t.c) +TEST_TAC := $(TEST_TC:%.t.c=%.t.a.c) +TEST_OBJ := $(TEST_TAC:%.t.a.c=%.t.a.o) +TEST_DEP := $(TEST_TAC:%.t.a.c=%.t.a.d) +TEST_CI := $(TEST_TAC:%.t.a.c=%.t.a.ci) TEST_GCNO := $(TEST_TAC:%.t.a.c=%.t.a.gcno) TEST_GCDA := $(TEST_TAC:%.t.a.c=%.t.a.gcda) +TEST_PERF := $(TEST_RUNNER:%=%.perf) BENCHES ?= $(wildcard benches/*.toml) BENCH_SRC ?= $(SRC) \ $(filter-out $(wildcard bd/*.*.c),$(wildcard bd/*.c)) \ runners/bench_runner.c -BENCH_BC := $(BENCHES:%.toml=$(BUILDDIR)%.b.c) $(BENCH_SRC:%.c=$(BUILDDIR)%.b.c) -BENCH_BAC := $(BENCH_BC:%.b.c=%.b.a.c) -BENCH_OBJ := $(BENCH_BAC:%.b.a.c=%.b.a.o) -BENCH_DEP := $(BENCH_BAC:%.b.a.c=%.b.a.d) -BENCH_CI := $(BENCH_BAC:%.b.a.c=%.b.a.ci) +BENCH_RUNNER ?= $(BUILDDIR)runners/bench_runner +BENCH_BC := $(BENCHES:%.toml=$(BUILDDIR)%.b.c) \ + $(BENCH_SRC:%.c=$(BUILDDIR)%.b.c) +BENCH_BAC := $(BENCH_BC:%.b.c=%.b.a.c) +BENCH_OBJ := $(BENCH_BAC:%.b.a.c=%.b.a.o) +BENCH_DEP := $(BENCH_BAC:%.b.a.c=%.b.a.d) +BENCH_CI := $(BENCH_BAC:%.b.a.c=%.b.a.ci) BENCH_GCNO := $(BENCH_BAC:%.b.a.c=%.b.a.gcno) BENCH_GCDA := $(BENCH_BAC:%.b.a.c=%.b.a.gcda) +BENCH_PERF := $(BENCH_RUNNER:%=%.perf) ifdef DEBUG override CFLAGS += -O0 @@ -71,40 +79,67 @@ override CFLAGS += -I. override CFLAGS += -std=c99 -Wall -pedantic override CFLAGS += -Wextra -Wshadow -Wjump-misses-init -Wundef override CFLAGS += -ftrack-macro-expansion=0 +ifdef YES_COVERAGE +override CFLAGS += --coverage +endif +ifdef YES_PERF +override CFLAGS += -fno-omit-frame-pointer +endif -override TESTFLAGS += -b -override BENCHFLAGS += -b -# forward -j flag -override TESTFLAGS += $(filter -j%,$(MAKEFLAGS)) -override BENCHFLAGS += $(filter -j%,$(MAKEFLAGS)) ifdef VERBOSE -override CODEFLAGS += -v -override DATAFLAGS += -v -override STACKFLAGS += -v -override STRUCTFLAGS += -v -override COVERAGEFLAGS += -v -override TESTFLAGS += -v -override TESTCFLAGS += -v -override BENCHFLAGS += -v -override BENCHCFLAGS += -v -endif -ifdef EXEC -override TESTFLAGS += --exec="$(EXEC)" -override BENCHFLAGS += --exec="$(EXEC)" -endif -ifdef BUILDDIR -override CODEFLAGS += --build-dir="$(BUILDDIR:/=)" -override DATAFLAGS += --build-dir="$(BUILDDIR:/=)" -override STACKFLAGS += --build-dir="$(BUILDDIR:/=)" -override STRUCTFLAGS += --build-dir="$(BUILDDIR:/=)" -override COVERAGEFLAGS += --build-dir="$(BUILDDIR:/=)" +override CODEFLAGS += -v +override DATAFLAGS += -v +override STACKFLAGS += -v +override STRUCTFLAGS += -v +override COVERAGEFLAGS += -v +override PERFFLAGS += -v endif ifneq ($(NM),nm) override CODEFLAGS += --nm-tool="$(NM)" override DATAFLAGS += --nm-tool="$(NM)" endif ifneq ($(OBJDUMP),objdump) +override CODEFLAGS += --objdump-tool="$(OBJDUMP)" +override DATAFLAGS += --objdump-tool="$(OBJDUMP)" override STRUCTFLAGS += --objdump-tool="$(OBJDUMP)" +override PERFFLAGS += --objdump-tool="$(OBJDUMP)" +endif +ifneq ($(PERF),perf) +override PERFFLAGS += --perf-tool="$(PERF)" +endif + +override TESTFLAGS += -b +override BENCHFLAGS += -b +# forward -j flag +override TESTFLAGS += $(filter -j%,$(MAKEFLAGS)) +override BENCHFLAGS += $(filter -j%,$(MAKEFLAGS)) +ifdef YES_PERF +override TESTFLAGS += --perf=$(TEST_PERF) +endif +ifndef NO_PERF +override BENCHFLAGS += --perf=$(BENCH_PERF) +endif +ifdef VERBOSE +override TESTFLAGS += -v +override TESTCFLAGS += -v +override BENCHFLAGS += -v +override BENCHCFLAGS += -v +endif +ifdef EXEC +override TESTFLAGS += --exec="$(EXEC)" +override BENCHFLAGS += --exec="$(EXEC)" +endif +ifneq ($(GDB),gdb) +override TESTFLAGS += --gdb-tool="$(GDB)" +override BENCHFLAGS += --gdb-tool="$(GDB)" +endif +ifneq ($(VALGRIND),valgrind) +override TESTFLAGS += --valgrind-tool="$(VALGRIND)" +override BENCHFLAGS += --valgrind-tool="$(VALGRIND)" +endif +ifneq ($(PERF),perf) +override TESTFLAGS += --perf-tool="$(PERF)" +override BENCHFLAGS += --perf-tool="$(PERF)" endif @@ -124,28 +159,50 @@ tags: $(CTAGS) --totals --c-types=+p $(shell find -H -name '*.h') $(SRC) .PHONY: test-runner build-test +ifndef NO_COVERAGE test-runner build-test: override CFLAGS+=--coverage -test-runner build-test: $(BUILDDIR)runners/test_runner +endif +ifdef YES_PERF +bench-runner build-bench: override CFLAGS+=-fno-omit-frame-pointer +endif +test-runner build-test: $(TEST_RUNNER) +ifndef NO_COVERAGE rm -f $(TEST_GCDA) +endif +ifdef YES_PERF + rm -f $(TEST_PERF) +endif .PHONY: test test: test-runner - ./scripts/test.py $(BUILDDIR)runners/test_runner $(TESTFLAGS) + ./scripts/test.py $(TEST_RUNNER) $(TESTFLAGS) .PHONY: test-list test-list: test-runner - ./scripts/test.py $(BUILDDIR)runners/test_runner $(TESTFLAGS) -l + ./scripts/test.py $(TEST_RUNNER) $(TESTFLAGS) -l .PHONY: bench-runner build-bench -bench-runner build-bench: $(BUILDDIR)runners/bench_runner +ifdef YES_COVERAGE +bench-runner build-bench: override CFLAGS+=--coverage +endif +ifndef NO_PERF +bench-runner build-bench: override CFLAGS+=-fno-omit-frame-pointer +endif +bench-runner build-bench: $(BENCH_RUNNER) +ifdef YES_COVERAGE + rm -f $(BENCH_GCDA) +endif +ifndef NO_PERF + rm -f $(BENCH_PERF) +endif .PHONY: bench bench: bench-runner - ./scripts/bench.py $(BUILDDIR)runners/bench_runner $(BENCHFLAGS) + ./scripts/bench.py $(BENCH_RUNNER) $(BENCHFLAGS) .PHONY: bench-list bench-list: bench-runner - ./scripts/bench.py $(BUILDDIR)runners/bench_runner $(BENCHFLAGS) -l + ./scripts/bench.py $(BENCH_RUNNER) $(BENCHFLAGS) -l .PHONY: code code: $(OBJ) @@ -165,7 +222,17 @@ struct: $(OBJ) .PHONY: coverage coverage: $(GCDA) - ./scripts/coverage.py $^ -slines -sbranches $(COVERAGEFLAGS) + $(strip ./scripts/coverage.py \ + $^ $(patsubst %,-F%,$(SRC)) \ + -slines -sbranches \ + $(COVERAGEFLAGS)) + +.PHONY: perf +perf: $(BENCH_PERF) + $(strip ./scripts/perf.py \ + $^ $(patsubst %,-F%,$(SRC)) \ + -scycles \ + $(PERFFLAGS)) .PHONY: summary sizes summary sizes: $(BUILDDIR)lfs.csv @@ -203,7 +270,10 @@ $(BUILDDIR)lfs.struct.csv: $(OBJ) ./scripts/struct_.py $^ -q $(CODEFLAGS) -o $@ $(BUILDDIR)lfs.coverage.csv: $(GCDA) - ./scripts/coverage.py $^ -q $(COVERAGEFLAGS) -o $@ + ./scripts/coverage.py $^ $(patsubst %,-F%,$(SRC)) -q $(COVERAGEFLAGS) -o $@ + +$(BUILDDIR)lfs.perf.csv: $(BENCH_PERF) + ./scripts/perf.py $^ $(patsubst %,-F%,$(SRC)) -q $(PERFFLAGS) -o $@ $(BUILDDIR)lfs.csv: \ $(BUILDDIR)lfs.code.csv \ @@ -255,13 +325,13 @@ clean: $(BUILDDIR)lfs.data.csv \ $(BUILDDIR)lfs.stack.csv \ $(BUILDDIR)lfs.struct.csv \ - $(BUILDDIR)lfs.coverage.csv) - rm -f $(BUILDDIR)runners/test_runner - rm -f $(BUILDDIR)runners/bench_runner + $(BUILDDIR)lfs.coverage.csv \ + $(BUILDDIR)lfs.perf.csv) rm -f $(OBJ) rm -f $(DEP) rm -f $(ASM) rm -f $(CI) + rm -f $(TEST_RUNNER) rm -f $(TEST_TC) rm -f $(TEST_TAC) rm -f $(TEST_OBJ) @@ -269,6 +339,8 @@ clean: rm -f $(TEST_CI) rm -f $(TEST_GCNO) rm -f $(TEST_GCDA) + rm -f $(TEST_PERF) + rm -f $(BENCH_RUNNER) rm -f $(BENCH_BC) rm -f $(BENCH_BAC) rm -f $(BENCH_OBJ) @@ -276,3 +348,4 @@ clean: rm -f $(BENCH_CI) rm -f $(BENCH_GCNO) rm -f $(BENCH_GCDA) + rm -f $(BENCH_PERF) diff --git a/scripts/bench.py b/scripts/bench.py index e401d7cc..61db83e4 100755 --- a/scripts/bench.py +++ b/scripts/bench.py @@ -27,9 +27,13 @@ import time import toml -RUNNER_PATH = 'runners/bench_runner' +RUNNER_PATH = './runners/bench_runner' HEADER_PATH = 'runners/bench_runner.h' +GDB_TOOL = ['gdb'] +VALGRIND_TOOL = ['valgrind'] +PERF_SCRIPT = ['./scripts/perf.py'] + def openio(path, mode='r', buffering=-1, nb=False): if path == '-': @@ -502,12 +506,25 @@ def find_runner(runner, **args): # run under valgrind? if args.get('valgrind'): - cmd[:0] = filter(None, [ - 'valgrind', + cmd[:0] = args['valgrind_tool'] + [ '--leak-check=full', '--track-origins=yes', '--error-exitcode=4', - '-q']) + '-q'] + + # run under perf? + if args.get('perf'): + cmd[:0] = args['perf_script'] + list(filter(None, [ + '-R', + '--perf-freq=%s' % args['perf_freq'] + if args.get('perf_freq') else None, + '--perf-period=%s' % args['perf_period'] + if args.get('perf_period') else None, + '--perf-events=%s' % args['perf_events'] + if args.get('perf_events') else None, + '--perf-tool=%s' % args['perf_tool'] + if args.get('perf_tool') else None, + '-o%s' % args['perf']])) # other context if args.get('geometry'): @@ -789,9 +806,9 @@ def run_stage(name, runner_, ids, output_, **args): try: line = mpty.readline() except OSError as e: - if e.errno == errno.EIO: - break - raise + if e.errno != errno.EIO: + raise + break if not line: break last_stdout.append(line) @@ -1126,24 +1143,24 @@ def run(runner, bench_ids=[], **args): cmd = runner_ + [failure.id] if args.get('gdb_main'): - cmd[:0] = ['gdb', + cmd[:0] = args['gdb_tool'] + [ '-ex', 'break main', '-ex', 'run', '--args'] elif args.get('gdb_case'): path, lineno = find_path(runner_, failure.id, **args) - cmd[:0] = ['gdb', + cmd[:0] = args['gdb_tool'] + [ '-ex', 'break %s:%d' % (path, lineno), '-ex', 'run', '--args'] elif failure.assert_ is not None: - cmd[:0] = ['gdb', + cmd[:0] = args['gdb_tool'] + [ '-ex', 'run', '-ex', 'frame function raise', '-ex', 'up 2', '--args'] else: - cmd[:0] = ['gdb', + cmd[:0] = args['gdb_tool'] + [ '-ex', 'run', '--args'] @@ -1187,6 +1204,7 @@ if __name__ == "__main__": argparse._ArgumentGroup._handle_conflict_ignore = lambda *_: None parser = argparse.ArgumentParser( description="Build and run benches.", + allow_abbrev=False, conflict_handler='ignore') parser.add_argument( '-v', '--verbose', @@ -1315,6 +1333,11 @@ if __name__ == "__main__": action='store_true', help="Drop into gdb on bench failure but stop at the beginning " "of main.") + bench_parser.add_argument( + '--gdb-tool', + type=lambda x: x.split(), + default=GDB_TOOL, + help="Path to gdb tool to use. Defaults to %r." % GDB_TOOL) bench_parser.add_argument( '--exec', type=lambda e: e.split(), @@ -1324,6 +1347,37 @@ if __name__ == "__main__": action='store_true', help="Run under Valgrind to find memory errors. Implicitly sets " "--isolate.") + bench_parser.add_argument( + '--valgrind-tool', + type=lambda x: x.split(), + default=VALGRIND_TOOL, + help="Path to Valgrind tool to use. Defaults to %r." % VALGRIND_TOOL) + bench_parser.add_argument( + '--perf', + help="Run under Linux's perf to sample performance counters, writing " + "samples to this file.") + bench_parser.add_argument( + '--perf-freq', + help="perf sampling frequency. This is passed directly to the perf " + "script.") + bench_parser.add_argument( + '--perf-period', + help="perf sampling period. This is passed directly to the perf " + "script.") + bench_parser.add_argument( + '--perf-events', + help="perf events to record. This is passed directly to the perf " + "script.") + bench_parser.add_argument( + '--perf-script', + type=lambda x: x.split(), + default=PERF_SCRIPT, + help="Path to the perf script to use. Defaults to %r." % PERF_SCRIPT) + bench_parser.add_argument( + '--perf-tool', + type=lambda x: x.split(), + help="Path to the perf tool to use. This is passed directly to the " + "perf script") # compilation flags comp_parser = parser.add_argument_group('compilation options') @@ -1348,7 +1402,7 @@ if __name__ == "__main__": '-o', '--output', help="Output file.") - # runner + bench_ids overlaps bench_paths, so we need to do some munging here + # runner/bench_paths overlap, so need to do some munging here args = parser.parse_intermixed_args() args.bench_paths = [' '.join(args.runner or [])] + args.bench_ids args.runner = args.runner or [RUNNER_PATH] diff --git a/scripts/code.py b/scripts/code.py index 6b373fcf..7e7e960c 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -5,7 +5,7 @@ # by Linux's Bloat-O-Meter. # # Example: -# ./scripts/code.py lfs.o lfs_util.o -S +# ./scripts/code.py lfs.o lfs_util.o -Ssize # # Copyright (c) 2022, The littlefs authors. # Copyright (c) 2020, Arm Limited. All rights reserved. @@ -14,6 +14,7 @@ import collections as co import csv +import difflib import glob import itertools as it import math as m @@ -25,7 +26,8 @@ import subprocess as sp OBJ_PATHS = ['*.o'] NM_TOOL = ['nm'] -TYPE = 'tTrRdD' +NM_TYPES = 'tTrRdD' +OBJDUMP_TOOL = ['objdump'] # integer fields @@ -135,21 +137,32 @@ def openio(path, mode='r'): def collect(paths, *, nm_tool=NM_TOOL, - type=TYPE, - build_dir=None, + nm_types=NM_TYPES, + objdump_tool=OBJDUMP_TOOL, + sources=None, everything=False, **args): - results = [] - pattern = re.compile( + size_pattern = re.compile( '^(?P[0-9a-fA-F]+)' + - ' (?P[%s])' % re.escape(type) + + ' (?P[%s])' % re.escape(nm_types) + ' (?P.+?)$') + line_pattern = re.compile( + '^\s+(?P[0-9]+)\s+' + '(?:(?P[0-9]+)\s+)?' + '.*\s+' + '(?P[^\s]+)$') + info_pattern = re.compile( + '^(?:.*(?PDW_TAG_[a-z_]+).*' + '|^.*DW_AT_name.*:\s*(?P[^:\s]+)\s*' + '|^.*DW_AT_decl_file.*:\s*(?P[0-9]+)\s*)$') + + results = [] for path in paths: - # map to source file - src_path = re.sub('\.o$', '.c', path) - if build_dir: - src_path = re.sub('%s/*' % re.escape(build_dir), '', - src_path) + # guess the source, if we have debug-info we'll replace this later + file = re.sub('(\.o)?$', '.c', path, 1) + + # find symbol sizes + results_ = [] # note nm-tool may contain extra args cmd = nm_tool + ['--size-sort', path] if args.get('verbose'): @@ -158,21 +171,18 @@ def collect(paths, *, stdout=sp.PIPE, stderr=sp.PIPE if not args.get('verbose') else None, universal_newlines=True, - errors='replace') + errors='replace', + close_fds=False) for line in proc.stdout: - m = pattern.match(line) + m = size_pattern.match(line) if m: func = m.group('func') # discard internal functions if not everything and func.startswith('__'): continue - # discard .8449 suffixes created by optimizer - func = re.sub('\.[0-9]+', '', func) - - results.append(CodeResult( - src_path, func, + results_.append(CodeResult( + file, func, int(m.group('size'), 16))) - proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -180,6 +190,121 @@ def collect(paths, *, sys.stdout.write(line) sys.exit(-1) + + # try to figure out the source file if we have debug-info + dirs = {} + files = {} + # note objdump-tool may contain extra args + cmd = objdump_tool + ['--dwarf=rawline', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + for line in proc.stdout: + # note that files contain references to dirs, which we + # dereference as soon as we see them as each file table follows a + # dir table + m = line_pattern.match(line) + if m: + if not m.group('dir'): + # found a directory entry + dirs[int(m.group('no'))] = m.group('path') + else: + # found a file entry + dir = int(m.group('dir')) + if dir in dirs: + files[int(m.group('no'))] = os.path.join( + dirs[dir], + m.group('path')) + else: + files[int(m.group('no'))] = m.group('path') + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + # do nothing on error, we don't need objdump to work, source files + # may just be inaccurate + pass + + defs = {} + is_func = False + f_name = None + f_file = None + # note objdump-tool may contain extra args + cmd = objdump_tool + ['--dwarf=info', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + for line in proc.stdout: + # state machine here to find definitions + m = info_pattern.match(line) + if m: + if m.group('tag'): + if is_func: + defs[f_name] = files.get(f_file, '?') + is_func = (m.group('tag') == 'DW_TAG_subprogram') + elif m.group('name'): + f_name = m.group('name') + elif m.group('file'): + f_file = int(m.group('file')) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + # do nothing on error, we don't need objdump to work, source files + # may just be inaccurate + pass + + for r in results_: + # find best matching debug symbol, this may be slightly different + # due to optimizations + if defs: + # exact match? avoid difflib if we can for speed + if r.function in defs: + file = defs[r.function] + else: + _, file = max( + defs.items(), + key=lambda d: difflib.SequenceMatcher(None, + d[0], + r.function, False).ratio()) + else: + file = r.file + + # ignore filtered sources + if sources is not None: + if not any( + os.path.abspath(file) == os.path.abspath(s) + for s in sources): + continue + else: + # default to only cwd + if not everything and not os.path.commonpath([ + os.getcwd(), + os.path.abspath(file)]) == os.getcwd(): + continue + + # simplify path + if os.path.commonpath([ + os.getcwd(), + os.path.abspath(file)]) == os.getcwd(): + file = os.path.relpath(file) + else: + file = os.path.abspath(file) + + results.append(CodeResult(file, r.function, r.size)) + return results @@ -437,7 +562,7 @@ def main(obj_paths, *, paths.append(path) if not paths: - print("error: no .obj files found in %r?" % obj_paths) + print("error: no .o files found in %r?" % obj_paths) sys.exit(-1) results = collect(paths, **args) @@ -469,13 +594,16 @@ def main(obj_paths, *, # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, CodeResult._by + writer = csv.DictWriter(f, + (by if by is not None else CodeResult._by) + ['code_'+k for k in CodeResult._fields]) writer.writeheader() for r in results: writer.writerow( - {k: getattr(r, k) for k in CodeResult._by} - | {'code_'+k: getattr(r, k) for k in CodeResult._fields}) + {k: getattr(r, k) + for k in (by if by is not None else CodeResult._by)} + | {'code_'+k: getattr(r, k) + for k in CodeResult._fields}) # find previous results? if args.get('diff'): @@ -512,7 +640,8 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Find code size at the function level.") + description="Find code size at the function level.", + allow_abbrev=False) parser.add_argument( 'obj_paths', nargs='*', @@ -579,23 +708,30 @@ if __name__ == "__main__": action='store_true', help="Only show the total.") parser.add_argument( - '-A', '--everything', + '-F', '--source', + dest='sources', + action='append', + help="Only consider definitions in this file. Defaults to anything " + "in the current directory.") + parser.add_argument( + '--everything', action='store_true', help="Include builtin and libc specific symbols.") parser.add_argument( - '--type', - default=TYPE, + '--nm-types', + default=NM_TYPES, help="Type of symbols to report, this uses the same single-character " - "type-names emitted by nm. Defaults to %r." % TYPE) + "type-names emitted by nm. Defaults to %r." % NM_TYPES) parser.add_argument( '--nm-tool', type=lambda x: x.split(), default=NM_TOOL, help="Path to the nm tool to use. Defaults to %r." % NM_TOOL) parser.add_argument( - '--build-dir', - help="Specify the relative build directory. Used to map object files " - "to the correct source files.") + '--objdump-tool', + type=lambda x: x.split(), + default=OBJDUMP_TOOL, + help="Path to the objdump tool to use. Defaults to %r." % OBJDUMP_TOOL) sys.exit(main(**{k: v for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) diff --git a/scripts/coverage.py b/scripts/coverage.py index 7d36a47e..f74dda82 100755 --- a/scripts/coverage.py +++ b/scripts/coverage.py @@ -3,7 +3,9 @@ # Script to find coverage info after running tests. # # Example: -# ./scripts/coverage.py lfs.t.a.gcda lfs_util.t.a.gcda -s +# ./scripts/coverage.py \ +# lfs.t.a.gcda lfs_util.t.a.gcda \ +# -Flfs.c -Flfs_util.c -slines # # Copyright (c) 2022, The littlefs authors. # Copyright (c) 2020, Arm Limited. All rights reserved. @@ -209,19 +211,13 @@ def openio(path, mode='r'): else: return open(path, mode) -def collect(paths, *, +def collect(gcda_paths, *, gcov_tool=GCOV_TOOL, - build_dir=None, + sources=None, everything=False, **args): results = [] - for path in paths: - # map to source file - src_path = re.sub('\.t\.a\.gcda$', '.c', path) - if build_dir: - src_path = re.sub('%s/*' % re.escape(build_dir), '', - src_path) - + for path in gcda_paths: # get coverage info through gcov's json output # note, gcov-tool may contain extra args cmd = GCOV_TOOL + ['-b', '-t', '--json-format', path] @@ -231,7 +227,8 @@ def collect(paths, *, stdout=sp.PIPE, stderr=sp.PIPE if not args.get('verbose') else None, universal_newlines=True, - errors='replace') + errors='replace', + close_fds=False) data = json.load(proc.stdout) proc.wait() if proc.returncode != 0: @@ -242,12 +239,30 @@ def collect(paths, *, # collect line/branch coverage for file in data['files']: - if file['file'] != src_path: - continue + # ignore filtered sources + if sources is not None: + if not any( + os.path.abspath(file['file']) == os.path.abspath(s) + for s in sources): + continue + else: + # default to only cwd + if not everything and not os.path.commonpath([ + os.getcwd(), + os.path.abspath(file['file'])]) == os.getcwd(): + continue + + # simplify path + if os.path.commonpath([ + os.getcwd(), + os.path.abspath(file['file'])]) == os.getcwd(): + file_name = os.path.relpath(file['file']) + else: + file_name = os.path.abspath(file['file']) for func in file['functions']: func_name = func.get('name', '(inlined)') - # discard internal function (this includes injected test cases) + # discard internal functions (this includes injected test cases) if not everything: if func_name.startswith('__'): continue @@ -255,7 +270,7 @@ def collect(paths, *, # go ahead and add functions, later folding will merge this if # there are other hits on this line results.append(CoverageResult( - src_path, func_name, func['start_line'], + file_name, func_name, func['start_line'], func['execution_count'], 0, Frac(1 if func['execution_count'] > 0 else 0, 1), 0, @@ -271,7 +286,7 @@ def collect(paths, *, # go ahead and add lines, later folding will merge this if # there are other hits on this line results.append(CoverageResult( - src_path, func_name, line['line_number'], + file_name, func_name, line['line_number'], 0, line['count'], 0, Frac(1 if line['count'] > 0 else 0, 1), @@ -519,31 +534,25 @@ def table(Result, results, diff_results=None, *, line[-1])) -def annotate(Result, results, paths, *, +def annotate(Result, results, *, annotate=False, lines=False, branches=False, - build_dir=None, **args): # if neither branches/lines specified, color both if annotate and not lines and not branches: lines, branches = True, True - for path in paths: - # map to source file - src_path = re.sub('\.t\.a\.gcda$', '.c', path) - if build_dir: - src_path = re.sub('%s/*' % re.escape(build_dir), '', - src_path) - + for path in co.OrderedDict.fromkeys(r.file for r in results).keys(): # flatten to line info results = fold(Result, results, by=['file', 'line']) - table = {r.line: r for r in results if r.file == src_path} + table = {r.line: r for r in results if r.file == path} # calculate spans to show if not annotate: spans = [] last = None + func = None for line, r in sorted(table.items()): if ((lines and int(r.hits) == 0) or (branches and r.branches.a < r.branches.b)): @@ -553,27 +562,29 @@ def annotate(Result, results, paths, *, line+1+args['context']) else: if last is not None: - spans.append(last) + spans.append((last, func)) last = range( line-args['context'], line+1+args['context']) + func = r.function if last is not None: - spans.append(last) + spans.append((last, func)) - with open(src_path) as f: + with open(path) as f: skipped = False for i, line in enumerate(f): # skip lines not in spans? - if not annotate and not any(i+1 in s for s in spans): + if not annotate and not any(i+1 in s for s, _ in spans): skipped = True continue if skipped: skipped = False - print('%s@@ %s:%d @@%s' % ( + print('%s@@ %s:%d: %s @@%s' % ( '\x1b[36m' if args['color'] else '', - src_path, + path, i+1, + next(iter(f for _, f in spans)), '\x1b[m' if args['color'] else '')) # build line @@ -659,12 +670,14 @@ def main(gcda_paths, *, # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, CoverageResult._by + writer = csv.DictWriter(f, + (by if by is not None else CoverageResult._by) + ['coverage_'+k for k in CoverageResult._fields]) writer.writeheader() for r in results: writer.writerow( - {k: getattr(r, k) for k in CoverageResult._by} + {k: getattr(r, k) + for k in (by if by is not None else CoverageResult._by)} | {'coverage_'+k: getattr(r, k) for k in CoverageResult._fields}) @@ -698,8 +711,7 @@ def main(gcda_paths, *, or args.get('lines') or args.get('branches')): # annotate sources - annotate(CoverageResult, results, paths, - **args) + annotate(CoverageResult, results, **args) else: # print table table(CoverageResult, results, @@ -724,7 +736,8 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Find coverage info after running tests.") + description="Find coverage info after running tests.", + allow_abbrev=False) parser.add_argument( 'gcda_paths', nargs='*', @@ -791,15 +804,21 @@ if __name__ == "__main__": action='store_true', help="Only show the total.") parser.add_argument( - '-A', '--everything', + '-F', '--source', + dest='sources', + action='append', + help="Only consider definitions in this file. Defaults to anything " + "in the current directory.") + parser.add_argument( + '--everything', action='store_true', help="Include builtin and libc specific symbols.") parser.add_argument( - '-H', '--hits', + '--hits', action='store_true', help="Show total hits instead of coverage.") parser.add_argument( - '-l', '--annotate', + '-A', '--annotate', action='store_true', help="Show source files annotated with coverage info.") parser.add_argument( @@ -814,7 +833,7 @@ if __name__ == "__main__": '-c', '--context', type=lambda x: int(x, 0), default=3, - help="Show a additional lines of context. Defaults to 3.") + help="Show n additional lines of context. Defaults to 3.") parser.add_argument( '-W', '--width', type=lambda x: int(x, 0), @@ -838,10 +857,6 @@ if __name__ == "__main__": default=GCOV_TOOL, type=lambda x: x.split(), help="Path to the gcov tool to use. Defaults to %r." % GCOV_TOOL) - parser.add_argument( - '--build-dir', - help="Specify the relative build directory. Used to map object files " - "to the correct source files.") sys.exit(main(**{k: v for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) diff --git a/scripts/data.py b/scripts/data.py index 05ef8681..f95540f3 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -5,7 +5,7 @@ # by Linux's Bloat-O-Meter. # # Example: -# ./scripts/data.py lfs.o lfs_util.o -S +# ./scripts/data.py lfs.o lfs_util.o -Ssize # # Copyright (c) 2022, The littlefs authors. # Copyright (c) 2020, Arm Limited. All rights reserved. @@ -14,6 +14,7 @@ import collections as co import csv +import difflib import glob import itertools as it import math as m @@ -25,7 +26,8 @@ import subprocess as sp OBJ_PATHS = ['*.o'] NM_TOOL = ['nm'] -TYPE = 'dDbB' +NM_TYPES = 'dDbB' +OBJDUMP_TOOL = ['objdump'] # integer fields @@ -135,21 +137,32 @@ def openio(path, mode='r'): def collect(paths, *, nm_tool=NM_TOOL, - type=TYPE, - build_dir=None, + nm_types=NM_TYPES, + objdump_tool=OBJDUMP_TOOL, + sources=None, everything=False, **args): - results = [] - pattern = re.compile( + size_pattern = re.compile( '^(?P[0-9a-fA-F]+)' + - ' (?P[%s])' % re.escape(type) + + ' (?P[%s])' % re.escape(nm_types) + ' (?P.+?)$') + line_pattern = re.compile( + '^\s+(?P[0-9]+)\s+' + '(?:(?P[0-9]+)\s+)?' + '.*\s+' + '(?P[^\s]+)$') + info_pattern = re.compile( + '^(?:.*(?PDW_TAG_[a-z_]+).*' + '|^.*DW_AT_name.*:\s*(?P[^:\s]+)\s*' + '|^.*DW_AT_decl_file.*:\s*(?P[0-9]+)\s*)$') + + results = [] for path in paths: - # map to source file - src_path = re.sub('\.o$', '.c', path) - if build_dir: - src_path = re.sub('%s/*' % re.escape(build_dir), '', - src_path) + # guess the source, if we have debug-info we'll replace this later + file = re.sub('(\.o)?$', '.c', path, 1) + + # find symbol sizes + results_ = [] # note nm-tool may contain extra args cmd = nm_tool + ['--size-sort', path] if args.get('verbose'): @@ -158,21 +171,18 @@ def collect(paths, *, stdout=sp.PIPE, stderr=sp.PIPE if not args.get('verbose') else None, universal_newlines=True, - errors='replace') + errors='replace', + close_fds=False) for line in proc.stdout: - m = pattern.match(line) + m = size_pattern.match(line) if m: func = m.group('func') # discard internal functions if not everything and func.startswith('__'): continue - # discard .8449 suffixes created by optimizer - func = re.sub('\.[0-9]+', '', func) - - results.append(DataResult( - src_path, func, + results_.append(DataResult( + file, func, int(m.group('size'), 16))) - proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -180,6 +190,121 @@ def collect(paths, *, sys.stdout.write(line) sys.exit(-1) + + # try to figure out the source file if we have debug-info + dirs = {} + files = {} + # note objdump-tool may contain extra args + cmd = objdump_tool + ['--dwarf=rawline', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + for line in proc.stdout: + # note that files contain references to dirs, which we + # dereference as soon as we see them as each file table follows a + # dir table + m = line_pattern.match(line) + if m: + if not m.group('dir'): + # found a directory entry + dirs[int(m.group('no'))] = m.group('path') + else: + # found a file entry + dir = int(m.group('dir')) + if dir in dirs: + files[int(m.group('no'))] = os.path.join( + dirs[dir], + m.group('path')) + else: + files[int(m.group('no'))] = m.group('path') + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + # do nothing on error, we don't need objdump to work, source files + # may just be inaccurate + pass + + defs = {} + is_func = False + f_name = None + f_file = None + # note objdump-tool may contain extra args + cmd = objdump_tool + ['--dwarf=info', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + for line in proc.stdout: + # state machine here to find definitions + m = info_pattern.match(line) + if m: + if m.group('tag'): + if is_func: + defs[f_name] = files.get(f_file, '?') + is_func = (m.group('tag') == 'DW_TAG_subprogram') + elif m.group('name'): + f_name = m.group('name') + elif m.group('file'): + f_file = int(m.group('file')) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + # do nothing on error, we don't need objdump to work, source files + # may just be inaccurate + pass + + for r in results_: + # find best matching debug symbol, this may be slightly different + # due to optimizations + if defs: + # exact match? avoid difflib if we can for speed + if r.function in defs: + file = defs[r.function] + else: + _, file = max( + defs.items(), + key=lambda d: difflib.SequenceMatcher(None, + d[0], + r.function, False).ratio()) + else: + file = r.file + + # ignore filtered sources + if sources is not None: + if not any( + os.path.abspath(file) == os.path.abspath(s) + for s in sources): + continue + else: + # default to only cwd + if not everything and not os.path.commonpath([ + os.getcwd(), + os.path.abspath(file)]) == os.getcwd(): + continue + + # simplify path + if os.path.commonpath([ + os.getcwd(), + os.path.abspath(file)]) == os.getcwd(): + file = os.path.relpath(file) + else: + file = os.path.abspath(file) + + results.append(DataResult(file, r.function, r.size)) + return results @@ -437,7 +562,7 @@ def main(obj_paths, *, paths.append(path) if not paths: - print("error: no .obj files found in %r?" % obj_paths) + print("error: no .o files found in %r?" % obj_paths) sys.exit(-1) results = collect(paths, **args) @@ -469,13 +594,16 @@ def main(obj_paths, *, # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, DataResult._by + writer = csv.DictWriter(f, + (by if by is not None else DataResult._by) + ['data_'+k for k in DataResult._fields]) writer.writeheader() for r in results: writer.writerow( - {k: getattr(r, k) for k in DataResult._by} - | {'data_'+k: getattr(r, k) for k in DataResult._fields}) + {k: getattr(r, k) + for k in (by if by is not None else DataResult._by)} + | {'data_'+k: getattr(r, k) + for k in DataResult._fields}) # find previous results? if args.get('diff'): @@ -512,7 +640,8 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Find data size at the function level.") + description="Find data size at the function level.", + allow_abbrev=False) parser.add_argument( 'obj_paths', nargs='*', @@ -579,23 +708,30 @@ if __name__ == "__main__": action='store_true', help="Only show the total.") parser.add_argument( - '-A', '--everything', + '-F', '--source', + dest='sources', + action='append', + help="Only consider definitions in this file. Defaults to anything " + "in the current directory.") + parser.add_argument( + '--everything', action='store_true', help="Include builtin and libc specific symbols.") parser.add_argument( - '--type', - default=TYPE, + '--nm-types', + default=NM_TYPES, help="Type of symbols to report, this uses the same single-character " - "type-names emitted by nm. Defaults to %r." % TYPE) + "type-names emitted by nm. Defaults to %r." % NM_TYPES) parser.add_argument( '--nm-tool', type=lambda x: x.split(), default=NM_TOOL, help="Path to the nm tool to use. Defaults to %r." % NM_TOOL) parser.add_argument( - '--build-dir', - help="Specify the relative build directory. Used to map object files " - "to the correct source files.") + '--objdump-tool', + type=lambda x: x.split(), + default=OBJDUMP_TOOL, + help="Path to the objdump tool to use. Defaults to %r." % OBJDUMP_TOOL) sys.exit(main(**{k: v for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) diff --git a/scripts/perf.py b/scripts/perf.py new file mode 100755 index 00000000..3eb3dbc2 --- /dev/null +++ b/scripts/perf.py @@ -0,0 +1,1263 @@ +#!/usr/bin/env python3 +# +# Script to aggregate and report Linux perf results. +# +# Example: +# ./scripts/perf.py -R -obench.perf ./runners/bench_runner +# ./scripts/perf.py bench.perf -Flfs.c -Flfs_util.c -Scycles +# +# Copyright (c) 2022, The littlefs authors. +# SPDX-License-Identifier: BSD-3-Clause +# + +import bisect +import collections as co +import csv +import errno +import fcntl +import functools as ft +import glob +import itertools as it +import math as m +import multiprocessing as mp +import os +import re +import shlex +import shutil +import subprocess as sp +import tempfile +import zipfile + + +PERF_PATHS = ['*.perf'] +PERF_TOOL = ['perf'] +PERF_EVENTS = 'cycles,branch-misses,branches,cache-misses,cache-references' +PERF_FREQ = 100 +OBJDUMP_TOOL = ['objdump'] +THRESHOLD = (0.5, 0.85) + + +# integer fields +class Int(co.namedtuple('Int', 'x')): + __slots__ = () + def __new__(cls, x=0): + if isinstance(x, Int): + return x + if isinstance(x, str): + try: + x = int(x, 0) + except ValueError: + # also accept +-∞ and +-inf + if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): + x = m.inf + elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): + x = -m.inf + else: + raise + assert isinstance(x, int) or m.isinf(x), x + return super().__new__(cls, x) + + def __str__(self): + if self.x == m.inf: + return '∞' + elif self.x == -m.inf: + return '-∞' + else: + return str(self.x) + + def __int__(self): + assert not m.isinf(self.x) + return self.x + + def __float__(self): + return float(self.x) + + none = '%7s' % '-' + def table(self): + return '%7s' % (self,) + + diff_none = '%7s' % '-' + diff_table = table + + def diff_diff(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + diff = new - old + if diff == +m.inf: + return '%7s' % '+∞' + elif diff == -m.inf: + return '%7s' % '-∞' + else: + return '%+7d' % diff + + def ratio(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + if m.isinf(new) and m.isinf(old): + return 0.0 + elif m.isinf(new): + return +m.inf + elif m.isinf(old): + return -m.inf + elif not old and not new: + return 0.0 + elif not old: + return 1.0 + else: + return (new-old) / old + + def __add__(self, other): + return self.__class__(self.x + other.x) + + def __sub__(self, other): + return self.__class__(self.x - other.x) + + def __mul__(self, other): + return self.__class__(self.x * other.x) + +# perf results +class PerfResult(co.namedtuple('PerfResult', [ + 'file', 'function', 'line', + 'self_cycles', + 'self_bmisses', 'self_branches', + 'self_cmisses', 'self_caches', + 'cycles', + 'bmisses', 'branches', + 'cmisses', 'caches', + 'children', 'parents'])): + _by = ['file', 'function', 'line'] + _fields = [ + 'self_cycles', + 'self_bmisses', 'self_branches', + 'self_cmisses', 'self_caches', + 'cycles', + 'bmisses', 'branches', + 'cmisses', 'caches'] + _types = { + 'self_cycles': Int, + 'self_bmisses': Int, 'self_branches': Int, + 'self_cmisses': Int, 'self_caches': Int, + 'cycles': Int, + 'bmisses': Int, 'branches': Int, + 'cmisses': Int, 'caches': Int} + + __slots__ = () + def __new__(cls, file='', function='', line=0, + self_cycles=0, + self_bmisses=0, self_branches=0, + self_cmisses=0, self_caches=0, + cycles=0, + bmisses=0, branches=0, + cmisses=0, caches=0, + children=set(), parents=set()): + return super().__new__(cls, file, function, int(Int(line)), + Int(self_cycles), + Int(self_bmisses), Int(self_branches), + Int(self_cmisses), Int(self_caches), + Int(cycles), + Int(bmisses), Int(branches), + Int(cmisses), Int(caches), + children, parents) + + def __add__(self, other): + return PerfResult(self.file, self.function, self.line, + self.self_cycles + other.self_cycles, + self.self_bmisses + other.self_bmisses, + self.self_branches + other.self_branches, + self.self_cmisses + other.self_cmisses, + self.self_caches + other.self_caches, + self.cycles + other.cycles, + self.bmisses + other.bmisses, + self.branches + other.branches, + self.cmisses + other.cmisses, + self.caches + other.caches, + self.children | other.children, + self.parents | other.parents) + + +def openio(path, mode='r'): + if path == '-': + if mode == 'r': + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) + +# run perf as a subprocess, storing measurements into a zip file +def record(command, *, + output=None, + perf_freq=PERF_FREQ, + perf_period=None, + perf_events=PERF_EVENTS, + perf_tool=PERF_TOOL, + **args): + if not command: + print('error: no command specified?') + sys.exit(-1) + + if not output: + print('error: no output file specified?') + sys.exit(-1) + + # create a temporary file for perf to write to, as far as I can tell + # this is strictly needed because perf's pipe-mode only works with stdout + with tempfile.NamedTemporaryFile('rb') as f: + # figure out our perf invocation + perf = perf_tool + list(filter(None, [ + 'record', + '-F%s' % perf_freq + if perf_freq is not None + and perf_period is None else None, + '-c%s' % perf_period + if perf_period is not None else None, + '-B', + '-g', + '--all-user', + '-e%s' % perf_events, + '-o%s' % f.name])) + + # run our command + try: + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in perf + command)) + err = sp.call(perf + command, close_fds=False) + + except KeyboardInterrupt: + err = errno.EOWNERDEAD + + # synchronize access + z = os.open(output, os.O_RDWR | os.O_CREAT) + fcntl.flock(z, fcntl.LOCK_EX) + + # copy measurements into our zip file + with os.fdopen(z, 'r+b') as z: + with zipfile.ZipFile(z, 'a', + compression=zipfile.ZIP_DEFLATED, + compresslevel=1) as z: + with z.open('perf.%d' % os.getpid(), 'w') as g: + shutil.copyfileobj(f, g) + + # forward the return code + return err + + +def collect_decompressed(path, *, + perf_tool=PERF_TOOL, + everything=False, + depth=0, + **args): + sample_pattern = re.compile( + '(?P\w+)' + '\s+(?P\w+)' + '\s+(?P