From 56a990336b1d214ecc1ba51ec7e9f3e764714f42 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 16 Apr 2022 13:32:35 -0500 Subject: [PATCH] Created new test_runner.c and test_.py This is to try a different design for testing, the goals are to make the test infrastructure a bit simpler, with clear stages for building and running, and faster, by avoiding rebuilding lfs.c n-times. --- Makefile | 33 +++++- runners/test_runner.c | 178 ++++++++++++++++++++++++++++ runners/test_runner.h | 26 ++++ scripts/test_.py | 269 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 505 insertions(+), 1 deletion(-) create mode 100644 runners/test_runner.c create mode 100644 runners/test_runner.h create mode 100755 scripts/test_.py diff --git a/Makefile b/Makefile index 7cc59f8a..9d3b1444 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ override BUILDDIR := $(BUILDDIR)/ $(if $(findstring n,$(MAKEFLAGS)),, $(shell mkdir -p \ $(BUILDDIR) \ $(BUILDDIR)bd \ + $(BUILDDIR)runners \ $(BUILDDIR)tests)) endif @@ -25,12 +26,19 @@ NM ?= nm OBJDUMP ?= objdump LCOV ?= lcov -SRC ?= $(wildcard *.c) +SRC ?= $(filter-out $(wildcard *.*.c),$(wildcard *.c)) OBJ := $(SRC:%.c=$(BUILDDIR)%.o) DEP := $(SRC:%.c=$(BUILDDIR)%.d) ASM := $(SRC:%.c=$(BUILDDIR)%.s) CGI := $(SRC:%.c=$(BUILDDIR)%.ci) +TESTS ?= $(wildcard tests_/*.toml) +TEST_TSRC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) \ + $(SRC:%.c=$(BUILDDIR)%.t.c) \ + $(BUILDDIR)runners/test_runner.t.c +TEST_TASRC := $(TEST_TSRC:%.t.c=%.t.a.c) +TEST_TAOBJ := $(TEST_TASRC:%.t.a.c=%.t.a.o) + ifdef DEBUG override CFLAGS += -O0 else @@ -103,6 +111,9 @@ test: test%: tests/test$$(firstword $$(subst \#, ,%)).toml ./scripts/test.py $@ $(TESTFLAGS) +.PHONY: test_ +test_: $(BUILDDIR)runners/test_runner + .PHONY: code code: $(OBJ) ./scripts/code.py $^ -S $(CODEFLAGS) @@ -131,6 +142,7 @@ summary: $(BUILDDIR)lfs.csv # rules -include $(DEP) .SUFFIXES: +.SECONDARY: $(BUILDDIR)lfs: $(OBJ) $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ @@ -147,6 +159,9 @@ $(BUILDDIR)lfs.csv: $(OBJ) $(CGI) ./scripts/coverage.py $(BUILDDIR)tests/*.toml.info \ -q -m $@ $(COVERAGEFLAGS) -o $@) +$(BUILDDIR)runners/test_runner: $(TEST_TAOBJ) + $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ + $(BUILDDIR)%.o: %.c $(CC) -c -MMD $(CFLAGS) $< -o $@ @@ -160,14 +175,30 @@ $(BUILDDIR)%.s: %.c $(BUILDDIR)%.ci: %.c | $(BUILDDIR)%.o $(CC) -c -MMD -fcallgraph-info=su $(CFLAGS) $< -o $| +$(BUILDDIR)%.a.c: %.c + ./scripts/explode_asserts.py $< -o $@ + +$(BUILDDIR)%.a.c: $(BUILDDIR)%.c + ./scripts/explode_asserts.py $< -o $@ + +$(BUILDDIR)%.t.c: %.toml + ./scripts/test_.py -c $< -o $@ + +$(BUILDDIR)%.t.c: %.c $(TESTS) + ./scripts/test_.py -c $(TESTS) -s $< -o $@ + # clean everything .PHONY: clean clean: rm -f $(BUILDDIR)lfs rm -f $(BUILDDIR)lfs.a rm -f $(BUILDDIR)lfs.csv + rm -f $(BUILDDIR)runners/test_runner rm -f $(OBJ) rm -f $(CGI) rm -f $(DEP) rm -f $(ASM) rm -f $(BUILDDIR)tests/*.toml.* + rm -f $(TEST_TSRC) + rm -f $(TEST_TASRC) + rm -f $(TEST_TAOBJ) diff --git a/runners/test_runner.c b/runners/test_runner.c new file mode 100644 index 00000000..758e1fe3 --- /dev/null +++ b/runners/test_runner.c @@ -0,0 +1,178 @@ + +#include "runners/test_runner.h" +#include + + +// disk geometries +struct test_geometry { + const char *name; + lfs_size_t read_size; + lfs_size_t prog_size; + lfs_size_t erase_size; + lfs_size_t erase_count; +}; + +const struct test_geometry test_geometries[] = { + // Made up geometries that works well for testing + {"small", 16, 16, 512, (1024*1024)/512}, + {"medium", 16, 16, 4096, (1024*1024)/4096}, + {"big", 16, 16, 32*1024, (1024*1024)/(32*1024)}, + // EEPROM/NVRAM + {"eeprom", 1, 1, 512, (1024*1024)/512}, + // SD/eMMC + {"emmc", 512, 512, 512, (1024*1024)/512}, + // NOR flash + {"nor", 1, 1, 4096, (1024*1024)/4096}, + // NAND flash + {"nand", 4096, 4096, 32*1024, (1024*1024)/(32*1024)}, +}; +const size_t test_geometry_count = ( + sizeof(test_geometries) / sizeof(test_geometries[0])); + + +// option handling +enum opt_flags { + OPT_HELP = 'h', + OPT_LIST = 'l', + OPT_LIST_PATHS = 1, + OPT_LIST_DEFINES = 2, + OPT_LIST_GEOMETRIES = 3, +}; + +const struct option long_opts[] = { + {"help", no_argument, NULL, OPT_HELP}, + {"list", no_argument, NULL, OPT_LIST}, + {"list-paths", no_argument, NULL, OPT_LIST_PATHS}, + {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, + {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, + {NULL, 0, NULL, 0}, +}; + +const char *const help_text[] = { + "Show this help message.", + "List test cases.", + "List the path for each test case.", + "List the defines for each test permutation.", + "List the disk geometries used for testing.", +}; + +int main(int argc, char **argv) { + bool list = false; + bool list_paths = false; + bool list_defines = false; + bool list_geometries = false; + + // parse options + while (true) { + int index = 0; + int c = getopt_long(argc, argv, "hl", long_opts, &index); + switch (c) { + // generate help message + case OPT_HELP: { + printf("usage: %s [options] [test_case]\n", argv[0]); + printf("\n"); + + printf("options:\n"); + size_t i = 0; + while (long_opts[i].name) { + size_t indent; + if (long_opts[i].val >= '0' && long_opts[i].val < 'z') { + printf(" -%c, --%-16s", + long_opts[i].val, + long_opts[i].name); + indent = 8+strlen(long_opts[i].name); + } else { + printf(" --%-20s", long_opts[i].name); + indent = 4+strlen(long_opts[i].name); + } + + // a quick, hacky, byte-level method for text wrapping + size_t len = strlen(help_text[i]); + size_t j = 0; + if (indent < 24) { + printf("%.80s\n", &help_text[i][j]); + j += 80; + } + + while (j < len) { + printf("%24s%.80s\n", "", &help_text[i][j]); + j += 80; + } + + i += 1; + } + + printf("\n"); + exit(0); + } + // list flags + case OPT_LIST: + list = true; + break; + case OPT_LIST_PATHS: + list_paths = true; + break; + case OPT_LIST_DEFINES: + list_defines = true; + break; + case OPT_LIST_GEOMETRIES: + list_geometries = true; + break; + // done parsing + case -1: + goto getopt_done; + // unknown arg, getopt prints a message for us + default: + exit(-1); + } + } +getopt_done: + + // what do we need to do? + if (list) { + printf("%-36s %-12s %-12s %7s %7s\n", + "id", "suite", "case", "type", "perms"); + for (size_t i = 0; i < test_suite_count; i++) { + for (size_t j = 0; j < test_suites[i]->case_count; j++) { + printf("%-36s %-12s %-12s %7s %7d\n", + test_suites[i]->cases[j]->id, + test_suites[i]->name, + test_suites[i]->cases[j]->name, + "n", // TODO + test_suites[i]->cases[j]->permutations); + } + } + + } else if (list_paths) { + printf("%-36s %-36s\n", "id", "path"); + for (size_t i = 0; i < test_suite_count; i++) { + for (size_t j = 0; j < test_suites[i]->case_count; j++) { + printf("%-36s %-36s\n", + test_suites[i]->cases[j]->id, + test_suites[i]->cases[j]->path); + } + } + } else if (list_defines) { + // TODO + } else if (list_geometries) { + printf("%-12s %7s %7s %7s %7s %7s\n", + "name", "read", "prog", "erase", "count", "size"); + for (size_t i = 0; i < test_geometry_count; i++) { + printf("%-12s %7d %7d %7d %7d %7d\n", + test_geometries[i].name, + test_geometries[i].read_size, + test_geometries[i].prog_size, + test_geometries[i].erase_size, + test_geometries[i].erase_count, + test_geometries[i].erase_size + * test_geometries[i].erase_count); + } + } else { + printf("remaining: "); + for (int i = optind; i < argc; i++) { + printf("%s ", argv[i]); + } + printf("\n"); + } +} + diff --git a/runners/test_runner.h b/runners/test_runner.h new file mode 100644 index 00000000..c830e36b --- /dev/null +++ b/runners/test_runner.h @@ -0,0 +1,26 @@ +#ifndef TEST_RUNNER_H +#define TEST_RUNNER_H + +#include "lfs.h" + + +struct test_case { + const char *id; + const char *name; + const char *path; + uint32_t permutations; + void (*run)(struct lfs_config *cfg, uint32_t perm); +}; + +struct test_suite { + const char *id; + const char *name; + const char *path; + const struct test_case *const *cases; + size_t case_count; +}; + +extern const struct test_suite *test_suites[]; +extern const size_t test_suite_count; + +#endif diff --git a/scripts/test_.py b/scripts/test_.py new file mode 100755 index 00000000..c411d0ca --- /dev/null +++ b/scripts/test_.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +# +# Script to compile and runs tests. +# + +import glob +import itertools as it +import os +import re +import shutil +import toml + +TEST_PATHS = ['tests_'] + +SUITE_PROLOGUE = """ +//////// AUTOGENERATED //////// +#include "runners/test_runner.h" +#include +""" +# TODO handle indention implicity? +# TODO change cfg to be not by value? maybe not? +CASE_PROLOGUE = """ + lfs_t lfs; + struct lfs_config cfg = *cfg_; +""" +CASE_EPILOGUE = """ +""" + + +# TODO +# def testpath(path): +# def testcase(path): +# def testperm(path): + +def testsuite(path): + name = os.path.basename(path) + if name.endswith('.toml'): + name = name[:-len('.toml')] + return name + +# TODO move this out in other files +def openio(path, mode='r'): + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) + +class TestCase: + # create a TestCase object from a config + def __init__(self, config, args={}): + self.name = config.pop('name') + self.path = config.pop('path') + self.suite = config.pop('suite') + self.lineno = config.pop('lineno', None) + self.code = config.pop('code') + self.code_lineno = config.pop('code_lineno', None) + + self.permutations = 1 + + for k in config.keys(): + print('warning: in %s, found unused key %r' % (self.id(), k), + file=sys.stderr) + + def id(self): + return '%s#%s' % (self.suite, self.name) + + +class TestSuite: + # create a TestSuite object from a toml file + def __init__(self, path, args={}): + self.name = testsuite(path) + self.path = path + + # load toml file and parse test cases + with open(self.path) as f: + # load tests + config = toml.load(f) + + # find line numbers + f.seek(0) + case_linenos = [] + code_linenos = [] + for i, line in enumerate(f): + match = re.match( + '(?P\[\s*cases\s*\.\s*(?P\w+)\s*\])' + + '|(?Pcode\s*=\s*(?:\'\'\'|"""))', + line) + if match and match.group('case'): + case_linenos.append((i+1, match.group('name'))) + elif match and match.group('code'): + code_linenos.append(i+2) + + # sort in case toml parsing did not retain order + case_linenos.sort() + + cases = config.pop('cases', []) + for (lineno, name), (nlineno, _) in it.zip_longest( + case_linenos, case_linenos[1:], + fillvalue=(float('inf'), None)): + code_lineno = min( + (l for l in code_linenos if l >= lineno and l < nlineno), + default=None) + cases[name]['lineno'] = lineno + cases[name]['code_lineno'] = code_lineno + + self.code = config.pop('code', None) + self.code_lineno = min( + (l for l in code_linenos + if not case_linenos or l < case_linenos[0][0]), + default=None) + + self.cases = [] + for name, case in sorted(cases.items(), + key=lambda c: c[1].get('lineno')): + self.cases.append(TestCase(config={ + 'name': name, + 'path': path + (':%d' % case['lineno'] + if 'lineno' in case else ''), + 'suite': self.name, + **case})) + + for k in config.keys(): + print('warning: in %s, found unused key %r' % (self.id(), k), + file=sys.stderr) + + def id(self): + return self.name + + + +def compile(**args): + # find .toml files + paths = [] + for path in args['test_paths']: + if os.path.isdir(path): + path = path + '/*.toml' + + for path in glob.glob(path): + paths.append(path) + + if not paths: + print('no test suites found in %r?' % args['test_paths']) + sys.exit(-1) + + if not args.get('source'): + if len(paths) > 1: + print('more than one test suite for compilation? (%r)' + % args['test_paths']) + sys.exit(-1) + + # write out a test suite + suite = TestSuite(paths[0]) + if 'output' in args: + with openio(args['output'], 'w') as f: + f.write(SUITE_PROLOGUE) + f.write('\n') + if suite.code is not None: + if suite.code_lineno is not None: + f.write('#line %d "%s"\n' + % (suite.code_lineno, suite.path)) + f.write(suite.code) + f.write('\n') + + # create test functions and case structs + for case in suite.cases: + f.write('void __test__%s__%s(' + '__attribute__((unused)) struct lfs_config *cfg_, ' + '__attribute__((unused)) uint32_t perm) {\n' + % (suite.name, case.name)) + f.write(CASE_PROLOGUE) + f.write('\n') + f.write(4*' '+'// test case %s\n' % case.id()) + if case.code_lineno is not None: + f.write(4*' '+'#line %d "%s"\n' + % (case.code_lineno, suite.path)) + f.write(case.code) + f.write('\n') + f.write(CASE_EPILOGUE) + f.write('}\n') + f.write('\n') + + f.write('const struct test_case __test__%s__%s__case = {\n' + % (suite.name, case.name)) + f.write(4*' '+'.id = "%s",\n' % case.id()) + f.write(4*' '+'.name = "%s",\n' % case.name) + f.write(4*' '+'.path = "%s",\n' % case.path) + f.write(4*' '+'.permutations = %d,\n' % case.permutations) + f.write(4*' '+'.run = __test__%s__%s,\n' + % (suite.name, case.name)) + f.write('};\n') + f.write('\n') + + # create suite struct + f.write('const struct test_suite __test__%s__suite = {\n' + % (suite.name)) + f.write(4*' '+'.id = "%s",\n' % suite.id()) + f.write(4*' '+'.name = "%s",\n' % suite.name) + f.write(4*' '+'.path = "%s",\n' % suite.path) + f.write(4*' '+'.cases = (const struct test_case *const []){\n') + for case in suite.cases: + f.write(8*' '+'&__test__%s__%s__case,\n' + % (suite.name, case.name)) + f.write(4*' '+'},\n') + f.write(4*' '+'.case_count = %d,\n' % len(suite.cases)) + f.write('};\n') + f.write('\n') + + else: + # load all suites + suites = [TestSuite(path) for path in paths] + suites.sort(key=lambda s: s.name) + + # write out a test source + if 'output' in args: + with openio(args['output'], 'w') as f: + f.write(SUITE_PROLOGUE) + f.write('\n') + f.write('#line 1 "%s"\n' % args['source']) + with open(args['source']) as sf: + shutil.copyfileobj(sf, f) + + # add suite info to test_runner.c + if args['source'] == 'runners/test_runner.c': + f.write('\n') + for suite in suites: + f.write('extern const struct test_suite ' + '__test__%s__suite;\n' % suite.name) + f.write('const struct test_suite *test_suites[] = {\n') + for suite in suites: + f.write(4*' '+'&__test__%s__suite,\n' % suite.name) + f.write('};\n') + f.write('const size_t test_suite_count = %d;\n' + % len(suites)) + +def run(**args): + pass + +def main(**args): + if args.get('compile'): + compile(**args) + else: + run(**args) + +if __name__ == "__main__": + import argparse + import sys + parser = argparse.ArgumentParser( + description="Build and run tests.") + # TODO document test case/perm specifier + parser.add_argument('test_paths', nargs='*', default=TEST_PATHS, + help="Description of test(s) to run. May be a directory, a path, or \ + test identifier. Defaults to all tests in %r." % TEST_PATHS) + # test flags + test_parser = parser.add_argument_group('test options') + # compilation flags + comp_parser = parser.add_argument_group('compilation options') + comp_parser.add_argument('-c', '--compile', action='store_true', + help="Compile a test suite or source file.") + comp_parser.add_argument('-s', '--source', + help="Source file to compile, possibly injecting internal tests.") + comp_parser.add_argument('-o', '--output', + help="Output file.") + # TODO apply this to other scripts? + sys.exit(main(**{k: v + for k, v in vars(parser.parse_args()).items() + if v is not None}))