diff --git a/Makefile b/Makefile index 943735bc..04fe8bdc 100644 --- a/Makefile +++ b/Makefile @@ -62,12 +62,15 @@ override DATAFLAGS += -v override STACKFLAGS += -v override STRUCTSFLAGS += -v override COVERAGEFLAGS += -v +override TESTFLAGS_ += -v +override TESTCFLAGS_ += -v endif ifdef EXEC -override TESTFLAGS += --exec="$(EXEC)" +override TESTFLAGS_ += --exec="$(EXEC)" endif ifdef COVERAGE override TESTFLAGS += --coverage +override TESTFLAGS_ += --coverage endif ifdef BUILDDIR override TESTFLAGS += --build-dir="$(BUILDDIR:/=)" @@ -85,6 +88,8 @@ endif ifneq ($(OBJDUMP),objdump) override STRUCTSFLAGS += --objdump-tool="$(OBJDUMP)" endif +# forward -j flag +override TESTFLAGS_ += $(filter -j%,$(MAKEFLAGS)) # commands @@ -115,6 +120,7 @@ test%: tests/test$$(firstword $$(subst \#, ,%)).toml .PHONY: test_ test_: $(BUILDDIR)runners/test_runner + ./scripts/test_.py --runner=$< $(TESTFLAGS_) .PHONY: code code: $(OBJ) @@ -185,10 +191,10 @@ $(BUILDDIR)%.a.c: $(BUILDDIR)%.c ./scripts/explode_asserts.py $< -o $@ $(BUILDDIR)%.t.c: %.toml - ./scripts/test_.py -c $< -o $@ + ./scripts/test_.py -c $< $(TESTCFLAGS_) -o $@ $(BUILDDIR)%.t.c: %.c $(TESTS) - ./scripts/test_.py -c $(TESTS) -s $< -o $@ + ./scripts/test_.py -c $(TESTS) -s $< $(TESTCFLAGS_) -o $@ # clean everything .PHONY: clean diff --git a/runners/test_runner.c b/runners/test_runner.c index 185fa8aa..1406574e 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -29,9 +29,7 @@ struct test_geometry { const struct test_geometry test_geometries[] = { // Made up geometry that works well for testing - TEST_GEOMETRY("small", 16, 16, 512, (1024*1024)/512), - TEST_GEOMETRY("medium", 16, 16, 4096, (1024*1024)/4096), - TEST_GEOMETRY("big", 16, 16, 32*1024, (1024*1024)/(32*1024)), + TEST_GEOMETRY("test", 16, 16, 512, (1024*1024)/512), // EEPROM/NVRAM TEST_GEOMETRY("eeprom", 1, 1, 512, (1024*1024)/512), // SD/eMMC @@ -585,7 +583,7 @@ const char *const help_text[] = { "Filter by geometry.", "Filter for normal tests. Can be combined.", "Filter for reentrant tests. Can be combined.", - "Filter for valgrind tests. Can be combined.", + "Filter for Valgrind tests. Can be combined.", "Skip the first n tests.", "Stop after n tests.", "Only run every n tests, calculated after --skip and --stop.", diff --git a/scripts/test_.py b/scripts/test_.py index e6458760..8d6e34ec 100755 --- a/scripts/test_.py +++ b/scripts/test_.py @@ -3,22 +3,30 @@ # Script to compile and runs tests. # +import collections as co +import errno import glob import itertools as it import math as m import os +import pty import re +import shlex import shutil +import subprocess as sp +import threading as th +import time import toml + TEST_PATHS = ['tests_'] +RUNNER_PATH = './runners/test_runner' SUITE_PROLOGUE = """ #include "runners/test_runner.h" #include """ CASE_PROLOGUE = """ -lfs_t lfs; """ CASE_EPILOGUE = """ """ @@ -194,13 +202,13 @@ class TestSuite: def id(self): return self.name - + def compile(**args): # find .toml files paths = [] - for path in args['test_paths']: + for path in args.get('test_paths', TEST_PATHS): if os.path.isdir(path): path = path + '/*.toml' @@ -429,26 +437,411 @@ def compile(**args): f.write('const size_t test_suite_count = %d;\n' % len(suites)) +def runner(**args): + cmd = args['runner'].copy() + # TODO multiple paths? + if 'test_paths' in args: + cmd.extend(args.get('test_paths')) + + if args.get('normal'): cmd.append('-n') + if args.get('reentrant'): cmd.append('-r') + if args.get('valgrind'): cmd.append('-V') + if args.get('geometry'): + cmd.append('-G%s' % args.get('geometry')) + if args.get('define'): + for define in args.get('define'): + cmd.append('-D%s' % define) + + return cmd + +def list_(**args): + cmd = runner(**args) + if args.get('summary'): cmd.append('--summary') + if args.get('list_suites'): cmd.append('--list-suites') + if args.get('list_cases'): cmd.append('--list-cases') + if args.get('list_paths'): cmd.append('--list-paths') + if args.get('list_defines'): cmd.append('--list-defines') + if args.get('list_geometries'): cmd.append('--list-geometries') + + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + sys.exit(sp.call(cmd)) + + +def find_cases(runner_, **args): + # first get suite/case/perm counts + cmd = runner_ + ['--list-cases'] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace') + expected_suite_perms = co.defaultdict(lambda: 0) + expected_case_perms = co.defaultdict(lambda: 0) + expected_perms = 0 + total_perms = 0 + pattern = re.compile( + '^(?P(?P[^#]+)#[^ #]+) +' + '[^ ]+ +[^ ]+ +[^ ]+ +' + '(?P[0-9]+)/(?P[0-9]+)$') + # skip the first line + next(proc.stdout) + for line in proc.stdout: + m = pattern.match(line) + if m: + filtered = int(m.group('filtered')) + expected_suite_perms[m.group('suite')] += filtered + expected_case_perms[m.group('id')] += filtered + expected_perms += filtered + total_perms += int(m.group('perms')) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return ( + expected_suite_perms, + expected_case_perms, + expected_perms, + total_perms) + +class TestFailure(Exception): + def __init__(self, id, returncode, stdout, assert_=None): + self.id = id + self.returncode = returncode + self.stdout = stdout + self.assert_ = assert_ + +def run_step(name, runner_, **args): + # get expected suite/case/perm counts + expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( + find_cases(runner_, **args)) + + # TODO persist/trace + # TODO valgrind/gdb/exec + passed_suite_perms = co.defaultdict(lambda: 0) + passed_case_perms = co.defaultdict(lambda: 0) + passed_perms = 0 + failures = [] + + pattern = re.compile('^(?:' + '(?Prunning|finished|skipped) ' + '(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)' + '|' '(?P[^:]+):(?P[0-9]+):(?Passert):' + ' *(?P.*)' ')$') + locals = th.local() + # TODO use process group instead of this set? + children = set() + + def run_runner(runner_): + nonlocal passed_suite_perms + nonlocal passed_case_perms + nonlocal passed_perms + nonlocal locals + + # run the tests! + cmd = runner_ + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + mpty, spty = pty.openpty() + proc = sp.Popen(cmd, stdout=spty, stderr=spty) + os.close(spty) + mpty = os.fdopen(mpty, 'r', 1) + children.add(proc) + + last_id = None + last_stdout = [] + last_assert = None + try: + while True: + # parse a line for state changes + try: + line = mpty.readline() + except OSError as e: + if e.errno == errno.EIO: + break + raise + if not line: + break + last_stdout.append(line) + if args.get('verbose'): + sys.stdout.write(line) + + m = pattern.match(line) + if m: + op = m.group('op') or m.group('op_') + if op == 'running': + locals.seen_perms += 1 + last_id = m.group('id') + last_stdout = [] + last_assert = None + elif op == 'finished': + passed_suite_perms[m.group('suite')] += 1 + passed_case_perms[m.group('case')] += 1 + passed_perms += 1 + elif op == 'skipped': + locals.seen_perms += 1 + elif op == 'assert': + last_assert = ( + m.group('path'), + int(m.group('lineno')), + m.group('message')) + # TODO why is kill _so_ much faster than terminate? + proc.kill() + except KeyboardInterrupt: + raise TestFailure(last_id, 1, last_stdout) + finally: + children.remove(proc) + mpty.close() + + proc.wait() + if proc.returncode != 0: + raise TestFailure( + last_id, + proc.returncode, + last_stdout, + last_assert) + + def run_job(runner, skip=None, every=None): + nonlocal failures + nonlocal locals + + while (skip or 0) < total_perms: + runner_ = runner.copy() + if skip is not None: + runner_.append('--skip=%d' % skip) + if every is not None: + runner_.append('--every=%d' % every) + + try: + # run the tests + locals.seen_perms = 0 + run_runner(runner_) + + except TestFailure as failure: + # race condition for multiple failures? + if failures and not args.get('keep_going'): + break + + failures.append(failure) + + if args.get('keep_going'): + # resume after failed test + skip = (skip or 0) + locals.seen_perms*(every or 1) + continue + else: + # stop other tests + for child in children: + # TODO why is kill _so_ much faster than terminate? + child.kill() + + break + + + # parallel jobs? + runners = [] + if 'jobs' in args: + for job in range(args['jobs']): + runners.append(th.Thread( + target=run_job, args=(runner_, job, args['jobs']))) + else: + runners.append(th.Thread( + target=run_job, args=(runner_, None, None))) + + for r in runners: + r.start() + + needs_newline = False + try: + while any(r.is_alive() for r in runners): + time.sleep(0.01) + + if not args.get('verbose'): + sys.stdout.write('\r\x1b[K' + 'running \x1b[%dm%s\x1b[m: ' + '%d/%d suites, %d/%d cases, %d/%d perms%s ' + % (32 if not failures else 31, + name, + sum(passed_suite_perms[k] == v + for k, v in expected_suite_perms.items()), + len(expected_suite_perms), + sum(passed_case_perms[k] == v + for k, v in expected_case_perms.items()), + len(expected_case_perms), + passed_perms, + expected_perms, + ', \x1b[31m%d/%d failures\x1b[m' + % (len(failures), expected_perms) + if failures else '')) + sys.stdout.flush() + needs_newline = True + finally: + if needs_newline: + print() + + for r in runners: + r.join() + + return ( + expected_perms, + passed_perms, + failures) + + def run(**args): - pass + start = time.time() + + runner_ = runner(**args) + print('using runner `%s`' + % ' '.join(shlex.quote(c) for c in runner_)) + expected_suite_perms, expected_case_perms, expected_perms, total_perms = ( + find_cases(runner_, **args)) + print('found %d suites, %d cases, %d/%d permutations' + % (len(expected_suite_perms), + len(expected_case_perms), + expected_perms, + total_perms)) + print() + + expected = 0 + passed = 0 + failures = [] + if args.get('by_suites'): + for type in ['normal', 'reentrant', 'valgrind']: + for suite in expected_suite_perms.keys(): + expected_, passed_, failures_ = run_step( + '%s %s' % (type, suite), + runner_ + ['--%s' % type, suite], + **args) + expected += expected_ + passed += passed_ + failures.extend(failures_) + if failures and not args.get('keep_going'): + break + if failures and not args.get('keep_going'): + break + elif args.get('by_cases'): + for type in ['normal', 'reentrant', 'valgrind']: + for case in expected_case_perms.keys(): + expected_, passed_, failures_ = run_step( + '%s %s' % (type, case), + runner_ + ['--%s' % type, case], + **args) + expected += expected_ + passed += passed_ + failures.extend(failures_) + if failures and not args.get('keep_going'): + break + if failures and not args.get('keep_going'): + break + else: + for type in ['normal', 'reentrant', 'valgrind']: + expected_, passed_, failures_ = run_step( + '%s tests' % type, + runner_ + ['--%s' % type], + **args) + expected += expected_ + passed += passed_ + failures.extend(failures_) + if failures and not args.get('keep_going'): + break + + # show summary + print() + print('\x1b[%dmdone\x1b[m: %d/%d passed, %d/%d failed, in %.2fs' + % (32 if not failures else 31, + passed, expected, len(failures), expected, + time.time()-start)) + print() + + # print each failure + # TODO get line, defines, path + for failure in failures: +# print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s failed' +# # TODO this should be the suite path and lineno +# % (failure.assert + + if failure.assert_ is not None: + path, lineno, message = failure.assert_ + print('\x1b[01m%s:%d:\x1b[01;31massert:\x1b[m %s' + % (path, lineno, message)) + with open(path) as f: + line = next(it.islice(f, lineno-1, None)).strip('\n') + print(line) + print() + + return 1 if failures else 0 + def main(**args): if args.get('compile'): compile(**args) + elif (args.get('summary') + or args.get('list_suites') + or args.get('list_cases') + or args.get('list_paths') + or args.get('list_defines') + or args.get('list_geometries')): + list_(**args) else: run(**args) + if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( description="Build and run tests.") # TODO document test case/perm specifier - parser.add_argument('test_paths', nargs='*', default=TEST_PATHS, - help="Description of test(s) to run. May be a directory, a path, or \ - test identifier. Defaults to all tests in %r." % TEST_PATHS) + parser.add_argument('test_paths', nargs='*', + help="Description of testis to run. May be a directory, path, or \ + test identifier. Defaults to %r." % TEST_PATHS) + parser.add_argument('-v', '--verbose', action='store_true', + help="Output commands that run behind the scenes.") # test flags test_parser = parser.add_argument_group('test options') + test_parser.add_argument('-Y', '--summary', action='store_true', + help="Show quick summary.") + test_parser.add_argument('-l', '--list-suites', action='store_true', + help="List test suites.") + test_parser.add_argument('-L', '--list-cases', action='store_true', + help="List test cases.") + test_parser.add_argument('--list-paths', action='store_true', + help="List the path for each test case.") + test_parser.add_argument('--list-defines', action='store_true', + help="List the defines for each test permutation.") + test_parser.add_argument('--list-geometries', action='store_true', + help="List the disk geometries used for testing.") + test_parser.add_argument('-D', '--define', action='append', + help="Override a test define.") + test_parser.add_argument('-G', '--geometry', + help="Filter by geometry.") + test_parser.add_argument('-n', '--normal', action='store_true', + help="Filter for normal tests. Can be combined.") + test_parser.add_argument('-r', '--reentrant', action='store_true', + help="Filter for reentrant tests. Can be combined.") + test_parser.add_argument('-V', '--valgrind', action='store_true', + help="Filter for Valgrind tests. Can be combined.") + test_parser.add_argument('-p', '--persist', + help="Persist the disk to this file.") + test_parser.add_argument('-t', '--trace', + help="Redirect trace output to this file.") + test_parser.add_argument('--runner', default=[RUNNER_PATH], + type=lambda x: x.split(), + help="Path to runner, defaults to %r" % RUNNER_PATH) + test_parser.add_argument('-j', '--jobs', nargs='?', type=int, + const=len(os.sched_getaffinity(0)), + help="Number of parallel runners to run.") + test_parser.add_argument('-k', '--keep-going', action='store_true', + help="Don't stop on first error.") + test_parser.add_argument('-b', '--by-suites', action='store_true', + help="Step through tests by suite.") + test_parser.add_argument('-B', '--by-cases', action='store_true', + help="Step through tests by case.") # compilation flags comp_parser = parser.add_argument_group('compilation options') comp_parser.add_argument('-c', '--compile', action='store_true', diff --git a/tests/test_files.toml b/tests/test_files.toml index 565e665b..54630546 100644 --- a/tests/test_files.toml +++ b/tests/test_files.toml @@ -65,7 +65,7 @@ code = ''' lfs_format(&lfs, &cfg) => 0; // write - lfs_mount(&lfs, &cfg) => 0; + lfs_mount(&lfs, &cfg) => 1; lfs_file_open(&lfs, &file, "avacado", LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; srand(1);