Github/littlefs
1
Fork 0
Code Actions Activity

Continued progress toward feature parity with new test-runner

Browse Source 
- Expanded test defines to allow for lists of configurations

  These are useful for changing multi-dimensional test configurations
  without leading to extremely large and less useful configuration
  combinations.

- Made warnings more visible durring test parsing

- Add lfs_testbd.h to implicit test includes

- Fixed issue with not closing files in ./scripts/explode_asserts.py

- Add `make test_runner` and `make test_list` build rules for
  convenience
This commit is contained in:
Christopher Haster
2022-05-01 11:06:34 -05:00
parent 5ee4b052ae
commit 4962829017
3 changed files with 94 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+9 -2
View File
@@ -119,9 +119,16 @@ test:
test%: tests/test$$(firstword $$(subst \#, ,%)).toml test%: tests/test$$(firstword $$(subst \#, ,%)).toml
./scripts/test.py $@ $(TESTFLAGS) ./scripts/test.py $@ $(TESTFLAGS)
.PHONY: test_runner
test_runner: $(BUILDDIR)runners/test_runner
.PHONY: test_ .PHONY: test_
test_: $(BUILDDIR)runners/test_runner test_: test_runner
./scripts/test_.py --runner=$< $(TESTFLAGS_) ./scripts/test_.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS_)
.PHONY: test_list
test_list: test_runner
./scripts/test_.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS_) -l
.PHONY: code .PHONY: code
code: $(OBJ) code: $(OBJ)
scripts/explode_asserts.py
+31 -23
View File
@@ -134,6 +134,15 @@ TYPE = {
} }
} }
def openio(path, mode='r'):
if path == '-':
if 'r' in mode:
return os.fdopen(os.dup(sys.stdin.fileno()), 'r')
else:
return os.fdopen(os.dup(sys.stdout.fileno()), 'w')
else:
return open(path, mode)
def mkdecls(outf, maxwidth=16): def mkdecls(outf, maxwidth=16):
outf.write("#include <stdio.h>\n") outf.write("#include <stdio.h>\n")
outf.write("#include <stdbool.h>\n") outf.write("#include <stdbool.h>\n")
@@ -341,32 +350,31 @@ def pstmt(p):
def main(args): def main(args):
inf = open(args.input, 'r') if args.input else sys.stdin with openio(args.input or '-', 'r') as inf:
outf = open(args.output, 'w') if args.output else sys.stdout with openio(args.output or '-', 'w') as outf:
lexemes = LEX.copy()
if args.pattern:
lexemes['assert'] = args.pattern
p = Parse(inf, lexemes)
lexemes = LEX.copy() # write extra verbose asserts
if args.pattern: mkdecls(outf, maxwidth=args.maxwidth)
lexemes['assert'] = args.pattern if args.input:
p = Parse(inf, lexemes) outf.write("#line %d \"%s\"\n" % (1, args.input))
# write extra verbose asserts # parse and write out stmt at a time
mkdecls(outf, maxwidth=args.maxwidth) try:
if args.input: while True:
outf.write("#line %d \"%s\"\n" % (1, args.input)) outf.write(pstmt(p))
if p.accept('sep'):
outf.write(p.m)
else:
break
except ParseFailure as f:
pass
# parse and write out stmt at a time for i in range(p.off, len(p.tokens)):
try: outf.write(p.tokens[i][1])
while True:
outf.write(pstmt(p))
if p.accept('sep'):
outf.write(p.m)
else:
break
except ParseFailure as f:
pass
for i in range(p.off, len(p.tokens)):
outf.write(p.tokens[i][1])
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
scripts/test_.py
+54 -42
View File
@@ -24,6 +24,7 @@ RUNNER_PATH = './runners/test_runner'
SUITE_PROLOGUE = """ SUITE_PROLOGUE = """
#include "runners/test_runner.h" #include "runners/test_runner.h"
#include "bd/lfs_testbd.h"
#include <stdio.h> #include <stdio.h>
""" """
CASE_PROLOGUE = """ CASE_PROLOGUE = """
@@ -78,20 +79,30 @@ class TestCase:
self.valgrind = config.pop('valgrind', self.valgrind = config.pop('valgrind',
config.pop('suite_valgrind', True)) config.pop('suite_valgrind', True))
# figure out defines and the number of resulting permutations # figure out defines and build possible permutations
self.defines = {} self.defines = set()
for k, v in ( self.permutations = []
config.pop('suite_defines', {})
| config.pop('defines', {})).items():
if not isinstance(v, list):
v = [v]
self.defines[k] = v suite_defines = config.pop('suite_defines', {})
if not isinstance(suite_defines, list):
suite_defines = [suite_defines]
defines = config.pop('defines', {})
if not isinstance(defines, list):
defines = [defines]
self.permutations = m.prod(len(v) for v in self.defines.values()) # build possible permutations
for suite_defines_ in suite_defines:
self.defines |= suite_defines_.keys()
for defines_ in defines:
self.defines |= defines_.keys()
self.permutations.extend(map(dict, it.product(*(
[(k, v) for v in (vs if isinstance(vs, list) else [vs])]
for k, vs in sorted(
(suite_defines_ | defines_).items())))))
for k in config.keys(): for k in config.keys():
print('warning: in %s, found unused key %r' % (self.id(), k), print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r'
% (self.id(), k),
file=sys.stderr) file=sys.stderr)
def id(self): def id(self):
@@ -130,7 +141,7 @@ class TestSuite:
# sort in case toml parsing did not retain order # sort in case toml parsing did not retain order
case_linenos.sort() case_linenos.sort()
cases = config.pop('cases', []) cases = config.pop('cases')
for (lineno, name), (nlineno, _) in it.zip_longest( for (lineno, name), (nlineno, _) in it.zip_longest(
case_linenos, case_linenos[1:], case_linenos, case_linenos[1:],
fillvalue=(float('inf'), None)): fillvalue=(float('inf'), None)):
@@ -179,8 +190,8 @@ class TestSuite:
**case})) **case}))
# combine per-case defines # combine per-case defines
self.defines = sorted( self.defines = set.union(*(
set.union(*(set(case.defines) for case in self.cases))) set(case.defines) for case in self.cases))
# combine other per-case things # combine other per-case things
self.normal = any(case.normal for case in self.cases) self.normal = any(case.normal for case in self.cases)
@@ -188,7 +199,8 @@ class TestSuite:
self.valgrind = any(case.valgrind for case in self.cases) self.valgrind = any(case.valgrind for case in self.cases)
for k in config.keys(): for k in config.keys():
print('warning: in %s, found unused key %r' % (self.id(), k), print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r'
% (self.id(), k),
file=sys.stderr) file=sys.stderr)
def id(self): def id(self):
@@ -266,7 +278,7 @@ def compile(**args):
% (f.lineno+1, args['output'])) % (f.lineno+1, args['output']))
f.writeln() f.writeln()
for i, define in enumerate(suite.defines): for i, define in enumerate(sorted(suite.defines)):
f.writeln('#ifndef %s' % define) f.writeln('#ifndef %s' % define)
f.writeln('#define %-24s test_define(%d)' % (define, i)) f.writeln('#define %-24s test_define(%d)' % (define, i))
f.writeln('#endif') f.writeln('#endif')
@@ -275,16 +287,13 @@ def compile(**args):
for case in suite.cases: for case in suite.cases:
# create case defines # create case defines
if case.defines: if case.defines:
sorted_defines = sorted(case.defines.items())
f.writeln('const test_define_t *const ' f.writeln('const test_define_t *const '
'__test__%s__%s__defines[] = {' '__test__%s__%s__defines[] = {'
% (suite.name, case.name)) % (suite.name, case.name))
for defines in it.product(*( for permutation in case.permutations:
[(k, v) for v in vs]
for k, vs in sorted_defines)):
f.writeln(4*' '+'(const test_define_t[]){%s},' f.writeln(4*' '+'(const test_define_t[]){%s},'
% ', '.join('%s' % v for _, v in defines)) % ', '.join(str(v) for _, v in sorted(
permutation.items())))
f.writeln('};') f.writeln('};')
f.writeln() f.writeln()
@@ -293,9 +302,9 @@ def compile(**args):
% (suite.name, case.name)) % (suite.name, case.name))
f.writeln(4*' '+'%s,' f.writeln(4*' '+'%s,'
% ', '.join( % ', '.join(
'%s' % [k for k, _ in sorted_defines].index(k) str(sorted(case.defines).index(k))
if k in case.defines else '0xff' if k in case.defines else '0xff'
for k in suite.defines)) for k in sorted(suite.defines)))
f.writeln('};') f.writeln('};')
f.writeln() f.writeln()
@@ -365,7 +374,8 @@ def compile(**args):
'TEST_NORMAL' if case.normal else None, 'TEST_NORMAL' if case.normal else None,
'TEST_REENTRANT' if case.reentrant else None, 'TEST_REENTRANT' if case.reentrant else None,
'TEST_VALGRIND' if case.valgrind else None]))) 'TEST_VALGRIND' if case.valgrind else None])))
f.writeln(4*' '+'.permutations = %d,' % case.permutations) f.writeln(4*' '+'.permutations = %d,'
% len(case.permutations))
if case.defines: if case.defines:
f.writeln(4*' '+'.defines = __test__%s__%s__defines,' f.writeln(4*' '+'.defines = __test__%s__%s__defines,'
% (suite.name, case.name)) % (suite.name, case.name))
@@ -381,12 +391,13 @@ def compile(**args):
f.writeln() f.writeln()
# create suite define names # create suite define names
f.writeln('const char *const __test__%s__define_names[] = {' if suite.defines:
% suite.name) f.writeln('const char *const __test__%s__define_names[] = {'
for k in suite.defines: % suite.name)
f.writeln(4*' '+'"%s",' % k) for k in sorted(suite.defines):
f.writeln('};') f.writeln(4*' '+'"%s",' % k)
f.writeln() f.writeln('};')
f.writeln()
# create suite struct # create suite struct
f.writeln('const struct test_suite __test__%s__suite = {' f.writeln('const struct test_suite __test__%s__suite = {'
@@ -399,8 +410,9 @@ def compile(**args):
'TEST_NORMAL' if suite.normal else None, 'TEST_NORMAL' if suite.normal else None,
'TEST_REENTRANT' if suite.reentrant else None, 'TEST_REENTRANT' if suite.reentrant else None,
'TEST_VALGRIND' if suite.valgrind else None]))) 'TEST_VALGRIND' if suite.valgrind else None])))
f.writeln(4*' '+'.define_names = __test__%s__define_names,' if suite.defines:
% suite.name) f.writeln(4*' '+'.define_names = __test__%s__define_names,'
% suite.name)
f.writeln(4*' '+'.define_count = %d,' % len(suite.defines)) f.writeln(4*' '+'.define_count = %d,' % len(suite.defines))
f.writeln(4*' '+'.cases = (const struct test_case *const []){') f.writeln(4*' '+'.cases = (const struct test_case *const []){')
for case in suite.cases: for case in suite.cases:
@@ -726,7 +738,7 @@ def run_stage(name, runner_, **args):
if not args.get('verbose'): if not args.get('verbose'):
sys.stdout.write('\r\x1b[K' sys.stdout.write('\r\x1b[K'
'running \x1b[%dm%s\x1b[m: ' 'running \x1b[%dm%s:\x1b[m '
'%d/%d suites, %d/%d cases, %d/%d perms%s ' '%d/%d suites, %d/%d cases, %d/%d perms%s '
% (32 if not failures else 31, % (32 if not failures else 31,
name, name,
@@ -779,12 +791,12 @@ def run(**args):
expected = 0 expected = 0
passed = 0 passed = 0
failures = [] failures = []
if args.get('by_suites'): if args.get('by_cases'):
for type in ['normal', 'reentrant', 'valgrind']: for type in ['normal', 'reentrant', 'valgrind']:
for suite in expected_suite_perms.keys(): for case in expected_case_perms.keys():
expected_, passed_, failures_, killed = run_stage( expected_, passed_, failures_, killed = run_stage(
'%s %s' % (type, suite), '%s %s' % (type, case),
runner_ + ['--%s' % type, suite], runner_ + ['--%s' % type, case],
**args) **args)
expected += expected_ expected += expected_
passed += passed_ passed += passed_
@@ -793,12 +805,12 @@ def run(**args):
break break
if (failures and not args.get('keep_going')) or killed: if (failures and not args.get('keep_going')) or killed:
break break
elif args.get('by_cases'): elif args.get('by_suites'):
for type in ['normal', 'reentrant', 'valgrind']: for type in ['normal', 'reentrant', 'valgrind']:
for case in expected_case_perms.keys(): for suite in expected_suite_perms.keys():
expected_, passed_, failures_, killed = run_stage( expected_, passed_, failures_, killed = run_stage(
'%s %s' % (type, case), '%s %s' % (type, suite),
runner_ + ['--%s' % type, case], runner_ + ['--%s' % type, suite],
**args) **args)
expected += expected_ expected += expected_
passed += passed_ passed += passed_
@@ -821,7 +833,7 @@ def run(**args):
# show summary # show summary
print() print()
print('\x1b[%dmdone\x1b[m: %d/%d passed, %d/%d failed, in %.2fs' print('\x1b[%dmdone:\x1b[m %d/%d passed, %d/%d failed, in %.2fs'
% (32 if not failures else 31, % (32 if not failures else 31,
passed, expected, len(failures), expected, passed, expected, len(failures), expected,
time.time()-start)) time.time()-start))