Github/littlefs
1
Fork 0
Code Actions Activity

Reworked how multi-layered defines work in the test-runner

Browse Source 
In the test-runner, defines are parameterized constants (limited
to integers) that are generated from the test suite tomls resulting
in many permutations of each test.

In order to make this efficient, these defines are implemented as
multi-layered lookup tables, using per-layer/per-scope indirect
mappings. This lets the test-runner and test suites define their
own defines with compile-time indexes independently. It also makes
building of the lookup tables very efficient, since they can be
incrementally populated as we expand the test permutations.

The four current define layers and when we need to build them:

layer                           defines         predefine_map   define_map
user-provided overrides         per-run         per-run         per-suite
per-permutation defines         per-perm        per-case        per-perm
per-geometry defines            per-perm        compile-time    -
default defines                 compile-time    compile-time    -
This commit is contained in:
Christopher Haster
2022-04-24 23:34:28 -05:00
parent 64436933e2
commit 5812d2b5cf
5 changed files with 541 additions and 373 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/test_.py
+264 -190
View File
@@ -31,30 +31,21 @@ CASE_PROLOGUE = """
CASE_EPILOGUE = """
"""
TEST_PREDEFINES = [
'READ_SIZE',
'PROG_SIZE',
'BLOCK_SIZE',
'BLOCK_COUNT',
'BLOCK_CYCLES',
'CACHE_SIZE',
'LOOKAHEAD_SIZE',
'ERASE_VALUE',
'ERASE_CYCLES',
'BADBLOCK_BEHAVIOR',
]
# TODO
# def testpath(path):
# def testcase(path):
# def testperm(path):
def testpath(path):
path, *_ = path.split('#', 1)
return path
def testsuite(path):
name = os.path.basename(path)
if name.endswith('.toml'):
name = name[:-len('.toml')]
return name
suite = testpath(path)
suite = os.path.basename(suite)
if suite.endswith('.toml'):
suite = suite[:-len('.toml')]
return suite
def testcase(path):
_, case, *_ = path.split('#', 2)
return '%s#%s' % (testsuite(path), case)
# TODO move this out in other files
def openio(path, mode='r'):
@@ -111,7 +102,7 @@ class TestSuite:
# create a TestSuite object from a toml file
def __init__(self, path, args={}):
self.name = testsuite(path)
self.path = path
self.path = testpath(path)
# load toml file and parse test cases
with open(self.path) as f:
@@ -125,9 +116,9 @@ class TestSuite:
code_linenos = []
for i, line in enumerate(f):
match = re.match(
'(?P<case>\[\s*cases\s*\.\s*(?P<name>\w+)\s*\])' +
'|(?P<if>if\s*=)'
'|(?P<code>code\s*=)',
'(?P<case>\[\s*cases\s*\.\s*(?P<name>\w+)\s*\])'
'|' '(?P<if>if\s*=)'
'|' '(?P<code>code\s*=)',
line)
if match and match.group('case'):
case_linenos.append((i+1, match.group('name')))
@@ -187,8 +178,8 @@ class TestSuite:
'suite_valgrind': valgrind,
**case}))
# combine pre-defines and per-case defines
self.defines = TEST_PREDEFINES + sorted(
# combine per-case defines
self.defines = sorted(
set.union(*(set(case.defines) for case in self.cases)))
# combine other per-case things
@@ -225,216 +216,222 @@ def compile(**args):
% args['test_paths'])
sys.exit(-1)
# write out a test suite
# load our suite
suite = TestSuite(paths[0])
if 'output' in args:
with openio(args['output'], 'w') as f:
# redirect littlefs tracing
f.write('#define LFS_TRACE_(fmt, ...) do { \\\n')
f.write(8*' '+'extern FILE *test_trace; \\\n')
f.write(8*' '+'if (test_trace) { \\\n')
f.write(12*' '+'fprintf(test_trace, '
'"%s:%d:trace: " fmt "%s\\n", \\\n')
f.write(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\\n')
f.write(8*' '+'} \\\n')
f.write(4*' '+'} while (0)\n')
f.write('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")\n')
f.write('#define LFS_TESTBD_TRACE(...) '
'LFS_TRACE_(__VA_ARGS__, "")\n')
f.write('\n')
else:
# load all suites
suites = [TestSuite(path) for path in paths]
suites.sort(key=lambda s: s.name)
f.write('%s\n' % SUITE_PROLOGUE.strip())
f.write('\n')
# write generated test source
if 'output' in args:
with openio(args['output'], 'w') as f:
_write = f.write
def write(s):
f.lineno += s.count('\n')
_write(s)
def writeln(s=''):
f.lineno += s.count('\n') + 1
_write(s)
_write('\n')
f.lineno = 1
f.write = write
f.writeln = writeln
# redirect littlefs tracing
f.writeln('#define LFS_TRACE_(fmt, ...) do { \\')
f.writeln(8*' '+'extern FILE *test_trace; \\')
f.writeln(8*' '+'if (test_trace) { \\')
f.writeln(12*' '+'fprintf(test_trace, '
'"%s:%d:trace: " fmt "%s\\n", \\')
f.writeln(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\')
f.writeln(8*' '+'} \\')
f.writeln(4*' '+'} while (0)')
f.writeln('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")')
f.writeln('#define LFS_TESTBD_TRACE(...) '
'LFS_TRACE_(__VA_ARGS__, "")')
f.writeln()
if not args.get('source'):
# write test suite prologue
f.writeln('%s' % SUITE_PROLOGUE.strip())
f.writeln()
if suite.code is not None:
if suite.code_lineno is not None:
f.write('#line %d "%s"\n'
f.writeln('#line %d "%s"'
% (suite.code_lineno, suite.path))
f.write(suite.code)
f.write('\n')
if suite.code_lineno is not None:
f.writeln('#line %d "%s"'
% (f.lineno+1, args['output']))
f.writeln()
for i, define in it.islice(
enumerate(suite.defines),
len(TEST_PREDEFINES), None):
f.write('#define %-24s test_define(%d)\n' % (define, i))
f.write('\n')
for i, define in enumerate(suite.defines):
f.writeln('#ifndef %s' % define)
f.writeln('#define %-24s test_define(%d)' % (define, i))
f.writeln('#endif')
f.writeln()
for case in suite.cases:
# create case defines
if case.defines:
sorted_defines = sorted(case.defines.items())
for perm, defines in enumerate(
it.product(*(
[(k, v) for v in vs]
for k, vs in sorted_defines))):
f.write('const test_define_t '
'__test__%s__%s__%d__defines[] = {\n'
% (suite.name, case.name, perm))
for k, v in defines:
f.write(4*' '+'%s,\n' % v)
f.write('};\n')
f.write('\n')
f.write('const test_define_t *const '
'__test__%s__%s__defines[] = {\n'
f.writeln('const test_define_t *const '
'__test__%s__%s__defines[] = {'
% (suite.name, case.name))
for perm in range(case.permutations):
f.write(4*' '+'__test__%s__%s__%d__defines,\n'
% (suite.name, case.name, perm))
f.write('};\n')
f.write('\n')
for defines in it.product(*(
[(k, v) for v in vs]
for k, vs in sorted_defines)):
f.writeln(4*' '+'(const test_define_t[]){%s},'
% ', '.join('%s' % v for _, v in defines))
f.writeln('};')
f.writeln()
f.write('const uint8_t '
'__test__%s__%s__define_map[] = {\n'
f.writeln('const uint8_t '
'__test__%s__%s__define_map[] = {'
% (suite.name, case.name))
for k in suite.defines:
f.write(4*' '+'%s,\n'
% ([k for k, _ in sorted_defines].index(k)
if k in case.defines else '0xff'))
f.write('};\n')
f.write('\n')
f.writeln(4*' '+'%s,'
% ', '.join(
'%s' % [k for k, _ in sorted_defines].index(k)
if k in case.defines else '0xff'
for k in suite.defines))
f.writeln('};')
f.writeln()
# create case filter function
if suite.if_ is not None or case.if_ is not None:
f.write('bool __test__%s__%s__filter('
'__attribute__((unused)) uint32_t perm) {\n'
f.writeln('bool __test__%s__%s__filter('
'__attribute__((unused)) uint32_t perm) {'
% (suite.name, case.name))
if suite.if_ is not None:
f.write(4*' '+'#line %d "%s"\n'
% (suite.if_lineno, suite.path))
f.write(4*' '+'if (!(%s)) {\n' % suite.if_)
f.write(8*' '+'return false;\n')
f.write(4*' '+'}\n')
f.write('\n')
if suite.if_lineno is not None:
f.writeln(4*' '+'#line %d "%s"'
% (suite.if_lineno, suite.path))
f.writeln(4*' '+'if (!(%s)) {' % suite.if_)
if suite.if_lineno is not None:
f.writeln(4*' '+'#line %d "%s"'
% (f.lineno+1, args['output']))
f.writeln(8*' '+'return false;')
f.writeln(4*' '+'}')
f.writeln()
if case.if_ is not None:
f.write(4*' '+'#line %d "%s"\n'
% (case.if_lineno, suite.path))
f.write(4*' '+'if (!(%s)) {\n' % case.if_)
f.write(8*' '+'return false;\n')
f.write(4*' '+'}\n')
f.write('\n')
f.write(4*' '+'return true;\n')
f.write('}\n')
f.write('\n')
if case.if_lineno is not None:
f.writeln(4*' '+'#line %d "%s"'
% (case.if_lineno, suite.path))
f.writeln(4*' '+'if (!(%s)) {' % case.if_)
if case.if_lineno is not None:
f.writeln(4*' '+'#line %d "%s"'
% (f.lineno+1, args['output']))
f.writeln(8*' '+'return false;')
f.writeln(4*' '+'}')
f.writeln()
f.writeln(4*' '+'return true;')
f.writeln('}')
f.writeln()
# create case run function
f.write('void __test__%s__%s__run('
f.writeln('void __test__%s__%s__run('
'__attribute__((unused)) struct lfs_config *cfg, '
'__attribute__((unused)) uint32_t perm) {\n'
'__attribute__((unused)) uint32_t perm) {'
% (suite.name, case.name))
f.write(4*' '+'%s\n'
% CASE_PROLOGUE.strip().replace('\n', '\n'+4*' '))
f.write('\n')
f.write(4*' '+'// test case %s\n' % case.id())
if CASE_PROLOGUE.strip():
f.writeln(4*' '+'%s'
% CASE_PROLOGUE.strip().replace('\n', '\n'+4*' '))
f.writeln()
f.writeln(4*' '+'// test case %s' % case.id())
if case.code_lineno is not None:
f.write(4*' '+'#line %d "%s"\n'
f.writeln(4*' '+'#line %d "%s"'
% (case.code_lineno, suite.path))
f.write(case.code)
f.write('\n')
f.write(4*' '+'%s\n'
% CASE_EPILOGUE.strip().replace('\n', '\n'+4*' '))
f.write('}\n')
f.write('\n')
if case.code_lineno is not None:
f.writeln(4*' '+'#line %d "%s"'
% (f.lineno+1, args['output']))
if CASE_EPILOGUE.strip():
f.writeln()
f.writeln(4*' '+'%s'
% CASE_EPILOGUE.strip().replace('\n', '\n'+4*' '))
f.writeln('}')
f.writeln()
# create case struct
f.write('const struct test_case __test__%s__%s__case = {\n'
f.writeln('const struct test_case __test__%s__%s__case = {'
% (suite.name, case.name))
f.write(4*' '+'.id = "%s",\n' % case.id())
f.write(4*' '+'.name = "%s",\n' % case.name)
f.write(4*' '+'.path = "%s",\n' % case.path)
f.write(4*' '+'.types = %s,\n'
f.writeln(4*' '+'.id = "%s",' % case.id())
f.writeln(4*' '+'.name = "%s",' % case.name)
f.writeln(4*' '+'.path = "%s",' % case.path)
f.writeln(4*' '+'.types = %s,'
% ' | '.join(filter(None, [
'TEST_NORMAL' if case.normal else None,
'TEST_REENTRANT' if case.reentrant else None,
'TEST_VALGRIND' if case.valgrind else None])))
f.write(4*' '+'.permutations = %d,\n' % case.permutations)
f.writeln(4*' '+'.permutations = %d,' % case.permutations)
if case.defines:
f.write(4*' '+'.defines = __test__%s__%s__defines,\n'
f.writeln(4*' '+'.defines = __test__%s__%s__defines,'
% (suite.name, case.name))
f.write(4*' '+'.define_map = '
'__test__%s__%s__define_map,\n'
f.writeln(4*' '+'.define_map = '
'__test__%s__%s__define_map,'
% (suite.name, case.name))
if suite.if_ is not None or case.if_ is not None:
f.write(4*' '+'.filter = __test__%s__%s__filter,\n'
f.writeln(4*' '+'.filter = __test__%s__%s__filter,'
% (suite.name, case.name))
f.write(4*' '+'.run = __test__%s__%s__run,\n'
f.writeln(4*' '+'.run = __test__%s__%s__run,'
% (suite.name, case.name))
f.write('};\n')
f.write('\n')
f.writeln('};')
f.writeln()
# create suite define names
f.write('const char *const __test__%s__define_names[] = {\n'
f.writeln('const char *const __test__%s__define_names[] = {'
% suite.name)
for k in suite.defines:
f.write(4*' '+'"%s",\n' % k)
f.write('};\n')
f.write('\n')
f.writeln(4*' '+'"%s",' % k)
f.writeln('};')
f.writeln()
# create suite struct
f.write('const struct test_suite __test__%s__suite = {\n'
f.writeln('const struct test_suite __test__%s__suite = {'
% suite.name)
f.write(4*' '+'.id = "%s",\n' % suite.id())
f.write(4*' '+'.name = "%s",\n' % suite.name)
f.write(4*' '+'.path = "%s",\n' % suite.path)
f.write(4*' '+'.types = %s,\n'
f.writeln(4*' '+'.id = "%s",' % suite.id())
f.writeln(4*' '+'.name = "%s",' % suite.name)
f.writeln(4*' '+'.path = "%s",' % suite.path)
f.writeln(4*' '+'.types = %s,'
% ' | '.join(filter(None, [
'TEST_NORMAL' if suite.normal else None,
'TEST_REENTRANT' if suite.reentrant else None,
'TEST_VALGRIND' if suite.valgrind else None])))
f.write(4*' '+'.define_names = __test__%s__define_names,\n'
f.writeln(4*' '+'.define_names = __test__%s__define_names,'
% suite.name)
f.write(4*' '+'.define_count = %d,\n' % len(suite.defines))
f.write(4*' '+'.cases = (const struct test_case *const []){\n')
f.writeln(4*' '+'.define_count = %d,' % len(suite.defines))
f.writeln(4*' '+'.cases = (const struct test_case *const []){')
for case in suite.cases:
f.write(8*' '+'&__test__%s__%s__case,\n'
f.writeln(8*' '+'&__test__%s__%s__case,'
% (suite.name, case.name))
f.write(4*' '+'},\n')
f.write(4*' '+'.case_count = %d,\n' % len(suite.cases))
f.write('};\n')
f.write('\n')
else:
# load all suites
suites = [TestSuite(path) for path in paths]
suites.sort(key=lambda s: s.name)
# write out a test source
if 'output' in args:
with openio(args['output'], 'w') as f:
# redirect littlefs tracing
f.write('#define LFS_TRACE_(fmt, ...) do { \\\n')
f.write(8*' '+'extern FILE *test_trace; \\\n')
f.write(8*' '+'if (test_trace) { \\\n')
f.write(12*' '+'fprintf(test_trace, '
'"%s:%d:trace: " fmt "%s\\n", \\\n')
f.write(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\\n')
f.write(8*' '+'} \\\n')
f.write(4*' '+'} while (0)\n')
f.write('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")\n')
f.write('#define LFS_TESTBD_TRACE(...) '
'LFS_TRACE_(__VA_ARGS__, "")\n')
f.write('\n')
f.writeln(4*' '+'},')
f.writeln(4*' '+'.case_count = %d,' % len(suite.cases))
f.writeln('};')
f.writeln()
else:
# copy source
f.write('#line 1 "%s"\n' % args['source'])
f.writeln('#line 1 "%s"' % args['source'])
with open(args['source']) as sf:
shutil.copyfileobj(sf, f)
f.write('\n')
f.writeln()
f.write(SUITE_PROLOGUE)
f.write('\n')
f.writeln()
# add suite info to test_runner.c
if args['source'] == 'runners/test_runner.c':
f.write('\n')
f.writeln()
for suite in suites:
f.write('extern const struct test_suite '
'__test__%s__suite;\n' % suite.name)
f.write('const struct test_suite *test_suites[] = {\n')
f.writeln('extern const struct test_suite '
'__test__%s__suite;' % suite.name)
f.writeln('const struct test_suite *test_suites[] = {')
for suite in suites:
f.write(4*' '+'&__test__%s__suite,\n' % suite.name)
f.write('};\n')
f.write('const size_t test_suite_count = %d;\n'
f.writeln(4*' '+'&__test__%s__suite,' % suite.name)
f.writeln('};')
f.writeln('const size_t test_suite_count = %d;'
% len(suites))
def runner(**args):
@@ -469,7 +466,7 @@ def list_(**args):
def find_cases(runner_, **args):
# first get suite/case/perm counts
# query from runner
cmd = runner_ + ['--list-cases']
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
@@ -483,9 +480,8 @@ def find_cases(runner_, **args):
expected_perms = 0
total_perms = 0
pattern = re.compile(
'^(?P<id>(?P<suite>[^#]+)#[^ #]+) +'
'[^ ]+ +[^ ]+ +[^ ]+ +'
'(?P<filtered>[0-9]+)/(?P<perms>[0-9]+)$')
'^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
'[^\s]+\s+(?P<filtered>\d+)/(?P<perms>\d+)')
# skip the first line
next(proc.stdout)
for line in proc.stdout:
@@ -509,11 +505,69 @@ def find_cases(runner_, **args):
expected_perms,
total_perms)
def find_paths(runner_, **args):
# query from runner
cmd = runner_ + ['--list-paths']
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd,
stdout=sp.PIPE,
stderr=sp.PIPE if not args.get('verbose') else None,
universal_newlines=True,
errors='replace')
paths = co.OrderedDict()
pattern = re.compile(
'^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
'(?P<path>[^:]+):(?P<lineno>\d+)')
# skip the first line
for line in proc.stdout:
m = pattern.match(line)
if m:
paths[m.group('id')] = (m.group('path'), int(m.group('lineno')))
proc.wait()
if proc.returncode != 0:
if not args.get('verbose'):
for line in proc.stderr:
sys.stdout.write(line)
sys.exit(-1)
return paths
def find_defines(runner_, **args):
# query from runner
cmd = runner_ + ['--list-defines']
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd,
stdout=sp.PIPE,
stderr=sp.PIPE if not args.get('verbose') else None,
universal_newlines=True,
errors='replace')
defines = co.OrderedDict()
pattern = re.compile(
'^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
'(?P<defines>(?:\w+=\w+\s*)+)')
# skip the first line
for line in proc.stdout:
m = pattern.match(line)
if m:
defines[m.group('id')] = {k: v
for k, v in re.findall('(\w+)=(\w+)', m.group('defines'))}
proc.wait()
if proc.returncode != 0:
if not args.get('verbose'):
for line in proc.stderr:
sys.stdout.write(line)
sys.exit(-1)
return defines
class TestFailure(Exception):
def __init__(self, id, returncode, stdout, assert_=None):
def __init__(self, id, returncode, output, assert_=None):
self.id = id
self.returncode = returncode
self.stdout = stdout
self.output = output
self.assert_ = assert_
def run_step(name, runner_, **args):
@@ -531,7 +585,7 @@ def run_step(name, runner_, **args):
pattern = re.compile('^(?:'
'(?P<op>running|finished|skipped) '
'(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)'
'|' '(?P<path>[^:]+):(?P<lineno>[0-9]+):(?P<op_>assert):'
'|' '(?P<path>[^:]+):(?P<lineno>\d+):(?P<op_>assert):'
' *(?P<message>.*)' ')$')
locals = th.local()
# TODO use process group instead of this set?
@@ -554,7 +608,7 @@ def run_step(name, runner_, **args):
children.add(proc)
last_id = None
last_stdout = []
last_output = []
last_assert = None
try:
while True:
@@ -567,7 +621,7 @@ def run_step(name, runner_, **args):
raise
if not line:
break
last_stdout.append(line)
last_output.append(line)
if args.get('verbose'):
sys.stdout.write(line)
@@ -577,7 +631,7 @@ def run_step(name, runner_, **args):
if op == 'running':
locals.seen_perms += 1
last_id = m.group('id')
last_stdout = []
last_output = []
last_assert = None
elif op == 'finished':
passed_suite_perms[m.group('suite')] += 1
@@ -590,10 +644,11 @@ def run_step(name, runner_, **args):
m.group('path'),
int(m.group('lineno')),
m.group('message'))
# TODO why is kill _so_ much faster than terminate?
proc.kill()
# go ahead and kill the process, aborting takes a while
if args.get('keep_going'):
proc.kill()
except KeyboardInterrupt:
raise TestFailure(last_id, 1, last_stdout)
raise TestFailure(last_id, 1, last_output)
finally:
children.remove(proc)
mpty.close()
@@ -603,7 +658,7 @@ def run_step(name, runner_, **args):
raise TestFailure(
last_id,
proc.returncode,
last_stdout,
last_output,
last_assert)
def run_job(runner, skip=None, every=None):
@@ -636,7 +691,6 @@ def run_step(name, runner_, **args):
else:
# stop other tests
for child in children:
# TODO why is kill _so_ much faster than terminate?
child.kill()
break
@@ -759,11 +813,28 @@ def run(**args):
print()
# print each failure
# TODO get line, defines, path
if failures:
# get some extra info from runner
runner_paths = find_paths(runner_, **args)
runner_defines = find_defines(runner_, **args)
for failure in failures:
# print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s failed'
# # TODO this should be the suite path and lineno
# % (failure.assert
# show summary of failure
path, lineno = runner_paths[testcase(failure.id)]
defines = runner_defines[failure.id]
print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s%s failed'
% (path, lineno, failure.id,
' (%s)' % ', '.join(
'%s=%s' % (k, v) for k, v in defines.items())
if defines else ''))
if failure.output:
output = failure.output
if failure.assert_ is not None:
output = output[:-1]
for line in output[-5:]:
sys.stdout.write(line)
if failure.assert_ is not None:
path, lineno, message = failure.assert_
@@ -785,7 +856,8 @@ def main(**args):
or args.get('list_cases')
or args.get('list_paths')
or args.get('list_defines')
or args.get('list_geometries')):
or args.get('list_geometries')
or args.get('list_defaults')):
list_(**args)
else:
run(**args)
@@ -816,6 +888,8 @@ if __name__ == "__main__":
help="List the defines for each test permutation.")
test_parser.add_argument('--list-geometries', action='store_true',
help="List the disk geometries used for testing.")
test_parser.add_argument('--list-defaults', action='store_true',
help="List the default defines in this test-runner.")
test_parser.add_argument('-D', '--define', action='append',
help="Override a test define.")
test_parser.add_argument('-G', '--geometry',