Github/littlefs
1
Fork 0
Code Actions Activity

Changed bench/test.py to error if explicit suite/case can't be found

Browse Source 
Previously no matches would noop, which, while consistent with an empty
test suite that contains no tests but shouldn't really error, this made
it easy to miss when a typo would cause tests to be missed.

Also added a bit of color to script-level errors in test/bench.py
This commit is contained in:
Christopher Haster
2023-06-01 16:50:38 -05:00
parent 2339e9865f
commit 82027f3d90
2 changed files with 71 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bench.py
+36 -18
View File
@@ -233,7 +233,10 @@ def compile(bench_paths, **args):
paths.append(path) paths.append(path)
if not paths: if not paths:
print('no bench suites found in %r?' % bench_paths) print('%serror:%s no bench suites found in %r?' % (
'\x1b[01;31m' if args['color'] else '',
'\x1b[m' if args['color'] else '',
bench_paths))
sys.exit(-1) sys.exit(-1)
# load the suites # load the suites
@@ -271,7 +274,10 @@ def compile(bench_paths, **args):
# we can only compile one bench suite at a time # we can only compile one bench suite at a time
if not args.get('source'): if not args.get('source'):
if len(suites) > 1: if len(suites) > 1:
print('more than one bench suite for compilation? (%r)' % bench_paths) print('%serror:%s compiling more than one bench suite? (%r)' % (
'\x1b[01;31m' if args['color'] else '',
'\x1b[m' if args['color'] else '',
bench_paths))
sys.exit(-1) sys.exit(-1)
suite = suites[0] suite = suites[0]
@@ -470,7 +476,7 @@ def compile(bench_paths, **args):
shutil.copyfileobj(sf, f) shutil.copyfileobj(sf, f)
f.writeln() f.writeln()
# write any internal benchs # write any internal benches
for suite in suites: for suite in suites:
for case in suite.cases: for case in suite.cases:
if (case.in_ is not None if (case.in_ is not None
@@ -716,13 +722,12 @@ def find_defines(runner, id, **args):
return defines return defines
def find_ids(runner, bench_ids=[], **args): def find_ids(runner, bench_ids=[], **args):
obench_ids = bench_ids # no ids => all ids, we don't need an extra lookup if no special
# behavior is requested
# we can avoid an extra lookup if all ids are explicit
if not (args.get('by_cases') if not (args.get('by_cases')
or args.get('by_suites') or args.get('by_suites')
or any('*' in id for id in bench_ids)): or bench_ids):
return bench_ids return []
# lookup suites/cases # lookup suites/cases
(suite_cases, (suite_cases,
@@ -731,24 +736,40 @@ def find_ids(runner, bench_ids=[], **args):
_, _,
_) = find_perms(runner, **args) _) = find_perms(runner, **args)
# no ids => all ids, only before globs! # no ids => all ids, before we evaluate globs
if not bench_ids and args.get('by_cases'): if not bench_ids and args.get('by_cases'):
return [case_ for case_ in expected_case_perms.keys()] return [case_ for case_ in expected_case_perms.keys()]
if not bench_ids and args.get('by_suites'): if not bench_ids and args.get('by_suites'):
return [suite for suite in expected_suite_perms.keys()] return [suite for suite in expected_suite_perms.keys()]
# first resolve globs # find suite/case by id
bench_ids_ = [] bench_ids_ = []
for id in bench_ids: for id in bench_ids:
bench_ids__ = []
# resolve globs
if '*' in id: if '*' in id:
bench_ids_.extend(suite bench_ids__.extend(suite
for suite in expected_suite_perms.keys() for suite in expected_suite_perms.keys()
if fnmatch.fnmatch(suite, id)) if fnmatch.fnmatch(suite, id))
bench_ids_.extend(case_ bench_ids__.extend(case_
for case_ in expected_case_perms.keys() for case_ in expected_case_perms.keys()
if fnmatch.fnmatch(case_, id)) if fnmatch.fnmatch(case_, id))
else: # literal suite
bench_ids_.append(id) elif id in expected_suite_perms:
bench_ids__.append(id)
# literal case
elif id in expected_case_perms:
bench_ids__.append(id)
# no suite/case found? error
if not bench_ids__:
print('%serror:%s no benches match id %r?' % (
'\x1b[01;31m' if args['color'] else '',
'\x1b[m' if args['color'] else '',
id))
sys.exit(-1)
bench_ids_.extend(bench_ids__)
bench_ids = bench_ids_ bench_ids = bench_ids_
# expand suites to cases? # expand suites to cases?
@@ -764,10 +785,7 @@ def find_ids(runner, bench_ids=[], **args):
bench_ids = bench_ids_ bench_ids = bench_ids_
# no bench ids found? return a garbage id for consistency # no bench ids found? return a garbage id for consistency
if not bench_ids: return bench_ids if bench_ids else ['?']
return '?'
return bench_ids
def list_(runner, bench_ids=[], **args): def list_(runner, bench_ids=[], **args):
scripts/test.py
+35 -17
View File
@@ -241,7 +241,10 @@ def compile(test_paths, **args):
paths.append(path) paths.append(path)
if not paths: if not paths:
print('no test suites found in %r?' % test_paths) print('%serror:%s no test suites found in %r?' % (
'\x1b[01;31m' if args['color'] else '',
'\x1b[m' if args['color'] else '',
test_paths))
sys.exit(-1) sys.exit(-1)
# load the suites # load the suites
@@ -279,7 +282,10 @@ def compile(test_paths, **args):
# we can only compile one test suite at a time # we can only compile one test suite at a time
if not args.get('source'): if not args.get('source'):
if len(suites) > 1: if len(suites) > 1:
print('more than one test suite for compilation? (%r)' % test_paths) print('%serror:%s compiling more than one test suite? (%r)' % (
'\x1b[01;31m' if args['color'] else '',
'\x1b[m' if args['color'] else '',
test_paths))
sys.exit(-1) sys.exit(-1)
suite = suites[0] suite = suites[0]
@@ -732,13 +738,12 @@ def find_defines(runner, id, **args):
return defines return defines
def find_ids(runner, test_ids=[], **args): def find_ids(runner, test_ids=[], **args):
otest_ids = test_ids # no ids => all ids, we don't need an extra lookup if no special
# behavior is requested
# we can avoid an extra lookup if all ids are explicit
if not (args.get('by_cases') if not (args.get('by_cases')
or args.get('by_suites') or args.get('by_suites')
or any('*' in id for id in test_ids)): or test_ids):
return test_ids return []
# lookup suites/cases # lookup suites/cases
(suite_cases, (suite_cases,
@@ -747,24 +752,40 @@ def find_ids(runner, test_ids=[], **args):
_, _,
_) = find_perms(runner, **args) _) = find_perms(runner, **args)
# no ids => all ids, only before globs! # no ids => all ids, before we evaluate globs
if not test_ids and args.get('by_cases'): if not test_ids and args.get('by_cases'):
return [case_ for case_ in expected_case_perms.keys()] return [case_ for case_ in expected_case_perms.keys()]
if not test_ids and args.get('by_suites'): if not test_ids and args.get('by_suites'):
return [suite for suite in expected_suite_perms.keys()] return [suite for suite in expected_suite_perms.keys()]
# first resolve globs # find suite/case by id
test_ids_ = [] test_ids_ = []
for id in test_ids: for id in test_ids:
test_ids__ = []
# resolve globs
if '*' in id: if '*' in id:
test_ids_.extend(suite test_ids__.extend(suite
for suite in expected_suite_perms.keys() for suite in expected_suite_perms.keys()
if fnmatch.fnmatch(suite, id)) if fnmatch.fnmatch(suite, id))
test_ids_.extend(case_ test_ids__.extend(case_
for case_ in expected_case_perms.keys() for case_ in expected_case_perms.keys()
if fnmatch.fnmatch(case_, id)) if fnmatch.fnmatch(case_, id))
else: # literal suite
test_ids_.append(id) elif id in expected_suite_perms:
test_ids__.append(id)
# literal case
elif id in expected_case_perms:
test_ids__.append(id)
# no suite/case found? error
if not test_ids__:
print('%serror:%s no tests match id %r?' % (
'\x1b[01;31m' if args['color'] else '',
'\x1b[m' if args['color'] else '',
id))
sys.exit(-1)
test_ids_.extend(test_ids__)
test_ids = test_ids_ test_ids = test_ids_
# expand suites to cases? # expand suites to cases?
@@ -780,10 +801,7 @@ def find_ids(runner, test_ids=[], **args):
test_ids = test_ids_ test_ids = test_ids_
# no test ids found? return a garbage id for consistency # no test ids found? return a garbage id for consistency
if not test_ids: return test_ids if test_ids else ['?']
return '?'
return test_ids
def list_(runner, test_ids=[], **args): def list_(runner, test_ids=[], **args):