Github/littlefs
1
Fork 0
Code Actions Activity

Added some rbyd benchmarks, fixed/tweaked some related scripts

Browse Source 
- Added both uattr (limited to 256) and id (limited to 65535) benchmarks
  covering the main rbyd operations

- Fixed issue where --defines gets passed to the test/bench runners when
  querying id-specific information. After changing the test/bench
  runners to prioritize explicit defines, this causes problems for
  recorded benchmark results and debug related things.

- In plot.py/plotmpl.py, made --by/-x/-y in subplots behave somewhat
  reasonably, contributing to a global dataset and the figure's legend,
  colors, etc, but only shown in the specified subplot. This is useful
  mainly for showing different -y values on different subplots.

- In plot.py/plotmpl.py, added --labels to allow explicit configuration
  of legend labels, much like --colors/--formats/--chars/etc. This
  removes one of the main annoying needs for modifying benchmark results.
This commit is contained in:
Christopher Haster
2023-01-20 01:54:49 -06:00
parent 27e4fbd3ad
commit 9a8e1d93c6
5 changed files with 926 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/test.py
+44 -31
View File
@@ -507,7 +507,7 @@ def compile(test_paths, **args):
f.writeln('#endif')
f.writeln()
def find_runner(runner, **args):
def find_runner(runner, id=None, **args):
cmd = runner.copy()
# run under some external command?
@@ -559,10 +559,18 @@ def find_runner(runner, **args):
cmd.append('--erase-sleep=%s' % args['erase_sleep'])
# defines?
if args.get('define'):
if args.get('define') and id is None:
for define in args.get('define'):
cmd.append('-D%s' % define)
# test id?
#
# note we disable defines above when id is explicit, defines override id
# in the test runner, which is not what we want when querying an explicit
# test id
if id is not None:
cmd.append(id)
return cmd
def list_(runner, test_ids=[], **args):
@@ -585,7 +593,8 @@ def list_(runner, test_ids=[], **args):
return sp.call(cmd)
def find_perms(runner_, ids=[], **args):
def find_perms(runner, ids=[], **args):
runner_ = find_runner(runner, **args)
case_suites = {}
expected_case_perms = co.defaultdict(lambda: 0)
expected_perms = 0
@@ -663,7 +672,8 @@ def find_perms(runner_, ids=[], **args):
expected_perms,
total_perms)
def find_path(runner_, id, **args):
def find_path(runner, id, **args):
runner_ = find_runner(runner, id, **args)
path = None
# query from runner
cmd = runner_ + ['--list-case-paths', id]
@@ -694,7 +704,8 @@ def find_path(runner_, id, **args):
return path
def find_defines(runner_, id, **args):
def find_defines(runner, id, **args):
runner_ = find_runner(runner, id, **args)
# query permutation defines from runner
cmd = runner_ + ['--list-permutation-defines', id]
if args.get('verbose'):
@@ -766,13 +777,13 @@ class TestFailure(Exception):
self.stdout = stdout
self.assert_ = assert_
def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
def run_stage(name, runner, ids, stdout_, trace_, output_, **args):
# get expected suite/case/perm counts
(case_suites,
expected_suite_perms,
expected_case_perms,
expected_perms,
total_perms) = find_perms(runner_, ids, **args)
total_perms) = find_perms(runner, ids, **args)
passed_suite_perms = co.defaultdict(lambda: 0)
passed_case_perms = co.defaultdict(lambda: 0)
@@ -790,7 +801,7 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
locals = th.local()
children = set()
def run_runner(runner_, ids=[]):
def run_runner(runner_):
nonlocal passed_suite_perms
nonlocal passed_case_perms
nonlocal passed_perms
@@ -798,7 +809,7 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
nonlocal locals
# run the tests!
cmd = runner_ + ids
cmd = runner_
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
@@ -850,7 +861,7 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
if output_:
# get defines and write to csv
defines = find_defines(
runner_, m.group('id'), **args)
runner, m.group('id'), **args)
output_.writerow({
'suite': suite,
'case': case,
@@ -880,7 +891,7 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
list(last_stdout),
last_assert)
def run_job(runner_, ids=[], start=None, step=None):
def run_job(start=None, step=None):
nonlocal failures
nonlocal killed
nonlocal locals
@@ -888,16 +899,18 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
start = start or 0
step = step or 1
while start < total_perms:
job_runner = runner_.copy()
runner_ = find_runner(runner, **args)
if args.get('isolate') or args.get('valgrind'):
job_runner.append('-s%s,%s,%s' % (start, start+step, step))
runner_.append('-s%s,%s,%s' % (start, start+step, step))
else:
job_runner.append('-s%s,,%s' % (start, step))
runner_.append('-s%s,,%s' % (start, step))
runner_.extend(ids)
try:
# run the tests
locals.seen_perms = 0
run_runner(job_runner, ids)
run_runner(runner_)
assert locals.seen_perms > 0
start += locals.seen_perms*step
@@ -907,7 +920,7 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
case, _ = failure.id.split(':', 1)
suite = case_suites[case]
# get defines and write to csv
defines = find_defines(runner_, failure.id, **args)
defines = find_defines(runner, failure.id, **args)
output_.writerow({
'suite': suite,
'case': case,
@@ -938,11 +951,11 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
if 'jobs' in args:
for job in range(args['jobs']):
runners.append(th.Thread(
target=run_job, args=(runner_, ids, job, args['jobs']),
target=run_job, args=(job, args['jobs']),
daemon=True))
else:
runners.append(th.Thread(
target=run_job, args=(runner_, ids, None, None),
target=run_job, args=(None, None),
daemon=True))
def print_update(done):
@@ -1005,13 +1018,13 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
def run(runner, test_ids=[], **args):
# query runner for tests
runner_ = find_runner(runner, **args)
print('using runner: %s' % ' '.join(shlex.quote(c) for c in runner_))
print('using runner: %s' % ' '.join(
shlex.quote(c) for c in find_runner(runner, **args)))
(_,
expected_suite_perms,
expected_case_perms,
expected_perms,
total_perms) = find_perms(runner_, test_ids, **args)
total_perms) = find_perms(runner, test_ids, **args)
print('found %d suites, %d cases, %d/%d permutations' % (
len(expected_suite_perms),
len(expected_case_perms),
@@ -1055,7 +1068,7 @@ def run(runner, test_ids=[], **args):
failures_,
killed) = run_stage(
by or 'tests',
runner_,
runner,
[by] if by is not None else [],
stdout,
trace,
@@ -1100,12 +1113,12 @@ def run(runner, test_ids=[], **args):
# print each failure
for failure in failures:
assert failure.id is not None, '%s broken? %r' % (
' '.join(shlex.quote(c) for c in runner_),
' '.join(shlex.quote(c) for c in find_runner(runner, **args)),
failure)
# get some extra info from runner
path, lineno = find_path(runner_, failure.id, **args)
defines = find_defines(runner_, failure.id, **args)
path, lineno = find_path(runner, failure.id, **args)
defines = find_defines(runner, failure.id, **args)
# show summary of failure
print('%s%s:%d:%sfailure:%s %s%s failed' % (
@@ -1145,25 +1158,25 @@ def run(runner, test_ids=[], **args):
or args.get('gdb_pl_before')
or args.get('gdb_pl_after')):
failure = failures[0]
cmd = runner_ + [failure.id]
cmd = find_runner(runner, failure.id, **args)
if args.get('gdb_main'):
# we don't really need the case breakpoint here, but it
# can be helpful
path, lineno = find_path(runner_, failure.id, **args)
path, lineno = find_path(runner, failure.id, **args)
cmd[:0] = args['gdb_path'] + [
'-ex', 'break main',
'-ex', 'break %s:%d' % (path, lineno),
'-ex', 'run',
'--args']
elif args.get('gdb_case'):
path, lineno = find_path(runner_, failure.id, **args)
path, lineno = find_path(runner, failure.id, **args)
cmd[:0] = args['gdb_path'] + [
'-ex', 'break %s:%d' % (path, lineno),
'-ex', 'run',
'--args']
elif args.get('gdb_pl') is not None:
path, lineno = find_path(runner_, failure.id, **args)
path, lineno = find_path(runner, failure.id, **args)
cmd[:0] = args['gdb_path'] + [
'-ex', 'break %s:%d' % (path, lineno),
'-ex', 'ignore 1 %d' % args['gdb_pl'],
@@ -1175,7 +1188,7 @@ def run(runner, test_ids=[], **args):
sum(1 for _ in re.finditer('[0-9a-f]',
failure.id.split(':', 2)[-1]))
if failure.id.count(':') >= 2 else 0)
path, lineno = find_path(runner_, failure.id, **args)
path, lineno = find_path(runner, failure.id, **args)
cmd[:0] = args['gdb_path'] + [
'-ex', 'break %s:%d' % (path, lineno),
'-ex', 'ignore 1 %d' % max(powerlosses-1, 0),
@@ -1187,7 +1200,7 @@ def run(runner, test_ids=[], **args):
sum(1 for _ in re.finditer('[0-9a-f]',
failure.id.split(':', 2)[-1]))
if failure.id.count(':') >= 2 else 0)
path, lineno = find_path(runner_, failure.id, **args)
path, lineno = find_path(runner, failure.id, **args)
cmd[:0] = args['gdb_path'] + [
'-ex', 'break %s:%d' % (path, lineno),
'-ex', 'ignore 1 %d' % powerlosses,