Github/littlefs
1
Fork 0
Code Actions Activity

Putting together rewritten test.py script

Browse Source
This commit is contained in:
Christopher Haster
2022-04-24 01:45:22 -05:00
parent 92a600a980
commit 64436933e2
4 changed files with 412 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+9 -3
View File
@@ -62,12 +62,15 @@ override DATAFLAGS += -v
override STACKFLAGS += -v
override STRUCTSFLAGS += -v
override COVERAGEFLAGS += -v
override TESTFLAGS_ += -v
override TESTCFLAGS_ += -v
endif
ifdef EXEC
override TESTFLAGS += --exec="$(EXEC)"
override TESTFLAGS_ += --exec="$(EXEC)"
endif
ifdef COVERAGE
override TESTFLAGS += --coverage
override TESTFLAGS_ += --coverage
endif
ifdef BUILDDIR
override TESTFLAGS += --build-dir="$(BUILDDIR:/=)"
@@ -85,6 +88,8 @@ endif
ifneq ($(OBJDUMP),objdump)
override STRUCTSFLAGS += --objdump-tool="$(OBJDUMP)"
endif
# forward -j flag
override TESTFLAGS_ += $(filter -j%,$(MAKEFLAGS))
# commands
@@ -115,6 +120,7 @@ test%: tests/test$$(firstword $$(subst \#, ,%)).toml
.PHONY: test_
test_: $(BUILDDIR)runners/test_runner
./scripts/test_.py --runner=$< $(TESTFLAGS_)
.PHONY: code
code: $(OBJ)
@@ -185,10 +191,10 @@ $(BUILDDIR)%.a.c: $(BUILDDIR)%.c
./scripts/explode_asserts.py $< -o $@
$(BUILDDIR)%.t.c: %.toml
./scripts/test_.py -c $< -o $@
./scripts/test_.py -c $< $(TESTCFLAGS_) -o $@
$(BUILDDIR)%.t.c: %.c $(TESTS)
./scripts/test_.py -c $(TESTS) -s $< -o $@
./scripts/test_.py -c $(TESTS) -s $< $(TESTCFLAGS_) -o $@
# clean everything
.PHONY: clean
runners/test_runner.c
+2 -4
View File
@@ -29,9 +29,7 @@ struct test_geometry {
const struct test_geometry test_geometries[] = {
// Made up geometry that works well for testing
TEST_GEOMETRY("small", 16, 16, 512, (1024*1024)/512),
TEST_GEOMETRY("medium", 16, 16, 4096, (1024*1024)/4096),
TEST_GEOMETRY("big", 16, 16, 32*1024, (1024*1024)/(32*1024)),
TEST_GEOMETRY("test", 16, 16, 512, (1024*1024)/512),
// EEPROM/NVRAM
TEST_GEOMETRY("eeprom", 1, 1, 512, (1024*1024)/512),
// SD/eMMC
@@ -585,7 +583,7 @@ const char *const help_text[] = {
"Filter by geometry.",
"Filter for normal tests. Can be combined.",
"Filter for reentrant tests. Can be combined.",
"Filter for valgrind tests. Can be combined.",
"Filter for Valgrind tests. Can be combined.",
"Skip the first n tests.",
"Stop after n tests.",
"Only run every n tests, calculated after --skip and --stop.",
scripts/test_.py
+399 -6
View File
@@ -3,22 +3,30 @@
# Script to compile and runs tests.
#
import collections as co
import errno
import glob
import itertools as it
import math as m
import os
import pty
import re
import shlex
import shutil
import subprocess as sp
import threading as th
import time
import toml
TEST_PATHS = ['tests_']
RUNNER_PATH = './runners/test_runner'
SUITE_PROLOGUE = """
#include "runners/test_runner.h"
#include <stdio.h>
"""
CASE_PROLOGUE = """
lfs_t lfs;
"""
CASE_EPILOGUE = """
"""
@@ -200,7 +208,7 @@ class TestSuite:
def compile(**args):
# find .toml files
paths = []
for path in args['test_paths']:
for path in args.get('test_paths', TEST_PATHS):
if os.path.isdir(path):
path = path + '/*.toml'
@@ -429,26 +437,411 @@ def compile(**args):
f.write('const size_t test_suite_count = %d;\n'
% len(suites))
def runner(**args):
cmd = args['runner'].copy()
# TODO multiple paths?
if 'test_paths' in args:
cmd.extend(args.get('test_paths'))
if args.get('normal'): cmd.append('-n')
if args.get('reentrant'): cmd.append('-r')
if args.get('valgrind'): cmd.append('-V')
if args.get('geometry'):
cmd.append('-G%s' % args.get('geometry'))
if args.get('define'):
for define in args.get('define'):
cmd.append('-D%s' % define)
return cmd
def list_(**args):
cmd = runner(**args)
if args.get('summary'): cmd.append('--summary')
if args.get('list_suites'): cmd.append('--list-suites')
if args.get('list_cases'): cmd.append('--list-cases')
if args.get('list_paths'): cmd.append('--list-paths')
if args.get('list_defines'): cmd.append('--list-defines')
if args.get('list_geometries'): cmd.append('--list-geometries')
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
sys.exit(sp.call(cmd))
def find_cases(runner_, **args):
# first get suite/case/perm counts
cmd = runner_ + ['--list-cases']
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd,
stdout=sp.PIPE,
stderr=sp.PIPE if not args.get('verbose') else None,
universal_newlines=True,
errors='replace')
expected_suite_perms = co.defaultdict(lambda: 0)
expected_case_perms = co.defaultdict(lambda: 0)
expected_perms = 0
total_perms = 0
pattern = re.compile(
'^(?P<id>(?P<suite>[^#]+)#[^ #]+) +'
'[^ ]+ +[^ ]+ +[^ ]+ +'
'(?P<filtered>[0-9]+)/(?P<perms>[0-9]+)$')
# skip the first line
next(proc.stdout)
for line in proc.stdout:
m = pattern.match(line)
if m:
filtered = int(m.group('filtered'))
expected_suite_perms[m.group('suite')] += filtered
expected_case_perms[m.group('id')] += filtered
expected_perms += filtered
total_perms += int(m.group('perms'))
proc.wait()
if proc.returncode != 0:
if not args.get('verbose'):
for line in proc.stderr:
sys.stdout.write(line)
sys.exit(-1)
return (
expected_suite_perms,
expected_case_perms,
expected_perms,
total_perms)
class TestFailure(Exception):
def __init__(self, id, returncode, stdout, assert_=None):
self.id = id
self.returncode = returncode
self.stdout = stdout
self.assert_ = assert_
def run_step(name, runner_, **args):
# get expected suite/case/perm counts
expected_suite_perms, expected_case_perms, expected_perms, total_perms = (
find_cases(runner_, **args))
# TODO persist/trace
# TODO valgrind/gdb/exec
passed_suite_perms = co.defaultdict(lambda: 0)
passed_case_perms = co.defaultdict(lambda: 0)
passed_perms = 0
failures = []
pattern = re.compile('^(?:'
'(?P<op>running|finished|skipped) '
'(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)'
'|' '(?P<path>[^:]+):(?P<lineno>[0-9]+):(?P<op_>assert):'
' *(?P<message>.*)' ')$')
locals = th.local()
# TODO use process group instead of this set?
children = set()
def run_runner(runner_):
nonlocal passed_suite_perms
nonlocal passed_case_perms
nonlocal passed_perms
nonlocal locals
# run the tests!
cmd = runner_
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
mpty, spty = pty.openpty()
proc = sp.Popen(cmd, stdout=spty, stderr=spty)
os.close(spty)
mpty = os.fdopen(mpty, 'r', 1)
children.add(proc)
last_id = None
last_stdout = []
last_assert = None
try:
while True:
# parse a line for state changes
try:
line = mpty.readline()
except OSError as e:
if e.errno == errno.EIO:
break
raise
if not line:
break
last_stdout.append(line)
if args.get('verbose'):
sys.stdout.write(line)
m = pattern.match(line)
if m:
op = m.group('op') or m.group('op_')
if op == 'running':
locals.seen_perms += 1
last_id = m.group('id')
last_stdout = []
last_assert = None
elif op == 'finished':
passed_suite_perms[m.group('suite')] += 1
passed_case_perms[m.group('case')] += 1
passed_perms += 1
elif op == 'skipped':
locals.seen_perms += 1
elif op == 'assert':
last_assert = (
m.group('path'),
int(m.group('lineno')),
m.group('message'))
# TODO why is kill _so_ much faster than terminate?
proc.kill()
except KeyboardInterrupt:
raise TestFailure(last_id, 1, last_stdout)
finally:
children.remove(proc)
mpty.close()
proc.wait()
if proc.returncode != 0:
raise TestFailure(
last_id,
proc.returncode,
last_stdout,
last_assert)
def run_job(runner, skip=None, every=None):
nonlocal failures
nonlocal locals
while (skip or 0) < total_perms:
runner_ = runner.copy()
if skip is not None:
runner_.append('--skip=%d' % skip)
if every is not None:
runner_.append('--every=%d' % every)
try:
# run the tests
locals.seen_perms = 0
run_runner(runner_)
except TestFailure as failure:
# race condition for multiple failures?
if failures and not args.get('keep_going'):
break
failures.append(failure)
if args.get('keep_going'):
# resume after failed test
skip = (skip or 0) + locals.seen_perms*(every or 1)
continue
else:
# stop other tests
for child in children:
# TODO why is kill _so_ much faster than terminate?
child.kill()
break
# parallel jobs?
runners = []
if 'jobs' in args:
for job in range(args['jobs']):
runners.append(th.Thread(
target=run_job, args=(runner_, job, args['jobs'])))
else:
runners.append(th.Thread(
target=run_job, args=(runner_, None, None)))
for r in runners:
r.start()
needs_newline = False
try:
while any(r.is_alive() for r in runners):
time.sleep(0.01)
if not args.get('verbose'):
sys.stdout.write('\r\x1b[K'
'running \x1b[%dm%s\x1b[m: '
'%d/%d suites, %d/%d cases, %d/%d perms%s '
% (32 if not failures else 31,
name,
sum(passed_suite_perms[k] == v
for k, v in expected_suite_perms.items()),
len(expected_suite_perms),
sum(passed_case_perms[k] == v
for k, v in expected_case_perms.items()),
len(expected_case_perms),
passed_perms,
expected_perms,
', \x1b[31m%d/%d failures\x1b[m'
% (len(failures), expected_perms)
if failures else ''))
sys.stdout.flush()
needs_newline = True
finally:
if needs_newline:
print()
for r in runners:
r.join()
return (
expected_perms,
passed_perms,
failures)
def run(**args):
pass
start = time.time()
runner_ = runner(**args)
print('using runner `%s`'
% ' '.join(shlex.quote(c) for c in runner_))
expected_suite_perms, expected_case_perms, expected_perms, total_perms = (
find_cases(runner_, **args))
print('found %d suites, %d cases, %d/%d permutations'
% (len(expected_suite_perms),
len(expected_case_perms),
expected_perms,
total_perms))
print()
expected = 0
passed = 0
failures = []
if args.get('by_suites'):
for type in ['normal', 'reentrant', 'valgrind']:
for suite in expected_suite_perms.keys():
expected_, passed_, failures_ = run_step(
'%s %s' % (type, suite),
runner_ + ['--%s' % type, suite],
**args)
expected += expected_
passed += passed_
failures.extend(failures_)
if failures and not args.get('keep_going'):
break
if failures and not args.get('keep_going'):
break
elif args.get('by_cases'):
for type in ['normal', 'reentrant', 'valgrind']:
for case in expected_case_perms.keys():
expected_, passed_, failures_ = run_step(
'%s %s' % (type, case),
runner_ + ['--%s' % type, case],
**args)
expected += expected_
passed += passed_
failures.extend(failures_)
if failures and not args.get('keep_going'):
break
if failures and not args.get('keep_going'):
break
else:
for type in ['normal', 'reentrant', 'valgrind']:
expected_, passed_, failures_ = run_step(
'%s tests' % type,
runner_ + ['--%s' % type],
**args)
expected += expected_
passed += passed_
failures.extend(failures_)
if failures and not args.get('keep_going'):
break
# show summary
print()
print('\x1b[%dmdone\x1b[m: %d/%d passed, %d/%d failed, in %.2fs'
% (32 if not failures else 31,
passed, expected, len(failures), expected,
time.time()-start))
print()
# print each failure
# TODO get line, defines, path
for failure in failures:
# print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s failed'
# # TODO this should be the suite path and lineno
# % (failure.assert
if failure.assert_ is not None:
path, lineno, message = failure.assert_
print('\x1b[01m%s:%d:\x1b[01;31massert:\x1b[m %s'
% (path, lineno, message))
with open(path) as f:
line = next(it.islice(f, lineno-1, None)).strip('\n')
print(line)
print()
return 1 if failures else 0
def main(**args):
if args.get('compile'):
compile(**args)
elif (args.get('summary')
or args.get('list_suites')
or args.get('list_cases')
or args.get('list_paths')
or args.get('list_defines')
or args.get('list_geometries')):
list_(**args)
else:
run(**args)
if __name__ == "__main__":
import argparse
import sys
parser = argparse.ArgumentParser(
description="Build and run tests.")
# TODO document test case/perm specifier
parser.add_argument('test_paths', nargs='*', default=TEST_PATHS,
help="Description of test(s) to run. May be a directory, a path, or \
test identifier. Defaults to all tests in %r." % TEST_PATHS)
parser.add_argument('test_paths', nargs='*',
help="Description of testis to run. May be a directory, path, or \
test identifier. Defaults to %r." % TEST_PATHS)
parser.add_argument('-v', '--verbose', action='store_true',
help="Output commands that run behind the scenes.")
# test flags
test_parser = parser.add_argument_group('test options')
test_parser.add_argument('-Y', '--summary', action='store_true',
help="Show quick summary.")
test_parser.add_argument('-l', '--list-suites', action='store_true',
help="List test suites.")
test_parser.add_argument('-L', '--list-cases', action='store_true',
help="List test cases.")
test_parser.add_argument('--list-paths', action='store_true',
help="List the path for each test case.")
test_parser.add_argument('--list-defines', action='store_true',
help="List the defines for each test permutation.")
test_parser.add_argument('--list-geometries', action='store_true',
help="List the disk geometries used for testing.")
test_parser.add_argument('-D', '--define', action='append',
help="Override a test define.")
test_parser.add_argument('-G', '--geometry',
help="Filter by geometry.")
test_parser.add_argument('-n', '--normal', action='store_true',
help="Filter for normal tests. Can be combined.")
test_parser.add_argument('-r', '--reentrant', action='store_true',
help="Filter for reentrant tests. Can be combined.")
test_parser.add_argument('-V', '--valgrind', action='store_true',
help="Filter for Valgrind tests. Can be combined.")
test_parser.add_argument('-p', '--persist',
help="Persist the disk to this file.")
test_parser.add_argument('-t', '--trace',
help="Redirect trace output to this file.")
test_parser.add_argument('--runner', default=[RUNNER_PATH],
type=lambda x: x.split(),
help="Path to runner, defaults to %r" % RUNNER_PATH)
test_parser.add_argument('-j', '--jobs', nargs='?', type=int,
const=len(os.sched_getaffinity(0)),
help="Number of parallel runners to run.")
test_parser.add_argument('-k', '--keep-going', action='store_true',
help="Don't stop on first error.")
test_parser.add_argument('-b', '--by-suites', action='store_true',
help="Step through tests by suite.")
test_parser.add_argument('-B', '--by-cases', action='store_true',
help="Step through tests by case.")
# compilation flags
comp_parser = parser.add_argument_group('compilation options')
comp_parser.add_argument('-c', '--compile', action='store_true',
tests/test_files.toml
+1 -1
View File
@@ -65,7 +65,7 @@ code = '''
lfs_format(&lfs, &cfg) => 0;
// write
lfs_mount(&lfs, &cfg) => 0;
lfs_mount(&lfs, &cfg) => 1;
lfs_file_open(&lfs, &file, "avacado",
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
srand(1);