Github/littlefs
1
Fork 0
Code Actions Activity

Reworked test defines a bit to use one common array layout

Browse Source 
Previously didn't think this would work without making test.py aware of
the number of implicit defines, which risks being incredibly fragile.
Fortunately it turns out we can defer the actual array size calculation
until the C preprocessor. This simplifies a few things.

Also a bitmap-based caching layer for the defines. Since the test
defines have been upgraded to callbacks recursive defines risk spending
a decent amount of time evaluating on every lookup. Some quick testing
shows 408015154 hits to 46160 misses so that's a good sign.

Also changed the geometries to be their own leb16-encoded part of the
test identifier. This means any geometry can be captured and reproduced
with just the test identifier. Here are the current test geometries:

./runners/test_runner --list-geometries
geometry                    read    prog   erase   count        size  leb16
d,default                     16      16     512    2048     1048576  g1gg2
e,eeprom                       1       1     512    2048     1048576  1gg2
E,emmc                       512     512     512    2048     1048576  gg2
n,nor                          1       1    4096     256     1048576  1ggg1
N,nand                      4096    4096   32768      32     1048576  ggg1ggg8
This commit is contained in:
Christopher Haster
2022-09-05 22:53:12 -05:00
parent 91200e6678
commit a208d848e5
3 changed files with 1075 additions and 586 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runners/test_runner.c
+933 -483
View File
File diff suppressed because it is too large Load Diff
runners/test_runner.h
+39 -49
View File
@@ -38,7 +38,7 @@ struct test_case {
test_flags_t flags; test_flags_t flags;
size_t permutations; size_t permutations;
intmax_t (*const *const *defines)(void); intmax_t (*const *const *defines)(size_t);
bool (*filter)(void); bool (*filter)(void);
void (*run)(struct lfs_config *cfg); void (*run)(struct lfs_config *cfg);
@@ -59,60 +59,50 @@ struct test_suite {
// access generated test defines // access generated test defines
intmax_t test_predefine(size_t define); //intmax_t test_predefine(size_t define);
intmax_t test_define(size_t define); intmax_t test_define(size_t define);
// a few preconfigured defines that control how tests run // a few preconfigured defines that control how tests run
#define READ_SIZE test_predefine(0)
#define PROG_SIZE test_predefine(1) #define READ_SIZE_i 0
#define BLOCK_SIZE test_predefine(2) #define PROG_SIZE_i 1
#define BLOCK_COUNT test_predefine(3) #define BLOCK_SIZE_i 2
#define CACHE_SIZE test_predefine(4) #define BLOCK_COUNT_i 3
#define LOOKAHEAD_SIZE test_predefine(5) #define CACHE_SIZE_i 4
#define BLOCK_CYCLES test_predefine(6) #define LOOKAHEAD_SIZE_i 5
#define ERASE_VALUE test_predefine(7) #define BLOCK_CYCLES_i 6
#define ERASE_CYCLES test_predefine(8) #define ERASE_VALUE_i 7
#define BADBLOCK_BEHAVIOR test_predefine(9) #define ERASE_CYCLES_i 8
#define POWERLOSS_BEHAVIOR test_predefine(10) #define BADBLOCK_BEHAVIOR_i 9
#define POWERLOSS_BEHAVIOR_i 10
#define TEST_PREDEFINE_NAMES { \ #define READ_SIZE test_define(READ_SIZE_i)
"READ_SIZE", \ #define PROG_SIZE test_define(PROG_SIZE_i)
"PROG_SIZE", \ #define BLOCK_SIZE test_define(BLOCK_SIZE_i)
"BLOCK_SIZE", \ #define BLOCK_COUNT test_define(BLOCK_COUNT_i)
"BLOCK_COUNT", \ #define CACHE_SIZE test_define(CACHE_SIZE_i)
"CACHE_SIZE", \ #define LOOKAHEAD_SIZE test_define(LOOKAHEAD_SIZE_i)
"LOOKAHEAD_SIZE", \ #define BLOCK_CYCLES test_define(BLOCK_CYCLES_i)
"BLOCK_CYCLES", \ #define ERASE_VALUE test_define(ERASE_VALUE_i)
"ERASE_VALUE", \ #define ERASE_CYCLES test_define(ERASE_CYCLES_i)
"ERASE_CYCLES", \ #define BADBLOCK_BEHAVIOR test_define(BADBLOCK_BEHAVIOR_i)
"BADBLOCK_BEHAVIOR", \ #define POWERLOSS_BEHAVIOR test_define(POWERLOSS_BEHAVIOR_i)
"POWERLOSS_BEHAVIOR", \
}
#define TEST_PREDEFINE_COUNT 11
#define TEST_IMPLICIT_DEFINES \
TEST_DEFINE(READ_SIZE, test_geometry->read_size) \
TEST_DEFINE(PROG_SIZE, test_geometry->prog_size) \
TEST_DEFINE(BLOCK_SIZE, test_geometry->block_size) \
TEST_DEFINE(BLOCK_COUNT, test_geometry->block_count) \
TEST_DEFINE(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \
TEST_DEFINE(LOOKAHEAD_SIZE, 16) \
TEST_DEFINE(BLOCK_CYCLES, -1) \
TEST_DEFINE(ERASE_VALUE, 0xff) \
TEST_DEFINE(ERASE_CYCLES, 0) \
TEST_DEFINE(BADBLOCK_BEHAVIOR, LFS_TESTBD_BADBLOCK_PROGERROR) \
TEST_DEFINE(POWERLOSS_BEHAVIOR, LFS_TESTBD_POWERLOSS_NOOP)
// default predefines #define TEST_GEOMETRY_DEFINE_COUNT 4
#define TEST_DEFAULTS { \ #define TEST_IMPLICIT_DEFINE_COUNT 11
/* LOOKAHEAD_SIZE */ 16, \
/* BLOCK_CYCLES */ -1, \
/* ERASE_VALUE */ 0xff, \
/* ERASE_CYCLES */ 0, \
/* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \
/* POWERLOSS_BEHAVIOR */ LFS_TESTBD_POWERLOSS_NOOP, \
}
#define TEST_DEFAULT_DEFINE_COUNT 5
// test geometries
#define TEST_GEOMETRIES { \
/*geometry, read, write, erase, count, cache */ \
{"test", { 16, 16, 512, (1024*1024)/512, 64}}, \
{"eeprom", { 1, 1, 512, (1024*1024)/512, 64}}, \
{"emmc", { 512, 512, 512, (1024*1024)/512, 512}}, \
{"nor", { 1, 1, 4096, (1024*1024)/4096, 64}}, \
{"nand", {4096, 4096, 32*1024, (1024*1024)/(32*1024), 4096}}, \
}
#define TEST_GEOMETRY_COUNT 5
#define TEST_GEOMETRY_DEFINE_COUNT 5
#endif #endif
scripts/test.py
+103 -54
View File
@@ -252,6 +252,7 @@ def compile(**args):
# include test_runner.h in every generated file # include test_runner.h in every generated file
f.writeln("#include \"%s\"" % HEADER_PATH) f.writeln("#include \"%s\"" % HEADER_PATH)
f.writeln()
# write out generated functions, this can end up in different # write out generated functions, this can end up in different
# files depending on the "in" attribute # files depending on the "in" attribute
@@ -270,20 +271,22 @@ def compile(**args):
name = ('__test__%s__%s__%s__%d' name = ('__test__%s__%s__%s__%d'
% (suite.name, case.name, k, i)) % (suite.name, case.name, k, i))
define_cbs[v] = name define_cbs[v] = name
f.writeln('intmax_t %s(void) {' % name) f.writeln('intmax_t %s('
'__attribute__((unused)) '
'size_t define) {' % name)
f.writeln(4*' '+'return %s;' % v) f.writeln(4*' '+'return %s;' % v)
f.writeln('}') f.writeln('}')
f.writeln() f.writeln()
f.writeln('intmax_t (*const *const ' f.writeln('intmax_t (*const *const '
'__test__%s__%s__defines[])(void) = {' '__test__%s__%s__defines[])(size_t) = {'
% (suite.name, case.name)) % (suite.name, case.name))
for defines in case.permutations: for defines in case.permutations:
f.writeln(4*' '+'(intmax_t (*const[])(void)){') f.writeln(4*' '+'(intmax_t (*const['
for define in sorted(suite.defines): 'TEST_IMPLICIT_DEFINE_COUNT+%d])(size_t)){' % (
f.writeln(8*' '+'%s,' % ( len(suite.defines)))
define_cbs[defines[define]] for k, v in sorted(defines.items()):
if define in defines f.writeln(8*' '+'[%-24s] = %s,' % (
else 'NULL')) k+'_i', define_cbs[v]))
f.writeln(4*' '+'},') f.writeln(4*' '+'},')
f.writeln('};') f.writeln('};')
f.writeln() f.writeln()
@@ -328,8 +331,10 @@ def compile(**args):
if suite.defines: if suite.defines:
for i, define in enumerate(sorted(suite.defines)): for i, define in enumerate(sorted(suite.defines)):
f.writeln('#ifndef %s' % define) f.writeln('#ifndef %s' % define)
f.writeln('#define %-24s test_define(%d)' f.writeln('#define %-24s '
% (define, i)) 'TEST_IMPLICIT_DEFINE_COUNT+%d' % (define+'_i', i))
f.writeln('#define %-24s '
'test_define(%s)' % (define, define+'_i'))
f.writeln('#endif') f.writeln('#endif')
f.writeln() f.writeln()
@@ -340,7 +345,7 @@ def compile(**args):
else: else:
if case.defines: if case.defines:
f.writeln('extern intmax_t (*const *const ' f.writeln('extern intmax_t (*const *const '
'__test__%s__%s__defines[])(void);' '__test__%s__%s__defines[])(size_t);'
% (suite.name, case.name)) % (suite.name, case.name))
if suite.if_ is not None or case.if_ is not None: if suite.if_ is not None or case.if_ is not None:
f.writeln('extern bool __test__%s__%s__filter(' f.writeln('extern bool __test__%s__%s__filter('
@@ -364,11 +369,14 @@ def compile(**args):
or 0)) or 0))
if suite.defines: if suite.defines:
# create suite define names # create suite define names
f.writeln(4*' '+'.define_names = (const char *const[]){') f.writeln(4*' '+'.define_names = (const char *const['
'TEST_IMPLICIT_DEFINE_COUNT+%d]){' % (
len(suite.defines)))
for k in sorted(suite.defines): for k in sorted(suite.defines):
f.writeln(8*' '+'"%s",' % k) f.writeln(8*' '+'[%-24s] = "%s",' % (k+'_i', k))
f.writeln(4*' '+'},') f.writeln(4*' '+'},')
f.writeln(4*' '+'.define_count = %d,' % len(suite.defines)) f.writeln(4*' '+'.define_count = '
'TEST_IMPLICIT_DEFINE_COUNT+%d,' % len(suite.defines))
f.writeln(4*' '+'.cases = (const struct test_case[]){') f.writeln(4*' '+'.cases = (const struct test_case[]){')
for case in suite.cases: for case in suite.cases:
# create case structs # create case structs
@@ -415,10 +423,15 @@ def compile(**args):
for i, define in enumerate( for i, define in enumerate(
sorted(suite.defines)): sorted(suite.defines)):
f.writeln('#ifndef %s' % define) f.writeln('#ifndef %s' % define)
f.writeln('#define %-24s test_define(%d)' f.writeln('#define %-24s '
% (define, i)) 'TEST_IMPLICIT_DEFINE_COUNT+%d' % (
f.writeln('#define __TEST__%s__NEEDS_UNDEF' define+'_i', i))
% define) f.writeln('#define %-24s '
'test_define(%s)' % (
define, define+'_i'))
f.writeln('#define '
'__TEST__%s__NEEDS_UNDEF' % (
define))
f.writeln('#endif') f.writeln('#endif')
f.writeln() f.writeln()
@@ -431,6 +444,7 @@ def compile(**args):
f.writeln('#undef __TEST__%s__NEEDS_UNDEF' f.writeln('#undef __TEST__%s__NEEDS_UNDEF'
% define) % define)
f.writeln('#undef %s' % define) f.writeln('#undef %s' % define)
f.writeln('#undef %s' % (define+'_i'))
f.writeln('#endif') f.writeln('#endif')
f.writeln() f.writeln()
@@ -469,9 +483,10 @@ def list_(**args):
if args.get('summary'): cmd.append('--summary') if args.get('summary'): cmd.append('--summary')
if args.get('list_suites'): cmd.append('--list-suites') if args.get('list_suites'): cmd.append('--list-suites')
if args.get('list_cases'): cmd.append('--list-cases') if args.get('list_cases'): cmd.append('--list-cases')
if args.get('list_paths'): cmd.append('--list-paths') if args.get('list_suite_paths'): cmd.append('--list-suite-paths')
if args.get('list_case_paths'): cmd.append('--list-case-paths')
if args.get('list_defines'): cmd.append('--list-defines') if args.get('list_defines'): cmd.append('--list-defines')
if args.get('list_defaults'): cmd.append('--list-defaults') if args.get('list_implicit'): cmd.append('--list-implicit')
if args.get('list_geometries'): cmd.append('--list-geometries') if args.get('list_geometries'): cmd.append('--list-geometries')
if args.get('list_powerlosses'): cmd.append('--list-powerlosses') if args.get('list_powerlosses'): cmd.append('--list-powerlosses')
@@ -503,10 +518,11 @@ def find_cases(runner_, **args):
m = pattern.match(line) m = pattern.match(line)
if m: if m:
filtered = int(m.group('filtered')) filtered = int(m.group('filtered'))
perms = int(m.group('perms'))
expected_suite_perms[m.group('suite')] += filtered expected_suite_perms[m.group('suite')] += filtered
expected_case_perms[m.group('id')] += filtered expected_case_perms[m.group('id')] += filtered
expected_perms += filtered expected_perms += filtered
total_perms += int(m.group('perms')) total_perms += perms
proc.wait() proc.wait()
if proc.returncode != 0: if proc.returncode != 0:
if not args.get('verbose'): if not args.get('verbose'):
@@ -520,9 +536,9 @@ def find_cases(runner_, **args):
expected_perms, expected_perms,
total_perms) total_perms)
def find_paths(runner_, **args): def find_path(runner_, id, **args):
# query from runner # query from runner
cmd = runner_ + ['--list-paths'] cmd = runner_ + ['--list-case-paths', id]
if args.get('verbose'): if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd)) print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, proc = sp.Popen(cmd,
@@ -531,14 +547,17 @@ def find_paths(runner_, **args):
universal_newlines=True, universal_newlines=True,
errors='replace', errors='replace',
close_fds=False) close_fds=False)
paths = co.OrderedDict() path = None
pattern = re.compile( pattern = re.compile(
'^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+' '^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
'(?P<path>[^:]+):(?P<lineno>\d+)') '(?P<path>[^:]+):(?P<lineno>\d+)')
for line in proc.stdout: # skip the first line
for line in it.islice(proc.stdout, 1, None):
m = pattern.match(line) m = pattern.match(line)
if m: if m and path is None:
paths[m.group('id')] = (m.group('path'), int(m.group('lineno'))) path_ = m.group('path')
lineno = int(m.group('lineno'))
path = (path_, lineno)
proc.wait() proc.wait()
if proc.returncode != 0: if proc.returncode != 0:
if not args.get('verbose'): if not args.get('verbose'):
@@ -546,11 +565,36 @@ def find_paths(runner_, **args):
sys.stdout.write(line) sys.stdout.write(line)
sys.exit(-1) sys.exit(-1)
return paths return path
def find_defines(runner_, **args): def find_defines(runner_, id, **args):
# query from runner # query implicit defines from runner
cmd = runner_ + ['--list-defines'] cmd = runner_ + ['--list-implicit']
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd,
stdout=sp.PIPE,
stderr=sp.PIPE if not args.get('verbose') else None,
universal_newlines=True,
errors='replace',
close_fds=False)
implicit_defines = co.OrderedDict()
pattern = re.compile('^(?P<define>\w+)=(?P<values>.+)')
for line in proc.stdout:
m = pattern.match(line)
if m:
define = m.group('define')
values = m.group('values').split(',')
implicit_defines[define] = set(values)
proc.wait()
if proc.returncode != 0:
if not args.get('verbose'):
for line in proc.stderr:
sys.stdout.write(line)
sys.exit(-1)
# query case defines from runner
cmd = runner_ + ['--list-defines', id]
if args.get('verbose'): if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd)) print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, proc = sp.Popen(cmd,
@@ -560,14 +604,15 @@ def find_defines(runner_, **args):
errors='replace', errors='replace',
close_fds=False) close_fds=False)
defines = co.OrderedDict() defines = co.OrderedDict()
pattern = re.compile( pattern = re.compile('^(?P<define>\w+)=(?P<value>.+)')
'^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
'(?P<defines>(?:\w+=\w+\s*)+)')
for line in proc.stdout: for line in proc.stdout:
m = pattern.match(line) m = pattern.match(line)
if m: if m:
defines[m.group('id')] = {k: v define = m.group('define')
for k, v in re.findall('(\w+)=(\w+)', m.group('defines'))} value = m.group('value')
if (define not in implicit_defines
or value not in implicit_defines[define]):
defines[define] = value
proc.wait() proc.wait()
if proc.returncode != 0: if proc.returncode != 0:
if not args.get('verbose'): if not args.get('verbose'):
@@ -887,16 +932,16 @@ def run(**args):
print() print()
# print each failure # print each failure
if failures:
# get some extra info from runner
runner_paths = find_paths(runner_, **args)
runner_defines = find_defines(runner_, **args)
for failure in failures: for failure in failures:
# show summary of failure assert failure.id is not None, '%s broken? %r' % (
path, lineno = runner_paths[testcase(failure.id)] ' '.join(shlex.quote(c) for c in runner_),
defines = runner_defines.get(failure.id, {}) failure)
# get some extra info from runner
path, lineno = find_path(runner_, failure.id, **args)
defines = find_defines(runner_, failure.id, **args)
# show summary of failure
print('%s%s:%d:%sfailure:%s %s%s failed' % ( print('%s%s:%d:%sfailure:%s %s%s failed' % (
'\x1b[01m' if color(**args) else '', '\x1b[01m' if color(**args) else '',
path, lineno, path, lineno,
@@ -969,9 +1014,10 @@ def main(**args):
elif (args.get('summary') elif (args.get('summary')
or args.get('list_suites') or args.get('list_suites')
or args.get('list_cases') or args.get('list_cases')
or args.get('list_paths') or args.get('list_suite_paths')
or args.get('list_case_paths')
or args.get('list_defines') or args.get('list_defines')
or args.get('list_defaults') or args.get('list_implicit')
or args.get('list_geometries') or args.get('list_geometries')
or args.get('list_powerlosses')): or args.get('list_powerlosses')):
list_(**args) list_(**args)
@@ -1005,20 +1051,23 @@ if __name__ == "__main__":
help="List test suites.") help="List test suites.")
test_parser.add_argument('-L', '--list-cases', action='store_true', test_parser.add_argument('-L', '--list-cases', action='store_true',
help="List test cases.") help="List test cases.")
test_parser.add_argument('--list-paths', action='store_true', test_parser.add_argument('--list-suite-paths', action='store_true',
help="List the path for each test case.") help="List the path for each test suite.")
test_parser.add_argument('--list-case-paths', action='store_true',
help="List the path and line number for each test case.")
test_parser.add_argument('--list-defines', action='store_true', test_parser.add_argument('--list-defines', action='store_true',
help="List the defines for each test permutation.") help="List all defines in this test-runner.")
test_parser.add_argument('--list-defaults', action='store_true', test_parser.add_argument('--list-implicit', action='store_true',
help="List the default defines in this test-runner.") help="List implicit defines in this test-runner.")
test_parser.add_argument('--list-geometries', action='store_true', test_parser.add_argument('--list-geometries', action='store_true',
help="List the disk geometries used for testing.") help="List the available disk geometries.")
test_parser.add_argument('--list-powerlosses', action='store_true', test_parser.add_argument('--list-powerlosses', action='store_true',
help="List the available power-loss scenarios.") help="List the available power-loss scenarios.")
test_parser.add_argument('-D', '--define', action='append', test_parser.add_argument('-D', '--define', action='append',
help="Override a test define.") help="Override a test define.")
test_parser.add_argument('-G', '--geometry', test_parser.add_argument('-g', '--geometry',
help="Filter by geometry.") help="Comma-separated list of disk geometries to test. \
Defaults to d,e,E,n,N.")
test_parser.add_argument('-p', '--powerloss', test_parser.add_argument('-p', '--powerloss',
help="Comma-separated list of power-loss scenarios to test. \ help="Comma-separated list of power-loss scenarios to test. \
Defaults to 0,l.") Defaults to 0,l.")