Github/littlefs
1
Fork 0
Code Actions Activity

Some more minor improvements to the test_runner

Browse Source 
- Indirect index map instead of bitmap+sparse array
- test_define_t and test_type_t
- Added back conditional filtering
- Added suite-level defines and filtering
This commit is contained in:
Christopher Haster
2022-04-18 00:09:01 -05:00
parent d683f1c76c
commit 4b0aa6272e
3 changed files with 147 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runners/test_runner.c
+71 -60
View File
@@ -8,12 +8,12 @@
// disk geometries
struct test_geometry {
const char *name;
const uintmax_t *defines;
const test_define_t *defines;
};
// Note this includes the default configuration for test pre-defines
#define TEST_GEOMETRY(name, read, prog, erase, count) \
{name, (const uintmax_t[]){ \
{name, (const test_define_t[]){ \
/* READ_SIZE */ read, \
/* PROG_SIZE */ prog, \
/* BLOCK_SIZE */ erase, \
@@ -46,14 +46,14 @@ const size_t test_geometry_count = (
// test define lookup and management
const uintmax_t *test_defines[3] = {NULL};
const bool *test_define_masks[2] = {NULL};
const test_define_t *test_defines[3] = {NULL};
const uint8_t *test_define_maps[2] = {NULL};
uintmax_t test_define(size_t define) {
if (test_define_masks[0] && test_define_masks[0][define]) {
return test_defines[0][define];
} else if (test_define_masks[1] && test_define_masks[1][define]) {
return test_defines[1][define];
test_define_t test_define(size_t define) {
if (test_define_maps[0] && test_define_maps[0][define] != 0xff) {
return test_defines[0][test_define_maps[0][define]];
} else if (test_define_maps[1] && test_define_maps[1][define] != 0xff) {
return test_defines[1][test_define_maps[1][define]];
} else {
return test_defines[2][define];
}
@@ -70,61 +70,54 @@ void test_define_geometry(const struct test_geometry *geometry) {
void test_define_case(const struct test_case *case_, size_t perm) {
if (case_ && case_->defines) {
test_defines[1] = case_->defines[perm];
test_define_masks[1] = case_->define_mask;
test_define_maps[1] = case_->define_map;
} else {
test_defines[1] = NULL;
test_define_masks[1] = NULL;
test_define_maps[1] = NULL;
}
}
struct override {
const char *name;
uintmax_t override;
};
void test_define_overrides(
const struct test_suite *suite,
const struct override *overrides,
const char *const *override_names,
const test_define_t *override_defines,
size_t override_count) {
if (overrides && override_count > 0) {
uintmax_t *defines = malloc(suite->define_count * sizeof(uintmax_t));
memset(defines, 0, suite->define_count * sizeof(uintmax_t));
bool *define_mask = malloc(suite->define_count * sizeof(bool));
memset(define_mask, 0, suite->define_count * sizeof(bool));
if (override_names && override_defines && override_count > 0) {
uint8_t *define_map = malloc(suite->define_count * sizeof(uint8_t));
memset(define_map, 0xff, suite->define_count * sizeof(bool));
// lookup each override in the suite defines, they may have a
// different index in each suite
for (size_t i = 0; i < override_count; i++) {
ssize_t index = -1;
for (size_t j = 0; j < suite->define_count; j++) {
if (strcmp(overrides[i].name, suite->define_names[j]) == 0) {
index = j;
size_t j = 0;
for (; j < suite->define_count; j++) {
if (strcmp(override_names[i], suite->define_names[j]) == 0) {
break;
}
}
if (index >= 0) {
defines[index] = overrides[i].override;
define_mask[index] = true;
if (j < suite->define_count) {
define_map[j] = i;
}
}
test_defines[0] = defines;
test_define_masks[0] = define_mask;
test_defines[0] = override_defines;
test_define_maps[0] = define_map;
} else {
free((uintmax_t *)test_defines[0]);
test_defines[0] = NULL;
free((bool *)test_define_masks[0]);
test_define_masks[0] = NULL;
free((uint8_t *)test_define_maps[0]);
test_define_maps[0] = NULL;
}
}
// operations we can do
void summary(
struct override *overrides,
const char *const *override_names,
const test_define_t *override_defines,
size_t override_count) {
(void)overrides;
(void)override_names;
(void)override_defines;
(void)override_count;
printf("%-36s %7s %7s %7s %7s\n",
"", "geoms", "suites", "cases", "perms");
@@ -147,9 +140,11 @@ void summary(
}
void list_suites(
struct override *overrides,
const char *const *override_names,
const test_define_t *override_defines,
size_t override_count) {
(void)overrides;
(void)override_names;
(void)override_defines;
(void)override_count;
printf("%-36s %-12s %7s %7s %7s\n",
"id", "suite", "types", "cases", "perms");
@@ -169,9 +164,11 @@ void list_suites(
}
void list_cases(
struct override *overrides,
const char *const *override_names,
const test_define_t *override_defines,
size_t override_count) {
(void)overrides;
(void)override_names;
(void)override_defines;
(void)override_count;
printf("%-36s %-12s %-12s %7s %7s\n",
"id", "suite", "case", "types", "perms");
@@ -189,9 +186,11 @@ void list_cases(
}
void list_paths(
struct override *overrides,
const char *const *override_names,
const test_define_t *override_defines,
size_t override_count) {
(void)overrides;
(void)override_names;
(void)override_defines;
(void)override_count;
printf("%-36s %-36s\n", "id", "path");
for (size_t i = 0; i < test_suite_count; i++) {
@@ -204,17 +203,21 @@ void list_paths(
}
void list_defines(
struct override *overrides,
const char *const *override_names,
const test_define_t *override_defines,
size_t override_count) {
(void)overrides;
(void)override_names;
(void)override_defines;
(void)override_count;
// TODO
}
void list_geometries(
struct override *overrides,
const char *const *override_names,
const test_define_t *override_defines,
size_t override_count) {
(void)overrides;
(void)override_names;
(void)override_defines;
(void)override_count;
printf("%-36s %7s %7s %7s %7s %7s\n",
"name", "read", "prog", "erase", "count", "size");
@@ -232,10 +235,13 @@ void list_geometries(
}
void run(
struct override *overrides,
const char *const *override_names,
const test_define_t *override_defines,
size_t override_count) {
for (size_t i = 0; i < test_suite_count; i++) {
test_define_overrides(test_suites[i], overrides, override_count);
test_define_overrides(
test_suites[i],
override_names, override_defines, override_count);
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
for (size_t perm = 0;
@@ -303,7 +309,7 @@ void run(
}
}
test_define_overrides(NULL, NULL, 0);
test_define_overrides(NULL, NULL, NULL, 0);
}
}
@@ -349,9 +355,11 @@ const char *const help_text[] = {
int main(int argc, char **argv) {
void (*op)(
struct override *overrides,
const char *const *override_names,
const test_define_t *override_defines,
size_t override_count) = run;
struct override *overrides = NULL;
const char **override_names = NULL;
test_define_t *override_defines = NULL;
size_t override_count = 0;
size_t override_cap = 0;
@@ -422,24 +430,26 @@ int main(int argc, char **argv) {
override_count += 1;
if (override_count > override_cap) {
override_cap = (2*override_cap > 4) ? 2*override_cap : 4;
overrides = realloc(overrides, override_cap
* sizeof(struct override));
override_names = realloc(override_names, override_cap
* sizeof(const char *));
override_defines = realloc(override_defines, override_cap
* sizeof(test_define_t));
}
// parse into string key/uintmax_t value, cannibalizing the
// parse into string key/test_define_t value, cannibalizing the
// arg in the process
char *sep = strchr(optarg, '=');
char *parsed = NULL;
if (!sep) {
goto invalid_define;
}
overrides[override_count-1].override
override_defines[override_count-1]
= strtoumax(sep+1, &parsed, 0);
if (parsed == sep+1) {
goto invalid_define;
}
overrides[override_count-1].name = optarg;
override_names[override_count-1] = optarg;
*sep = '\0';
break;
@@ -458,15 +468,16 @@ invalid_define:
getopt_done:
for (size_t i = 0; i < override_count; i++) {
printf("define: %s %ju\n", overrides[i].name, overrides[i].override);
printf("define: %s %ju\n", override_names[i], override_defines[i]);
}
// do the thing
op(
overrides,
override_names,
override_defines,
override_count);
// cleanup (need to be done for valgrind testing)
free(overrides);
}
free(override_names);
free(override_defines);}
runners/test_runner.h
+8 -5
View File
@@ -5,21 +5,24 @@
// generated test configurations
enum test_type {
enum test_types {
TEST_NORMAL = 0x1,
TEST_REENTRANT = 0x2,
TEST_VALGRIND = 0x4,
};
typedef uint8_t test_types_t;
typedef uintmax_t test_define_t;
struct test_case {
const char *id;
const char *name;
const char *path;
uint8_t types;
test_types_t types;
size_t permutations;
const uintmax_t *const *defines;
const bool *define_mask;
const test_define_t *const *defines;
const uint8_t *define_map;
bool (*filter)(struct lfs_config *cfg, uint32_t perm);
void (*run)(struct lfs_config *cfg, uint32_t perm);
@@ -43,7 +46,7 @@ extern const size_t test_suite_count;
// access generated test defines
uintmax_t test_define(size_t define);
test_define_t test_define(size_t define);
// a few preconfigured defines that control how tests run
#define READ_SIZE test_define(0)
scripts/test_.py
+59 -27
View File
@@ -66,23 +66,19 @@ class TestCase:
self.path = config.pop('path')
self.suite = config.pop('suite')
self.lineno = config.pop('lineno', None)
self.if_ = config.pop('if', None)
if isinstance(self.if_, bool):
self.if_ = 'true' if self.if_ else 'false'
self.if_lineno = config.pop('if_lineno', None)
self.code = config.pop('code')
self.code_lineno = config.pop('code_lineno', None)
# figure out defines and the number of resulting permutations
self.defines = {}
for k, v in config.pop('defines', {}).items():
try:
v = eval(v)
except:
v = v
if not isinstance(v, str):
try:
v = list(v)
except:
v = [v]
else:
for k, v in (
config.pop('suite_defines', {})
| config.pop('defines', {})).items():
if not isinstance(v, list):
v = [v]
self.defines[k] = v
@@ -111,14 +107,18 @@ class TestSuite:
# find line numbers
f.seek(0)
case_linenos = []
if_linenos = []
code_linenos = []
for i, line in enumerate(f):
match = re.match(
'(?P<case>\[\s*cases\s*\.\s*(?P<name>\w+)\s*\])' +
'|(?P<code>code\s*=\s*(?:\'\'\'|"""))',
'|(?P<if>if\s*=)'
'|(?P<code>code\s*=)',
line)
if match and match.group('case'):
case_linenos.append((i+1, match.group('name')))
elif match and match.group('if'):
if_linenos.append(i+1)
elif match and match.group('code'):
code_linenos.append(i+2)
@@ -129,18 +129,33 @@ class TestSuite:
for (lineno, name), (nlineno, _) in it.zip_longest(
case_linenos, case_linenos[1:],
fillvalue=(float('inf'), None)):
if_lineno = min(
(l for l in if_linenos if l >= lineno and l < nlineno),
default=None)
code_lineno = min(
(l for l in code_linenos if l >= lineno and l < nlineno),
default=None)
cases[name]['lineno'] = lineno
cases[name]['if_lineno'] = if_lineno
cases[name]['code_lineno'] = code_lineno
self.if_ = config.pop('if', None)
if isinstance(self.if_, bool):
self.if_ = 'true' if self.if_ else 'false'
self.if_lineno = min(
(l for l in if_linenos
if not case_linenos or l < case_linenos[0][0]),
default=None)
self.code = config.pop('code', None)
self.code_lineno = min(
(l for l in code_linenos
if not case_linenos or l < case_linenos[0][0]),
default=None)
# a couple of these we just forward to all cases
defines = config.pop('defines', {})
self.cases = []
for name, case in sorted(cases.items(),
key=lambda c: c[1].get('lineno')):
@@ -149,6 +164,7 @@ class TestSuite:
'path': path + (':%d' % case['lineno']
if 'lineno' in case else ''),
'suite': self.name,
'suite_defines': defines,
**case}))
# combine pre-defines and per-case defines
@@ -164,8 +180,6 @@ class TestSuite:
def compile(**args):
# find .toml files
paths = []
@@ -208,20 +222,21 @@ def compile(**args):
for case in suite.cases:
# create case defines
if case.defines:
sorted_defines = sorted(case.defines.items())
for perm, defines in enumerate(
it.product(*(
[(k, v) for v in vs]
for k, vs in case.defines.items()))):
f.write('const uintmax_t '
for k, vs in sorted_defines))):
f.write('const test_define_t '
'__test__%s__%s__%d__defines[] = {\n'
% (suite.name, case.name, perm))
for k, v in sorted(defines):
f.write(4*' '+'[%d] = %s,\n'
% (suite.defines.index(k), v))
for k, v in defines:
f.write(4*' '+'%s,\n' % v)
f.write('};\n')
f.write('\n')
f.write('const uintmax_t *const '
f.write('const test_define_t *const '
'__test__%s__%s__defines[] = {\n'
% (suite.name, case.name))
for perm in range(case.permutations):
@@ -230,20 +245,36 @@ def compile(**args):
f.write('};\n')
f.write('\n')
f.write('const bool '
'__test__%s__%s__define_mask[] = {\n'
f.write('const uint8_t '
'__test__%s__%s__define_map[] = {\n'
% (suite.name, case.name))
for i, k in enumerate(suite.defines):
for k in suite.defines:
f.write(4*' '+'%s,\n'
% ('true' if k in case.defines else 'false'))
% ([k for k, _ in sorted_defines].index(k)
if k in case.defines else '0xff'))
f.write('};\n')
f.write('\n')
# create case filter function
if suite.if_ is not None or case.if_ is not None:
f.write('bool __test__%s__%s__filter('
'__attribute__((unused)) struct lfs_config *cfg, '
'__attribute__((unused)) uint32_t perm) {\n'
% (suite.name, case.name))
if suite.if_ is not None:
f.write(4*' '+'#line %d "%s"\n'
% (suite.if_lineno, suite.path))
f.write(4*' '+'if (!(%s)) {\n' % suite.if_)
f.write(8*' '+'return false;\n')
f.write(4*' '+'}\n')
f.write('\n')
if case.if_ is not None:
f.write(4*' '+'#line %d "%s"\n'
% (case.if_lineno, suite.path))
f.write(4*' '+'if (!(%s)) {\n' % case.if_)
f.write(8*' '+'return false;\n')
f.write(4*' '+'}\n')
f.write('\n')
f.write(4*' '+'return true;\n')
f.write('}\n')
f.write('\n')
@@ -278,9 +309,10 @@ def compile(**args):
if case.defines:
f.write(4*' '+'.defines = __test__%s__%s__defines,\n'
% (suite.name, case.name))
f.write(4*' '+'.define_mask = '
'__test__%s__%s__define_mask,\n'
f.write(4*' '+'.define_map = '
'__test__%s__%s__define_map,\n'
% (suite.name, case.name))
if suite.if_ is not None or case.if_ is not None:
f.write(4*' '+'.filter = __test__%s__%s__filter,\n'
% (suite.name, case.name))
f.write(4*' '+'.run = __test__%s__%s__run,\n'