diff --git a/runners/test_runner.c b/runners/test_runner.c index 4a31ccfc..cb7dd9f4 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -274,18 +274,17 @@ void test_define_geometry(const test_geometry_t *geometry) { // override updates typedef struct test_override { const char *name; - intmax_t define; + const intmax_t *defines; + size_t permutations; } test_override_t; const test_override_t *test_overrides = NULL; size_t test_override_count = 0; -void test_define_overrides( - const test_override_t *overrides, - size_t override_count) { - test_overrides = overrides; - test_override_count = override_count; -} +test_define_t *test_override_defines = NULL; +size_t test_override_define_count = 0; +size_t test_override_define_permutations = 1; +size_t test_override_define_capacity = 0; // suite/perm updates void test_define_suite(const struct test_suite *suite) { @@ -308,35 +307,63 @@ void test_define_suite(const struct test_suite *suite) { // map any overrides if (test_override_count > 0) { + // first figure out the total size of override permutations + size_t count = 0; + size_t permutations = 1; + for (size_t i = 0; i < test_override_count; i++) { + for (size_t d = 0; + d < lfs_max( + suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + // define name match? + const char *name = test_define_name(d); + if (name && strcmp(name, test_overrides[i].name) == 0) { + count = d+1; + permutations *= test_overrides[i].permutations; + break; + } + } + } + test_override_define_count = count; + test_override_define_permutations = permutations; + // make sure our override arrays are big enough - if (suite->define_count - > test_define_maps[TEST_DEFINE_MAP_OVERRIDE].count) { + if (count * permutations > test_override_define_capacity) { // align to power of two to avoid any superlinear growth - size_t ncount = 1 << lfs_npw2(suite->define_count); - test_define_maps[TEST_DEFINE_MAP_OVERRIDE].defines = realloc( - (test_define_t*)test_define_maps[ - TEST_DEFINE_MAP_OVERRIDE].defines, - ncount*sizeof(test_define_t)); - test_define_maps[TEST_DEFINE_MAP_OVERRIDE].count = ncount; + size_t ncapacity = 1 << lfs_npw2(count * permutations); + test_override_defines = realloc( + test_override_defines, + sizeof(test_define_t)*ncapacity); + test_override_define_capacity = ncapacity; } - for (size_t i = 0; - i < test_define_maps[TEST_DEFINE_MAP_OVERRIDE].count; - i++) { - ((test_define_t*)test_define_maps[ - TEST_DEFINE_MAP_OVERRIDE].defines)[i] - = (test_define_t){NULL}; + // zero unoverridden defines + memset(test_override_defines, 0, + sizeof(test_define_t) * count * permutations); - const char *name = test_define_name(i); - if (!name) { - continue; - } + // compute permutations + size_t p = 1; + for (size_t i = 0; i < test_override_count; i++) { + for (size_t d = 0; + d < lfs_max( + suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + // define name match? + const char *name = test_define_name(d); + if (name && strcmp(name, test_overrides[i].name) == 0) { + // scatter the define permutations based on already + // seen permutations + for (size_t j = 0; j < permutations; j++) { + test_override_defines[j*count + d] + = (test_define_t)TEST_LIT( + test_overrides[i].defines[(j/p) + % test_overrides[i].permutations]); + } - for (size_t j = 0; j < test_override_count; j++) { - if (strcmp(name, test_overrides[j].name) == 0) { - ((test_define_t*)test_define_maps[ - TEST_DEFINE_MAP_OVERRIDE].defines)[i] - = (test_define_t)TEST_LIT(test_overrides[j].define); + // keep track of how many permutations we've seen so far + p *= test_overrides[i].permutations; break; } } @@ -350,13 +377,20 @@ void test_define_perm( size_t perm) { if (case_->defines) { test_define_maps[TEST_DEFINE_MAP_PERMUTATION] = (test_define_map_t){ - case_->defines[perm], suite->define_count}; + case_->defines + perm*suite->define_count, + suite->define_count}; } else { test_define_maps[TEST_DEFINE_MAP_PERMUTATION] = (test_define_map_t){ NULL, 0}; } } +void test_define_override(size_t perm) { + test_define_maps[TEST_DEFINE_MAP_OVERRIDE] = (test_define_map_t){ + test_override_defines + perm*test_override_define_count, + test_override_define_count}; +} + void test_define_explicit( const test_define_t *defines, size_t define_count) { @@ -368,7 +402,7 @@ void test_define_cleanup(void) { // test define management can allocate a few things free(test_define_cache); free(test_define_cache_mask); - free((test_define_t*)test_define_maps[TEST_DEFINE_MAP_OVERRIDE].defines); + free(test_override_defines); } @@ -522,25 +556,30 @@ static void case_forperm( // define permutation test_define_perm(suite, case_, k); - for (size_t g = 0; g < test_geometry_count; g++) { - // define geometry - test_define_geometry(&test_geometries[g]); - test_define_flush(); + for (size_t v = 0; v < test_override_define_permutations; v++) { + // define override permutation + test_define_override(v); - if (cycles) { - cb(data, suite, case_, &(test_powerloss_t){ - .run=run_powerloss_cycles, - .cycles=cycles, - .cycle_count=cycle_count}); - } else { - for (size_t p = 0; p < test_powerloss_count; p++) { - // skip non-reentrant tests when powerloss testing - if (test_powerlosses[p].short_name != '0' - && !(case_->flags & TEST_REENTRANT)) { - continue; + for (size_t g = 0; g < test_geometry_count; g++) { + // define geometry + test_define_geometry(&test_geometries[g]); + test_define_flush(); + + if (cycles) { + cb(data, suite, case_, &(test_powerloss_t){ + .run=run_powerloss_cycles, + .cycles=cycles, + .cycle_count=cycle_count}); + } else { + for (size_t p = 0; p < test_powerloss_count; p++) { + // skip non-reentrant tests when powerloss testing + if (test_powerlosses[p].short_name != '0' + && !(case_->flags & TEST_REENTRANT)) { + continue; + } + + cb(data, suite, case_, &test_powerlosses[p]); } - - cb(data, suite, case_, &test_powerlosses[p]); } } } @@ -832,11 +871,31 @@ void perm_list_defines( d < lfs_max(suite->define_count, TEST_IMPLICIT_DEFINE_COUNT); d++) { - if (!test_define_ispermutation(d)) { - continue; + if (d < TEST_IMPLICIT_DEFINE_COUNT + || test_define_ispermutation(d)) { + list_defines_add(defines, d); } + } +} - list_defines_add(defines, d); +void perm_list_permutation_defines( + void *data, + const struct test_suite *suite, + const struct test_case *case_, + const test_powerloss_t *powerloss) { + struct list_defines_defines *defines = data; + (void)suite; + (void)case_; + (void)powerloss; + + // collect permutation_defines + for (size_t d = 0; + d < lfs_max(suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + if (test_define_ispermutation(d)) { + list_defines_add(defines, d); + } } } @@ -845,22 +904,7 @@ extern const test_geometry_t builtin_geometries[]; static void list_defines(void) { struct list_defines_defines defines = {NULL, 0, 0}; - // yes we do need to define a suite, this does a bit of bookeeping - // such as setting up the define cache - test_define_suite(&(const struct test_suite){0}); - - // make sure to include builtin geometries here - for (size_t g = 0; builtin_geometries[g].long_name; g++) { - test_define_geometry(&builtin_geometries[g]); - test_define_flush(); - - // add implicit defines - for (size_t d = 0; d < TEST_IMPLICIT_DEFINE_COUNT; d++) { - list_defines_add(&defines, d); - } - } - - // add permutation defines + // add defines for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { if (test_ids[t].suite && strcmp( @@ -932,7 +976,7 @@ static void list_permutation_defines(void) { test_ids[t].define_count, test_ids[t].cycles, test_ids[t].cycle_count, - perm_list_defines, + perm_list_permutation_defines, &defines); } } @@ -1700,10 +1744,7 @@ const char *const help_text[] = { int main(int argc, char **argv) { void (*op)(void) = run; - test_override_t *overrides = NULL; - size_t override_count = 0; - size_t override_capacity = 0; - + size_t test_override_capacity = 0; size_t test_geometry_capacity = 0; size_t test_powerloss_capacity = 0; size_t test_id_capacity = 0; @@ -1803,10 +1844,10 @@ int main(int argc, char **argv) { case OPT_DEFINE: { // allocate space test_override_t *override = mappend( - (void**)&overrides, + (void**)&test_overrides, sizeof(test_override_t), - &override_count, - &override_capacity); + &test_override_count, + &test_override_capacity); // parse into string key/intmax_t value, cannibalizing the // arg in the process @@ -1815,13 +1856,112 @@ int main(int argc, char **argv) { if (!sep) { goto invalid_define; } - override->define = strtoumax(sep+1, &parsed, 0); - if (parsed == sep+1) { - goto invalid_define; - } - - override->name = optarg; *sep = '\0'; + override->name = optarg; + optarg = sep+1; + + // parse comma-separated permutations + { + override->defines = NULL; + override->permutations = 0; + size_t override_capacity = 0; + while (true) { + optarg += strspn(optarg, " "); + + if (strncmp(optarg, "range", strlen("range")) == 0) { + // range of values + optarg += strlen("range"); + optarg += strspn(optarg, " "); + if (*optarg != '(') { + goto invalid_define; + } + optarg += 1; + + intmax_t start = strtoumax(optarg, &parsed, 0); + intmax_t stop = -1; + intmax_t step = 1; + // allow empty string for start=0 + if (parsed == optarg) { + start = 0; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != ')') { + goto invalid_define; + } + + if (*optarg == ',') { + optarg += 1; + stop = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=end + if (parsed == optarg) { + stop = -1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ',' && *optarg != ')') { + goto invalid_define; + } + + if (*optarg == ',') { + optarg += 1; + step = strtoumax(optarg, &parsed, 0); + // allow empty string for stop=1 + if (parsed == optarg) { + step = 1; + } + optarg = parsed + strspn(parsed, " "); + + if (*optarg != ')') { + goto invalid_define; + } + } + } else { + // single value = stop only + stop = start; + start = 0; + } + + if (*optarg != ')') { + goto invalid_define; + } + optarg += 1; + + // calculate the range of values + assert(step != 0); + for (intmax_t i = start; + (step < 0) + ? i > stop + : (uintmax_t)i < (uintmax_t)stop; + i += step) { + *(intmax_t*)mappend( + (void**)&override->defines, + sizeof(intmax_t), + &override->permutations, + &override_capacity) = i; + } + } else if (*optarg != '\0') { + // single value + intmax_t define = strtoimax(optarg, &parsed, 0); + if (parsed == optarg) { + goto invalid_define; + } + *(intmax_t*)mappend( + (void**)&override->defines, + sizeof(intmax_t), + &override->permutations, + &override_capacity) = define; + } else { + break; + } + + optarg = parsed + strspn(parsed, " "); + if (*optarg == ',') { + optarg += 1; + } + } + } + assert(override->permutations > 0); break; invalid_define: @@ -2117,10 +2257,12 @@ powerloss_next: } case OPT_STEP: { char *parsed = NULL; - size_t start = strtoumax(optarg, &parsed, 0); + test_step_start = strtoumax(optarg, &parsed, 0); + test_step_stop = -1; + test_step_step = 1; // allow empty string for start=0 - if (parsed != optarg) { - test_step_start = start; + if (parsed == optarg) { + test_step_start = 0; } optarg = parsed + strspn(parsed, " "); @@ -2130,10 +2272,10 @@ powerloss_next: if (*optarg == ',') { optarg += 1; - size_t stop = strtoumax(optarg, &parsed, 0); + test_step_stop = strtoumax(optarg, &parsed, 0); // allow empty string for stop=end - if (parsed != optarg) { - test_step_stop = stop; + if (parsed == optarg) { + test_step_stop = -1; } optarg = parsed + strspn(parsed, " "); @@ -2143,10 +2285,10 @@ powerloss_next: if (*optarg == ',') { optarg += 1; - size_t step = strtoumax(optarg, &parsed, 0); + test_step_step = strtoumax(optarg, &parsed, 0); // allow empty string for stop=1 - if (parsed != optarg) { - test_step_step = step; + if (parsed == optarg) { + test_step_step = 1; } optarg = parsed + strspn(parsed, " "); @@ -2154,6 +2296,10 @@ powerloss_next: goto step_unknown; } } + } else { + // single value = stop only + test_step_stop = test_step_start; + test_step_start = 0; } break; @@ -2317,30 +2463,31 @@ getopt_done: ; }; } - // register overrides - test_define_overrides(overrides, override_count); - // do the thing op(); // cleanup (need to be done for valgrind testing) test_define_cleanup(); - free(overrides); - + if (test_overrides) { + for (size_t i = 0; i < test_override_count; i++) { + free((void*)test_overrides[i].defines); + } + free((void*)test_overrides); + } if (test_geometry_capacity) { - free((test_geometry_t*)test_geometries); + free((void*)test_geometries); } if (test_powerloss_capacity) { for (size_t i = 0; i < test_powerloss_count; i++) { - free((lfs_testbd_powercycles_t*)test_powerlosses[i].cycles); + free((void*)test_powerlosses[i].cycles); } - free((test_powerloss_t*)test_powerlosses); + free((void*)test_powerlosses); } if (test_id_capacity) { for (size_t i = 0; i < test_id_count; i++) { - free((test_geometry_t*)test_ids[i].defines); - free((lfs_testbd_powercycles_t*)test_ids[i].cycles); + free((void*)test_ids[i].defines); + free((void*)test_ids[i].cycles); } - free((test_id_t*)test_ids); + free((void*)test_ids); } } diff --git a/runners/test_runner.h b/runners/test_runner.h index 813dd8b0..c27b9856 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -43,7 +43,7 @@ struct test_case { test_flags_t flags; size_t permutations; - const test_define_t *const *defines; + const test_define_t *defines; bool (*filter)(void); void (*run)(struct lfs_config *cfg); diff --git a/scripts/test.py b/scripts/test.py index 170307cd..fa41af3b 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -62,6 +62,7 @@ class TestCase: self.defines = set() self.permutations = [] + # defines can be a dict or a list or dicts suite_defines = config.pop('suite_defines', {}) if not isinstance(suite_defines, list): suite_defines = [suite_defines] @@ -69,15 +70,63 @@ class TestCase: if not isinstance(defines, list): defines = [defines] + def csplit(v): + # split commas but only outside of parens + parens = 0 + i_ = 0 + for i in range(len(v)): + if v[i] == ',' and parens == 0: + yield v[i_:i] + i_ = i+1 + elif v[i] in '([{': + parens += 1 + elif v[i] in '}])': + parens -= 1 + if v[i_:].strip(): + yield v[i_:] + + def parse_define(v): + # a define entry can be a list + if isinstance(v, list): + for v_ in v: + yield from parse_define(v_) + # or a string + elif isinstance(v, str): + # which can be comma-separated values, with optional + # range statements. This matches the runtime define parser in + # the runner itself. + for v_ in csplit(v): + m = re.search(r'\brange\b\s*\(' + '(?P[^,\s]*)' + '\s*(?:,\s*(?P[^,\s]*)' + '\s*(?:,\s*(?P[^,\s]*)\s*)?)?\)', + v_) + if m: + start = (int(m.group('start'), 0) + if m.group('start') else 0) + stop = (int(m.group('stop'), 0) + if m.group('stop') else None) + step = (int(m.group('step'), 0) + if m.group('step') else 1) + if m.lastindex <= 1: + start, stop = 0, start + for x in range(start, stop, step): + yield from parse_define('%s(%d)%s' % ( + v_[:m.start()], x, v_[m.end():])) + else: + yield v_ + # or a literal value + else: + yield v + # build possible permutations for suite_defines_ in suite_defines: self.defines |= suite_defines_.keys() for defines_ in defines: self.defines |= defines_.keys() - self.permutations.extend(map(dict, it.product(*( - [(k, v) for v in (vs if isinstance(vs, list) else [vs])] - for k, vs in sorted( - (suite_defines_ | defines_).items()))))) + self.permutations.extend(dict(perm) for perm in it.product(*( + [(k, v) for v in parse_define(vs)] + for k, vs in sorted((suite_defines_ | defines_).items())))) for k in config.keys(): print('%swarning:%s in %s, found unused key %r' % ( @@ -254,13 +303,12 @@ def compile(test_paths, **args): f.writeln(4*' '+'return %s;' % v) f.writeln('}') f.writeln() - f.writeln('const test_define_t *const ' - '__test__%s__%s__defines[] = {' - % (suite.name, case.name)) + f.writeln('const test_define_t ' + '__test__%s__%s__defines[][' + 'TEST_IMPLICIT_DEFINE_COUNT+%d] = {' + % (suite.name, case.name, len(suite.defines))) for defines in case.permutations: - f.writeln(4*' '+'(const test_define_t[' - 'TEST_IMPLICIT_DEFINE_COUNT+%d]){' % ( - len(suite.defines))) + f.writeln(4*' '+'{') for k, v in sorted(defines.items()): f.writeln(8*' '+'[%-24s] = {%s, NULL},' % ( k+'_i', define_cbs[v])) @@ -321,9 +369,10 @@ def compile(test_paths, **args): write_case_functions(f, suite, case) else: if case.defines: - f.writeln('extern const test_define_t *const ' - '__test__%s__%s__defines[];' - % (suite.name, case.name)) + f.writeln('extern const test_define_t ' + '__test__%s__%s__defines[][' + 'TEST_IMPLICIT_DEFINE_COUNT+%d];' + % (suite.name, case.name, len(suite.defines))) if suite.if_ is not None or case.if_ is not None: f.writeln('extern bool __test__%s__%s__filter(' 'void);' @@ -368,7 +417,8 @@ def compile(test_paths, **args): f.writeln(12*' '+'.permutations = %d,' % len(case.permutations)) if case.defines: - f.writeln(12*' '+'.defines = __test__%s__%s__defines,' + f.writeln(12*' '+'.defines ' + '= (const test_define_t*)__test__%s__%s__defines,' % (suite.name, case.name)) if suite.if_ is not None or case.if_ is not None: f.writeln(12*' '+'.filter = __test__%s__%s__filter,' @@ -1088,7 +1138,7 @@ if __name__ == "__main__": help="Output file.") # runner + test_ids overlaps test_paths, so we need to do some munging here - args = parser.parse_args() + args = parser.parse_intermixed_args() args.test_paths = [' '.join(args.runner or [])] + args.test_ids args.runner = args.runner or [RUNNER_PATH] diff --git a/tests/test_dirs.toml b/tests/test_dirs.toml index 60346c0c..07107885 100644 --- a/tests/test_dirs.toml +++ b/tests/test_dirs.toml @@ -18,7 +18,7 @@ code = ''' ''' [cases.many_dir_creation] -defines.N = [3,6,9,12,21,33,57,66,72,93,99] +defines.N = 'range(3, 100, 3)' if = 'N < BLOCK_COUNT/2' code = ''' lfs_t lfs; @@ -55,7 +55,7 @@ code = ''' ''' [cases.many_dir_removal] -defines.N = [3,6,9,12,21,33,57,66,72,93,99] +defines.N = 'range(3, 100, 11)' if = 'N < BLOCK_COUNT/2' code = ''' lfs_t lfs; @@ -112,7 +112,7 @@ code = ''' ''' [cases.many_dir_rename] -defines.N = [3,6,9,12,21,33,57,66,72,93,99] +defines.N = 'range(3, 100, 11)' if = 'N < BLOCK_COUNT/2' code = ''' lfs_t lfs; @@ -266,7 +266,7 @@ code = ''' ''' [cases.file_creation] -defines.N = [3,6,9,12,21,33,57,66,72,93,99] +defines.N = 'range(3, 100, 11)' if = 'N < BLOCK_COUNT/2' code = ''' lfs_t lfs; @@ -306,7 +306,7 @@ code = ''' ''' [cases.file_removal] -defines.N = [3,6,9,12,21,33,57,66,72,93,99] +defines.N = 'range(3, 100, 11)' if = 'N < BLOCK_COUNT/2' code = ''' lfs_t lfs; @@ -366,7 +366,7 @@ code = ''' ''' [cases.file_rename] -defines.N = [3,6,9,12,21,33,57,66,72,93,99] +defines.N = 'range(3, 100, 11)' if = 'N < BLOCK_COUNT/2' code = ''' lfs_t lfs; diff --git a/tests/test_move.toml b/tests/test_move.toml index f1825e48..dc2623e3 100644 --- a/tests/test_move.toml +++ b/tests/test_move.toml @@ -1586,7 +1586,7 @@ code = ''' # move fix in relocation [cases.move_fix_relocation] in = "lfs.c" -defines.RELOCATIONS = [0x0, 0x1, 0x2, 0x3] +defines.RELOCATIONS = 'range(4)' defines.ERASE_CYCLES = 0xffffffff code = ''' lfs_t lfs; @@ -1731,7 +1731,7 @@ code = ''' # move fix in relocation with predecessor [cases.move_fix_relocation_predecessor] in = "lfs.c" -defines.RELOCATIONS = [0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7] +defines.RELOCATIONS = 'range(8)' defines.ERASE_CYCLES = 0xffffffff code = ''' lfs_t lfs; diff --git a/tests/test_truncate.toml b/tests/test_truncate.toml index fc83ce37..80e250fe 100644 --- a/tests/test_truncate.toml +++ b/tests/test_truncate.toml @@ -285,7 +285,7 @@ code = ''' # more aggressive general truncation tests [cases.aggressive_truncate] -defines.CONFIG = [0,1,2,3,4,5] +defines.CONFIG = 'range(6)' defines.SMALLSIZE = 32 defines.MEDIUMSIZE = 2048 defines.LARGESIZE = 8192