diff --git a/runners/bench_runner.c b/runners/bench_runner.c index db4263fe..ef90244f 100644 --- a/runners/bench_runner.c +++ b/runners/bench_runner.c @@ -112,32 +112,21 @@ static uintmax_t leb16_parse(const char *s, char **tail) { typedef struct bench_id { const char *name; - const bench_define_t *defines; + bench_define_t *defines; size_t define_count; } bench_id_t; // bench define management -typedef struct bench_define_map { - const bench_define_t *defines; - size_t count; -} bench_define_map_t; -typedef struct bench_define_names { - const char *const *names; - size_t count; -} bench_define_names_t; +// implicit defines declared here +#define BENCH_DEFINE(k, v) \ + intmax_t k; -intmax_t bench_define_lit(void *data, size_t i) { - (void)i; - return (intptr_t)data; -} + BENCH_IMPLICIT_DEFINES +#undef BENCH_DEFINE -#define BENCH_CONST(x) {bench_define_lit, (void*)(uintptr_t)(x), 1} -#define BENCH_LIT(x) ((bench_define_t)BENCH_CONST(x)) - - -#define BENCH_DEF(k, v) \ +#define BENCH_DEFINE(k, v) \ intmax_t bench_define_##k(void *data, size_t i) { \ (void)data; \ (void)i; \ @@ -145,195 +134,244 @@ intmax_t bench_define_lit(void *data, size_t i) { } BENCH_IMPLICIT_DEFINES -#undef BENCH_DEF +#undef BENCH_DEFINE -#define BENCH_DEFINE_MAP_OVERRIDE 0 -#define BENCH_DEFINE_MAP_EXPLICIT 1 -#define BENCH_DEFINE_MAP_CASE 2 -#define BENCH_DEFINE_MAP_IMPLICIT 3 +const bench_define_t bench_implicit_defines[] = { + #define BENCH_DEFINE(k, v) \ + {#k, &k, bench_define_##k, NULL, 1}, -#define BENCH_DEFINE_MAP_COUNT 4 - -bench_define_map_t bench_define_maps[BENCH_DEFINE_MAP_COUNT] = { - [BENCH_DEFINE_MAP_IMPLICIT] = { - (const bench_define_t[BENCH_IMPLICIT_DEFINE_COUNT]) { - #define BENCH_DEF(k, v) \ - [k##_i] = {bench_define_##k, NULL, 1}, - - BENCH_IMPLICIT_DEFINES - #undef BENCH_DEF - }, - BENCH_IMPLICIT_DEFINE_COUNT, - }, + BENCH_IMPLICIT_DEFINES + #undef BENCH_DEFINE }; +const size_t bench_implicit_define_count + = sizeof(bench_implicit_defines) / sizeof(bench_define_t); -#define BENCH_DEFINE_NAMES_SUITE 0 -#define BENCH_DEFINE_NAMES_IMPLICIT 1 -#define BENCH_DEFINE_NAMES_COUNT 2 +// some helpers +intmax_t bench_define_lit(void *data, size_t i) { + (void)i; + return (intptr_t)data; +} -bench_define_names_t bench_define_names[BENCH_DEFINE_NAMES_COUNT] = { - [BENCH_DEFINE_NAMES_IMPLICIT] = { - (const char *const[BENCH_IMPLICIT_DEFINE_COUNT]){ - #define BENCH_DEF(k, v) \ - [k##_i] = #k, +#define BENCH_LIT(name, v) ((bench_define_t){ \ + name, NULL, bench_define_lit, (void*)(uintptr_t)(v), 1}) - BENCH_IMPLICIT_DEFINES - #undef BENCH_DEF - }, - BENCH_IMPLICIT_DEFINE_COUNT, - }, -}; -size_t bench_define_count; +// define mapping +const bench_define_t **bench_defines = NULL; +size_t bench_define_count = 0; +size_t bench_define_capacity = 0; -typedef struct bench_define_cache_entry { - // - >=0 => not cached - // - -1 => cached - ssize_t permutation; - union { - intmax_t value; - const bench_define_t *define; - } u; -} bench_define_cache_entry_t; +const bench_define_t **bench_suite_defines = NULL; +size_t bench_suite_define_count = 0; +ssize_t *bench_suite_define_map = NULL; -bench_define_cache_entry_t *bench_define_cache; -size_t bench_define_cache_capacity; +bench_define_t *bench_override_defines = NULL; +size_t bench_override_define_count = 0; -const char *bench_define_name(size_t define) { - // lookup in our bench names - for (size_t i = 0; i < BENCH_DEFINE_NAMES_COUNT; i++) { - if (define < bench_define_names[i].count - && bench_define_names[i].names - && bench_define_names[i].names[define]) { - return bench_define_names[i].names[define]; +size_t bench_define_depth = 1000; + + +static inline bool bench_define_isdefined(const bench_define_t *define) { + return define->cb; +} + +static inline bool bench_define_ispermutation(const bench_define_t *define) { + // permutation defines are basically anything that's not implicit + return bench_define_isdefined(define) + && !(define >= bench_implicit_defines + && define + < bench_implicit_defines + + bench_implicit_define_count); +} + + +void bench_define_suite( + const bench_id_t *id, + const struct bench_suite *suite) { + // reset our mapping + bench_define_count = 0; + bench_suite_define_count = 0; + + // make sure we have space for everything, just assume the worst case + if (bench_implicit_define_count + suite->define_count + > bench_define_capacity) { + bench_define_capacity + = bench_implicit_define_count + suite->define_count; + bench_defines = realloc( + bench_defines, + bench_define_capacity*sizeof(const bench_define_t*)); + bench_suite_defines = realloc( + bench_suite_defines, + bench_define_capacity*sizeof(const bench_define_t*)); + bench_suite_define_map = realloc( + bench_suite_define_map, + bench_define_capacity*sizeof(ssize_t)); + } + + // first map our implicit defines + for (size_t i = 0; i < bench_implicit_define_count; i++) { + bench_suite_defines[i] = &bench_implicit_defines[i]; + } + bench_suite_define_count = bench_implicit_define_count; + + // build a mapping from suite defines to bench defines + // + // we will use this for both suite and case defines + memset(bench_suite_define_map, -1, + bench_suite_define_count*sizeof(size_t)); + + for (size_t i = 0; i < suite->define_count; i++) { + // assume suite defines are unique so we only need to compare + // against implicit defines, this avoids a O(n^2) + for (size_t j = 0; j < bench_implicit_define_count; j++) { + if (bench_suite_defines[j]->define == suite->defines[i].define) { + bench_suite_define_map[j] = i; + + // don't override implicit defines if we're not defined + if (bench_define_isdefined(&suite->defines[i])) { + bench_suite_defines[j] = &suite->defines[i]; + } + goto next_suite_define; + } + } + + // map a new suite define + bench_suite_define_map[bench_suite_define_count] = i; + bench_suite_defines[bench_suite_define_count] = &suite->defines[i]; + bench_suite_define_count += 1; +next_suite_define:; + } + + // map any explicit defines + // + // we ignore any out-of-bounds defines here, even though it's likely + // an error + if (id && id->defines) { + for (size_t i = 0; + i < id->define_count && i < bench_suite_define_count; + i++) { + if (bench_define_isdefined(&id->defines[i])) { + // update name/addr + id->defines[i].name = bench_suite_defines[i]->name; + id->defines[i].define = bench_suite_defines[i]->define; + // map and override suite mapping + bench_suite_defines[i] = &id->defines[i]; + bench_suite_define_map[i] = -1; + } } } - return NULL; + // map any override defines + // + // note it's not an error to override a define that doesn't exist + for (size_t i = 0; i < bench_override_define_count; i++) { + for (size_t j = 0; j < bench_suite_define_count; j++) { + if (strcmp( + bench_suite_defines[j]->name, + bench_override_defines[i].name) == 0) { + // update addr + bench_override_defines[i].define + = bench_suite_defines[j]->define; + // map and override suite mapping + bench_suite_defines[j] = &bench_override_defines[i]; + bench_suite_define_map[j] = -1; + goto next_override_define; + } + } +next_override_define:; + } } -bool bench_define_ispermutation(size_t define) { - // is this define specific to the permutation? - for (size_t i = 0; i < BENCH_DEFINE_MAP_IMPLICIT; i++) { - if (define < bench_define_maps[i].count - && bench_define_maps[i].defines[define].cb) { - return true; +void bench_define_case( + const bench_id_t *id, + const struct bench_suite *suite, + const struct bench_case *case_, + size_t perm) { + (void)id; + + // copy over suite defines + for (size_t i = 0; i < bench_suite_define_count; i++) { + // map case define if case define is defined + if (case_->defines + && bench_suite_define_map[i] != -1 + && bench_define_isdefined(&case_->defines[ + perm*suite->define_count + + bench_suite_define_map[i]])) { + bench_defines[i] = &case_->defines[ + perm*suite->define_count + + bench_suite_define_map[i]]; + } else { + bench_defines[i] = bench_suite_defines[i]; } } - - return false; + bench_define_count = bench_suite_define_count; } -size_t bench_define_permutations(size_t define) { - for (size_t i = 0; i < BENCH_DEFINE_MAP_COUNT; i++) { - if (define < bench_define_maps[i].count - && bench_define_maps[i].defines[define].cb) { - return (bench_define_maps[i].defines[define].permutations) - ? bench_define_maps[i].defines[define].permutations - : 1; - } +void bench_define_permutation(size_t perm) { + // first zero everything, we really don't want reproducibility issues + for (size_t i = 0; i < bench_define_count; i++) { + *bench_defines[i]->define = 0; } - return 0; + // defines may be mutually recursive, which makes evaluation a bit tricky + // + // Rather than doing any clever, we just repeatedly evaluate the + // permutation until values stabilize. If things don't stabilize after + // some number of iterations, error, this likely means defines were + // stuck in a cycle + // + size_t attempt = 0; + while (true) { + const bench_define_t *changed = NULL; + // define-specific permutations are encoded in the case permutation + size_t perm_ = perm; + for (size_t i = 0; i < bench_define_count; i++) { + if (bench_defines[i]->cb) { + intmax_t v = bench_defines[i]->cb( + bench_defines[i]->data, + perm_ % bench_defines[i]->permutations); + if (v != *bench_defines[i]->define) { + *bench_defines[i]->define = v; + changed = bench_defines[i]; + } + + perm_ /= bench_defines[i]->permutations; + } + } + + // stabilized? + if (!changed) { + break; + } + + attempt += 1; + if (bench_define_depth && attempt >= bench_define_depth+1) { + fprintf(stderr, "error: could not resolve recursive defines: %s\n", + changed->name); + exit(-1); + } + } } -size_t bench_define_permutationpermutations(void) { +void bench_define_cleanup(void) { + // bench define management can allocate a few things + free(bench_defines); + free(bench_suite_defines); + free(bench_suite_define_map); +} + +size_t bench_define_permutations(void) { size_t prod = 1; - for (size_t d = 0; d < bench_define_count; d++) { - size_t permutations = bench_define_permutations(d); - if (permutations > 0) { - prod *= permutations; - } + for (size_t i = 0; i < bench_define_count; i++) { + prod *= (bench_defines[i]->permutations > 0) + ? bench_defines[i]->permutations + : 1; } return prod; } -intmax_t bench_define(size_t define) { - // cached? - if (bench_define_cache[define].permutation == -1) { - return bench_define_cache[define].u.value; - // lazily defined? - } else if (bench_define_cache[define].u.define) { - // evaluate and store in cache - bench_define_cache[define].u.value - = bench_define_cache[define].u.define->cb( - bench_define_cache[define].u.define->data, - bench_define_cache[define].permutation); - bench_define_cache[define].permutation = -1; - return bench_define_cache[define].u.value; - - // not defined? - } else { - const char *name = bench_define_name(define); - fprintf(stderr, "error: undefined define %s (%zd)\n", - (name) ? name : "(unknown)", - define); - assert(false); - exit(-1); - } -} - -// permutation updates -void bench_define_permutation(size_t perm) { - // We can't completely precompute the defines easily, since they may be - // mutually recursive. But we can precompute the permutations, which is - // expensive otherwise. - // - // Note that it's not really worth it to make define lookup completely - // lazy, the first thing we do is evaluate all defines for 1. deduplication - // and 2. logging. - - if (bench_define_cache_capacity < bench_define_count) { - // align to power of two to avoid any superlinear growth - bench_define_cache_capacity = 1 << lfs_npw2(bench_define_count); - bench_define_cache = realloc( - bench_define_cache, - bench_define_cache_capacity*sizeof(bench_define_cache_entry_t)); - } - - for (size_t d = 0; d < bench_define_count; d++) { - // lookup our bench defines - for (size_t i = 0; i < BENCH_DEFINE_MAP_COUNT; i++) { - if (d < bench_define_maps[i].count - && bench_define_maps[i].defines[d].cb) { - // note we can't precompute these due to mutual recursion - const bench_define_t *define = &bench_define_maps[i].defines[d]; - bench_define_cache[d] = (bench_define_cache_entry_t){ - .permutation = perm % define->permutations, - .u.define = define, - }; - perm /= bench_define_maps[i].defines[d].permutations; - goto next; - } - } - - // default to a null value, these should be unreachable - bench_define_cache[d] = (bench_define_cache_entry_t){0}; - next:; - } -} - -// case updates -void bench_define_case( - const struct bench_suite *suite, - const struct bench_case *case_, - size_t perm) { - if (case_->defines) { - bench_define_maps[BENCH_DEFINE_MAP_CASE] = (bench_define_map_t){ - &case_->defines[perm*suite->define_count], - suite->define_count}; - } else { - bench_define_maps[BENCH_DEFINE_MAP_CASE] = (bench_define_map_t){ - NULL, 0}; - } -} - -// override updates -typedef struct bench_override { - const char *name; - bench_define_t define; -} bench_override_t; +// override define stuff typedef struct bench_override_value { intmax_t start; @@ -378,63 +416,6 @@ intmax_t bench_override_cb(void *data, size_t i) { __builtin_unreachable(); } -const bench_override_t *bench_overrides = NULL; -size_t bench_override_count = 0; - -bench_define_t *bench_override_defines = NULL; -size_t bench_override_define_capacity = 0; - -// suite/perm updates -void bench_define_suite(const struct bench_suite *suite) { - // set define names - bench_define_names[BENCH_DEFINE_NAMES_SUITE] = (bench_define_names_t){ - suite->define_names, suite->define_count}; - - // set define count - bench_define_count = (suite->define_count > BENCH_IMPLICIT_DEFINE_COUNT) - ? suite->define_count - : BENCH_IMPLICIT_DEFINE_COUNT; - - // map any overrides - if (bench_override_count > 0) { - if (bench_define_count > bench_override_define_capacity) { - // align to power of two to avoid any superlinear growth - bench_override_define_capacity = 1 << lfs_npw2(bench_define_count); - bench_override_defines = realloc( - bench_override_defines, - bench_override_define_capacity*sizeof(bench_define_t)); - } - - memset(bench_override_defines, 0, - bench_define_count*sizeof(bench_define_t)); - for (size_t i = 0; i < bench_override_count; i++) { - for (size_t d = 0; d < bench_define_count; d++) { - // name match? - const char *name = bench_define_name(d); - if (name && strcmp(name, bench_overrides[i].name) == 0) { - bench_override_defines[d] = bench_overrides[i].define; - } - } - } - - bench_define_maps[BENCH_DEFINE_MAP_OVERRIDE] = (bench_define_map_t){ - bench_override_defines, bench_define_count}; - } -} - -void bench_define_explicit( - const bench_define_t *defines, - size_t define_count) { - bench_define_maps[BENCH_DEFINE_MAP_EXPLICIT] = (bench_define_map_t){ - defines, define_count}; -} - -void bench_define_cleanup(void) { - // bench define management can allocate a few things - free(bench_define_cache); - free(bench_override_defines); -} - // bench state @@ -731,9 +712,9 @@ static void perm_printid( // case[:permutation] printf("%s:", case_->name); for (size_t d = 0; d < bench_define_count; d++) { - if (bench_define_ispermutation(d)) { + if (bench_define_ispermutation(bench_defines[d])) { leb16_print(d); - leb16_print(BENCH_DEFINE(d)); + leb16_print(*bench_defines[d]->define); } } } @@ -755,12 +736,14 @@ bool bench_seen_insert(bench_seen_t *seen) { bool was_seen = true; for (size_t d = 0; d < bench_define_count; d++) { // treat unpermuted defines the same as 0 - intmax_t define = bench_define_ispermutation(d) ? BENCH_DEFINE(d) : 0; + intmax_t v = bench_define_ispermutation(bench_defines[d]) + ? *bench_defines[d]->define + : 0; // already seen? struct bench_seen_branch *branch = NULL; for (size_t i = 0; i < seen->branch_count; i++) { - if (seen->branches[i].define == define) { + if (seen->branches[i].define == v) { branch = &seen->branches[i]; break; } @@ -774,7 +757,7 @@ bool bench_seen_insert(bench_seen_t *seen) { sizeof(struct bench_seen_branch), &seen->branch_count, &seen->branch_capacity); - branch->define = define; + branch->define = v; branch->branch = (bench_seen_t){NULL, 0, 0}; } @@ -793,9 +776,9 @@ void bench_seen_cleanup(bench_seen_t *seen) { // iterate through permutations in a bench case static void case_forperm( + const bench_id_t *id, const struct bench_suite *suite, const struct bench_case *case_, - const bench_id_t *id, void (*cb)( void *data, const struct bench_suite *suite, @@ -803,9 +786,10 @@ static void case_forperm( void *data) { // explicit permutation? if (id && id->defines) { - bench_define_explicit(id->defines, id->define_count); + // define case permutation, the exact case perm doesn't matter here + bench_define_case(id, suite, case_, 0); - size_t permutations = bench_define_permutationpermutations(); + size_t permutations = bench_define_permutations(); for (size_t p = 0; p < permutations; p++) { // define permutation permutation bench_define_permutation(p); @@ -827,9 +811,9 @@ static void case_forperm( k < ((case_->permutations) ? case_->permutations : 1); k++) { // define case permutation - bench_define_case(suite, case_, k); + bench_define_case(id, suite, case_, k); - size_t permutations = bench_define_permutationpermutations(); + size_t permutations = bench_define_permutations(); for (size_t p = 0; p < permutations; p++) { // define permutation permutation bench_define_permutation(p); @@ -882,7 +866,7 @@ static void summary(void) { for (size_t t = 0; t < bench_id_count; t++) { for (size_t i = 0; i < bench_suite_count; i++) { - bench_define_suite(bench_suites[i]); + bench_define_suite(&bench_ids[t], bench_suites[i]); for (size_t j = 0; j < bench_suites[i]->case_count; j++) { // does neither suite nor case name match? @@ -896,9 +880,9 @@ static void summary(void) { cases += 1; case_forperm( + &bench_ids[t], bench_suites[i], &bench_suites[i]->cases[j], - &bench_ids[t], perm_count, &perms); } @@ -937,7 +921,7 @@ static void list_suites(void) { name_width, "suite", "flags", "cases", "perms"); for (size_t t = 0; t < bench_id_count; t++) { for (size_t i = 0; i < bench_suite_count; i++) { - bench_define_suite(bench_suites[i]); + bench_define_suite(&bench_ids[t], bench_suites[i]); size_t cases = 0; struct perm_count_state perms = {0, 0}; @@ -954,9 +938,9 @@ static void list_suites(void) { cases += 1; case_forperm( + &bench_ids[t], bench_suites[i], &bench_suites[i]->cases[j], - &bench_ids[t], perm_count, &perms); } @@ -998,7 +982,7 @@ static void list_cases(void) { printf("%-*s %7s %15s\n", name_width, "case", "flags", "perms"); for (size_t t = 0; t < bench_id_count; t++) { for (size_t i = 0; i < bench_suite_count; i++) { - bench_define_suite(bench_suites[i]); + bench_define_suite(&bench_ids[t], bench_suites[i]); for (size_t j = 0; j < bench_suites[i]->case_count; j++) { // does neither suite nor case name match? @@ -1012,9 +996,9 @@ static void list_cases(void) { struct perm_count_state perms = {0, 0}; case_forperm( + &bench_ids[t], bench_suites[i], &bench_suites[i]->cases[j], - &bench_ids[t], perm_count, &perms); @@ -1128,16 +1112,16 @@ struct list_defines_defines { static void list_defines_add( struct list_defines_defines *defines, - size_t d) { - const char *name = bench_define_name(d); - intmax_t value = BENCH_DEFINE(d); + const bench_define_t *define) { + const char *name = define->name; + intmax_t v = *define->define; // define already in defines? for (size_t i = 0; i < defines->define_count; i++) { if (strcmp(defines->defines[i].name, name) == 0) { // value already in values? for (size_t j = 0; j < defines->defines[i].value_count; j++) { - if (defines->defines[i].values[j] == value) { + if (defines->defines[i].values[j] == v) { return; } } @@ -1146,23 +1130,23 @@ static void list_defines_add( (void**)&defines->defines[i].values, sizeof(intmax_t), &defines->defines[i].value_count, - &defines->defines[i].value_capacity) = value; + &defines->defines[i].value_capacity) = v; return; } } // new define? - struct list_defines_define *define = mappend( + struct list_defines_define *define_ = mappend( (void**)&defines->defines, sizeof(struct list_defines_define), &defines->define_count, &defines->define_capacity); - define->name = name; - define->values = malloc(sizeof(intmax_t)); - define->values[0] = value; - define->value_count = 1; - define->value_capacity = 1; + define_->name = name; + define_->values = malloc(sizeof(intmax_t)); + define_->values[0] = v; + define_->value_count = 1; + define_->value_capacity = 1; } void perm_list_defines( @@ -1175,9 +1159,8 @@ void perm_list_defines( // collect defines for (size_t d = 0; d < bench_define_count; d++) { - if (d < BENCH_IMPLICIT_DEFINE_COUNT - || bench_define_ispermutation(d)) { - list_defines_add(defines, d); + if (bench_define_isdefined(bench_defines[d])) { + list_defines_add(defines, bench_defines[d]); } } } @@ -1192,8 +1175,8 @@ void perm_list_permutation_defines( // collect permutation_defines for (size_t d = 0; d < bench_define_count; d++) { - if (bench_define_ispermutation(d)) { - list_defines_add(defines, d); + if (bench_define_ispermutation(bench_defines[d])) { + list_defines_add(defines, bench_defines[d]); } } } @@ -1204,7 +1187,7 @@ static void list_defines(void) { // add defines for (size_t t = 0; t < bench_id_count; t++) { for (size_t i = 0; i < bench_suite_count; i++) { - bench_define_suite(bench_suites[i]); + bench_define_suite(&bench_ids[t], bench_suites[i]); for (size_t j = 0; j < bench_suites[i]->case_count; j++) { // does neither suite nor case name match? @@ -1217,9 +1200,9 @@ static void list_defines(void) { } case_forperm( + &bench_ids[t], bench_suites[i], &bench_suites[i]->cases[j], - &bench_ids[t], perm_list_defines, &defines); } @@ -1249,7 +1232,7 @@ static void list_permutation_defines(void) { // add permutation defines for (size_t t = 0; t < bench_id_count; t++) { for (size_t i = 0; i < bench_suite_count; i++) { - bench_define_suite(bench_suites[i]); + bench_define_suite(&bench_ids[t], bench_suites[i]); for (size_t j = 0; j < bench_suites[i]->case_count; j++) { // does neither suite nor case name match? @@ -1262,9 +1245,9 @@ static void list_permutation_defines(void) { } case_forperm( + &bench_ids[t], bench_suites[i], &bench_suites[i]->cases[j], - &bench_ids[t], perm_list_permutation_defines, &defines); } @@ -1291,14 +1274,24 @@ static void list_permutation_defines(void) { static void list_implicit_defines(void) { struct list_defines_defines defines = {NULL, 0, 0}; - // yes we do need to define a suite, this does a bit of bookeeping - // such as setting up the define cache - bench_define_suite(&(const struct bench_suite){0}); - bench_define_permutation(0); + // yes we do need to define a suite/case, these do a bit of bookeeping + // around mapping defines + bench_define_suite(NULL, + &(const struct bench_suite){0}); + bench_define_case(NULL, + &(const struct bench_suite){0}, + &(const struct bench_case){0}, + 0); - // add implicit defines - for (size_t d = 0; d < BENCH_IMPLICIT_DEFINE_COUNT; d++) { - list_defines_add(&defines, d); + size_t permutations = bench_define_permutations(); + for (size_t p = 0; p < permutations; p++) { + // define permutation permutation + bench_define_permutation(p); + + // add implicit defines + for (size_t d = 0; d < bench_define_count; d++) { + list_defines_add(&defines, bench_defines[d]); + } } for (size_t i = 0; i < defines.define_count; i++) { @@ -1398,7 +1391,7 @@ static void run(void) { for (size_t t = 0; t < bench_id_count; t++) { for (size_t i = 0; i < bench_suite_count; i++) { - bench_define_suite(bench_suites[i]); + bench_define_suite(&bench_ids[t], bench_suites[i]); for (size_t j = 0; j < bench_suites[i]->case_count; j++) { // does neither suite nor case name match? @@ -1411,9 +1404,9 @@ static void run(void) { } case_forperm( + &bench_ids[t], bench_suites[i], &bench_suites[i]->cases[j], - &bench_ids[t], perm_run, NULL); } @@ -1435,15 +1428,16 @@ enum opt_flags { OPT_LIST_PERMUTATION_DEFINES = 4, OPT_LIST_IMPLICIT_DEFINES = 5, OPT_DEFINE = 'D', + OPT_DEFINE_DEPTH = 6, OPT_STEP = 's', OPT_DISK = 'd', OPT_TRACE = 't', - OPT_TRACE_BACKTRACE = 6, - OPT_TRACE_PERIOD = 7, - OPT_TRACE_FREQ = 8, - OPT_READ_SLEEP = 9, - OPT_PROG_SLEEP = 10, - OPT_ERASE_SLEEP = 11, + OPT_TRACE_BACKTRACE = 7, + OPT_TRACE_PERIOD = 8, + OPT_TRACE_FREQ = 9, + OPT_READ_SLEEP = 10, + OPT_PROG_SLEEP = 11, + OPT_ERASE_SLEEP = 12, }; const char *short_opts = "hYlLD:s:d:t:"; @@ -1461,6 +1455,7 @@ const struct option long_opts[] = { {"list-implicit-defines", no_argument, NULL, OPT_LIST_IMPLICIT_DEFINES}, {"define", required_argument, NULL, OPT_DEFINE}, + {"define-depth", required_argument, NULL, OPT_DEFINE_DEPTH}, {"step", required_argument, NULL, OPT_STEP}, {"disk", required_argument, NULL, OPT_DISK}, {"trace", required_argument, NULL, OPT_TRACE}, @@ -1484,6 +1479,7 @@ const char *const help_text[] = { "List explicit defines in this bench-runner.", "List implicit defines in this bench-runner.", "Override a bench define.", + "How deep to evaluate recursive defines before erroring.", "Comma-separated range of bench permutations to run (start,stop,step).", "Direct block device operations to this file.", "Direct trace output to this file.", @@ -1498,7 +1494,7 @@ const char *const help_text[] = { int main(int argc, char **argv) { void (*op)(void) = run; - size_t bench_override_capacity = 0; + size_t bench_override_define_capacity = 0; size_t bench_id_capacity = 0; // parse options @@ -1597,11 +1593,11 @@ int main(int argc, char **argv) { // configuration case OPT_DEFINE:; // allocate space - bench_override_t *override = mappend( - (void**)&bench_overrides, - sizeof(bench_override_t), - &bench_override_count, - &bench_override_capacity); + bench_define_t *override = mappend( + (void**)&bench_override_defines, + sizeof(bench_define_t), + &bench_override_define_count, + &bench_override_define_capacity); // parse into string key/intmax_t value, cannibalizing the // arg in the process @@ -1728,15 +1724,16 @@ int main(int argc, char **argv) { } } - override->define.cb = bench_override_cb; - override->define.data = malloc( - sizeof(bench_override_data_t)); - *(bench_override_data_t*)override->define.data + // define should be patched in bench_define_suite + override->define = NULL; + override->cb = bench_override_cb; + override->data = malloc(sizeof(bench_override_data_t)); + *(bench_override_data_t*)override->data = (bench_override_data_t){ .values = override_values, .value_count = override_value_count, }; - override->define.permutations = override_permutations; + override->permutations = override_permutations; } break; @@ -1744,6 +1741,15 @@ int main(int argc, char **argv) { fprintf(stderr, "error: invalid define: %s\n", optarg); exit(-1); + case OPT_DEFINE_DEPTH:; + parsed = NULL; + bench_define_depth = strtoumax(optarg, &parsed, 0); + if (parsed == optarg) { + fprintf(stderr, "error: invalid define-depth: %s\n", optarg); + exit(-1); + } + break; + case OPT_STEP:; parsed = NULL; bench_step_start = strtoumax(optarg, &parsed, 0); @@ -1919,7 +1925,8 @@ getopt_done: ; (ncount-define_count)*sizeof(bench_define_t)); define_count = ncount; } - defines[d] = BENCH_LIT(v); + // name/define should be patched in bench_define_suite + defines[d] = BENCH_LIT(NULL, v); } } @@ -1940,11 +1947,11 @@ getopt_done: ; // cleanup (need to be done for valgrind benching) bench_define_cleanup(); - if (bench_overrides) { - for (size_t i = 0; i < bench_override_count; i++) { - free((void*)bench_overrides[i].define.data); + if (bench_override_defines) { + for (size_t i = 0; i < bench_override_define_count; i++) { + free((void*)bench_override_defines[i].data); } - free((void*)bench_overrides); + free((void*)bench_override_defines); } if (bench_id_capacity) { for (size_t i = 0; i < bench_id_count; i++) { diff --git a/runners/bench_runner.h b/runners/bench_runner.h index c6a9efd0..f891d81a 100644 --- a/runners/bench_runner.h +++ b/runners/bench_runner.h @@ -59,6 +59,8 @@ enum bench_flags { typedef uint8_t bench_flags_t; typedef struct bench_define { + const char *name; + intmax_t *define; intmax_t (*cb)(void *data, size_t i); void *data; size_t permutations; @@ -81,7 +83,7 @@ struct bench_suite { const char *path; bench_flags_t flags; - const char *const *define_names; + const bench_define_t *defines; size_t define_count; const struct bench_case *cases; @@ -105,68 +107,35 @@ void bench_permutation(size_t i, uint32_t *buffer, size_t size); #define BENCH_PERMUTATION(i, buffer, size) bench_permutation(i, buffer, size) -// access generated bench defines -intmax_t bench_define(size_t define); - -#define BENCH_DEFINE(i) bench_define(i) - // a few preconfigured defines that control how benches run - -#define READ_SIZE_i 0 -#define PROG_SIZE_i 1 -#define BLOCK_SIZE_i 2 -#define BLOCK_COUNT_i 3 -#define DISK_SIZE_i 4 -#define CACHE_SIZE_i 5 -#define INLINE_SIZE_i 6 -#define SHRUB_SIZE_i 7 -#define FRAGMENT_SIZE_i 8 -#define CRYSTAL_THRESH_i 9 -#define LOOKAHEAD_SIZE_i 10 -#define BLOCK_CYCLES_i 11 -#define ERASE_VALUE_i 12 -#define ERASE_CYCLES_i 13 -#define BADBLOCK_BEHAVIOR_i 14 -#define POWERLOSS_BEHAVIOR_i 15 - -#define BENCH_IMPLICIT_DEFINE_COUNT 16 - -#define READ_SIZE bench_define(READ_SIZE_i) -#define PROG_SIZE bench_define(PROG_SIZE_i) -#define BLOCK_SIZE bench_define(BLOCK_SIZE_i) -#define BLOCK_COUNT bench_define(BLOCK_COUNT_i) -#define DISK_SIZE bench_define(DISK_SIZE_i) -#define CACHE_SIZE bench_define(CACHE_SIZE_i) -#define INLINE_SIZE bench_define(INLINE_SIZE_i) -#define SHRUB_SIZE bench_define(SHRUB_SIZE_i) -#define FRAGMENT_SIZE bench_define(FRAGMENT_SIZE_i) -#define CRYSTAL_THRESH bench_define(CRYSTAL_THRESH_i) -#define LOOKAHEAD_SIZE bench_define(LOOKAHEAD_SIZE_i) -#define BLOCK_CYCLES bench_define(BLOCK_CYCLES_i) -#define ERASE_VALUE bench_define(ERASE_VALUE_i) -#define ERASE_CYCLES bench_define(ERASE_CYCLES_i) -#define BADBLOCK_BEHAVIOR bench_define(BADBLOCK_BEHAVIOR_i) -#define POWERLOSS_BEHAVIOR bench_define(POWERLOSS_BEHAVIOR_i) - #define BENCH_IMPLICIT_DEFINES \ /* name value (overridable) */ \ - BENCH_DEF(READ_SIZE, 1 ) \ - BENCH_DEF(PROG_SIZE, 1 ) \ - BENCH_DEF(BLOCK_SIZE, 4096 ) \ - BENCH_DEF(BLOCK_COUNT, DISK_SIZE/BLOCK_SIZE ) \ - BENCH_DEF(DISK_SIZE, 1024*1024 ) \ - BENCH_DEF(CACHE_SIZE, lfs_max(16, lfs_max(READ_SIZE, PROG_SIZE))) \ - BENCH_DEF(INLINE_SIZE, BLOCK_SIZE/4 ) \ - BENCH_DEF(SHRUB_SIZE, INLINE_SIZE ) \ - BENCH_DEF(FRAGMENT_SIZE, CACHE_SIZE ) \ - BENCH_DEF(CRYSTAL_THRESH, BLOCK_SIZE/8 ) \ - BENCH_DEF(LOOKAHEAD_SIZE, 16 ) \ - BENCH_DEF(BLOCK_CYCLES, -1 ) \ - BENCH_DEF(ERASE_VALUE, 0xff ) \ - BENCH_DEF(ERASE_CYCLES, 0 ) \ - BENCH_DEF(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR ) \ - BENCH_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP ) + BENCH_DEFINE(READ_SIZE, 1 ) \ + BENCH_DEFINE(PROG_SIZE, 1 ) \ + BENCH_DEFINE(BLOCK_SIZE, 4096 ) \ + BENCH_DEFINE(BLOCK_COUNT, DISK_SIZE/BLOCK_SIZE ) \ + BENCH_DEFINE(DISK_SIZE, 1024*1024 ) \ + BENCH_DEFINE(CACHE_SIZE, \ + lfs_max(16, lfs_max(READ_SIZE, PROG_SIZE))) \ + BENCH_DEFINE(INLINE_SIZE, BLOCK_SIZE/4 ) \ + BENCH_DEFINE(SHRUB_SIZE, INLINE_SIZE ) \ + BENCH_DEFINE(FRAGMENT_SIZE, CACHE_SIZE ) \ + BENCH_DEFINE(CRYSTAL_THRESH, BLOCK_SIZE/8 ) \ + BENCH_DEFINE(LOOKAHEAD_SIZE, 16 ) \ + BENCH_DEFINE(BLOCK_CYCLES, -1 ) \ + BENCH_DEFINE(ERASE_VALUE, 0xff ) \ + BENCH_DEFINE(ERASE_CYCLES, 0 ) \ + BENCH_DEFINE(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR ) \ + BENCH_DEFINE(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP ) +// declare defines as global intmax_ts +#define BENCH_DEFINE(k, v) \ + extern intmax_t k; + + BENCH_IMPLICIT_DEFINES +#undef BENCH_DEFINE + +// map defines to cfg struct fields #define BENCH_CFG \ .read_size = READ_SIZE, \ .prog_size = PROG_SIZE, \ diff --git a/runners/test_runner.c b/runners/test_runner.c index 9faaf9b1..8553d697 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -122,33 +122,22 @@ typedef struct test_powerloss { typedef struct test_id { const char *name; - const test_define_t *defines; + test_define_t *defines; size_t define_count; test_powerloss_t powerloss; } test_id_t; // test define management -typedef struct test_define_map { - const test_define_t *defines; - size_t count; -} test_define_map_t; -typedef struct test_define_names { - const char *const *names; - size_t count; -} test_define_names_t; +// implicit defines declared here +#define TEST_DEFINE(k, v) \ + intmax_t k; -intmax_t test_define_lit(void *data, size_t i) { - (void)i; - return (intptr_t)data; -} + TEST_IMPLICIT_DEFINES +#undef TEST_DEFINE -#define TEST_CONST(x) {test_define_lit, (void*)(uintptr_t)(x), 1} -#define TEST_LIT(x) ((test_define_t)TEST_CONST(x)) - - -#define TEST_DEF(k, v) \ +#define TEST_DEFINE(k, v) \ intmax_t test_define_##k(void *data, size_t i) { \ (void)data; \ (void)i; \ @@ -156,196 +145,244 @@ intmax_t test_define_lit(void *data, size_t i) { } TEST_IMPLICIT_DEFINES -#undef TEST_DEF +#undef TEST_DEFINE -#define TEST_DEFINE_MAP_OVERRIDE 0 -#define TEST_DEFINE_MAP_EXPLICIT 1 -#define TEST_DEFINE_MAP_CASE 2 -#define TEST_DEFINE_MAP_IMPLICIT 3 +const test_define_t test_implicit_defines[] = { + #define TEST_DEFINE(k, v) \ + {#k, &k, test_define_##k, NULL, 1}, -#define TEST_DEFINE_MAP_COUNT 4 - -test_define_map_t test_define_maps[TEST_DEFINE_MAP_COUNT] = { - [TEST_DEFINE_MAP_IMPLICIT] = { - (const test_define_t[TEST_IMPLICIT_DEFINE_COUNT]) { - #define TEST_DEF(k, v) \ - [k##_i] = {test_define_##k, NULL, 1}, - - TEST_IMPLICIT_DEFINES - #undef TEST_DEF - }, - TEST_IMPLICIT_DEFINE_COUNT, - }, + TEST_IMPLICIT_DEFINES + #undef TEST_DEFINE }; +const size_t test_implicit_define_count + = sizeof(test_implicit_defines) / sizeof(test_define_t); -#define TEST_DEFINE_NAMES_SUITE 0 -#define TEST_DEFINE_NAMES_IMPLICIT 1 +// some helpers +intmax_t test_define_lit(void *data, size_t i) { + (void)i; + return (intptr_t)data; +} -#define TEST_DEFINE_NAMES_COUNT 2 +#define TEST_LIT(name, v) ((test_define_t){ \ + name, NULL, test_define_lit, (void*)(uintptr_t)(v), 1}) -test_define_names_t test_define_names[TEST_DEFINE_NAMES_COUNT] = { - [TEST_DEFINE_NAMES_IMPLICIT] = { - (const char *const[TEST_IMPLICIT_DEFINE_COUNT]){ - #define TEST_DEF(k, v) \ - [k##_i] = #k, - TEST_IMPLICIT_DEFINES - #undef TEST_DEF - }, - TEST_IMPLICIT_DEFINE_COUNT, - }, -}; +// define mapping +const test_define_t **test_defines = NULL; +size_t test_define_count = 0; +size_t test_define_capacity = 0; -size_t test_define_count; +const test_define_t **test_suite_defines = NULL; +size_t test_suite_define_count = 0; +ssize_t *test_suite_define_map = NULL; -typedef struct test_define_cache_entry { - // - >=0 => not cached - // - -1 => cached - ssize_t permutation; - union { - intmax_t value; - const test_define_t *define; - } u; -} test_define_cache_entry_t; +test_define_t *test_override_defines = NULL; +size_t test_override_define_count = 0; -test_define_cache_entry_t *test_define_cache; -size_t test_define_cache_capacity; +size_t test_define_depth = 1000; -const char *test_define_name(size_t define) { - // lookup in our test names - for (size_t i = 0; i < TEST_DEFINE_NAMES_COUNT; i++) { - if (define < test_define_names[i].count - && test_define_names[i].names - && test_define_names[i].names[define]) { - return test_define_names[i].names[define]; + +static inline bool test_define_isdefined(const test_define_t *define) { + return define->cb; +} + +static inline bool test_define_ispermutation(const test_define_t *define) { + // permutation defines are basically anything that's not implicit + return test_define_isdefined(define) + && !(define >= test_implicit_defines + && define + < test_implicit_defines + + test_implicit_define_count); +} + + +void test_define_suite( + const test_id_t *id, + const struct test_suite *suite) { + // reset our mapping + test_define_count = 0; + test_suite_define_count = 0; + + // make sure we have space for everything, just assume the worst case + if (test_implicit_define_count + suite->define_count + > test_define_capacity) { + test_define_capacity + = test_implicit_define_count + suite->define_count; + test_defines = realloc( + test_defines, + test_define_capacity*sizeof(const test_define_t*)); + test_suite_defines = realloc( + test_suite_defines, + test_define_capacity*sizeof(const test_define_t*)); + test_suite_define_map = realloc( + test_suite_define_map, + test_define_capacity*sizeof(ssize_t)); + } + + // first map our implicit defines + for (size_t i = 0; i < test_implicit_define_count; i++) { + test_suite_defines[i] = &test_implicit_defines[i]; + } + test_suite_define_count = test_implicit_define_count; + + // build a mapping from suite defines to test defines + // + // we will use this for both suite and case defines + memset(test_suite_define_map, -1, + test_suite_define_count*sizeof(size_t)); + + for (size_t i = 0; i < suite->define_count; i++) { + // assume suite defines are unique so we only need to compare + // against implicit defines, this avoids a O(n^2) + for (size_t j = 0; j < test_implicit_define_count; j++) { + if (test_suite_defines[j]->define == suite->defines[i].define) { + test_suite_define_map[j] = i; + + // don't override implicit defines if we're not defined + if (test_define_isdefined(&suite->defines[i])) { + test_suite_defines[j] = &suite->defines[i]; + } + goto next_suite_define; + } + } + + // map a new suite define + test_suite_define_map[test_suite_define_count] = i; + test_suite_defines[test_suite_define_count] = &suite->defines[i]; + test_suite_define_count += 1; +next_suite_define:; + } + + // map any explicit defines + // + // we ignore any out-of-bounds defines here, even though it's likely + // an error + if (id && id->defines) { + for (size_t i = 0; + i < id->define_count && i < test_suite_define_count; + i++) { + if (test_define_isdefined(&id->defines[i])) { + // update name/addr + id->defines[i].name = test_suite_defines[i]->name; + id->defines[i].define = test_suite_defines[i]->define; + // map and override suite mapping + test_suite_defines[i] = &id->defines[i]; + test_suite_define_map[i] = -1; + } } } - return NULL; + // map any override defines + // + // note it's not an error to override a define that doesn't exist + for (size_t i = 0; i < test_override_define_count; i++) { + for (size_t j = 0; j < test_suite_define_count; j++) { + if (strcmp( + test_suite_defines[j]->name, + test_override_defines[i].name) == 0) { + // update addr + test_override_defines[i].define + = test_suite_defines[j]->define; + // map and override suite mapping + test_suite_defines[j] = &test_override_defines[i]; + test_suite_define_map[j] = -1; + goto next_override_define; + } + } +next_override_define:; + } } -bool test_define_ispermutation(size_t define) { - // is this define specific to the permutation? - for (size_t i = 0; i < TEST_DEFINE_MAP_IMPLICIT; i++) { - if (define < test_define_maps[i].count - && test_define_maps[i].defines[define].cb) { - return true; +void test_define_case( + const test_id_t *id, + const struct test_suite *suite, + const struct test_case *case_, + size_t perm) { + (void)id; + + // copy over suite defines + for (size_t i = 0; i < test_suite_define_count; i++) { + // map case define if case define is defined + if (case_->defines + && test_suite_define_map[i] != -1 + && test_define_isdefined(&case_->defines[ + perm*suite->define_count + + test_suite_define_map[i]])) { + test_defines[i] = &case_->defines[ + perm*suite->define_count + + test_suite_define_map[i]]; + } else { + test_defines[i] = test_suite_defines[i]; } } - - return false; + test_define_count = test_suite_define_count; } -size_t test_define_permutations(size_t define) { - for (size_t i = 0; i < TEST_DEFINE_MAP_COUNT; i++) { - if (define < test_define_maps[i].count - && test_define_maps[i].defines[define].cb) { - return (test_define_maps[i].defines[define].permutations) - ? test_define_maps[i].defines[define].permutations - : 1; - } +void test_define_permutation(size_t perm) { + // first zero everything, we really don't want reproducibility issues + for (size_t i = 0; i < test_define_count; i++) { + *test_defines[i]->define = 0; } - return 0; + // defines may be mutually recursive, which makes evaluation a bit tricky + // + // Rather than doing any clever, we just repeatedly evaluate the + // permutation until values stabilize. If things don't stabilize after + // some number of iterations, error, this likely means defines were + // stuck in a cycle + // + size_t attempt = 0; + while (true) { + const test_define_t *changed = NULL; + // define-specific permutations are encoded in the case permutation + size_t perm_ = perm; + for (size_t i = 0; i < test_define_count; i++) { + if (test_defines[i]->cb) { + intmax_t v = test_defines[i]->cb( + test_defines[i]->data, + perm_ % test_defines[i]->permutations); + if (v != *test_defines[i]->define) { + *test_defines[i]->define = v; + changed = test_defines[i]; + } + + perm_ /= test_defines[i]->permutations; + } + } + + // stabilized? + if (!changed) { + break; + } + + attempt += 1; + if (test_define_depth && attempt >= test_define_depth+1) { + fprintf(stderr, "error: could not resolve recursive defines: %s\n", + changed->name); + exit(-1); + } + } } -size_t test_define_permutationpermutations(void) { +void test_define_cleanup(void) { + // test define management can allocate a few things + free(test_defines); + free(test_suite_defines); + free(test_suite_define_map); +} + +size_t test_define_permutations(void) { size_t prod = 1; - for (size_t d = 0; d < test_define_count; d++) { - size_t permutations = test_define_permutations(d); - if (permutations > 0) { - prod *= permutations; - } + for (size_t i = 0; i < test_define_count; i++) { + prod *= (test_defines[i]->permutations > 0) + ? test_defines[i]->permutations + : 1; } return prod; } -intmax_t test_define(size_t define) { - // cached? - if (test_define_cache[define].permutation == -1) { - return test_define_cache[define].u.value; - // lazily defined? - } else if (test_define_cache[define].u.define) { - // evaluate and store in cache - test_define_cache[define].u.value - = test_define_cache[define].u.define->cb( - test_define_cache[define].u.define->data, - test_define_cache[define].permutation); - test_define_cache[define].permutation = -1; - return test_define_cache[define].u.value; - - // not defined? - } else { - const char *name = test_define_name(define); - fprintf(stderr, "error: undefined define %s (%zd)\n", - (name) ? name : "(unknown)", - define); - assert(false); - exit(-1); - } -} - -// permutation updates -void test_define_permutation(size_t perm) { - // We can't completely precompute the defines easily, since they may be - // mutually recursive. But we can precompute the permutations, which is - // expensive otherwise. - // - // Note that it's not really worth it to make define lookup completely - // lazy, the first thing we do is evaluate all defines for 1. deduplication - // and 2. logging. - - if (test_define_cache_capacity < test_define_count) { - // align to power of two to avoid any superlinear growth - test_define_cache_capacity = 1 << lfs_npw2(test_define_count); - test_define_cache = realloc( - test_define_cache, - test_define_cache_capacity*sizeof(test_define_cache_entry_t)); - } - - for (size_t d = 0; d < test_define_count; d++) { - // lookup our test defines - for (size_t i = 0; i < TEST_DEFINE_MAP_COUNT; i++) { - if (d < test_define_maps[i].count - && test_define_maps[i].defines[d].cb) { - // note we can't precompute these due to mutual recursion - const test_define_t *define = &test_define_maps[i].defines[d]; - test_define_cache[d] = (test_define_cache_entry_t){ - .permutation = perm % define->permutations, - .u.define = define, - }; - perm /= test_define_maps[i].defines[d].permutations; - goto next; - } - } - - // default to a null value, these should be unreachable - test_define_cache[d] = (test_define_cache_entry_t){0}; - next:; - } -} - -// case updates -void test_define_case( - const struct test_suite *suite, - const struct test_case *case_, - size_t perm) { - if (case_->defines) { - test_define_maps[TEST_DEFINE_MAP_CASE] = (test_define_map_t){ - &case_->defines[perm*suite->define_count], - suite->define_count}; - } else { - test_define_maps[TEST_DEFINE_MAP_CASE] = (test_define_map_t){ - NULL, 0}; - } -} - -// override updates -typedef struct test_override { - const char *name; - test_define_t define; -} test_override_t; +// override define stuff typedef struct test_override_value { intmax_t start; @@ -390,63 +427,6 @@ intmax_t test_override_cb(void *data, size_t i) { __builtin_unreachable(); } -const test_override_t *test_overrides = NULL; -size_t test_override_count = 0; - -test_define_t *test_override_defines = NULL; -size_t test_override_define_capacity = 0; - -// suite/perm updates -void test_define_suite(const struct test_suite *suite) { - // set define names - test_define_names[TEST_DEFINE_NAMES_SUITE] = (test_define_names_t){ - suite->define_names, suite->define_count}; - - // set define count - test_define_count = (suite->define_count > TEST_IMPLICIT_DEFINE_COUNT) - ? suite->define_count - : TEST_IMPLICIT_DEFINE_COUNT; - - // map any overrides - if (test_override_count > 0) { - if (test_define_count > test_override_define_capacity) { - // align to power of two to avoid any superlinear growth - test_override_define_capacity = 1 << lfs_npw2(test_define_count); - test_override_defines = realloc( - test_override_defines, - test_override_define_capacity*sizeof(test_define_t)); - } - - memset(test_override_defines, 0, - test_define_count*sizeof(test_define_t)); - for (size_t i = 0; i < test_override_count; i++) { - for (size_t d = 0; d < test_define_count; d++) { - // name match? - const char *name = test_define_name(d); - if (name && strcmp(name, test_overrides[i].name) == 0) { - test_override_defines[d] = test_overrides[i].define; - } - } - } - - test_define_maps[TEST_DEFINE_MAP_OVERRIDE] = (test_define_map_t){ - test_override_defines, test_define_count}; - } -} - -void test_define_explicit( - const test_define_t *defines, - size_t define_count) { - test_define_maps[TEST_DEFINE_MAP_EXPLICIT] = (test_define_map_t){ - defines, define_count}; -} - -void test_define_cleanup(void) { - // test define management can allocate a few things - free(test_define_cache); - free(test_override_defines); -} - // test state @@ -472,7 +452,7 @@ lfs_emubd_sleep_t test_read_sleep = 0.0; lfs_emubd_sleep_t test_prog_sleep = 0.0; lfs_emubd_sleep_t test_erase_sleep = 0.0; -volatile size_t test_pls = 0; +volatile size_t TEST_PLS = 0; extern const test_powerloss_t *test_powerlosses; extern size_t test_powerloss_count; @@ -647,9 +627,9 @@ static void perm_printid( // case[:permutation[:powercycles]] printf("%s:", case_->name); for (size_t d = 0; d < test_define_count; d++) { - if (test_define_ispermutation(d)) { + if (test_define_ispermutation(test_defines[d])) { leb16_print(d); - leb16_print(TEST_DEFINE(d)); + leb16_print(*test_defines[d]->define); } } @@ -680,12 +660,14 @@ bool test_seen_insert(test_seen_t *seen) { bool was_seen = true; for (size_t d = 0; d < test_define_count; d++) { // treat unpermuted defines the same as 0 - intmax_t define = test_define_ispermutation(d) ? TEST_DEFINE(d) : 0; + intmax_t v = test_define_ispermutation(test_defines[d]) + ? *test_defines[d]->define + : 0; // already seen? struct test_seen_branch *branch = NULL; for (size_t i = 0; i < seen->branch_count; i++) { - if (seen->branches[i].define == define) { + if (seen->branches[i].define == v) { branch = &seen->branches[i]; break; } @@ -699,7 +681,7 @@ bool test_seen_insert(test_seen_t *seen) { sizeof(struct test_seen_branch), &seen->branch_count, &seen->branch_capacity); - branch->define = define; + branch->define = v; branch->branch = (test_seen_t){NULL, 0, 0}; } @@ -727,9 +709,9 @@ static void run_powerloss_cycles( // iterate through permutations in a test case static void case_forperm( + const test_id_t *id, const struct test_suite *suite, const struct test_case *case_, - const test_id_t *id, void (*cb)( void *data, const struct test_suite *suite, @@ -738,9 +720,10 @@ static void case_forperm( void *data) { // explicit permutation? if (id && id->defines) { - test_define_explicit(id->defines, id->define_count); + // define case permutation, the exact case perm doesn't matter here + test_define_case(id, suite, case_, 0); - size_t permutations = test_define_permutationpermutations(); + size_t permutations = test_define_permutations(); for (size_t p = 0; p < permutations; p++) { // define permutation permutation test_define_permutation(p); @@ -775,9 +758,9 @@ static void case_forperm( k < ((case_->permutations) ? case_->permutations : 1); k++) { // define case permutation - test_define_case(suite, case_, k); + test_define_case(id, suite, case_, k); - size_t permutations = test_define_permutationpermutations(); + size_t permutations = test_define_permutations(); for (size_t p = 0; p < permutations; p++) { // define permutation permutation test_define_permutation(p); @@ -826,7 +809,7 @@ void perm_count( state->total += 1; // set pls to 1 if running under powerloss so it useful for if predicates - test_pls = (powerloss->run != run_powerloss_none); + TEST_PLS = (powerloss->run != run_powerloss_none); if (case_->if_ && !case_->if_()) { return; } @@ -846,7 +829,7 @@ static void summary(void) { for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < test_suite_count; i++) { - test_define_suite(test_suites[i]); + test_define_suite(&test_ids[t], test_suites[i]); for (size_t j = 0; j < test_suites[i]->case_count; j++) { // does neither suite nor case name match? @@ -860,9 +843,9 @@ static void summary(void) { cases += 1; case_forperm( + &test_ids[t], test_suites[i], &test_suites[i]->cases[j], - &test_ids[t], perm_count, &perms); } @@ -902,7 +885,7 @@ static void list_suites(void) { name_width, "suite", "flags", "cases", "perms"); for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < test_suite_count; i++) { - test_define_suite(test_suites[i]); + test_define_suite(&test_ids[t], test_suites[i]); size_t cases = 0; struct perm_count_state perms = {0, 0}; @@ -919,9 +902,9 @@ static void list_suites(void) { cases += 1; case_forperm( + &test_ids[t], test_suites[i], &test_suites[i]->cases[j], - &test_ids[t], perm_count, &perms); } @@ -964,7 +947,7 @@ static void list_cases(void) { printf("%-*s %7s %15s\n", name_width, "case", "flags", "perms"); for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < test_suite_count; i++) { - test_define_suite(test_suites[i]); + test_define_suite(&test_ids[t], test_suites[i]); for (size_t j = 0; j < test_suites[i]->case_count; j++) { // does neither suite nor case name match? @@ -978,9 +961,9 @@ static void list_cases(void) { struct perm_count_state perms = {0, 0}; case_forperm( + &test_ids[t], test_suites[i], &test_suites[i]->cases[j], - &test_ids[t], perm_count, &perms); @@ -1096,16 +1079,16 @@ struct list_defines_defines { static void list_defines_add( struct list_defines_defines *defines, - size_t d) { - const char *name = test_define_name(d); - intmax_t value = TEST_DEFINE(d); + const test_define_t *define) { + const char *name = define->name; + intmax_t v = *define->define; // define already in defines? for (size_t i = 0; i < defines->define_count; i++) { if (strcmp(defines->defines[i].name, name) == 0) { // value already in values? for (size_t j = 0; j < defines->defines[i].value_count; j++) { - if (defines->defines[i].values[j] == value) { + if (defines->defines[i].values[j] == v) { return; } } @@ -1114,23 +1097,23 @@ static void list_defines_add( (void**)&defines->defines[i].values, sizeof(intmax_t), &defines->defines[i].value_count, - &defines->defines[i].value_capacity) = value; + &defines->defines[i].value_capacity) = v; return; } } // new define? - struct list_defines_define *define = mappend( + struct list_defines_define *define_ = mappend( (void**)&defines->defines, sizeof(struct list_defines_define), &defines->define_count, &defines->define_capacity); - define->name = name; - define->values = malloc(sizeof(intmax_t)); - define->values[0] = value; - define->value_count = 1; - define->value_capacity = 1; + define_->name = name; + define_->values = malloc(sizeof(intmax_t)); + define_->values[0] = v; + define_->value_count = 1; + define_->value_capacity = 1; } void perm_list_defines( @@ -1145,9 +1128,8 @@ void perm_list_defines( // collect defines for (size_t d = 0; d < test_define_count; d++) { - if (d < TEST_IMPLICIT_DEFINE_COUNT - || test_define_ispermutation(d)) { - list_defines_add(defines, d); + if (test_define_isdefined(test_defines[d])) { + list_defines_add(defines, test_defines[d]); } } } @@ -1164,8 +1146,8 @@ void perm_list_permutation_defines( // collect permutation_defines for (size_t d = 0; d < test_define_count; d++) { - if (test_define_ispermutation(d)) { - list_defines_add(defines, d); + if (test_define_ispermutation(test_defines[d])) { + list_defines_add(defines, test_defines[d]); } } } @@ -1176,7 +1158,7 @@ static void list_defines(void) { // add defines for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < test_suite_count; i++) { - test_define_suite(test_suites[i]); + test_define_suite(&test_ids[t], test_suites[i]); for (size_t j = 0; j < test_suites[i]->case_count; j++) { // does neither suite nor case name match? @@ -1189,9 +1171,9 @@ static void list_defines(void) { } case_forperm( + &test_ids[t], test_suites[i], &test_suites[i]->cases[j], - &test_ids[t], perm_list_defines, &defines); } @@ -1221,7 +1203,7 @@ static void list_permutation_defines(void) { // add permutation defines for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < test_suite_count; i++) { - test_define_suite(test_suites[i]); + test_define_suite(&test_ids[t], test_suites[i]); for (size_t j = 0; j < test_suites[i]->case_count; j++) { // does neither suite nor case name match? @@ -1234,9 +1216,9 @@ static void list_permutation_defines(void) { } case_forperm( + &test_ids[t], test_suites[i], &test_suites[i]->cases[j], - &test_ids[t], perm_list_permutation_defines, &defines); } @@ -1263,14 +1245,24 @@ static void list_permutation_defines(void) { static void list_implicit_defines(void) { struct list_defines_defines defines = {NULL, 0, 0}; - // yes we do need to define a suite, this does a bit of bookeeping - // such as setting up the define cache - test_define_suite(&(const struct test_suite){0}); - test_define_permutation(0); + // yes we do need to define a suite/case, these do a bit of bookeeping + // around mapping defines + test_define_suite(NULL, + &(const struct test_suite){0}); + test_define_case(NULL, + &(const struct test_suite){0}, + &(const struct test_case){0}, + 0); - // add implicit defines - for (size_t d = 0; d < TEST_IMPLICIT_DEFINE_COUNT; d++) { - list_defines_add(&defines, d); + size_t permutations = test_define_permutations(); + for (size_t p = 0; p < permutations; p++) { + // define permutation permutation + test_define_permutation(p); + + // add implicit defines + for (size_t d = 0; d < test_define_count; d++) { + list_defines_add(&defines, test_defines[d]); + } } for (size_t i = 0; i < defines.define_count; i++) { @@ -1332,7 +1324,7 @@ static void run_powerloss_none( printf("\n"); // zero pls - test_pls = 0; + TEST_PLS = 0; case_->run(&cfg); @@ -1358,7 +1350,7 @@ static void run_powerloss_linear( const struct test_suite *suite, const struct test_case *case_) { // zero pls - test_pls = 0; + TEST_PLS = 0; // create block device and configuration lfs_emubd_t bd; @@ -1378,8 +1370,8 @@ static void run_powerloss_linear( .read_sleep = test_read_sleep, .prog_sleep = test_prog_sleep, .erase_sleep = test_erase_sleep, - .power_cycles = (test_pls < powerloss->cycle_count) - ? test_pls+1 + .power_cycles = (TEST_PLS < powerloss->cycle_count) + ? TEST_PLS+1 : 0, .powerloss_behavior = POWERLOSS_BEHAVIOR, .powerloss_cb = powerloss_longjmp, @@ -1409,13 +1401,13 @@ static void run_powerloss_linear( printf("powerloss "); perm_printid(suite, case_, NULL, 0); printf(":x"); - leb16_print(test_pls+1); + leb16_print(TEST_PLS+1); printf("\n"); // increment pls - test_pls += 1; - lfs_emubd_setpowercycles(&cfg, (test_pls < powerloss->cycle_count) - ? test_pls+1 + TEST_PLS += 1; + lfs_emubd_setpowercycles(&cfg, (TEST_PLS < powerloss->cycle_count) + ? TEST_PLS+1 : 0); } @@ -1436,7 +1428,7 @@ static void run_powerloss_log( const struct test_suite *suite, const struct test_case *case_) { // zero pls - test_pls = 0; + TEST_PLS = 0; // create block device and configuration lfs_emubd_t bd; @@ -1456,8 +1448,8 @@ static void run_powerloss_log( .read_sleep = test_read_sleep, .prog_sleep = test_prog_sleep, .erase_sleep = test_erase_sleep, - .power_cycles = (test_pls < powerloss->cycle_count) - ? 1 << test_pls + .power_cycles = (TEST_PLS < powerloss->cycle_count) + ? 1 << TEST_PLS : 0, .powerloss_behavior = POWERLOSS_BEHAVIOR, .powerloss_cb = powerloss_longjmp, @@ -1487,13 +1479,13 @@ static void run_powerloss_log( printf("powerloss "); perm_printid(suite, case_, NULL, 0); printf(":y"); - leb16_print(test_pls+1); + leb16_print(TEST_PLS+1); printf("\n"); // increment pls - test_pls += 1; - lfs_emubd_setpowercycles(&cfg, (test_pls < powerloss->cycle_count) - ? 1 << test_pls + TEST_PLS += 1; + lfs_emubd_setpowercycles(&cfg, (TEST_PLS < powerloss->cycle_count) + ? 1 << TEST_PLS : 0); } @@ -1514,7 +1506,7 @@ static void run_powerloss_cycles( const struct test_suite *suite, const struct test_case *case_) { // zero pls - test_pls = 0; + TEST_PLS = 0; // create block device and configuration lfs_emubd_t bd; @@ -1534,8 +1526,8 @@ static void run_powerloss_cycles( .read_sleep = test_read_sleep, .prog_sleep = test_prog_sleep, .erase_sleep = test_erase_sleep, - .power_cycles = (test_pls < powerloss->cycle_count) - ? powerloss->cycles[test_pls] + .power_cycles = (TEST_PLS < powerloss->cycle_count) + ? powerloss->cycles[TEST_PLS] : 0, .powerloss_behavior = POWERLOSS_BEHAVIOR, .powerloss_cb = powerloss_longjmp, @@ -1562,15 +1554,15 @@ static void run_powerloss_cycles( } // power-loss! - assert(test_pls <= powerloss->cycle_count); + assert(TEST_PLS <= powerloss->cycle_count); printf("powerloss "); - perm_printid(suite, case_, powerloss->cycles, test_pls+1); + perm_printid(suite, case_, powerloss->cycles, TEST_PLS+1); printf("\n"); // increment pls - test_pls += 1; - lfs_emubd_setpowercycles(&cfg, (test_pls < powerloss->cycle_count) - ? powerloss->cycles[test_pls] + TEST_PLS += 1; + lfs_emubd_setpowercycles(&cfg, (TEST_PLS < powerloss->cycle_count) + ? powerloss->cycles[TEST_PLS] : 0); } @@ -1640,7 +1632,7 @@ static void run_powerloss_exhaustive_layer( }; // make the number of pls currently seen available to tests/debugging - test_pls = pls; + TEST_PLS = pls; // run through the test without additional powerlosses, collecting possible // branches as we do so @@ -1812,7 +1804,7 @@ void perm_run( test_step += 1; // set pls to 1 if running under powerloss so it useful for if predicates - test_pls = (powerloss->run != run_powerloss_none); + TEST_PLS = (powerloss->run != run_powerloss_none); // filter? if (case_->if_ && !case_->if_()) { printf("skipped "); @@ -1831,7 +1823,7 @@ static void run(void) { for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < test_suite_count; i++) { - test_define_suite(test_suites[i]); + test_define_suite(&test_ids[t], test_suites[i]); for (size_t j = 0; j < test_suites[i]->case_count; j++) { // does neither suite nor case name match? @@ -1844,9 +1836,9 @@ static void run(void) { } case_forperm( + &test_ids[t], test_suites[i], &test_suites[i]->cases[j], - &test_ids[t], perm_run, NULL); } @@ -1869,16 +1861,17 @@ enum opt_flags { OPT_LIST_IMPLICIT_DEFINES = 5, OPT_LIST_POWERLOSSES = 6, OPT_DEFINE = 'D', + OPT_DEFINE_DEPTH = 7, OPT_POWERLOSS = 'P', OPT_STEP = 's', OPT_DISK = 'd', OPT_TRACE = 't', - OPT_TRACE_BACKTRACE = 7, - OPT_TRACE_PERIOD = 8, - OPT_TRACE_FREQ = 9, - OPT_READ_SLEEP = 10, - OPT_PROG_SLEEP = 11, - OPT_ERASE_SLEEP = 12, + OPT_TRACE_BACKTRACE = 8, + OPT_TRACE_PERIOD = 9, + OPT_TRACE_FREQ = 10, + OPT_READ_SLEEP = 11, + OPT_PROG_SLEEP = 12, + OPT_ERASE_SLEEP = 13, }; const char *short_opts = "hYlLD:P:s:d:t:"; @@ -1897,6 +1890,7 @@ const struct option long_opts[] = { no_argument, NULL, OPT_LIST_IMPLICIT_DEFINES}, {"list-powerlosses", no_argument, NULL, OPT_LIST_POWERLOSSES}, {"define", required_argument, NULL, OPT_DEFINE}, + {"define-depth", required_argument, NULL, OPT_DEFINE_DEPTH}, {"powerloss", required_argument, NULL, OPT_POWERLOSS}, {"step", required_argument, NULL, OPT_STEP}, {"disk", required_argument, NULL, OPT_DISK}, @@ -1922,6 +1916,7 @@ const char *const help_text[] = { "List implicit defines in this test-runner.", "List the available power-loss scenarios.", "Override a test define.", + "How deep to evaluate recursive defines before erroring.", "Comma-separated list of power-loss scenarios to test.", "Comma-separated range of test permutations to run (start,stop,step).", "Direct block device operations to this file.", @@ -1937,7 +1932,7 @@ const char *const help_text[] = { int main(int argc, char **argv) { void (*op)(void) = run; - size_t test_override_capacity = 0; + size_t test_override_define_capacity = 0; size_t test_powerloss_capacity = 0; size_t test_id_capacity = 0; @@ -2041,11 +2036,11 @@ int main(int argc, char **argv) { // configuration case OPT_DEFINE:; // allocate space - test_override_t *override = mappend( - (void**)&test_overrides, - sizeof(test_override_t), - &test_override_count, - &test_override_capacity); + test_define_t *override = mappend( + (void**)&test_override_defines, + sizeof(test_define_t), + &test_override_define_count, + &test_override_define_capacity); // parse into string key/intmax_t value, cannibalizing the // arg in the process @@ -2172,15 +2167,16 @@ int main(int argc, char **argv) { } } - override->define.cb = test_override_cb; - override->define.data = malloc( - sizeof(test_override_data_t)); - *(test_override_data_t*)override->define.data + // define should be patched in test_define_suite + override->define = NULL; + override->cb = test_override_cb; + override->data = malloc(sizeof(test_override_data_t)); + *(test_override_data_t*)override->data = (test_override_data_t){ .values = override_values, .value_count = override_value_count, }; - override->define.permutations = override_permutations; + override->permutations = override_permutations; } break; @@ -2188,6 +2184,15 @@ int main(int argc, char **argv) { fprintf(stderr, "error: invalid define: %s\n", optarg); exit(-1); + case OPT_DEFINE_DEPTH:; + parsed = NULL; + test_define_depth = strtoumax(optarg, &parsed, 0); + if (parsed == optarg) { + fprintf(stderr, "error: invalid define-depth: %s\n", optarg); + exit(-1); + } + break; + case OPT_POWERLOSS:; // reset our powerloss scenarios if (test_powerloss_capacity > 0) { @@ -2531,7 +2536,8 @@ getopt_done:; (ncount-define_count)*sizeof(test_define_t)); define_count = ncount; } - defines[d] = TEST_LIT(v); + // name/define should be patched in test_define_suite + defines[d] = TEST_LIT(NULL, v); } // special case for linear power cycles @@ -2619,11 +2625,11 @@ getopt_done:; // cleanup (need to be done for valgrind testing) test_define_cleanup(); - if (test_overrides) { - for (size_t i = 0; i < test_override_count; i++) { - free((void*)test_overrides[i].define.data); + if (test_override_defines) { + for (size_t i = 0; i < test_override_define_count; i++) { + free((void*)test_override_defines[i].data); } - free((void*)test_overrides); + free((void*)test_override_defines); } if (test_powerloss_capacity) { for (size_t i = 0; i < test_powerloss_count; i++) { diff --git a/runners/test_runner.h b/runners/test_runner.h index 0274fda7..40b06346 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -39,6 +39,8 @@ enum test_flags { typedef uint8_t test_flags_t; typedef struct test_define { + const char *name; + intmax_t *define; intmax_t (*cb)(void *data, size_t i); void *data; size_t permutations; @@ -61,7 +63,7 @@ struct test_suite { const char *path; test_flags_t flags; - const char *const *define_names; + const test_define_t *defines; size_t define_count; const struct test_case *cases; @@ -74,9 +76,7 @@ extern const size_t test_suite_count; // this variable tracks the number of powerlosses triggered during the // current test permutation, this is useful for both tests and debugging -extern volatile size_t test_pls; - -#define TEST_PLS test_pls +extern volatile size_t TEST_PLS; // deterministic prng for pseudo-randomness in tests uint32_t test_prng(uint32_t *state); @@ -91,79 +91,46 @@ void test_permutation(size_t i, uint32_t *buffer, size_t size); #define TEST_PERMUTATION(i, buffer, size) test_permutation(i, buffer, size) -// access generated test defines -intmax_t test_define(size_t define); - -#define TEST_DEFINE(i) test_define(i) - // a few preconfigured defines that control how tests run - -#define READ_SIZE_i 0 -#define PROG_SIZE_i 1 -#define BLOCK_SIZE_i 2 -#define BLOCK_COUNT_i 3 -#define DISK_SIZE_i 4 -#define CACHE_SIZE_i 5 -#define INLINE_SIZE_i 6 -#define SHRUB_SIZE_i 7 -#define FRAGMENT_SIZE_i 8 -#define CRYSTAL_THRESH_i 9 -#define LOOKAHEAD_SIZE_i 10 -#define BLOCK_CYCLES_i 11 -#define ERASE_VALUE_i 12 -#define ERASE_CYCLES_i 13 -#define BADBLOCK_BEHAVIOR_i 14 -#define POWERLOSS_BEHAVIOR_i 15 - -#define TEST_IMPLICIT_DEFINE_COUNT 16 - -#define READ_SIZE TEST_DEFINE(READ_SIZE_i) -#define PROG_SIZE TEST_DEFINE(PROG_SIZE_i) -#define BLOCK_SIZE TEST_DEFINE(BLOCK_SIZE_i) -#define BLOCK_COUNT TEST_DEFINE(BLOCK_COUNT_i) -#define DISK_SIZE TEST_DEFINE(DISK_SIZE_i) -#define CACHE_SIZE TEST_DEFINE(CACHE_SIZE_i) -#define INLINE_SIZE TEST_DEFINE(INLINE_SIZE_i) -#define SHRUB_SIZE TEST_DEFINE(SHRUB_SIZE_i) -#define FRAGMENT_SIZE TEST_DEFINE(FRAGMENT_SIZE_i) -#define CRYSTAL_THRESH TEST_DEFINE(CRYSTAL_THRESH_i) -#define LOOKAHEAD_SIZE TEST_DEFINE(LOOKAHEAD_SIZE_i) -#define BLOCK_CYCLES TEST_DEFINE(BLOCK_CYCLES_i) -#define ERASE_VALUE TEST_DEFINE(ERASE_VALUE_i) -#define ERASE_CYCLES TEST_DEFINE(ERASE_CYCLES_i) -#define BADBLOCK_BEHAVIOR TEST_DEFINE(BADBLOCK_BEHAVIOR_i) -#define POWERLOSS_BEHAVIOR TEST_DEFINE(POWERLOSS_BEHAVIOR_i) - #define TEST_IMPLICIT_DEFINES \ - /* name value (overridable) */ \ - TEST_DEF(READ_SIZE, 1 ) \ - TEST_DEF(PROG_SIZE, 1 ) \ - TEST_DEF(BLOCK_SIZE, 4096 ) \ - TEST_DEF(BLOCK_COUNT, DISK_SIZE/BLOCK_SIZE ) \ - TEST_DEF(DISK_SIZE, 1024*1024 ) \ - TEST_DEF(CACHE_SIZE, lfs_max(16, lfs_max(READ_SIZE, PROG_SIZE)) ) \ - TEST_DEF(INLINE_SIZE, BLOCK_SIZE/4 ) \ - TEST_DEF(SHRUB_SIZE, INLINE_SIZE ) \ - TEST_DEF(FRAGMENT_SIZE, CACHE_SIZE ) \ - TEST_DEF(CRYSTAL_THRESH, BLOCK_SIZE/8 ) \ - TEST_DEF(LOOKAHEAD_SIZE, 16 ) \ - TEST_DEF(BLOCK_CYCLES, -1 ) \ - TEST_DEF(ERASE_VALUE, 0xff ) \ - TEST_DEF(ERASE_CYCLES, 0 ) \ - TEST_DEF(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR ) \ - TEST_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP ) + /* name value (overridable) */ \ + TEST_DEFINE(READ_SIZE, 1 ) \ + TEST_DEFINE(PROG_SIZE, 1 ) \ + TEST_DEFINE(BLOCK_SIZE, 4096 ) \ + TEST_DEFINE(BLOCK_COUNT, DISK_SIZE/BLOCK_SIZE ) \ + TEST_DEFINE(DISK_SIZE, 1024*1024 ) \ + TEST_DEFINE(CACHE_SIZE, \ + lfs_max(16, lfs_max(READ_SIZE, PROG_SIZE)) ) \ + TEST_DEFINE(INLINE_SIZE, BLOCK_SIZE/4 ) \ + TEST_DEFINE(SHRUB_SIZE, INLINE_SIZE ) \ + TEST_DEFINE(FRAGMENT_SIZE, CACHE_SIZE ) \ + TEST_DEFINE(CRYSTAL_THRESH, BLOCK_SIZE/8 ) \ + TEST_DEFINE(LOOKAHEAD_SIZE, 16 ) \ + TEST_DEFINE(BLOCK_CYCLES, -1 ) \ + TEST_DEFINE(ERASE_VALUE, 0xff ) \ + TEST_DEFINE(ERASE_CYCLES, 0 ) \ + TEST_DEFINE(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR ) \ + TEST_DEFINE(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP ) +// declare defines as global intmax_ts +#define TEST_DEFINE(k, v) \ + extern intmax_t k; + + TEST_IMPLICIT_DEFINES +#undef TEST_DEFINE + +// map defines to cfg struct fields #define TEST_CFG \ - .read_size = READ_SIZE, \ - .prog_size = PROG_SIZE, \ - .block_size = BLOCK_SIZE, \ - .block_count = BLOCK_COUNT, \ - .block_cycles = BLOCK_CYCLES, \ - .cache_size = CACHE_SIZE, \ - .inline_size = INLINE_SIZE, \ - .shrub_size = SHRUB_SIZE, \ - .fragment_size = FRAGMENT_SIZE, \ - .crystal_thresh = CRYSTAL_THRESH, \ + .read_size = READ_SIZE, \ + .prog_size = PROG_SIZE, \ + .block_size = BLOCK_SIZE, \ + .block_count = BLOCK_COUNT, \ + .block_cycles = BLOCK_CYCLES, \ + .cache_size = CACHE_SIZE, \ + .inline_size = INLINE_SIZE, \ + .shrub_size = SHRUB_SIZE, \ + .fragment_size = FRAGMENT_SIZE, \ + .crystal_thresh = CRYSTAL_THRESH, \ .lookahead_size = LOOKAHEAD_SIZE, #define TEST_BDCFG \ diff --git a/scripts/bench.py b/scripts/bench.py index eabfdc8f..f5747ec9 100755 --- a/scripts/bench.py +++ b/scripts/bench.py @@ -245,7 +245,7 @@ class BenchSuite: file=sys.stderr) def __repr__(self): - return '' % self.name + return '' % self.name def __lt__(self, other): # sort by name @@ -429,13 +429,9 @@ def compile(bench_paths, **args): if not args.get('source'): # write any suite defines if suite.defines: - for i, define in enumerate(sorted(suite.defines)): - f.writeln('#ifndef %s' % define) - f.writeln('#define %-24s ' - 'BENCH_IMPLICIT_DEFINE_COUNT+%d' % (define+'_i', i)) - f.writeln('#define %-24s ' - 'BENCH_DEFINE(%s)' % (define, define+'_i')) - f.writeln('#endif') + for define in sorted(suite.defines): + f.writeln('__attribute__((weak)) intmax_t %s;' + % define) f.writeln() # write any suite code @@ -477,17 +473,14 @@ def compile(bench_paths, **args): % (' | '.join(filter(None, [ 'BENCH_INTERNAL' if suite.internal else None])) or 0)) + # create suite defines if suite.defines: - # create suite define names - f.writeln(4*' '+'.define_names = (const char *const[' - 'BENCH_IMPLICIT_DEFINE_COUNT+%d]){' - % (len(suite.defines))) + f.writeln(4*' '+'.defines = (const bench_define_t[]){') for k in sorted(suite.defines): - f.writeln(8*' '+'[%-24s] = "%s",' % (k+'_i', k)) + f.writeln(8*' '+'{"%s", &%s, NULL, NULL, 0},' + % (k, k)) f.writeln(4*' '+'},') - f.writeln(4*' '+'.define_count = ' - 'BENCH_IMPLICIT_DEFINE_COUNT+%d,' - % len(suite.defines)) + f.writeln(4*' '+'.define_count = %d,' % len(suite.defines)) if suite.cases: f.writeln(4*' '+'.cases = (const struct bench_case[]){') for case in suite.cases: @@ -499,18 +492,20 @@ def compile(bench_paths, **args): % (' | '.join(filter(None, [ 'BENCH_INTERNAL' if suite.internal else None])) or 0)) + # create case defines if case.defines: - f.writeln(12*' '+'.defines = ' - '(const bench_define_t*)(const bench_define_t[][' - 'BENCH_IMPLICIT_DEFINE_COUNT+%d]){' + f.writeln(12*' '+'.defines' + ' = (const bench_define_t*)' + '(const bench_define_t[][%d]){' % (len(suite.defines))) for i, permutation in enumerate(case.permutations): f.writeln(16*' '+'{') for k, vs in sorted(permutation.items()): - f.writeln(20*' ' - +'[%-24s] = {__bench__%s__%s__%d, NULL, ' - '%d},' - % (k+'_i', case.name, k, i, + f.writeln(20*' '+'[%d] = {' + '"%s", &%s, ' + '__bench__%s__%s__%d, NULL, %d},' + % (sorted(suite.defines).index(k), + k, k, case.name, k, i, sum(len(v) if isinstance(v, range) else 1 @@ -537,30 +532,25 @@ def compile(bench_paths, **args): shutil.copyfileobj(sf, f) f.writeln() + # merge all defines we need, otherwise we will run into + # redefinition errors + defines = ({define + for suite in suites + if suite.isin(args['source']) + for define in suite.defines} + | {define + for suite in suites + for case in suite.cases + if case.isin(args['source']) + for define in case.defines}) + if defines: + for define in sorted(defines): + f.writeln('__attribute__((weak)) intmax_t %s;' + % define) + f.writeln() + # write any internal benches for suite in suites: - if (suite.isin(args['source']) - or any(case.isin(args['source']) - for case in suite.cases)): - # write defines, but note we need to undef any - # new defines since we're in someone else's file - if suite.defines: - for i, define in enumerate( - sorted(suite.defines)): - f.writeln('#ifndef %s' % define) - f.writeln('#define %-24s ' - 'BENCH_IMPLICIT_DEFINE_COUNT+%d' % ( - define+'_i', i)) - f.writeln('#define %-24s ' - 'BENCH_DEFINE(%s)' % ( - define, define+'_i')) - f.writeln('#define ' - '__BENCH__%s__NEEDS_UNDEF' % ( - define)) - f.writeln('#endif') - f.writeln() - - # write any internal suite code if suite.isin(args['source']): if suite.code_lineno is not None: f.writeln('#line %d "%s"' @@ -575,19 +565,6 @@ def compile(bench_paths, **args): if case.isin(args['source']): write_case_functions(f, suite, case) - if (suite.isin(args['source']) - or any(case.isin(args['source']) - for case in suite.cases)): - for define in sorted(suite.defines): - f.writeln('#ifdef __BENCH__%s__NEEDS_UNDEF' - % define) - f.writeln('#undef __BENCH__%s__NEEDS_UNDEF' - % define) - f.writeln('#undef %s' % define) - f.writeln('#undef %s' % (define+'_i')) - f.writeln('#endif') - f.writeln() - # declare our bench suites # # by declaring these as weak we can write these to every @@ -640,6 +617,8 @@ def find_runner(runner, id=None, **args): '-o%s' % args['perf']])) # other context + if args.get('define_depth'): + cmd.append('--define-depth=%s' % args['define_depth']) if args.get('disk'): cmd.append('-d%s' % args['disk']) if args.get('trace'): @@ -662,11 +641,11 @@ def find_runner(runner, id=None, **args): for define in args.get('define'): cmd.append('-D%s' % define) - # test id? + # bench id? # # note we disable defines above when id is explicit, defines override id - # in the test runner, which is not what we want when querying an explicit - # test id + # in the bench runner, which is not what we want when querying an explicit + # bench id if id is not None: cmd.append(id) @@ -1511,6 +1490,9 @@ if __name__ == "__main__": '-D', '--define', action='append', help="Override a bench define.") + bench_parser.add_argument( + '--define-depth', + help="How deep to evaluate recursive defines before erroring.") bench_parser.add_argument( '-d', '--disk', help="Direct block device operations to this file.") @@ -1568,7 +1550,7 @@ if __name__ == "__main__": '-F', '--failures', type=lambda x: int(x, 0), default=3, - help="Show this many test failures. Defaults to 3.") + help="Show this many bench failures. Defaults to 3.") bench_parser.add_argument( '-C', '--context', type=lambda x: int(x, 0), diff --git a/scripts/test.py b/scripts/test.py index 01589e07..20b49e46 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -434,13 +434,9 @@ def compile(test_paths, **args): if not args.get('source'): # write any suite defines if suite.defines: - for i, define in enumerate(sorted(suite.defines)): - f.writeln('#ifndef %s' % define) - f.writeln('#define %-24s ' - 'TEST_IMPLICIT_DEFINE_COUNT+%d' % (define+'_i', i)) - f.writeln('#define %-24s ' - 'TEST_DEFINE(%s)' % (define, define+'_i')) - f.writeln('#endif') + for define in sorted(suite.defines): + f.writeln('__attribute__((weak)) intmax_t %s;' + % define) f.writeln() # write any suite code @@ -483,17 +479,14 @@ def compile(test_paths, **args): 'TEST_INTERNAL' if suite.internal else None, 'TEST_REENTRANT' if suite.reentrant else None])) or 0)) + # create suite defines if suite.defines: - # create suite define names - f.writeln(4*' '+'.define_names = (const char *const[' - 'TEST_IMPLICIT_DEFINE_COUNT+%d]){' - % (len(suite.defines))) + f.writeln(4*' '+'.defines = (const test_define_t[]){') for k in sorted(suite.defines): - f.writeln(8*' '+'[%-24s] = "%s",' % (k+'_i', k)) + f.writeln(8*' '+'{"%s", &%s, NULL, NULL, 0},' + % (k, k)) f.writeln(4*' '+'},') - f.writeln(4*' '+'.define_count = ' - 'TEST_IMPLICIT_DEFINE_COUNT+%d,' - % len(suite.defines)) + f.writeln(4*' '+'.define_count = %d,' % len(suite.defines)) if suite.cases: f.writeln(4*' '+'.cases = (const struct test_case[]){') for case in suite.cases: @@ -506,18 +499,20 @@ def compile(test_paths, **args): 'TEST_INTERNAL' if case.internal else None, 'TEST_REENTRANT' if case.reentrant else None])) or 0)) + # create case defines if case.defines: - f.writeln(12*' '+'.defines = ' - '(const test_define_t*)(const test_define_t[][' - 'TEST_IMPLICIT_DEFINE_COUNT+%d]){' + f.writeln(12*' '+'.defines' + ' = (const test_define_t*)' + '(const test_define_t[][%d]){' % (len(suite.defines))) for i, permutation in enumerate(case.permutations): f.writeln(16*' '+'{') for k, vs in sorted(permutation.items()): - f.writeln(20*' ' - +'[%-24s] = {__test__%s__%s__%d, NULL, ' - '%d},' - % (k+'_i', case.name, k, i, + f.writeln(20*' '+'[%d] = {' + '"%s", &%s, ' + '__test__%s__%s__%d, NULL, %d},' + % (sorted(suite.defines).index(k), + k, k, case.name, k, i, sum(len(v) if isinstance(v, range) else 1 @@ -544,30 +539,25 @@ def compile(test_paths, **args): shutil.copyfileobj(sf, f) f.writeln() + # merge all defines we need, otherwise we will run into + # redefinition errors + defines = ({define + for suite in suites + if suite.isin(args['source']) + for define in suite.defines} + | {define + for suite in suites + for case in suite.cases + if case.isin(args['source']) + for define in case.defines}) + if defines: + for define in sorted(defines): + f.writeln('__attribute__((weak)) intmax_t %s;' + % define) + f.writeln() + # write any internal tests for suite in suites: - if (suite.isin(args['source']) - or any(case.isin(args['source']) - for case in suite.cases)): - # write defines, but note we need to undef any - # new defines since we're in someone else's file - if suite.defines: - for i, define in enumerate( - sorted(suite.defines)): - f.writeln('#ifndef %s' % define) - f.writeln('#define %-24s ' - 'TEST_IMPLICIT_DEFINE_COUNT+%d' % ( - define+'_i', i)) - f.writeln('#define %-24s ' - 'TEST_DEFINE(%s)' % ( - define, define+'_i')) - f.writeln('#define ' - '__TEST__%s__NEEDS_UNDEF' % ( - define)) - f.writeln('#endif') - f.writeln() - - # write any internal suite code if suite.isin(args['source']): if suite.code_lineno is not None: f.writeln('#line %d "%s"' @@ -582,19 +572,6 @@ def compile(test_paths, **args): if case.isin(args['source']): write_case_functions(f, suite, case) - if (suite.isin(args['source']) - or any(case.isin(args['source']) - for case in suite.cases)): - for define in sorted(suite.defines): - f.writeln('#ifdef __TEST__%s__NEEDS_UNDEF' - % define) - f.writeln('#undef __TEST__%s__NEEDS_UNDEF' - % define) - f.writeln('#undef %s' % define) - f.writeln('#undef %s' % (define+'_i')) - f.writeln('#endif') - f.writeln() - # declare our test suites # # by declaring these as weak we can write these to every @@ -647,6 +624,8 @@ def find_runner(runner, id=None, **args): '-o%s' % args['perf']])) # other context + if args.get('define_depth'): + cmd.append('--define-depth=%s' % args['define_depth']) if args.get('powerloss'): cmd.append('-P%s' % args['powerloss']) if args.get('disk'): @@ -1525,6 +1504,9 @@ if __name__ == "__main__": '-D', '--define', action='append', help="Override a test define.") + test_parser.add_argument( + '--define-depth', + help="How deep to evaluate recursive defines before erroring.") test_parser.add_argument( '-P', '--powerloss', help="Comma-separated list of power-loss scenarios to test.")