diff --git a/runners/test_runner.c b/runners/test_runner.c index eea4b5f3..e505e04e 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -16,197 +16,89 @@ #include -// test suites in a custom ld section -extern struct test_suite __start__test_suites; -extern struct test_suite __stop__test_suites; +// some helpers -const struct test_suite *test_suites = &__start__test_suites; -#define TEST_SUITE_COUNT \ - ((size_t)(&__stop__test_suites - &__start__test_suites)) +// append to an array with amortized doubling +void *mappend(void **p, + size_t size, + size_t *count, + size_t *capacity) { + uint8_t *p_ = *p; + size_t count_ = *count; + size_t capacity_ = *capacity; -// test geometries -struct test_geometry { - const char *name; - intmax_t defines[TEST_GEOMETRY_DEFINE_COUNT]; -}; + count_ += 1; + if (count_ > capacity_) { + capacity_ = (2*capacity_ < 4) ? 4 : 2*capacity_; -const struct test_geometry test_geometries[TEST_GEOMETRY_COUNT] - = TEST_GEOMETRIES; - -// test define lookup and management -const intmax_t *test_override_defines; -intmax_t (*const *test_case_defines)(void); -const intmax_t *test_geometry_defines; -const intmax_t test_default_defines[TEST_PREDEFINE_COUNT] - = TEST_DEFAULTS; - -uint8_t test_override_predefine_map[TEST_PREDEFINE_COUNT]; -uint8_t test_override_define_map[256]; -uint8_t test_case_predefine_map[TEST_PREDEFINE_COUNT]; - -const char *const *test_override_names; -size_t test_override_count; - -const char *const test_predefine_names[TEST_PREDEFINE_COUNT] - = TEST_PREDEFINE_NAMES; - -const char *const *test_define_names; -size_t test_define_count; - - -intmax_t test_predefine(size_t define) { - if (test_override_defines - && test_override_predefine_map[define] != 0xff) { - return test_override_defines[test_override_predefine_map[define]]; - } else if (test_case_defines - && test_case_predefine_map[define] != 0xff - && test_case_defines[test_case_predefine_map[define]]) { - return test_case_defines[test_case_predefine_map[define]](); - } else if (define < TEST_GEOMETRY_DEFINE_COUNT) { - return test_geometry_defines[define]; - } else { - return test_default_defines[define-TEST_GEOMETRY_DEFINE_COUNT]; - } -} - -intmax_t test_define(size_t define) { - if (test_override_defines - && test_override_define_map[define] != 0xff) { - return test_override_defines[test_override_define_map[define]]; - } else if (test_case_defines - && test_case_defines[define]) { - return test_case_defines[define](); - } - - fprintf(stderr, "error: undefined define %s\n", - test_define_names[define]); - assert(false); - exit(-1); -} - -static void define_geometry(const struct test_geometry *geometry) { - test_geometry_defines = geometry->defines; -} - -static void test_define_overrides( - const char *const *override_names, - const intmax_t *override_defines, - size_t override_count) { - test_override_defines = override_defines; - test_override_names = override_names; - test_override_count = override_count; - - // map any override predefines - memset(test_override_predefine_map, 0xff, TEST_PREDEFINE_COUNT); - for (size_t i = 0; i < test_override_count; i++) { - for (size_t j = 0; j < TEST_PREDEFINE_COUNT; j++) { - if (strcmp(test_override_names[i], test_predefine_names[j]) == 0) { - test_override_predefine_map[j] = i; - } - } - } -} - -static void define_suite(const struct test_suite *suite) { - test_define_names = suite->define_names; - test_define_count = suite->define_count; - - // map any override defines - memset(test_override_define_map, 0xff, suite->define_count); - for (size_t i = 0; i < test_override_count; i++) { - for (size_t j = 0; j < suite->define_count; j++) { - if (strcmp(test_override_names[i], suite->define_names[j]) == 0) { - test_override_define_map[j] = i; - } + p_ = realloc(p_, capacity_*size); + if (!p_) { + return NULL; } } - // map any suite/case predefines - memset(test_case_predefine_map, 0xff, TEST_PREDEFINE_COUNT); - for (size_t i = 0; i < suite->define_count; i++) { - for (size_t j = 0; j < TEST_PREDEFINE_COUNT; j++) { - if (strcmp(suite->define_names[i], test_predefine_names[j]) == 0) { - test_case_predefine_map[j] = i; - } + *p = p_; + *count = count_; + *capacity = capacity_; + return &p_[(count_-1)*size]; +} + +// a quick self-terminating text-safe varint scheme +static void leb16_print(uintmax_t x) { + while (true) { + lfs_testbd_powercycles_t nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0); + printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10); + if (x <= 0xf) { + break; } + x >>= 4; } } -static void define_perm( - const struct test_suite *suite, - const struct test_case *case_, - size_t perm) { - (void)suite; - if (case_->defines) { - test_case_defines = case_->defines[perm]; - } else { - test_case_defines = NULL; - } -} - - -// a quick encoding scheme for sequences of power-loss -static void leb16_print( - const lfs_testbd_powercycles_t *cycles, - size_t cycle_count) { - for (size_t i = 0; i < cycle_count; i++) { - lfs_testbd_powercycles_t x = cycles[i]; - while (true) { - lfs_testbd_powercycles_t nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0); - printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10); - if (x <= 0xf) { - break; - } - x >>= 4; - } - } -} - -static size_t leb16_parse(const char *s, char **tail, - lfs_testbd_powercycles_t **cycles) { - // first lets count how many number we're dealing with - size_t count = 0; - size_t len = 0; - for (size_t i = 0;; i++) { - if ((s[i] >= '0' && s[i] <= '9') - || (s[i] >= 'a' && s[i] <= 'f')) { - len = i+1; - count += 1; - } else if ((s[i] >= 'g' && s[i] <= 'v')) { - // do nothing +static uintmax_t leb16_parse(const char *s, char **tail) { + uintmax_t x = 0; + size_t i = 0; + while (true) { + uintmax_t nibble = s[i]; + if (nibble >= '0' && nibble <= '9') { + nibble = nibble - '0'; + } else if (nibble >= 'a' && nibble <= 'v') { + nibble = nibble - 'a' + 10; } else { + // invalid? + if (tail) { + *tail = (char*)s; + } + return 0; + } + + x |= (nibble & 0xf) << (4*i); + i += 1; + if (!(nibble & 0x10)) { break; } } - // then parse - lfs_testbd_powercycles_t *cycles_ = malloc( - count * sizeof(lfs_testbd_powercycles_t)); - size_t i = 0; - lfs_testbd_powercycles_t x = 0; - size_t k = 0; - for (size_t j = 0; j < len; j++) { - lfs_testbd_powercycles_t nibble = s[j]; - nibble = (nibble < 'a') ? nibble-'0' : nibble-'a'+10; - x |= (nibble & 0xf) << (4*k); - k += 1; - if (!(nibble & 0x10)) { - cycles_[i] = x; - i += 1; - x = 0; - k = 0; - } - } - if (tail) { - *tail = (char*)s + len; + *tail = (char*)s + i; } - *cycles = cycles_; - return count; + return x; } -// test state + +// test_runner types + +typedef struct test_geometry { + char short_name; + const char *long_name; + + lfs_size_t read_size; + lfs_size_t prog_size; + lfs_size_t block_size; + lfs_size_t block_count; +} test_geometry_t; + typedef struct test_powerloss { char short_name; const char *long_name; @@ -221,34 +113,270 @@ typedef struct test_powerloss { size_t cycle_count; } test_powerloss_t; -static void run_powerloss_none( - const struct test_suite *suite, - const struct test_case *case_, - size_t perm, - const lfs_testbd_powercycles_t *cycles, - size_t cycle_count); -const test_powerloss_t *test_powerlosses = (const test_powerloss_t[]){ - {'0', "none", run_powerloss_none, NULL, 0}, -}; -size_t test_powerloss_count = 1; - - typedef struct test_id { const char *suite; const char *case_; size_t perm; + const test_geometry_t *geometry; const lfs_testbd_powercycles_t *cycles; size_t cycle_count; } test_id_t; +static void print_id( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm, + const lfs_testbd_powercycles_t *cycles, + size_t cycle_count) { + (void)suite; + // suite[#case[#perm[#geometry[#powercycles]]]] + printf("%s#%zu#", case_->id, perm); + + // reduce duplication in geometry, this is appended to every test + if (READ_SIZE != BLOCK_SIZE || PROG_SIZE != BLOCK_SIZE) { + if (READ_SIZE != PROG_SIZE) { + leb16_print(READ_SIZE); + } + leb16_print(PROG_SIZE); + } + leb16_print(BLOCK_SIZE); + if (BLOCK_COUNT*BLOCK_SIZE != 1024*1024) { + leb16_print(BLOCK_COUNT); + } + + // only print power-cycles if any occured + if (cycles) { + printf("#"); + for (size_t i = 0; i < cycle_count; i++) { + leb16_print(cycles[i]); + } + } +} + + +// test suites are linked into a custom ld section +extern struct test_suite __start__test_suites; +extern struct test_suite __stop__test_suites; + +const struct test_suite *test_suites = &__start__test_suites; +#define TEST_SUITE_COUNT \ + ((size_t)(&__stop__test_suites - &__start__test_suites)) + + +// test define management +typedef struct test_define_map { + intmax_t (*const *defines)(size_t); + const char *const *names; + size_t count; +} test_define_map_t; + +extern const test_geometry_t *test_geometry; + +#define TEST_DEFINE(k, v) \ + intmax_t test_define_##k(__attribute__((unused)) size_t define) { \ + return v; \ + } + + TEST_IMPLICIT_DEFINES +#undef TEST_DEFINE + +#define TEST_DEFINE_MAP_COUNT 3 +test_define_map_t test_define_maps[TEST_DEFINE_MAP_COUNT] = { + {NULL, NULL, 0}, + {NULL, NULL, 0}, + { + (intmax_t (*const[TEST_IMPLICIT_DEFINE_COUNT])(size_t)){ + #define TEST_DEFINE(k, v) \ + [k##_i] = test_define_##k, + + TEST_IMPLICIT_DEFINES + #undef TEST_DEFINE + }, + (const char *const[TEST_IMPLICIT_DEFINE_COUNT]){ + #define TEST_DEFINE(k, v) \ + [k##_i] = #k, + + TEST_IMPLICIT_DEFINES + #undef TEST_DEFINE + }, + TEST_IMPLICIT_DEFINE_COUNT, + }, +}; + +intmax_t *test_define_cache; +size_t test_define_cache_count; +unsigned *test_define_cache_mask; + +const char *test_define_name(size_t define) { + // lookup in our test defines + for (size_t i = 0; i < TEST_DEFINE_MAP_COUNT; i++) { + if (define < test_define_maps[i].count + && test_define_maps[i].names + && test_define_maps[i].names[define]) { + return test_define_maps[i].names[define]; + } + } + + return NULL; +} + +intmax_t test_define(size_t define) { + // is the define in our cache? + if (define < test_define_cache_count + && (test_define_cache_mask[define/(8*sizeof(unsigned))] + & (1 << (define%(8*sizeof(unsigned)))))) { + return test_define_cache[define]; + } + + // lookup in our test defines + for (size_t i = 0; i < TEST_DEFINE_MAP_COUNT; i++) { + if (define < test_define_maps[i].count + && test_define_maps[i].defines[define]) { + intmax_t v = test_define_maps[i].defines[define](define); + + // insert into cache! + test_define_cache[define] = v; + test_define_cache_mask[define / (8*sizeof(unsigned))] + |= 1 << (define%(8*sizeof(unsigned))); + + return v; + } + } + + // not found? + const char *name = test_define_name(define); + fprintf(stderr, "error: undefined define %s (%zd)\n", + name ? name : "(unknown)", + define); + assert(false); + exit(-1); +} + +void test_define_flush(void) { + // clear cache between permutations + memset(test_define_cache_mask, 0, + sizeof(unsigned)*( + (test_define_cache_count+(8*sizeof(unsigned))-1) + / (8*sizeof(unsigned)))); +} + +// geometry updates +const test_geometry_t *test_geometry = NULL; + +void test_define_geometry(const test_geometry_t *geometry) { + test_geometry = geometry; +} + +// override updates +typedef struct test_override { + const char *name; + intmax_t define; +} test_override_t; + +const test_override_t *test_overrides = NULL; +size_t test_override_count = 0; +intmax_t *test_override_map = NULL; + +intmax_t test_define_override(size_t define) { + return test_override_map[define]; +} + +void test_define_overrides( + const test_override_t *overrides, + size_t override_count) { + test_overrides = overrides; + test_override_count = override_count; +} + +// suite/perm updates +void test_define_suite(const struct test_suite *suite) { + test_define_maps[1].names = suite->define_names; + test_define_maps[1].count = suite->define_count; + + // make sure our cache is large enough + if (lfs_max(suite->define_count, TEST_IMPLICIT_DEFINE_COUNT) + > test_define_cache_count) { + // align to power of two to avoid any superlinear growth + size_t ncount = 1 << lfs_npw2( + lfs_max(suite->define_count, TEST_IMPLICIT_DEFINE_COUNT)); + test_define_cache = realloc(test_define_cache, ncount*sizeof(intmax_t)); + test_define_cache_mask = realloc(test_define_cache_mask, + sizeof(unsigned)*( + (ncount+(8*sizeof(unsigned))-1) + / (8*sizeof(unsigned)))); + test_define_cache_count = ncount; + } + + // map any overrides + if (test_override_count > 0) { + // make sure our override arrays are big enough + if (suite->define_count > test_define_maps[0].count) { + // align to power of two to avoid any superlinear growth + size_t ncount = 1 << lfs_npw2(suite->define_count); + test_define_maps[0].defines = realloc( + (intmax_t (**)(size_t))test_define_maps[0].defines, + ncount*sizeof(intmax_t (*)(size_t))); + test_override_map = realloc( + test_override_map, + ncount*sizeof(intmax_t)); + test_define_maps[0].count = ncount; + } + + for (size_t i = 0; i < test_define_maps[0].count; i++) { + ((intmax_t (**)(size_t))test_define_maps[0].defines)[i] = NULL; + + const char *name = test_define_name(i); + if (!name) { + continue; + } + + for (size_t j = 0; j < test_override_count; j++) { + if (strcmp(name, test_overrides[j].name) == 0) { + test_override_map[i] = test_overrides[j].define; + ((intmax_t (**)(size_t))test_define_maps[0].defines)[i] + = test_define_override; + break; + } + } + } + } +} + +void test_define_perm( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm) { + if (case_->defines) { + test_define_maps[1].defines = case_->defines[perm]; + test_define_maps[1].count = suite->define_count; + } else { + test_define_maps[1].defines = NULL; + test_define_maps[1].count = 0; + } +} + +void test_define_cleanup(void) { + // test define management can allocate a few things + free(test_define_cache); + free(test_define_cache_mask); + free(test_override_map); + free((intmax_t (**)(size_t))test_define_maps[0].defines); +} + + + +// test state +extern const test_geometry_t *test_geometries; +extern size_t test_geometry_count; + +extern const test_powerloss_t *test_powerlosses; +extern size_t test_powerloss_count; + const test_id_t *test_ids = (const test_id_t[]) { - {NULL, NULL, -1, NULL, 0}, + {NULL, NULL, -1, NULL, NULL, 0}, }; size_t test_id_count = 1; - -const char *test_geometry = NULL; - size_t test_start = 0; size_t test_stop = -1; size_t test_step = 1; @@ -317,6 +445,7 @@ static void count_perms( const struct test_suite *suite, const struct test_case *case_, size_t perm, + const test_geometry_t *geometry, const lfs_testbd_powercycles_t *cycles, size_t cycle_count, size_t *perms, @@ -325,33 +454,29 @@ static void count_perms( size_t perms_ = 0; size_t filtered_ = 0; - for (size_t p = 0; p < (cycles ? 1 : test_powerloss_count); p++) { - if (!cycles - && test_powerlosses[p].short_name != '0' - && !(case_->flags & TEST_REENTRANT)) { + for (size_t k = 0; k < case_->permutations; k++) { + if (perm != (size_t)-1 && k != perm) { continue; } - size_t perm_ = 0; - for (size_t g = 0; g < TEST_GEOMETRY_COUNT; g++) { - if (test_geometry && strcmp( - test_geometries[g].name, test_geometry) != 0) { - continue; - } + // define permutation + test_define_perm(suite, case_, k); - for (size_t k = 0; k < case_->permutations; k++) { - perm_ += 1; + for (size_t g = 0; g < (geometry ? 1 : test_geometry_count); g++) { + // define geometry + test_define_geometry(geometry ? geometry : &test_geometries[g]); + test_define_flush(); - if (perm != (size_t)-1 && perm_ != perm) { + for (size_t p = 0; p < (cycles ? 1 : test_powerloss_count); p++) { + // skip non-reentrant tests when powerloss testing + if (!cycles + && test_powerlosses[p].short_name != '0' + && !(case_->flags & TEST_REENTRANT)) { continue; } perms_ += 1; - // setup defines - define_perm(suite, case_, k); - define_geometry(&test_geometries[g]); - if (case_->filter && !case_->filter()) { continue; } @@ -370,6 +495,7 @@ static void count_perms( static void summary(void) { printf("%-36s %7s %7s %7s %11s\n", "", "flags", "suites", "cases", "perms"); + size_t suites = 0; size_t cases = 0; test_flags_t flags = 0; size_t perms = 0; @@ -382,7 +508,7 @@ static void summary(void) { continue; } - define_suite(&test_suites[i]); + test_define_suite(&test_suites[i]); for (size_t j = 0; j < test_suites[i].case_count; j++) { if (test_ids[t].case_ && strcmp( @@ -390,14 +516,16 @@ static void summary(void) { continue; } + cases += 1; count_perms(&test_suites[i], &test_suites[i].cases[j], test_ids[t].perm, + test_ids[t].geometry, test_ids[t].cycles, test_ids[t].cycle_count, &perms, &filtered); } - cases += test_suites[i].case_count; + suites += 1; flags |= test_suites[i].flags; } } @@ -411,7 +539,7 @@ static void summary(void) { printf("%-36s %7s %7zu %7zu %11s\n", "TOTAL", flag_buf, - TEST_SUITE_COUNT, + suites, cases, perm_buf); } @@ -426,8 +554,9 @@ static void list_suites(void) { continue; } - define_suite(&test_suites[i]); + test_define_suite(&test_suites[i]); + size_t cases = 0; size_t perms = 0; size_t filtered = 0; @@ -437,8 +566,10 @@ static void list_suites(void) { continue; } + cases += 1; count_perms(&test_suites[i], &test_suites[i].cases[j], test_ids[t].perm, + test_ids[t].geometry, test_ids[t].cycles, test_ids[t].cycle_count, &perms, &filtered); @@ -453,7 +584,7 @@ static void list_suites(void) { printf("%-36s %7s %7zu %11s\n", test_suites[i].id, flag_buf, - test_suites[i].case_count, + cases, perm_buf); } } @@ -469,7 +600,7 @@ static void list_cases(void) { continue; } - define_suite(&test_suites[i]); + test_define_suite(&test_suites[i]); for (size_t j = 0; j < test_suites[i].case_count; j++) { if (test_ids[t].case_ && strcmp( @@ -482,6 +613,7 @@ static void list_cases(void) { count_perms(&test_suites[i], &test_suites[i].cases[j], test_ids[t].perm, + test_ids[t].geometry, test_ids[t].cycles, test_ids[t].cycle_count, &perms, &filtered); @@ -503,7 +635,26 @@ static void list_cases(void) { } } -static void list_paths(void) { +static void list_suite_paths(void) { + printf("%-36s %s\n", "suite", "path"); + + for (size_t t = 0; t < test_id_count; t++) { + for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { + if (test_ids[t].suite && strcmp( + test_suites[i].name, test_ids[t].suite) != 0) { + continue; + } + + printf("%-36s %s\n", + test_suites[i].id, + test_suites[i].path); + } + } +} + +static void list_case_paths(void) { + printf("%-36s %s\n", "case", "path"); + for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { if (test_ids[t].suite && strcmp( @@ -517,7 +668,7 @@ static void list_paths(void) { continue; } - printf("%-36s %-36s\n", + printf("%-36s %s\n", test_suites[i].cases[j].id, test_suites[i].cases[j].path); } @@ -525,7 +676,101 @@ static void list_paths(void) { } } +struct list_define { + const char *name; + intmax_t *values; + size_t value_count; + size_t value_capacity; +}; + +static void list_defines_perms( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm, + const test_geometry_t *geometry, + struct list_define **defines, + size_t *define_count, + size_t *define_capacity) { + struct list_define *defines_ = *defines; + size_t define_count_ = *define_count; + size_t define_capacity_ = *define_capacity; + + for (size_t k = 0; k < case_->permutations; k++) { + if (perm != (size_t)-1 && k != perm) { + continue; + } + + // define permutation + test_define_perm(suite, case_, k); + + for (size_t g = 0; g < (geometry ? 1 : test_geometry_count); g++) { + // define geometry + test_define_geometry(geometry ? geometry : &test_geometries[g]); + test_define_flush(); + + // collect defines + for (size_t d = 0; + d < lfs_max(suite->define_count, + TEST_IMPLICIT_DEFINE_COUNT); + d++) { + if (!(d < TEST_IMPLICIT_DEFINE_COUNT || ( + case_->defines + && case_->defines[k] + && case_->defines[k][d]))) { + continue; + } + const char *name = test_define_name(d); + intmax_t value = test_define(d); + + // define already in defines? + for (size_t i = 0; i < define_count_; i++) { + if (strcmp(defines_[i].name, name) == 0) { + // value already in values? + for (size_t j = 0; j < defines_[i].value_count; j++) { + if (defines_[i].values[j] == value) { + goto next_define; + } + } + + *(intmax_t*)mappend( + (void**)&defines_[i].values, + sizeof(intmax_t), + &defines_[i].value_count, + &defines_[i].value_capacity) = value; + + goto next_define; + } + } + + { + // new define? + struct list_define *define = mappend( + (void**)&defines_, + sizeof(struct list_define), + &define_count_, + &define_capacity_); + define->name = name; + define->values = malloc(sizeof(intmax_t)); + define->values[0] = value; + define->value_count = 1; + define->value_capacity = 1; + } + + next_define:; + } + } + } + + *defines = defines_; + *define_count = define_count_; + *define_capacity = define_capacity_; +} + static void list_defines(void) { + struct list_define *defines = NULL; + size_t define_count = 0; + size_t define_capacity = 0; + for (size_t t = 0; t < test_id_count; t++) { for (size_t i = 0; i < TEST_SUITE_COUNT; i++) { if (test_ids[t].suite && strcmp( @@ -533,7 +778,7 @@ static void list_defines(void) { continue; } - define_suite(&test_suites[i]); + test_define_suite(&test_suites[i]); for (size_t j = 0; j < test_suites[i].case_count; j++) { if (test_ids[t].case_ && strcmp( @@ -541,104 +786,123 @@ static void list_defines(void) { continue; } - for (size_t p = 0; - p < (test_ids[t].cycles ? 1 : test_powerloss_count); - p++) { - if (!test_ids[t].cycles - && test_powerlosses[p].short_name != '0' - && !(test_suites[i].cases[j].flags - & TEST_REENTRANT)) { - continue; - } - - size_t perm_ = 0; - for (size_t g = 0; g < TEST_GEOMETRY_COUNT; g++) { - if (test_geometry && strcmp( - test_geometries[g].name, test_geometry) != 0) { - continue; - } - - for (size_t k = 0; - k < test_suites[i].cases[j].permutations; - k++) { - perm_ += 1; - - if (test_ids[t].perm != (size_t)-1 - && perm_ != test_ids[t].perm) { - continue; - } - - // setup defines - define_perm(&test_suites[i], - &test_suites[i].cases[j], - k); - define_geometry(&test_geometries[g]); - - // print the case - char id_buf[256]; - sprintf(id_buf, "%s#%zu", - test_suites[i].cases[j].id, perm_); - printf("%-36s ", id_buf); - - // special case for the current geometry - printf("GEOMETRY=%s ", test_geometries[g].name); - - // print each define - for (size_t l = 0; - l < test_suites[i].define_count; - l++) { - if (test_suites[i].cases[j].defines - && test_suites[i].cases[j] - .defines[k][l]) { - printf("%s=%jd ", - test_suites[i].define_names[l], - test_define(l)); - } - } - printf("\n"); - } - } - } + list_defines_perms(&test_suites[i], &test_suites[i].cases[j], + test_ids[t].perm, + test_ids[t].geometry, + &defines, + &define_count, + &define_capacity); } } } -} -static void list_geometries(void) { - for (size_t i = 0; i < TEST_GEOMETRY_COUNT; i++) { - if (test_geometry && strcmp( - test_geometries[i].name, - test_geometry) != 0) { - continue; - } - - define_geometry(&test_geometries[i]); - - printf("%-36s ", test_geometries[i].name); - // print each define - for (size_t k = 0; k < TEST_GEOMETRY_DEFINE_COUNT; k++) { - printf("%s=%jd ", - test_predefine_names[k], - test_predefine(k)); + for (size_t i = 0; i < define_count; i++) { + printf("%s=", defines[i].name); + for (size_t j = 0; j < defines[i].value_count; j++) { + printf("%jd", defines[i].values[j]); + if (j != defines[i].value_count-1) { + printf(","); + } } printf("\n"); - } + + for (size_t i = 0; i < define_count; i++) { + free(defines[i].values); + } + free(defines); } -static void list_defaults(void) { - printf("%-36s ", "defaults"); - // print each define - for (size_t k = 0; k < TEST_DEFAULT_DEFINE_COUNT; k++) { - printf("%s=%jd ", - test_predefine_names[k+TEST_GEOMETRY_DEFINE_COUNT], - test_predefine(k+TEST_GEOMETRY_DEFINE_COUNT)); +static void list_implicit(void) { + struct list_define *defines = NULL; + size_t define_count = 0; + size_t define_capacity = 0; + + for (size_t t = 0; t < test_id_count; t++) { + // yes we do need to define a suite, this does a bit of bookeeping + // such as setting up the define cache + test_define_suite(&(const struct test_suite){0}); + list_defines_perms( + &(const struct test_suite){0}, + &(const struct test_case){.permutations=1}, + -1, + test_ids[t].geometry, + &defines, + &define_count, + &define_capacity); } - printf("\n"); + + for (size_t i = 0; i < define_count; i++) { + printf("%s=", defines[i].name); + for (size_t j = 0; j < defines[i].value_count; j++) { + printf("%jd", defines[i].values[j]); + if (j != defines[i].value_count-1) { + printf(","); + } + } + printf("\n"); + } + + for (size_t i = 0; i < define_count; i++) { + free(defines[i].values); + } + free(defines); } +// geometries to test + +const test_geometry_t builtin_geometries[] = { + {'d', "default", 16, 16, 512, (1024*1024)/512}, + {'e', "eeprom", 1, 1, 512, (1024*1024)/512}, + {'E', "emmc", 512, 512, 512, (1024*1024)/512}, + {'n', "nor", 1, 1, 4096, (1024*1024)/4096}, + {'N', "nand", 4096, 4096, 32768, (1024*1024)/(32*1024)}, + {0, NULL, 0, 0, 0, 0}, +}; + +const test_geometry_t *test_geometries = (const test_geometry_t[]){ + {'d', "default", 16, 16, 512, (1024*1024)/512}, + {'e', "eeprom", 1, 1, 512, (1024*1024)/512}, + {'E', "emmc", 512, 512, 512, (1024*1024)/512}, + {'n', "nor", 1, 1, 4096, (1024*1024)/4096}, + {'N', "nand", 4096, 4096, 32768, (1024*1024)/(32*1024)}, +}; +size_t test_geometry_count = 5; + +static void list_geometries(void) { + printf("%-24s %7s %7s %7s %7s %11s %s\n", + "geometry", "read", "prog", "erase", "count", "size", "leb16"); + size_t i = 0; + for (; builtin_geometries[i].long_name; i++) { + uintmax_t read_size = builtin_geometries[i].read_size; + uintmax_t prog_size = builtin_geometries[i].prog_size; + uintmax_t block_size = builtin_geometries[i].block_size; + uintmax_t block_count = builtin_geometries[i].block_count; + printf("%c,%-22s %7ju %7ju %7ju %7ju %11ju ", + builtin_geometries[i].short_name, + builtin_geometries[i].long_name, + read_size, + prog_size, + block_size, + block_count, + block_size*block_count); + if (read_size != block_size || prog_size != block_size) { + if (read_size != prog_size) { + leb16_print(read_size); + } + leb16_print(prog_size); + } + leb16_print(block_size); + if (block_count*block_size != 1024*1024) { + leb16_print(block_count); + } + printf("\n"); + } +} + + // scenarios to run tests under power-loss static void run_powerloss_none( @@ -686,11 +950,15 @@ static void run_powerloss_none( } // run the test - printf("running %s#%zu\n", case_->id, perm); + printf("running "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); case_->run(&cfg); - printf("finished %s#%zu\n", case_->id, perm); + printf("finished "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); // cleanup err = lfs_testbd_destroy(&cfg); @@ -756,7 +1024,9 @@ static void run_powerloss_linear( } // run the test, increasing power-cycles as power-loss events occur - printf("running %s#%zu\n", case_->id, perm); + printf("running "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); while (true) { if (!setjmp(powerloss_jmp)) { @@ -766,9 +1036,11 @@ static void run_powerloss_linear( } // power-loss! - printf("powerloss %s#%zu#", case_->id, perm); + printf("powerloss "); + print_id(suite, case_, perm, NULL, 0); + printf("#"); for (lfs_testbd_powercycles_t j = 1; j <= i; j++) { - leb16_print(&j, 1); + leb16_print(j); } printf("\n"); @@ -776,7 +1048,9 @@ static void run_powerloss_linear( lfs_testbd_setpowercycles(&cfg, i); } - printf("finished %s#%zu\n", case_->id, perm); + printf("finished "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); // cleanup err = lfs_testbd_destroy(&cfg); @@ -837,7 +1111,9 @@ static void run_powerloss_exponential( } // run the test, increasing power-cycles as power-loss events occur - printf("running %s#%zu\n", case_->id, perm); + printf("running "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); while (true) { if (!setjmp(powerloss_jmp)) { @@ -847,9 +1123,11 @@ static void run_powerloss_exponential( } // power-loss! - printf("powerloss %s#%zu#", case_->id, perm); + printf("powerloss "); + print_id(suite, case_, perm, NULL, 0); + printf("#"); for (lfs_testbd_powercycles_t j = 1; j <= i; j *= 2) { - leb16_print(&j, 1); + leb16_print(j); } printf("\n"); @@ -857,7 +1135,9 @@ static void run_powerloss_exponential( lfs_testbd_setpowercycles(&cfg, i); } - printf("finished %s#%zu\n", case_->id, perm); + printf("finished "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); // cleanup err = lfs_testbd_destroy(&cfg); @@ -916,7 +1196,9 @@ static void run_powerloss_cycles( } // run the test, increasing power-cycles as power-loss events occur - printf("running %s#%zu\n", case_->id, perm); + printf("running "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); while (true) { if (!setjmp(powerloss_jmp)) { @@ -927,8 +1209,8 @@ static void run_powerloss_cycles( // power-loss! assert(i <= cycle_count); - printf("powerloss %s#%zu#", case_->id, perm); - leb16_print(cycles, i+1); + printf("powerloss "); + print_id(suite, case_, perm, cycles, i+1); printf("\n"); i += 1; @@ -936,7 +1218,9 @@ static void run_powerloss_cycles( (i < cycle_count) ? cycles[i] : 0); } - printf("finished %s#%zu\n", case_->id, perm); + printf("finished "); + print_id(suite, case_, perm, NULL, 0); + printf("\n"); // cleanup err = lfs_testbd_destroy(&cfg); @@ -961,24 +1245,20 @@ struct powerloss_exhaustive_cycles { }; static void powerloss_exhaustive_branch(void *c) { - // append to branches struct powerloss_exhaustive_state *state = c; - state->branch_count += 1; - if (state->branch_count > state->branch_capacity) { - state->branch_capacity = (2*state->branch_capacity > 4) - ? 2*state->branch_capacity - : 4; - state->branches = realloc(state->branches, - state->branch_capacity * sizeof(lfs_testbd_t)); - if (!state->branches) { - fprintf(stderr, "error: exhaustive: out of memory\n"); - exit(-1); - } + // append to branches + lfs_testbd_t *branch = mappend( + (void**)&state->branches, + sizeof(lfs_testbd_t), + &state->branch_count, + &state->branch_capacity); + if (!branch) { + fprintf(stderr, "error: exhaustive: out of memory\n"); + exit(-1); } // create copy-on-write copy - int err = lfs_testbd_copy(state->cfg, - &state->branches[state->branch_count-1]); + int err = lfs_testbd_copy(state->cfg, branch); if (err) { fprintf(stderr, "error: exhaustive: could not create bd copy\n"); exit(-1); @@ -1023,22 +1303,19 @@ static void run_powerloss_exhaustive_layer( // recurse into each branch for (size_t i = 0; i < state.branch_count; i++) { // first push and print the branch - cycles->cycle_count += 1; - if (cycles->cycle_count > cycles->cycle_capacity) { - cycles->cycle_capacity = (2*cycles->cycle_capacity > 4) - ? 2*cycles->cycle_capacity - : 4; - cycles->cycles = realloc(cycles->cycles, - cycles->cycle_capacity * sizeof(lfs_testbd_powercycles_t)); - if (!cycles->cycles) { - fprintf(stderr, "error: exhaustive: out of memory\n"); - exit(-1); - } + lfs_testbd_powercycles_t *cycle = mappend( + (void**)&cycles->cycles, + sizeof(lfs_testbd_powercycles_t), + &cycles->cycle_count, + &cycles->cycle_capacity); + if (!cycle) { + fprintf(stderr, "error: exhaustive: out of memory\n"); + exit(-1); } - cycles->cycles[cycles->cycle_count-1] = i; + *cycle = i; - printf("powerloss %s#%zu#", case_->id, perm); - leb16_print(cycles->cycles, cycles->cycle_count); + printf("powerloss "); + print_id(suite, case_, perm, cycles->cycles, cycles->cycle_count); printf("\n"); // now recurse @@ -1122,14 +1399,19 @@ const test_powerloss_t builtin_powerlosses[] = { const char *const builtin_powerlosses_help[] = { "Run with no power-losses.", - "Run with linearly-decreasing power-losses.", "Run with exponentially-decreasing power-losses.", + "Run with linearly-decreasing power-losses.", "Run a all permutations of power-losses, this may take a while.", "Run a all permutations of n power-losses.", "Run a custom comma-separated set of power-losses.", "Run a custom leb16-encoded set of power-losses.", }; +const test_powerloss_t *test_powerlosses = (const test_powerloss_t[]){ + {'0', "none", run_powerloss_none, NULL, 0}, +}; +size_t test_powerloss_count = 1; + static void list_powerlosses(void) { printf("%-24s %s\n", "scenario", "description"); size_t i = 0; @@ -1148,33 +1430,34 @@ static void list_powerlosses(void) { // global test step count -static size_t step = 0; +size_t step = 0; // run the tests static void run_perms( const struct test_suite *suite, const struct test_case *case_, size_t perm, + const test_geometry_t *geometry, const lfs_testbd_powercycles_t *cycles, size_t cycle_count) { - for (size_t p = 0; p < (cycles ? 1 : test_powerloss_count); p++) { - if (!cycles - && test_powerlosses[p].short_name != '0' - && !(case_->flags & TEST_REENTRANT)) { + for (size_t k = 0; k < case_->permutations; k++) { + if (perm != (size_t)-1 && k != perm) { continue; } - size_t perm_ = 0; - for (size_t g = 0; g < TEST_GEOMETRY_COUNT; g++) { - if (test_geometry && strcmp( - test_geometries[g].name, test_geometry) != 0) { - continue; - } + // define permutation + test_define_perm(suite, case_, k); - for (size_t k = 0; k < case_->permutations; k++) { - perm_ += 1; + for (size_t g = 0; g < (geometry ? 1 : test_geometry_count); g++) { + // define geometry + test_define_geometry(geometry ? geometry : &test_geometries[g]); + test_define_flush(); - if (perm != (size_t)-1 && perm_ != perm) { + for (size_t p = 0; p < (cycles ? 1 : test_powerloss_count); p++) { + // skip non-reentrant tests when powerloss testing + if (!cycles + && test_powerlosses[p].short_name != '0' + && !(case_->flags & TEST_REENTRANT)) { continue; } @@ -1186,24 +1469,20 @@ static void run_perms( } step += 1; - // setup defines - define_perm(suite, case_, k); - define_geometry(&test_geometries[g]); - // filter? if (case_->filter && !case_->filter()) { - printf("skipped %s#%zu\n", case_->id, perm_); + printf("skipped %s#%zu\n", case_->id, k); continue; } if (cycles) { run_powerloss_cycles( - suite, case_, perm_, + suite, case_, k, cycles, cycle_count); } else { test_powerlosses[p].run( - suite, case_, perm_, + suite, case_, k, test_powerlosses[p].cycles, test_powerlosses[p].cycle_count); } @@ -1223,7 +1502,7 @@ static void run(void) { continue; } - define_suite(&test_suites[i]); + test_define_suite(&test_suites[i]); for (size_t j = 0; j < test_suites[i].case_count; j++) { if (test_ids[t].case_ && strcmp( @@ -1233,6 +1512,7 @@ static void run(void) { run_perms(&test_suites[i], &test_suites[i].cases[j], test_ids[t].perm, + test_ids[t].geometry, test_ids[t].cycles, test_ids[t].cycle_count); } @@ -1248,35 +1528,37 @@ enum opt_flags { OPT_SUMMARY = 'Y', OPT_LIST_SUITES = 'l', OPT_LIST_CASES = 'L', - OPT_LIST_PATHS = 1, - OPT_LIST_DEFINES = 2, - OPT_LIST_GEOMETRIES = 3, - OPT_LIST_DEFAULTS = 4, - OPT_LIST_POWERLOSSES = 5, + OPT_LIST_SUITE_PATHS = 1, + OPT_LIST_CASE_PATHS = 2, + OPT_LIST_DEFINES = 3, + OPT_LIST_IMPLICIT = 4, + OPT_LIST_GEOMETRIES = 5, + OPT_LIST_POWERLOSSES = 6, OPT_DEFINE = 'D', - OPT_GEOMETRY = 'G', + OPT_GEOMETRY = 'g', OPT_POWERLOSS = 'p', - OPT_START = 6, - OPT_STEP = 7, - OPT_STOP = 8, + OPT_START = 7, + OPT_STEP = 8, + OPT_STOP = 9, OPT_DISK = 'd', OPT_TRACE = 't', - OPT_READ_SLEEP = 9, - OPT_PROG_SLEEP = 10, - OPT_ERASE_SLEEP = 11, + OPT_READ_SLEEP = 10, + OPT_PROG_SLEEP = 11, + OPT_ERASE_SLEEP = 12, }; -const char *short_opts = "hYlLD:G:p:nrVd:t:"; +const char *short_opts = "hYlLD:g:p:d:t:"; const struct option long_opts[] = { {"help", no_argument, NULL, OPT_HELP}, {"summary", no_argument, NULL, OPT_SUMMARY}, {"list-suites", no_argument, NULL, OPT_LIST_SUITES}, {"list-cases", no_argument, NULL, OPT_LIST_CASES}, - {"list-paths", no_argument, NULL, OPT_LIST_PATHS}, + {"list-suite-paths", no_argument, NULL, OPT_LIST_SUITE_PATHS}, + {"list-case-paths", no_argument, NULL, OPT_LIST_CASE_PATHS}, {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, + {"list-implicit", no_argument, NULL, OPT_LIST_IMPLICIT}, {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, - {"list-defaults", no_argument, NULL, OPT_LIST_DEFAULTS}, {"list-powerlosses", no_argument, NULL, OPT_LIST_POWERLOSSES}, {"define", required_argument, NULL, OPT_DEFINE}, {"geometry", required_argument, NULL, OPT_GEOMETRY}, @@ -1297,13 +1579,14 @@ const char *const help_text[] = { "Show quick summary.", "List test suites.", "List test cases.", - "List the path for each test case.", - "List the defines for each test permutation.", - "List the disk geometries used for testing.", - "List the default defines in this test-runner.", + "List the path for each test suite.", + "List the path and line number for each test case.", + "List all defines in this test-runner.", + "List implicit defines in this test-runner.", + "List the available disk geometries.", "List the available power-loss scenarios.", "Override a test define.", - "Filter by geometry.", + "Comma-separated list of disk geometries to test. Defaults to d,e,E,n,N.", "Comma-separated list of power-loss scenarios to test. Defaults to 0,l.", "Start at the nth test.", "Stop before the nth test.", @@ -1318,11 +1601,11 @@ const char *const help_text[] = { int main(int argc, char **argv) { void (*op)(void) = run; - const char **override_names = NULL; - intmax_t *override_defines = NULL; + test_override_t *overrides = NULL; size_t override_count = 0; size_t override_capacity = 0; + size_t test_geometry_capacity = 0; size_t test_powerloss_capacity = 0; size_t test_id_capacity = 0; @@ -1396,41 +1679,32 @@ int main(int argc, char **argv) { case OPT_LIST_CASES: op = list_cases; break; - case OPT_LIST_PATHS: - op = list_paths; + case OPT_LIST_SUITE_PATHS: + op = list_suite_paths; + break; + case OPT_LIST_CASE_PATHS: + op = list_case_paths; break; case OPT_LIST_DEFINES: op = list_defines; break; + case OPT_LIST_IMPLICIT: + op = list_implicit; + break; case OPT_LIST_GEOMETRIES: op = list_geometries; break; - case OPT_LIST_DEFAULTS: - op = list_defaults; - break; case OPT_LIST_POWERLOSSES: op = list_powerlosses; break; // configuration case OPT_DEFINE: { - // special case for -DGEOMETRY=, we treat this the same - // as --geometry= - if (strncmp(optarg, "GEOMETRY=", strlen("GEOMETRY=")) == 0) { - test_geometry = &optarg[strlen("GEOMETRY=")]; - break; - } - - // realloc if necessary - override_count += 1; - if (override_count > override_capacity) { - override_capacity = (2*override_capacity > 4) - ? 2*override_capacity - : 4; - override_names = realloc(override_names, - override_capacity * sizeof(const char *)); - override_defines = realloc(override_defines, - override_capacity * sizeof(intmax_t)); - } + // allocate space + test_override_t *override = mappend( + (void**)&overrides, + sizeof(test_override_t), + &override_count, + &override_capacity); // parse into string key/intmax_t value, cannibalizing the // arg in the process @@ -1439,13 +1713,12 @@ int main(int argc, char **argv) { if (!sep) { goto invalid_define; } - override_defines[override_count-1] - = strtoumax(sep+1, &parsed, 0); + override->define = strtoumax(sep+1, &parsed, 0); if (parsed == sep+1) { goto invalid_define; } - override_names[override_count-1] = optarg; + override->name = optarg; *sep = '\0'; break; @@ -1453,9 +1726,136 @@ invalid_define: fprintf(stderr, "error: invalid define: %s\n", optarg); exit(-1); } - case OPT_GEOMETRY: - test_geometry = optarg; + case OPT_GEOMETRY: { + // reset our geometry scenarios + if (test_geometry_capacity > 0) { + free((test_geometry_t*)test_geometries); + } + test_geometries = NULL; + test_geometry_count = 0; + test_geometry_capacity = 0; + + // parse the comma separated list of disk geometries + while (*optarg) { + // allocate space + test_geometry_t *geometry = mappend( + (void**)&test_geometries, + sizeof(test_geometry_t), + &test_geometry_count, + &test_geometry_capacity); + + // parse the disk geometry + optarg += strspn(optarg, " "); + + // named disk geometry + size_t len = strcspn(optarg, " ,"); + for (size_t i = 0; builtin_geometries[i].long_name; i++) { + if ((len == 1 + && *optarg == builtin_geometries[i].short_name) + || (len == strlen( + builtin_geometries[i].long_name) + && memcmp(optarg, + builtin_geometries[i].long_name, + len) == 0)) { + *geometry = builtin_geometries[i]; + optarg += len; + goto geometry_next; + } + } + + // comma-separated read/prog/erase/count + if (*optarg == '{') { + lfs_size_t sizes[4]; + size_t count = 0; + + char *s = optarg + 1; + while (count < 4) { + char *parsed = NULL; + sizes[count] = strtoumax(s, &parsed, 0); + count += 1; + + s = parsed + strspn(parsed, " "); + if (*s == ',') { + s += 1; + continue; + } else if (*s == '}') { + s += 1; + break; + } else { + goto geometry_unknown; + } + } + + // allow implicit r=p and p=e for common geometries + geometry->read_size = sizes[0]; + geometry->prog_size + = count >= 3 ? sizes[1] + : sizes[0]; + geometry->block_size + = count >= 3 ? sizes[2] + : count >= 2 ? sizes[1] + : sizes[0]; + // if no block_count, figure out 1 MiB total size + geometry->block_count + = count >= 4 ? sizes[3] + : (1024*1024) / geometry->block_size; + optarg = s; + goto geometry_next; + } + + // leb16-encoded read/prog/erase/count + if (*optarg == '#') { + lfs_size_t sizes[4]; + size_t count = 0; + + char *s = optarg + 1; + while (true) { + char *parsed = NULL; + uintmax_t x = leb16_parse(s, &parsed); + if (parsed == s || count >= 4) { + break; + } + + sizes[count] = x; + count += 1; + s = parsed; + } + + // allow implicit r=p and p=e for common geometries + geometry->read_size = sizes[0]; + geometry->prog_size + = count >= 3 ? sizes[1] + : sizes[0]; + geometry->block_size + = count >= 3 ? sizes[2] + : count >= 2 ? sizes[1] + : sizes[0]; + // if no block_count, figure out 1 MiB total size + geometry->block_count + = count >= 4 ? sizes[3] + : (1024*1024) / geometry->block_size; + optarg = s; + goto geometry_next; + } + +geometry_unknown: + // unknown scenario? + fprintf(stderr, "error: unknown disk geometry: %s\n", + optarg); + exit(-1); + +geometry_next: + optarg += strspn(optarg, " "); + if (*optarg == ',') { + optarg += 1; + } else if (*optarg == '\0') { + break; + } else { + goto geometry_unknown; + } + } break; + } case OPT_POWERLOSS: { // reset our powerloss scenarios if (test_powerloss_capacity > 0) { @@ -1468,17 +1868,11 @@ invalid_define: // parse the comma separated list of power-loss scenarios while (*optarg) { // allocate space - test_powerloss_count += 1; - if (test_powerloss_count > test_powerloss_capacity) { - test_powerloss_capacity - = (2*test_powerloss_capacity > 4) - ? 2*test_powerloss_capacity - : 4; - test_powerlosses = realloc( - (test_powerloss_t*)test_powerlosses, - test_powerloss_capacity - * sizeof(test_powerloss_t)); - } + test_powerloss_t *powerloss = mappend( + (void**)&test_powerlosses, + sizeof(test_powerloss_t), + &test_powerloss_count, + &test_powerloss_capacity); // parse the power-loss scenario optarg += strspn(optarg, " "); @@ -1493,9 +1887,7 @@ invalid_define: && memcmp(optarg, builtin_powerlosses[i].long_name, len) == 0)) { - ((test_powerloss_t*)test_powerlosses)[ - test_powerloss_count-1] - = builtin_powerlosses[i]; + *powerloss = builtin_powerlosses[i]; optarg += len; goto powerloss_next; } @@ -1503,27 +1895,19 @@ invalid_define: // comma-separated permutation if (*optarg == '{') { - // how many cycles? - size_t count = 1; - for (size_t i = 0; optarg[i]; i++) { - if (optarg[i] == ',') { - count += 1; - } - } + lfs_testbd_powercycles_t *cycles = NULL; + size_t cycle_count = 0; + size_t cycle_capacity = 0; - // parse cycles - lfs_testbd_powercycles_t *cycles = malloc( - count * sizeof(lfs_testbd_powercycles_t)); - size_t i = 0; char *s = optarg + 1; while (true) { char *parsed = NULL; - cycles[i] = strtoumax(s, &parsed, 0); - if (parsed == s) { - count -= 1; - i -= 1; - } - i += 1; + *(lfs_testbd_powercycles_t*)mappend( + (void**)&cycles, + sizeof(lfs_testbd_powercycles_t), + &cycle_count, + &cycle_capacity) + = strtoumax(s, &parsed, 0); s = parsed + strspn(parsed, " "); if (*s == ',') { @@ -1537,11 +1921,10 @@ invalid_define: } } - ((test_powerloss_t*)test_powerlosses)[ - test_powerloss_count-1] = (test_powerloss_t){ + *powerloss = (test_powerloss_t){ .run = run_powerloss_cycles, .cycles = cycles, - .cycle_count = count, + .cycle_count = cycle_count, }; optarg = s; goto powerloss_next; @@ -1549,20 +1932,32 @@ invalid_define: // leb16-encoded permutation if (*optarg == '#') { - lfs_testbd_powercycles_t *cycles; - char *parsed = NULL; - size_t count = leb16_parse(optarg+1, &parsed, &cycles); - if (parsed == optarg+1) { - goto powerloss_unknown; + lfs_testbd_powercycles_t *cycles = NULL; + size_t cycle_count = 0; + size_t cycle_capacity = 0; + + char *s = optarg + 1; + while (true) { + char *parsed = NULL; + uintmax_t x = leb16_parse(s, &parsed); + if (parsed == s) { + break; + } + + *(lfs_testbd_powercycles_t*)mappend( + (void**)&cycles, + sizeof(lfs_testbd_powercycles_t), + &cycle_count, + &cycle_capacity) = x; + s = parsed; } - ((test_powerloss_t*)test_powerlosses)[ - test_powerloss_count-1] = (test_powerloss_t){ + *powerloss = (test_powerloss_t){ .run = run_powerloss_cycles, .cycles = cycles, - .cycle_count = count, + .cycle_count = cycle_count, }; - optarg = (char*)parsed; + optarg = s; goto powerloss_next; } @@ -1573,8 +1968,7 @@ invalid_define: if (parsed == optarg) { goto powerloss_unknown; } - ((test_powerloss_t*)test_powerlosses)[ - test_powerloss_count-1] = (test_powerloss_t){ + *powerloss = (test_powerloss_t){ .run = run_powerloss_exhaustive, .cycles = NULL, .cycle_count = count, @@ -1585,15 +1979,18 @@ invalid_define: powerloss_unknown: // unknown scenario? - fprintf(stderr, "error: " - "unknown power-loss scenario: %s\n", + fprintf(stderr, "error: unknown power-loss scenario: %s\n", optarg); exit(-1); powerloss_next: - optarg += strcspn(optarg, ","); + optarg += strspn(optarg, " "); if (*optarg == ',') { optarg += 1; + } else if (*optarg == '\0') { + break; + } else { + goto powerloss_unknown; } } break; @@ -1684,6 +2081,7 @@ getopt_done: ; char *suite = argv[optind]; char *case_ = strchr(suite, '#'); size_t perm = -1; + test_geometry_t *geometry = NULL; lfs_testbd_powercycles_t *cycles = NULL; size_t cycle_count = 0; @@ -1697,19 +2095,69 @@ getopt_done: ; *perm_ = '\0'; perm_ += 1; - // parse power cycles - char *cycles_ = strchr(perm_, '#'); - if (cycles_) { - *cycles_ = '\0'; - cycles_ += 1; + // parse geometry + char *geometry_ = strchr(perm_, '#'); + if (geometry_) { + *geometry_ = '\0'; + geometry_ += 1; - char *parsed = NULL; - cycle_count = leb16_parse(cycles_, &parsed, &cycles); - if (parsed == cycles_) { - fprintf(stderr, "error: " - "could not parse test cycles: %s\n", cycles_); - exit(-1); + // parse power cycles + char *cycles_ = strchr(geometry_, '#'); + if (cycles_) { + *cycles_ = '\0'; + cycles_ += 1; + + size_t cycle_capacity = 0; + while (*cycles_ != '\0') { + char *parsed = NULL; + *(lfs_testbd_powercycles_t*)mappend( + (void**)&cycles, + sizeof(lfs_testbd_powercycles_t), + &cycle_count, + &cycle_capacity) + = leb16_parse(cycles_, &parsed); + if (parsed == cycles_) { + fprintf(stderr, "error: " + "could not parse test cycles: %s\n", + cycles_); + exit(-1); + } + cycles_ = parsed; + } } + + geometry = malloc(sizeof(test_geometry_t)); + lfs_size_t sizes[4]; + size_t count = 0; + + while (*geometry_ != '\0') { + char *parsed = NULL; + uintmax_t x = leb16_parse(geometry_, &parsed); + if (parsed == geometry_ || count >= 4) { + fprintf(stderr, "error: " + "count not parse test geometry: %s\n", + geometry_); + exit(-1); + } + + sizes[count] = x; + count += 1; + geometry_ = parsed; + } + + // allow implicit r=p and p=e for common geometries + geometry->read_size = sizes[0]; + geometry->prog_size + = count >= 3 ? sizes[1] + : sizes[0]; + geometry->block_size + = count >= 3 ? sizes[2] + : count >= 2 ? sizes[1] + : sizes[0]; + // if no block_count, figure out 1 MiB total size + geometry->block_count + = count >= 4 ? sizes[3] + : (1024*1024) / geometry->block_size; } char *parsed = NULL; @@ -1734,32 +2182,33 @@ getopt_done: ; } // append to identifier list - test_id_count += 1; - if (test_id_count > test_id_capacity) { - test_id_capacity = (2*test_id_capacity > 4) - ? 2*test_id_capacity - : 4; - test_ids = realloc((test_id_t*)test_ids, - test_id_capacity * sizeof(test_id_t)); - } - ((test_id_t*)test_ids)[test_id_count-1] = (test_id_t){ + *(test_id_t*)mappend( + (void**)&test_ids, + sizeof(test_id_t), + &test_id_count, + &test_id_capacity) = (test_id_t){ .suite = suite, .case_ = case_, .perm = perm, + .geometry = geometry, .cycles = cycles, .cycle_count = cycle_count, }; } // register overrides - test_define_overrides(override_names, override_defines, override_count); + test_define_overrides(overrides, override_count); // do the thing op(); // cleanup (need to be done for valgrind testing) - free(override_names); - free(override_defines); + test_define_cleanup(); + free(overrides); + + if (test_geometry_capacity) { + free((test_geometry_t*)test_geometries); + } if (test_powerloss_capacity) { for (size_t i = 0; i < test_powerloss_count; i++) { free((lfs_testbd_powercycles_t*)test_powerlosses[i].cycles); @@ -1768,6 +2217,7 @@ getopt_done: ; } if (test_id_capacity) { for (size_t i = 0; i < test_id_count; i++) { + free((test_geometry_t*)test_ids[i].geometry); free((lfs_testbd_powercycles_t*)test_ids[i].cycles); } free((test_id_t*)test_ids); diff --git a/runners/test_runner.h b/runners/test_runner.h index 4459a5c9..810c7df2 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -38,7 +38,7 @@ struct test_case { test_flags_t flags; size_t permutations; - intmax_t (*const *const *defines)(void); + intmax_t (*const *const *defines)(size_t); bool (*filter)(void); void (*run)(struct lfs_config *cfg); @@ -59,60 +59,50 @@ struct test_suite { // access generated test defines -intmax_t test_predefine(size_t define); +//intmax_t test_predefine(size_t define); intmax_t test_define(size_t define); // a few preconfigured defines that control how tests run -#define READ_SIZE test_predefine(0) -#define PROG_SIZE test_predefine(1) -#define BLOCK_SIZE test_predefine(2) -#define BLOCK_COUNT test_predefine(3) -#define CACHE_SIZE test_predefine(4) -#define LOOKAHEAD_SIZE test_predefine(5) -#define BLOCK_CYCLES test_predefine(6) -#define ERASE_VALUE test_predefine(7) -#define ERASE_CYCLES test_predefine(8) -#define BADBLOCK_BEHAVIOR test_predefine(9) -#define POWERLOSS_BEHAVIOR test_predefine(10) + +#define READ_SIZE_i 0 +#define PROG_SIZE_i 1 +#define BLOCK_SIZE_i 2 +#define BLOCK_COUNT_i 3 +#define CACHE_SIZE_i 4 +#define LOOKAHEAD_SIZE_i 5 +#define BLOCK_CYCLES_i 6 +#define ERASE_VALUE_i 7 +#define ERASE_CYCLES_i 8 +#define BADBLOCK_BEHAVIOR_i 9 +#define POWERLOSS_BEHAVIOR_i 10 -#define TEST_PREDEFINE_NAMES { \ - "READ_SIZE", \ - "PROG_SIZE", \ - "BLOCK_SIZE", \ - "BLOCK_COUNT", \ - "CACHE_SIZE", \ - "LOOKAHEAD_SIZE", \ - "BLOCK_CYCLES", \ - "ERASE_VALUE", \ - "ERASE_CYCLES", \ - "BADBLOCK_BEHAVIOR", \ - "POWERLOSS_BEHAVIOR", \ -} -#define TEST_PREDEFINE_COUNT 11 +#define READ_SIZE test_define(READ_SIZE_i) +#define PROG_SIZE test_define(PROG_SIZE_i) +#define BLOCK_SIZE test_define(BLOCK_SIZE_i) +#define BLOCK_COUNT test_define(BLOCK_COUNT_i) +#define CACHE_SIZE test_define(CACHE_SIZE_i) +#define LOOKAHEAD_SIZE test_define(LOOKAHEAD_SIZE_i) +#define BLOCK_CYCLES test_define(BLOCK_CYCLES_i) +#define ERASE_VALUE test_define(ERASE_VALUE_i) +#define ERASE_CYCLES test_define(ERASE_CYCLES_i) +#define BADBLOCK_BEHAVIOR test_define(BADBLOCK_BEHAVIOR_i) +#define POWERLOSS_BEHAVIOR test_define(POWERLOSS_BEHAVIOR_i) +#define TEST_IMPLICIT_DEFINES \ + TEST_DEFINE(READ_SIZE, test_geometry->read_size) \ + TEST_DEFINE(PROG_SIZE, test_geometry->prog_size) \ + TEST_DEFINE(BLOCK_SIZE, test_geometry->block_size) \ + TEST_DEFINE(BLOCK_COUNT, test_geometry->block_count) \ + TEST_DEFINE(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ + TEST_DEFINE(LOOKAHEAD_SIZE, 16) \ + TEST_DEFINE(BLOCK_CYCLES, -1) \ + TEST_DEFINE(ERASE_VALUE, 0xff) \ + TEST_DEFINE(ERASE_CYCLES, 0) \ + TEST_DEFINE(BADBLOCK_BEHAVIOR, LFS_TESTBD_BADBLOCK_PROGERROR) \ + TEST_DEFINE(POWERLOSS_BEHAVIOR, LFS_TESTBD_POWERLOSS_NOOP) -// default predefines -#define TEST_DEFAULTS { \ - /* LOOKAHEAD_SIZE */ 16, \ - /* BLOCK_CYCLES */ -1, \ - /* ERASE_VALUE */ 0xff, \ - /* ERASE_CYCLES */ 0, \ - /* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \ - /* POWERLOSS_BEHAVIOR */ LFS_TESTBD_POWERLOSS_NOOP, \ -} -#define TEST_DEFAULT_DEFINE_COUNT 5 - -// test geometries -#define TEST_GEOMETRIES { \ - /*geometry, read, write, erase, count, cache */ \ - {"test", { 16, 16, 512, (1024*1024)/512, 64}}, \ - {"eeprom", { 1, 1, 512, (1024*1024)/512, 64}}, \ - {"emmc", { 512, 512, 512, (1024*1024)/512, 512}}, \ - {"nor", { 1, 1, 4096, (1024*1024)/4096, 64}}, \ - {"nand", {4096, 4096, 32*1024, (1024*1024)/(32*1024), 4096}}, \ -} -#define TEST_GEOMETRY_COUNT 5 -#define TEST_GEOMETRY_DEFINE_COUNT 5 +#define TEST_GEOMETRY_DEFINE_COUNT 4 +#define TEST_IMPLICIT_DEFINE_COUNT 11 #endif diff --git a/scripts/test.py b/scripts/test.py index c7c6b0f7..d224f9e9 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -252,6 +252,7 @@ def compile(**args): # include test_runner.h in every generated file f.writeln("#include \"%s\"" % HEADER_PATH) + f.writeln() # write out generated functions, this can end up in different # files depending on the "in" attribute @@ -270,20 +271,22 @@ def compile(**args): name = ('__test__%s__%s__%s__%d' % (suite.name, case.name, k, i)) define_cbs[v] = name - f.writeln('intmax_t %s(void) {' % name) + f.writeln('intmax_t %s(' + '__attribute__((unused)) ' + 'size_t define) {' % name) f.writeln(4*' '+'return %s;' % v) f.writeln('}') f.writeln() f.writeln('intmax_t (*const *const ' - '__test__%s__%s__defines[])(void) = {' + '__test__%s__%s__defines[])(size_t) = {' % (suite.name, case.name)) for defines in case.permutations: - f.writeln(4*' '+'(intmax_t (*const[])(void)){') - for define in sorted(suite.defines): - f.writeln(8*' '+'%s,' % ( - define_cbs[defines[define]] - if define in defines - else 'NULL')) + f.writeln(4*' '+'(intmax_t (*const[' + 'TEST_IMPLICIT_DEFINE_COUNT+%d])(size_t)){' % ( + len(suite.defines))) + for k, v in sorted(defines.items()): + f.writeln(8*' '+'[%-24s] = %s,' % ( + k+'_i', define_cbs[v])) f.writeln(4*' '+'},') f.writeln('};') f.writeln() @@ -328,8 +331,10 @@ def compile(**args): if suite.defines: for i, define in enumerate(sorted(suite.defines)): f.writeln('#ifndef %s' % define) - f.writeln('#define %-24s test_define(%d)' - % (define, i)) + f.writeln('#define %-24s ' + 'TEST_IMPLICIT_DEFINE_COUNT+%d' % (define+'_i', i)) + f.writeln('#define %-24s ' + 'test_define(%s)' % (define, define+'_i')) f.writeln('#endif') f.writeln() @@ -340,7 +345,7 @@ def compile(**args): else: if case.defines: f.writeln('extern intmax_t (*const *const ' - '__test__%s__%s__defines[])(void);' + '__test__%s__%s__defines[])(size_t);' % (suite.name, case.name)) if suite.if_ is not None or case.if_ is not None: f.writeln('extern bool __test__%s__%s__filter(' @@ -364,11 +369,14 @@ def compile(**args): or 0)) if suite.defines: # create suite define names - f.writeln(4*' '+'.define_names = (const char *const[]){') + f.writeln(4*' '+'.define_names = (const char *const[' + 'TEST_IMPLICIT_DEFINE_COUNT+%d]){' % ( + len(suite.defines))) for k in sorted(suite.defines): - f.writeln(8*' '+'"%s",' % k) + f.writeln(8*' '+'[%-24s] = "%s",' % (k+'_i', k)) f.writeln(4*' '+'},') - f.writeln(4*' '+'.define_count = %d,' % len(suite.defines)) + f.writeln(4*' '+'.define_count = ' + 'TEST_IMPLICIT_DEFINE_COUNT+%d,' % len(suite.defines)) f.writeln(4*' '+'.cases = (const struct test_case[]){') for case in suite.cases: # create case structs @@ -415,10 +423,15 @@ def compile(**args): for i, define in enumerate( sorted(suite.defines)): f.writeln('#ifndef %s' % define) - f.writeln('#define %-24s test_define(%d)' - % (define, i)) - f.writeln('#define __TEST__%s__NEEDS_UNDEF' - % define) + f.writeln('#define %-24s ' + 'TEST_IMPLICIT_DEFINE_COUNT+%d' % ( + define+'_i', i)) + f.writeln('#define %-24s ' + 'test_define(%s)' % ( + define, define+'_i')) + f.writeln('#define ' + '__TEST__%s__NEEDS_UNDEF' % ( + define)) f.writeln('#endif') f.writeln() @@ -431,6 +444,7 @@ def compile(**args): f.writeln('#undef __TEST__%s__NEEDS_UNDEF' % define) f.writeln('#undef %s' % define) + f.writeln('#undef %s' % (define+'_i')) f.writeln('#endif') f.writeln() @@ -469,9 +483,10 @@ def list_(**args): if args.get('summary'): cmd.append('--summary') if args.get('list_suites'): cmd.append('--list-suites') if args.get('list_cases'): cmd.append('--list-cases') - if args.get('list_paths'): cmd.append('--list-paths') + if args.get('list_suite_paths'): cmd.append('--list-suite-paths') + if args.get('list_case_paths'): cmd.append('--list-case-paths') if args.get('list_defines'): cmd.append('--list-defines') - if args.get('list_defaults'): cmd.append('--list-defaults') + if args.get('list_implicit'): cmd.append('--list-implicit') if args.get('list_geometries'): cmd.append('--list-geometries') if args.get('list_powerlosses'): cmd.append('--list-powerlosses') @@ -503,10 +518,11 @@ def find_cases(runner_, **args): m = pattern.match(line) if m: filtered = int(m.group('filtered')) + perms = int(m.group('perms')) expected_suite_perms[m.group('suite')] += filtered expected_case_perms[m.group('id')] += filtered expected_perms += filtered - total_perms += int(m.group('perms')) + total_perms += perms proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -520,9 +536,9 @@ def find_cases(runner_, **args): expected_perms, total_perms) -def find_paths(runner_, **args): +def find_path(runner_, id, **args): # query from runner - cmd = runner_ + ['--list-paths'] + cmd = runner_ + ['--list-case-paths', id] if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, @@ -531,14 +547,17 @@ def find_paths(runner_, **args): universal_newlines=True, errors='replace', close_fds=False) - paths = co.OrderedDict() + path = None pattern = re.compile( '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' '(?P[^:]+):(?P\d+)') - for line in proc.stdout: + # skip the first line + for line in it.islice(proc.stdout, 1, None): m = pattern.match(line) - if m: - paths[m.group('id')] = (m.group('path'), int(m.group('lineno'))) + if m and path is None: + path_ = m.group('path') + lineno = int(m.group('lineno')) + path = (path_, lineno) proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -546,11 +565,36 @@ def find_paths(runner_, **args): sys.stdout.write(line) sys.exit(-1) - return paths + return path -def find_defines(runner_, **args): - # query from runner - cmd = runner_ + ['--list-defines'] +def find_defines(runner_, id, **args): + # query implicit defines from runner + cmd = runner_ + ['--list-implicit'] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + implicit_defines = co.OrderedDict() + pattern = re.compile('^(?P\w+)=(?P.+)') + for line in proc.stdout: + m = pattern.match(line) + if m: + define = m.group('define') + values = m.group('values').split(',') + implicit_defines[define] = set(values) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + # query case defines from runner + cmd = runner_ + ['--list-defines', id] if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) proc = sp.Popen(cmd, @@ -560,14 +604,15 @@ def find_defines(runner_, **args): errors='replace', close_fds=False) defines = co.OrderedDict() - pattern = re.compile( - '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' - '(?P(?:\w+=\w+\s*)+)') + pattern = re.compile('^(?P\w+)=(?P.+)') for line in proc.stdout: m = pattern.match(line) if m: - defines[m.group('id')] = {k: v - for k, v in re.findall('(\w+)=(\w+)', m.group('defines'))} + define = m.group('define') + value = m.group('value') + if (define not in implicit_defines + or value not in implicit_defines[define]): + defines[define] = value proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -887,16 +932,16 @@ def run(**args): print() # print each failure - if failures: - # get some extra info from runner - runner_paths = find_paths(runner_, **args) - runner_defines = find_defines(runner_, **args) - for failure in failures: - # show summary of failure - path, lineno = runner_paths[testcase(failure.id)] - defines = runner_defines.get(failure.id, {}) + assert failure.id is not None, '%s broken? %r' % ( + ' '.join(shlex.quote(c) for c in runner_), + failure) + # get some extra info from runner + path, lineno = find_path(runner_, failure.id, **args) + defines = find_defines(runner_, failure.id, **args) + + # show summary of failure print('%s%s:%d:%sfailure:%s %s%s failed' % ( '\x1b[01m' if color(**args) else '', path, lineno, @@ -969,9 +1014,10 @@ def main(**args): elif (args.get('summary') or args.get('list_suites') or args.get('list_cases') - or args.get('list_paths') + or args.get('list_suite_paths') + or args.get('list_case_paths') or args.get('list_defines') - or args.get('list_defaults') + or args.get('list_implicit') or args.get('list_geometries') or args.get('list_powerlosses')): list_(**args) @@ -1005,20 +1051,23 @@ if __name__ == "__main__": help="List test suites.") test_parser.add_argument('-L', '--list-cases', action='store_true', help="List test cases.") - test_parser.add_argument('--list-paths', action='store_true', - help="List the path for each test case.") + test_parser.add_argument('--list-suite-paths', action='store_true', + help="List the path for each test suite.") + test_parser.add_argument('--list-case-paths', action='store_true', + help="List the path and line number for each test case.") test_parser.add_argument('--list-defines', action='store_true', - help="List the defines for each test permutation.") - test_parser.add_argument('--list-defaults', action='store_true', - help="List the default defines in this test-runner.") + help="List all defines in this test-runner.") + test_parser.add_argument('--list-implicit', action='store_true', + help="List implicit defines in this test-runner.") test_parser.add_argument('--list-geometries', action='store_true', - help="List the disk geometries used for testing.") + help="List the available disk geometries.") test_parser.add_argument('--list-powerlosses', action='store_true', help="List the available power-loss scenarios.") test_parser.add_argument('-D', '--define', action='append', help="Override a test define.") - test_parser.add_argument('-G', '--geometry', - help="Filter by geometry.") + test_parser.add_argument('-g', '--geometry', + help="Comma-separated list of disk geometries to test. \ + Defaults to d,e,E,n,N.") test_parser.add_argument('-p', '--powerloss', help="Comma-separated list of power-loss scenarios to test. \ Defaults to 0,l.")