diff --git a/.gitignore b/.gitignore index 6bee32df..abd7eb8a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ test.c tests/*.toml.* scripts/__pycache__ .gdb_history +runners/test_runner diff --git a/Makefile b/Makefile index 04fe8bdc..513187dc 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,9 @@ override CFLAGS += -I. override CFLAGS += -std=c99 -Wall -pedantic override CFLAGS += -Wextra -Wshadow -Wjump-misses-init -Wundef +override TESTFLAGS_ += -b +# forward -j flag +override TESTFLAGS_ += $(filter -j%,$(MAKEFLAGS)) ifdef VERBOSE override TESTFLAGS += -v override CALLSFLAGS += -v @@ -88,8 +91,6 @@ endif ifneq ($(OBJDUMP),objdump) override STRUCTSFLAGS += --objdump-tool="$(OBJDUMP)" endif -# forward -j flag -override TESTFLAGS_ += $(filter -j%,$(MAKEFLAGS)) # commands diff --git a/runners/test_runner.c b/runners/test_runner.c index 1406574e..1c9b2550 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -6,106 +6,151 @@ #include #include -// disk geometries + +// test geometries struct test_geometry { const char *name; - const test_define_t *defines; + test_define_t defines[TEST_GEOMETRY_DEFINE_COUNT]; }; -// Note this includes the default configuration for test pre-defines -#define TEST_GEOMETRY(name, read, prog, erase, count) \ - {name, (const test_define_t[]){ \ - /* READ_SIZE */ read, \ - /* PROG_SIZE */ prog, \ - /* BLOCK_SIZE */ erase, \ - /* BLOCK_COUNT */ count, \ - /* BLOCK_CYCLES */ -1, \ - /* CACHE_SIZE */ (64 % (prog) == 0) ? 64 : (prog), \ - /* LOOKAHEAD_SIZE */ 16, \ - /* ERASE_VALUE */ 0xff, \ - /* ERASE_CYCLES */ 0, \ - /* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \ - }} - -const struct test_geometry test_geometries[] = { - // Made up geometry that works well for testing - TEST_GEOMETRY("test", 16, 16, 512, (1024*1024)/512), - // EEPROM/NVRAM - TEST_GEOMETRY("eeprom", 1, 1, 512, (1024*1024)/512), - // SD/eMMC - TEST_GEOMETRY("emmc", 512, 512, 512, (1024*1024)/512), - // NOR flash - TEST_GEOMETRY("nor", 1, 1, 4096, (1024*1024)/4096), - // NAND flash - TEST_GEOMETRY("nand", 4096, 4096, 32*1024, (1024*1024)/(32*1024)), -}; - -const size_t test_geometry_count = ( - sizeof(test_geometries) / sizeof(test_geometries[0])); - +const struct test_geometry test_geometries[TEST_GEOMETRY_COUNT] + = TEST_GEOMETRIES; // test define lookup and management -const test_define_t *test_defines[3] = {NULL}; -const uint8_t *test_define_maps[2] = {NULL}; +#define TEST_DEFINE_LAYERS 4 +const test_define_t *test_defines[TEST_DEFINE_LAYERS] = { + NULL, + NULL, + NULL, + (const test_define_t[TEST_DEFAULT_COUNT])TEST_DEFAULTS, +}; + +const uint8_t *test_predefine_maps[TEST_DEFINE_LAYERS] = { + NULL, + NULL, + (const uint8_t[TEST_PREDEFINE_COUNT])TEST_GEOMETRY_DEFINE_MAP, + (const uint8_t[TEST_PREDEFINE_COUNT])TEST_DEFAULT_MAP, +}; + +const uint8_t *test_define_maps[TEST_DEFINE_LAYERS] = { + NULL, + NULL, + NULL, + NULL, +}; + +uint8_t test_override_predefine_map[TEST_PREDEFINE_COUNT]; +uint8_t test_override_define_map[256]; +uint8_t test_case_predefine_map[TEST_PREDEFINE_COUNT]; + +const char *const *test_override_names; +size_t test_override_count; + +const char *const test_predefine_names[TEST_PREDEFINE_COUNT] + = TEST_PREDEFINE_NAMES; + +const char *const *test_define_names; +size_t test_define_count; + + +test_define_t test_predefine(size_t define) { + for (int i = 0; i < TEST_DEFINE_LAYERS; i++) { + if (test_defines[i] + && test_predefine_maps[i] + && test_predefine_maps[i][define] != 0xff) { + return test_defines[i][test_predefine_maps[i][define]]; + } + } + + fprintf(stderr, "error: undefined predefine %s\n", + test_predefine_names[define]); + assert(false); + exit(-1); +} test_define_t test_define(size_t define) { - if (test_define_maps[0] && test_define_maps[0][define] != 0xff) { - return test_defines[0][test_define_maps[0][define]]; - } else if (test_define_maps[1] && test_define_maps[1][define] != 0xff) { - return test_defines[1][test_define_maps[1][define]]; - } else { - return test_defines[2][define]; + for (int i = 0; i < TEST_DEFINE_LAYERS; i++) { + if (test_defines[i] + && test_define_maps[i] + && test_define_maps[i][define] != 0xff) { + return test_defines[i][test_define_maps[i][define]]; + } } + + fprintf(stderr, "error: undefined define %s\n", + test_define_names[define]); + assert(false); + exit(-1); } static void test_define_geometry(const struct test_geometry *geometry) { - if (geometry) { - test_defines[2] = geometry->defines; - } else { - test_defines[2] = NULL; - } -} - -static void test_define_case(const struct test_case *case_, size_t perm) { - if (case_ && case_->defines) { - test_defines[1] = case_->defines[perm]; - test_define_maps[1] = case_->define_map; - } else { - test_defines[1] = NULL; - test_define_maps[1] = NULL; - } + test_defines[2] = geometry->defines; } static void test_define_overrides( - const struct test_suite *suite, const char *const *override_names, const test_define_t *override_defines, size_t override_count) { - if (override_names && override_defines && override_count > 0) { - uint8_t *define_map = malloc(suite->define_count * sizeof(uint8_t)); - memset(define_map, 0xff, suite->define_count * sizeof(bool)); + test_defines[0] = override_defines; + test_override_names = override_names; + test_override_count = override_count; - // lookup each override in the suite defines, they may have a - // different index in each suite - for (size_t i = 0; i < override_count; i++) { - size_t j = 0; - for (; j < suite->define_count; j++) { - if (strcmp(override_names[i], suite->define_names[j]) == 0) { - break; - } - } - - if (j < suite->define_count) { - define_map[j] = i; + // map any predefines + memset(test_override_predefine_map, 0xff, TEST_PREDEFINE_COUNT); + for (size_t i = 0; i < override_count; i++) { + for (size_t j = 0; j < TEST_PREDEFINE_COUNT; j++) { + if (strcmp(override_names[i], test_predefine_names[j]) == 0) { + test_override_predefine_map[j] = i; } } + } + test_predefine_maps[0] = test_override_predefine_map; +} - test_defines[0] = override_defines; - test_define_maps[0] = define_map; +static void test_define_suite(const struct test_suite *suite) { + test_define_names = suite->define_names; + test_define_count = suite->define_count; + + // map any defines + memset(test_override_define_map, 0xff, suite->define_count); + for (size_t i = 0; i < test_override_count; i++) { + for (size_t j = 0; j < suite->define_count; j++) { + if (strcmp(test_override_names[i], suite->define_names[j]) == 0) { + test_override_define_map[j] = i; + } + } + } + test_define_maps[0] = test_override_define_map; +} + +static void test_define_case( + const struct test_suite *suite, + const struct test_case *case_) { + (void)suite; + // case_->define_map is already correct, but we need to do + // some fixup for the predefine map + test_define_maps[1] = case_->define_map; + + memset(test_case_predefine_map, 0xff, TEST_PREDEFINE_COUNT); + for (size_t i = 0; i < test_define_count; i++) { + for (size_t j = 0; j < TEST_PREDEFINE_COUNT; j++) { + if (strcmp(test_define_names[i], test_predefine_names[j]) == 0) { + test_case_predefine_map[j] = case_->define_map[i]; + } + } + } + test_predefine_maps[1] = test_case_predefine_map; +} + +static void test_define_perm( + const struct test_suite *suite, + const struct test_case *case_, + size_t perm) { + (void)suite; + if (case_->defines) { + test_defines[1] = case_->defines[perm]; } else { - test_defines[0] = NULL; - free((uint8_t *)test_define_maps[0]); - test_define_maps[0] = NULL; + test_defines[1] = NULL; } } @@ -120,11 +165,6 @@ static size_t test_skip = 0; static size_t test_count = -1; static size_t test_every = 1; -static const char **override_names = NULL; -static test_define_t *override_defines = NULL; -static size_t override_count = 0; -static size_t override_cap = 0; - static const char *test_persist = NULL; FILE *test_trace = NULL; @@ -140,7 +180,7 @@ static bool test_case_skip(const struct test_case *case_) { } static bool test_perm_skip(size_t perm) { - size_t geom_perm = perm % test_geometry_count; + size_t geom_perm = perm % TEST_GEOMETRY_COUNT; return (test_perm != (size_t)-1 && perm != test_perm) || (test_geometry && (strcmp( test_geometries[geom_perm].name, @@ -153,7 +193,8 @@ static bool test_step_skip(size_t step) { && (step-test_skip) % test_every == 0); } -static void test_case_sumpermutations( +static void test_case_permcount( + const struct test_suite *suite, const struct test_case *case_, size_t *perms, size_t *filtered) { @@ -161,7 +202,7 @@ static void test_case_sumpermutations( size_t filtered_ = 0; for (size_t perm = 0; - perm < test_geometry_count + perm < TEST_GEOMETRY_COUNT * case_->permutations; perm++) { if (test_perm_skip(perm)) { @@ -171,23 +212,18 @@ static void test_case_sumpermutations( perms_ += 1; // setup defines - size_t case_perm = perm / test_geometry_count; - size_t geom_perm = perm % test_geometry_count; + size_t case_perm = perm / TEST_GEOMETRY_COUNT; + size_t geom_perm = perm % TEST_GEOMETRY_COUNT; + test_define_perm(suite, case_, case_perm); test_define_geometry(&test_geometries[geom_perm]); - test_define_case(case_, case_perm); if (case_->filter) { if (!case_->filter(case_perm)) { - test_define_geometry(NULL); - test_define_case(NULL, 0); continue; } } filtered_ += 1; - - test_define_geometry(NULL); - test_define_case(NULL, 0); } *perms += perms_; @@ -208,21 +244,18 @@ static void summary(void) { continue; } - test_define_overrides( - test_suites[i], - override_names, override_defines, override_count); + test_define_suite(test_suites[i]); for (size_t j = 0; j < test_suites[i]->case_count; j++) { if (test_case_skip(test_suites[i]->cases[j])) { continue; } - test_case_sumpermutations(test_suites[i]->cases[j], + test_define_case(test_suites[i], test_suites[i]->cases[j]); + test_case_permcount(test_suites[i], test_suites[i]->cases[j], &perms, &filtered); } - test_define_overrides(NULL, NULL, NULL, 0); - cases += test_suites[i]->case_count; types |= test_suites[i]->types; } @@ -243,16 +276,13 @@ static void summary(void) { } static void list_suites(void) { - printf("%-36s %-12s %7s %7s %11s\n", - "id", "suite", "types", "cases", "perms"); + printf("%-36s %7s %7s %11s\n", "suite", "types", "cases", "perms"); for (size_t i = 0; i < test_suite_count; i++) { if (test_suite_skip(test_suites[i])) { continue; } - test_define_overrides( - test_suites[i], - override_names, override_defines, override_count); + test_define_suite(test_suites[i]); size_t perms = 0; size_t filtered = 0; @@ -261,12 +291,11 @@ static void list_suites(void) { continue; } - test_case_sumpermutations(test_suites[i]->cases[j], + test_define_case(test_suites[i], test_suites[i]->cases[j]); + test_case_permcount(test_suites[i], test_suites[i]->cases[j], &perms, &filtered); } - test_define_overrides(NULL, NULL, NULL, 0); - char perm_buf[64]; sprintf(perm_buf, "%zu/%zu", filtered, perms); char type_buf[64]; @@ -274,9 +303,8 @@ static void list_suites(void) { (test_suites[i]->types & TEST_NORMAL) ? "n" : "", (test_suites[i]->types & TEST_REENTRANT) ? "r" : "", (test_suites[i]->types & TEST_VALGRIND) ? "V" : ""); - printf("%-36s %-12s %7s %7zu %11s\n", + printf("%-36s %7s %7zu %11s\n", test_suites[i]->id, - test_suites[i]->name, type_buf, test_suites[i]->case_count, perm_buf); @@ -284,25 +312,24 @@ static void list_suites(void) { } static void list_cases(void) { - printf("%-36s %-12s %-12s %7s %11s\n", - "id", "suite", "case", "types", "perms"); + printf("%-36s %7s %11s\n", "case", "types", "perms"); for (size_t i = 0; i < test_suite_count; i++) { if (test_suite_skip(test_suites[i])) { continue; } - test_define_overrides( - test_suites[i], - override_names, override_defines, override_count); + test_define_suite(test_suites[i]); for (size_t j = 0; j < test_suites[i]->case_count; j++) { if (test_case_skip(test_suites[i]->cases[j])) { continue; } + test_define_case(test_suites[i], test_suites[i]->cases[j]); + size_t perms = 0; size_t filtered = 0; - test_case_sumpermutations(test_suites[i]->cases[j], + test_case_permcount(test_suites[i], test_suites[i]->cases[j], &perms, &filtered); test_types_t types = test_suites[i]->cases[j]->types; @@ -313,20 +340,15 @@ static void list_cases(void) { (types & TEST_NORMAL) ? "n" : "", (types & TEST_REENTRANT) ? "r" : "", (types & TEST_VALGRIND) ? "V" : ""); - printf("%-36s %-12s %-12s %7s %11s\n", + printf("%-36s %7s %11s\n", test_suites[i]->cases[j]->id, - test_suites[i]->name, - test_suites[i]->cases[j]->name, type_buf, perm_buf); } - - test_define_overrides(NULL, NULL, NULL, 0); } } static void list_paths(void) { - printf("%-36s %-36s\n", "id", "path"); for (size_t i = 0; i < test_suite_count; i++) { if (test_suite_skip(test_suites[i])) { continue; @@ -345,23 +367,22 @@ static void list_paths(void) { } static void list_defines(void) { - printf("%-36s %s\n", "id", "defines"); for (size_t i = 0; i < test_suite_count; i++) { if (test_suite_skip(test_suites[i])) { continue; } - test_define_overrides( - test_suites[i], - override_names, override_defines, override_count); + test_define_suite(test_suites[i]); for (size_t j = 0; j < test_suites[i]->case_count; j++) { if (test_case_skip(test_suites[i]->cases[j])) { continue; } + test_define_case(test_suites[i], test_suites[i]->cases[j]); + for (size_t perm = 0; - perm < test_geometry_count + perm < TEST_GEOMETRY_COUNT * test_suites[i]->cases[j]->permutations; perm++) { if (test_perm_skip(perm)) { @@ -369,39 +390,38 @@ static void list_defines(void) { } // setup defines - size_t case_perm = perm / test_geometry_count; - size_t geom_perm = perm % test_geometry_count; + size_t case_perm = perm / TEST_GEOMETRY_COUNT; + size_t geom_perm = perm % TEST_GEOMETRY_COUNT; + test_define_perm(test_suites[i], + test_suites[i]->cases[j], case_perm); test_define_geometry(&test_geometries[geom_perm]); - test_define_case(test_suites[i]->cases[j], case_perm); - // print each define + // print the case char id_buf[256]; sprintf(id_buf, "%s#%zu", test_suites[i]->cases[j]->id, perm); printf("%-36s ", id_buf); - for (size_t k = 0; k < test_suites[i]->define_count; k++) { - if (k >= TEST_PREDEFINE_COUNT && ( - !test_suites[i]->cases[j]->define_map - || test_suites[i]->cases[j]->define_map[k] - == 0xff)) { - continue; - } - printf("%s=%jd ", - test_suites[i]->define_names[k], - test_define(k)); + // special case for the current geometry + printf("GEOMETRY=%s ", test_geometries[geom_perm].name); + + // print each define + for (size_t k = 0; k < test_suites[i]->define_count; k++) { + if (test_suites[i]->cases[j]->define_map + && test_suites[i]->cases[j]->define_map[k] + != 0xff) { + printf("%s=%jd ", + test_suites[i]->define_names[k], + test_define(k)); + } } printf("\n"); } } - - test_define_overrides(NULL, NULL, NULL, 0); } } static void list_geometries(void) { - printf("%-36s %7s %7s %7s %7s %7s\n", - "name", "read", "prog", "erase", "count", "size"); - for (size_t i = 0; i < test_geometry_count; i++) { + for (size_t i = 0; i < TEST_GEOMETRY_COUNT; i++) { if (test_geometry && strcmp( test_geometries[i].name, test_geometry) != 0) { @@ -410,16 +430,33 @@ static void list_geometries(void) { test_define_geometry(&test_geometries[i]); - printf("%-36s %7ju %7ju %7ju %7ju %7ju\n", - test_geometries[i].name, - READ_SIZE, - PROG_SIZE, - BLOCK_SIZE, - BLOCK_COUNT, - BLOCK_SIZE*BLOCK_COUNT); + printf("%-36s ", test_geometries[i].name); + // print each define + for (size_t k = 0; k < TEST_PREDEFINE_COUNT; k++) { + if (test_predefine_maps[2][k] != 0xff) { + printf("%s=%jd ", + test_predefine_names[k], + test_predefine(k)); + } + } + printf("\n"); + } } +static void list_defaults(void) { + printf("%-36s ", "defaults"); + // print each define + for (size_t k = 0; k < TEST_PREDEFINE_COUNT; k++) { + if (test_predefine_maps[3][k] != 0xff) { + printf("%s=%jd ", + test_predefine_names[k], + test_predefine(k)); + } + } + printf("\n"); +} + static void run(void) { size_t step = 0; for (size_t i = 0; i < test_suite_count; i++) { @@ -427,23 +464,22 @@ static void run(void) { continue; } - test_define_overrides( - test_suites[i], - override_names, override_defines, override_count); + test_define_suite(test_suites[i]); for (size_t j = 0; j < test_suites[i]->case_count; j++) { if (test_case_skip(test_suites[i]->cases[j])) { continue; } + test_define_case(test_suites[i], test_suites[i]->cases[j]); + for (size_t perm = 0; - perm < test_geometry_count + perm < TEST_GEOMETRY_COUNT * test_suites[i]->cases[j]->permutations; perm++) { if (test_perm_skip(perm)) { continue; } - if (test_step_skip(step)) { step += 1; continue; @@ -451,10 +487,11 @@ static void run(void) { step += 1; // setup defines - size_t case_perm = perm / test_geometry_count; - size_t geom_perm = perm % test_geometry_count; + size_t case_perm = perm / TEST_GEOMETRY_COUNT; + size_t geom_perm = perm % TEST_GEOMETRY_COUNT; + test_define_perm(test_suites[i], + test_suites[i]->cases[j], case_perm); test_define_geometry(&test_geometries[geom_perm]); - test_define_case(test_suites[i]->cases[j], case_perm); // filter? if (test_suites[i]->cases[j]->filter) { @@ -462,8 +499,6 @@ static void run(void) { printf("skipped %s#%zu\n", test_suites[i]->cases[j]->id, perm); - test_define_geometry(NULL); - test_define_case(NULL, 0); continue; } } @@ -514,13 +549,8 @@ static void run(void) { "could not destroy block device: %d\n", err); exit(-1); } - - test_define_geometry(NULL); - test_define_case(NULL, 0); } } - - test_define_overrides(NULL, NULL, NULL, 0); } } @@ -536,14 +566,15 @@ enum opt_flags { OPT_LIST_PATHS = 1, OPT_LIST_DEFINES = 2, OPT_LIST_GEOMETRIES = 3, + OPT_LIST_DEFAULTS = 4, OPT_DEFINE = 'D', OPT_GEOMETRY = 'G', OPT_NORMAL = 'n', OPT_REENTRANT = 'r', OPT_VALGRIND = 'V', - OPT_SKIP = 4, - OPT_COUNT = 5, - OPT_EVERY = 6, + OPT_SKIP = 5, + OPT_COUNT = 6, + OPT_EVERY = 7, OPT_PERSIST = 'p', OPT_TRACE = 't', }; @@ -558,6 +589,7 @@ const struct option long_opts[] = { {"list-paths", no_argument, NULL, OPT_LIST_PATHS}, {"list-defines", no_argument, NULL, OPT_LIST_DEFINES}, {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES}, + {"list-defaults", no_argument, NULL, OPT_LIST_DEFAULTS}, {"define", required_argument, NULL, OPT_DEFINE}, {"geometry", required_argument, NULL, OPT_GEOMETRY}, {"normal", no_argument, NULL, OPT_NORMAL}, @@ -579,6 +611,7 @@ const char *const help_text[] = { "List the path for each test case.", "List the defines for each test permutation.", "List the disk geometries used for testing.", + "List the default defines in this test-runner.", "Override a test define.", "Filter by geometry.", "Filter for normal tests. Can be combined.", @@ -594,6 +627,11 @@ const char *const help_text[] = { int main(int argc, char **argv) { void (*op)(void) = run; + static const char **override_names = NULL; + static test_define_t *override_defines = NULL; + static size_t override_count = 0; + static size_t override_cap = 0; + // parse options while (true) { int c = getopt_long(argc, argv, short_opts, long_opts, NULL); @@ -673,8 +711,18 @@ int main(int argc, char **argv) { case OPT_LIST_GEOMETRIES: op = list_geometries; break; + case OPT_LIST_DEFAULTS: + op = list_defaults; + break; // configuration case OPT_DEFINE: { + // special case for -DGEOMETRY=, we treat this the same + // as --geometry= + if (strncmp(optarg, "GEOMETRY=", strlen("GEOMETRY=")) == 0) { + test_geometry = &optarg[strlen("GEOMETRY=")]; + break; + } + // realloc if necessary override_count += 1; if (override_count > override_cap) { @@ -816,6 +864,9 @@ getopt_done: ; test_suite = suite; } + // register overrides + test_define_overrides(override_names, override_defines, override_count); + // do the thing op(); diff --git a/runners/test_runner.h b/runners/test_runner.h index 92d8d312..c413a85a 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -41,27 +41,68 @@ struct test_suite { size_t case_count; }; -// TODO remove this indirection extern const struct test_suite *test_suites[]; extern const size_t test_suite_count; // access generated test defines +test_define_t test_predefine(size_t define); test_define_t test_define(size_t define); // a few preconfigured defines that control how tests run -#define READ_SIZE test_define(0) -#define PROG_SIZE test_define(1) -#define BLOCK_SIZE test_define(2) -#define BLOCK_COUNT test_define(3) -#define BLOCK_CYCLES test_define(4) -#define CACHE_SIZE test_define(5) -#define LOOKAHEAD_SIZE test_define(6) -#define ERASE_VALUE test_define(7) -#define ERASE_CYCLES test_define(8) -#define BADBLOCK_BEHAVIOR test_define(9) +#define READ_SIZE test_predefine(0) +#define PROG_SIZE test_predefine(1) +#define BLOCK_SIZE test_predefine(2) +#define BLOCK_COUNT test_predefine(3) +#define CACHE_SIZE test_predefine(4) +#define LOOKAHEAD_SIZE test_predefine(5) +#define BLOCK_CYCLES test_predefine(6) +#define ERASE_VALUE test_predefine(7) +#define ERASE_CYCLES test_predefine(8) +#define BADBLOCK_BEHAVIOR test_predefine(9) +#define TEST_PREDEFINE_NAMES { \ + "READ_SIZE", \ + "PROG_SIZE", \ + "BLOCK_SIZE", \ + "BLOCK_COUNT", \ + "CACHE_SIZE", \ + "LOOKAHEAD_SIZE", \ + "BLOCK_CYCLES", \ + "ERASE_VALUE", \ + "ERASE_CYCLES", \ + "BADBLOCK_BEHAVIOR", \ +} #define TEST_PREDEFINE_COUNT 10 +// default predefines +#define TEST_DEFAULTS { \ + /* LOOKAHEAD_SIZE */ 16, \ + /* BLOCK_CYCLES */ -1, \ + /* ERASE_VALUE */ 0xff, \ + /* ERASE_CYCLES */ 0, \ + /* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \ +} +#define TEST_DEFAULT_COUNT 5 +#define TEST_DEFAULT_MAP { \ + 0xff, 0xff, 0xff, 0xff, 0xff, 0, 1, 2, 3, 4 \ +} + +// test geometries +#define TEST_GEOMETRIES { \ + /*geometry, read, write, erase, count, cache */ \ + {"test", { 16, 16, 512, (1024*1024)/512, 64}}, \ + {"eeprom", { 1, 1, 512, (1024*1024)/512, 64}}, \ + {"emmc", { 512, 512, 512, (1024*1024)/512, 512}}, \ + {"nor", { 1, 1, 4096, (1024*1024)/4096, 64}}, \ + {"nand", {4096, 4096, 32*1024, (1024*1024)/(32*1024), 4096}}, \ +} +#define TEST_GEOMETRY_COUNT 5 +#define TEST_GEOMETRY_DEFINE_COUNT 5 +#define TEST_GEOMETRY_DEFINE_MAP { \ + 0, 1, 2, 3, 4, 0xff, 0xff, 0xff, 0xff, 0xff \ +} + + #endif diff --git a/scripts/test_.py b/scripts/test_.py index 8d6e34ec..2e5b9841 100755 --- a/scripts/test_.py +++ b/scripts/test_.py @@ -31,30 +31,21 @@ CASE_PROLOGUE = """ CASE_EPILOGUE = """ """ -TEST_PREDEFINES = [ - 'READ_SIZE', - 'PROG_SIZE', - 'BLOCK_SIZE', - 'BLOCK_COUNT', - 'BLOCK_CYCLES', - 'CACHE_SIZE', - 'LOOKAHEAD_SIZE', - 'ERASE_VALUE', - 'ERASE_CYCLES', - 'BADBLOCK_BEHAVIOR', -] - -# TODO -# def testpath(path): -# def testcase(path): -# def testperm(path): +def testpath(path): + path, *_ = path.split('#', 1) + return path def testsuite(path): - name = os.path.basename(path) - if name.endswith('.toml'): - name = name[:-len('.toml')] - return name + suite = testpath(path) + suite = os.path.basename(suite) + if suite.endswith('.toml'): + suite = suite[:-len('.toml')] + return suite + +def testcase(path): + _, case, *_ = path.split('#', 2) + return '%s#%s' % (testsuite(path), case) # TODO move this out in other files def openio(path, mode='r'): @@ -111,7 +102,7 @@ class TestSuite: # create a TestSuite object from a toml file def __init__(self, path, args={}): self.name = testsuite(path) - self.path = path + self.path = testpath(path) # load toml file and parse test cases with open(self.path) as f: @@ -125,9 +116,9 @@ class TestSuite: code_linenos = [] for i, line in enumerate(f): match = re.match( - '(?P\[\s*cases\s*\.\s*(?P\w+)\s*\])' + - '|(?Pif\s*=)' - '|(?Pcode\s*=)', + '(?P\[\s*cases\s*\.\s*(?P\w+)\s*\])' + '|' '(?Pif\s*=)' + '|' '(?Pcode\s*=)', line) if match and match.group('case'): case_linenos.append((i+1, match.group('name'))) @@ -187,8 +178,8 @@ class TestSuite: 'suite_valgrind': valgrind, **case})) - # combine pre-defines and per-case defines - self.defines = TEST_PREDEFINES + sorted( + # combine per-case defines + self.defines = sorted( set.union(*(set(case.defines) for case in self.cases))) # combine other per-case things @@ -225,216 +216,222 @@ def compile(**args): % args['test_paths']) sys.exit(-1) - # write out a test suite + # load our suite suite = TestSuite(paths[0]) - if 'output' in args: - with openio(args['output'], 'w') as f: - # redirect littlefs tracing - f.write('#define LFS_TRACE_(fmt, ...) do { \\\n') - f.write(8*' '+'extern FILE *test_trace; \\\n') - f.write(8*' '+'if (test_trace) { \\\n') - f.write(12*' '+'fprintf(test_trace, ' - '"%s:%d:trace: " fmt "%s\\n", \\\n') - f.write(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\\n') - f.write(8*' '+'} \\\n') - f.write(4*' '+'} while (0)\n') - f.write('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")\n') - f.write('#define LFS_TESTBD_TRACE(...) ' - 'LFS_TRACE_(__VA_ARGS__, "")\n') - f.write('\n') + else: + # load all suites + suites = [TestSuite(path) for path in paths] + suites.sort(key=lambda s: s.name) - f.write('%s\n' % SUITE_PROLOGUE.strip()) - f.write('\n') + # write generated test source + if 'output' in args: + with openio(args['output'], 'w') as f: + _write = f.write + def write(s): + f.lineno += s.count('\n') + _write(s) + def writeln(s=''): + f.lineno += s.count('\n') + 1 + _write(s) + _write('\n') + f.lineno = 1 + f.write = write + f.writeln = writeln + + # redirect littlefs tracing + f.writeln('#define LFS_TRACE_(fmt, ...) do { \\') + f.writeln(8*' '+'extern FILE *test_trace; \\') + f.writeln(8*' '+'if (test_trace) { \\') + f.writeln(12*' '+'fprintf(test_trace, ' + '"%s:%d:trace: " fmt "%s\\n", \\') + f.writeln(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\') + f.writeln(8*' '+'} \\') + f.writeln(4*' '+'} while (0)') + f.writeln('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")') + f.writeln('#define LFS_TESTBD_TRACE(...) ' + 'LFS_TRACE_(__VA_ARGS__, "")') + f.writeln() + + if not args.get('source'): + # write test suite prologue + f.writeln('%s' % SUITE_PROLOGUE.strip()) + f.writeln() if suite.code is not None: if suite.code_lineno is not None: - f.write('#line %d "%s"\n' + f.writeln('#line %d "%s"' % (suite.code_lineno, suite.path)) f.write(suite.code) - f.write('\n') + if suite.code_lineno is not None: + f.writeln('#line %d "%s"' + % (f.lineno+1, args['output'])) + f.writeln() - for i, define in it.islice( - enumerate(suite.defines), - len(TEST_PREDEFINES), None): - f.write('#define %-24s test_define(%d)\n' % (define, i)) - f.write('\n') + for i, define in enumerate(suite.defines): + f.writeln('#ifndef %s' % define) + f.writeln('#define %-24s test_define(%d)' % (define, i)) + f.writeln('#endif') + f.writeln() for case in suite.cases: # create case defines if case.defines: sorted_defines = sorted(case.defines.items()) - for perm, defines in enumerate( - it.product(*( - [(k, v) for v in vs] - for k, vs in sorted_defines))): - f.write('const test_define_t ' - '__test__%s__%s__%d__defines[] = {\n' - % (suite.name, case.name, perm)) - for k, v in defines: - f.write(4*' '+'%s,\n' % v) - f.write('};\n') - f.write('\n') - - f.write('const test_define_t *const ' - '__test__%s__%s__defines[] = {\n' + f.writeln('const test_define_t *const ' + '__test__%s__%s__defines[] = {' % (suite.name, case.name)) - for perm in range(case.permutations): - f.write(4*' '+'__test__%s__%s__%d__defines,\n' - % (suite.name, case.name, perm)) - f.write('};\n') - f.write('\n') + for defines in it.product(*( + [(k, v) for v in vs] + for k, vs in sorted_defines)): + f.writeln(4*' '+'(const test_define_t[]){%s},' + % ', '.join('%s' % v for _, v in defines)) + f.writeln('};') + f.writeln() - f.write('const uint8_t ' - '__test__%s__%s__define_map[] = {\n' + f.writeln('const uint8_t ' + '__test__%s__%s__define_map[] = {' % (suite.name, case.name)) - for k in suite.defines: - f.write(4*' '+'%s,\n' - % ([k for k, _ in sorted_defines].index(k) - if k in case.defines else '0xff')) - f.write('};\n') - f.write('\n') + f.writeln(4*' '+'%s,' + % ', '.join( + '%s' % [k for k, _ in sorted_defines].index(k) + if k in case.defines else '0xff' + for k in suite.defines)) + f.writeln('};') + f.writeln() # create case filter function if suite.if_ is not None or case.if_ is not None: - f.write('bool __test__%s__%s__filter(' - '__attribute__((unused)) uint32_t perm) {\n' + f.writeln('bool __test__%s__%s__filter(' + '__attribute__((unused)) uint32_t perm) {' % (suite.name, case.name)) if suite.if_ is not None: - f.write(4*' '+'#line %d "%s"\n' - % (suite.if_lineno, suite.path)) - f.write(4*' '+'if (!(%s)) {\n' % suite.if_) - f.write(8*' '+'return false;\n') - f.write(4*' '+'}\n') - f.write('\n') + if suite.if_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (suite.if_lineno, suite.path)) + f.writeln(4*' '+'if (!(%s)) {' % suite.if_) + if suite.if_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (f.lineno+1, args['output'])) + f.writeln(8*' '+'return false;') + f.writeln(4*' '+'}') + f.writeln() if case.if_ is not None: - f.write(4*' '+'#line %d "%s"\n' - % (case.if_lineno, suite.path)) - f.write(4*' '+'if (!(%s)) {\n' % case.if_) - f.write(8*' '+'return false;\n') - f.write(4*' '+'}\n') - f.write('\n') - f.write(4*' '+'return true;\n') - f.write('}\n') - f.write('\n') + if case.if_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (case.if_lineno, suite.path)) + f.writeln(4*' '+'if (!(%s)) {' % case.if_) + if case.if_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (f.lineno+1, args['output'])) + f.writeln(8*' '+'return false;') + f.writeln(4*' '+'}') + f.writeln() + f.writeln(4*' '+'return true;') + f.writeln('}') + f.writeln() # create case run function - f.write('void __test__%s__%s__run(' + f.writeln('void __test__%s__%s__run(' '__attribute__((unused)) struct lfs_config *cfg, ' - '__attribute__((unused)) uint32_t perm) {\n' + '__attribute__((unused)) uint32_t perm) {' % (suite.name, case.name)) - f.write(4*' '+'%s\n' - % CASE_PROLOGUE.strip().replace('\n', '\n'+4*' ')) - f.write('\n') - f.write(4*' '+'// test case %s\n' % case.id()) + if CASE_PROLOGUE.strip(): + f.writeln(4*' '+'%s' + % CASE_PROLOGUE.strip().replace('\n', '\n'+4*' ')) + f.writeln() + f.writeln(4*' '+'// test case %s' % case.id()) if case.code_lineno is not None: - f.write(4*' '+'#line %d "%s"\n' + f.writeln(4*' '+'#line %d "%s"' % (case.code_lineno, suite.path)) f.write(case.code) - f.write('\n') - f.write(4*' '+'%s\n' - % CASE_EPILOGUE.strip().replace('\n', '\n'+4*' ')) - f.write('}\n') - f.write('\n') + if case.code_lineno is not None: + f.writeln(4*' '+'#line %d "%s"' + % (f.lineno+1, args['output'])) + if CASE_EPILOGUE.strip(): + f.writeln() + f.writeln(4*' '+'%s' + % CASE_EPILOGUE.strip().replace('\n', '\n'+4*' ')) + f.writeln('}') + f.writeln() # create case struct - f.write('const struct test_case __test__%s__%s__case = {\n' + f.writeln('const struct test_case __test__%s__%s__case = {' % (suite.name, case.name)) - f.write(4*' '+'.id = "%s",\n' % case.id()) - f.write(4*' '+'.name = "%s",\n' % case.name) - f.write(4*' '+'.path = "%s",\n' % case.path) - f.write(4*' '+'.types = %s,\n' + f.writeln(4*' '+'.id = "%s",' % case.id()) + f.writeln(4*' '+'.name = "%s",' % case.name) + f.writeln(4*' '+'.path = "%s",' % case.path) + f.writeln(4*' '+'.types = %s,' % ' | '.join(filter(None, [ 'TEST_NORMAL' if case.normal else None, 'TEST_REENTRANT' if case.reentrant else None, 'TEST_VALGRIND' if case.valgrind else None]))) - f.write(4*' '+'.permutations = %d,\n' % case.permutations) + f.writeln(4*' '+'.permutations = %d,' % case.permutations) if case.defines: - f.write(4*' '+'.defines = __test__%s__%s__defines,\n' + f.writeln(4*' '+'.defines = __test__%s__%s__defines,' % (suite.name, case.name)) - f.write(4*' '+'.define_map = ' - '__test__%s__%s__define_map,\n' + f.writeln(4*' '+'.define_map = ' + '__test__%s__%s__define_map,' % (suite.name, case.name)) if suite.if_ is not None or case.if_ is not None: - f.write(4*' '+'.filter = __test__%s__%s__filter,\n' + f.writeln(4*' '+'.filter = __test__%s__%s__filter,' % (suite.name, case.name)) - f.write(4*' '+'.run = __test__%s__%s__run,\n' + f.writeln(4*' '+'.run = __test__%s__%s__run,' % (suite.name, case.name)) - f.write('};\n') - f.write('\n') + f.writeln('};') + f.writeln() # create suite define names - f.write('const char *const __test__%s__define_names[] = {\n' + f.writeln('const char *const __test__%s__define_names[] = {' % suite.name) for k in suite.defines: - f.write(4*' '+'"%s",\n' % k) - f.write('};\n') - f.write('\n') + f.writeln(4*' '+'"%s",' % k) + f.writeln('};') + f.writeln() # create suite struct - f.write('const struct test_suite __test__%s__suite = {\n' + f.writeln('const struct test_suite __test__%s__suite = {' % suite.name) - f.write(4*' '+'.id = "%s",\n' % suite.id()) - f.write(4*' '+'.name = "%s",\n' % suite.name) - f.write(4*' '+'.path = "%s",\n' % suite.path) - f.write(4*' '+'.types = %s,\n' + f.writeln(4*' '+'.id = "%s",' % suite.id()) + f.writeln(4*' '+'.name = "%s",' % suite.name) + f.writeln(4*' '+'.path = "%s",' % suite.path) + f.writeln(4*' '+'.types = %s,' % ' | '.join(filter(None, [ 'TEST_NORMAL' if suite.normal else None, 'TEST_REENTRANT' if suite.reentrant else None, 'TEST_VALGRIND' if suite.valgrind else None]))) - f.write(4*' '+'.define_names = __test__%s__define_names,\n' + f.writeln(4*' '+'.define_names = __test__%s__define_names,' % suite.name) - f.write(4*' '+'.define_count = %d,\n' % len(suite.defines)) - f.write(4*' '+'.cases = (const struct test_case *const []){\n') + f.writeln(4*' '+'.define_count = %d,' % len(suite.defines)) + f.writeln(4*' '+'.cases = (const struct test_case *const []){') for case in suite.cases: - f.write(8*' '+'&__test__%s__%s__case,\n' + f.writeln(8*' '+'&__test__%s__%s__case,' % (suite.name, case.name)) - f.write(4*' '+'},\n') - f.write(4*' '+'.case_count = %d,\n' % len(suite.cases)) - f.write('};\n') - f.write('\n') - - else: - # load all suites - suites = [TestSuite(path) for path in paths] - suites.sort(key=lambda s: s.name) - - # write out a test source - if 'output' in args: - with openio(args['output'], 'w') as f: - # redirect littlefs tracing - f.write('#define LFS_TRACE_(fmt, ...) do { \\\n') - f.write(8*' '+'extern FILE *test_trace; \\\n') - f.write(8*' '+'if (test_trace) { \\\n') - f.write(12*' '+'fprintf(test_trace, ' - '"%s:%d:trace: " fmt "%s\\n", \\\n') - f.write(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\\n') - f.write(8*' '+'} \\\n') - f.write(4*' '+'} while (0)\n') - f.write('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")\n') - f.write('#define LFS_TESTBD_TRACE(...) ' - 'LFS_TRACE_(__VA_ARGS__, "")\n') - f.write('\n') + f.writeln(4*' '+'},') + f.writeln(4*' '+'.case_count = %d,' % len(suite.cases)) + f.writeln('};') + f.writeln() + else: # copy source - f.write('#line 1 "%s"\n' % args['source']) + f.writeln('#line 1 "%s"' % args['source']) with open(args['source']) as sf: shutil.copyfileobj(sf, f) - f.write('\n') + f.writeln() f.write(SUITE_PROLOGUE) - f.write('\n') + f.writeln() # add suite info to test_runner.c if args['source'] == 'runners/test_runner.c': - f.write('\n') + f.writeln() for suite in suites: - f.write('extern const struct test_suite ' - '__test__%s__suite;\n' % suite.name) - f.write('const struct test_suite *test_suites[] = {\n') + f.writeln('extern const struct test_suite ' + '__test__%s__suite;' % suite.name) + f.writeln('const struct test_suite *test_suites[] = {') for suite in suites: - f.write(4*' '+'&__test__%s__suite,\n' % suite.name) - f.write('};\n') - f.write('const size_t test_suite_count = %d;\n' + f.writeln(4*' '+'&__test__%s__suite,' % suite.name) + f.writeln('};') + f.writeln('const size_t test_suite_count = %d;' % len(suites)) def runner(**args): @@ -469,7 +466,7 @@ def list_(**args): def find_cases(runner_, **args): - # first get suite/case/perm counts + # query from runner cmd = runner_ + ['--list-cases'] if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) @@ -483,9 +480,8 @@ def find_cases(runner_, **args): expected_perms = 0 total_perms = 0 pattern = re.compile( - '^(?P(?P[^#]+)#[^ #]+) +' - '[^ ]+ +[^ ]+ +[^ ]+ +' - '(?P[0-9]+)/(?P[0-9]+)$') + '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' + '[^\s]+\s+(?P\d+)/(?P\d+)') # skip the first line next(proc.stdout) for line in proc.stdout: @@ -509,11 +505,69 @@ def find_cases(runner_, **args): expected_perms, total_perms) +def find_paths(runner_, **args): + # query from runner + cmd = runner_ + ['--list-paths'] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace') + paths = co.OrderedDict() + pattern = re.compile( + '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' + '(?P[^:]+):(?P\d+)') + # skip the first line + for line in proc.stdout: + m = pattern.match(line) + if m: + paths[m.group('id')] = (m.group('path'), int(m.group('lineno'))) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return paths + +def find_defines(runner_, **args): + # query from runner + cmd = runner_ + ['--list-defines'] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace') + defines = co.OrderedDict() + pattern = re.compile( + '^(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)\s+' + '(?P(?:\w+=\w+\s*)+)') + # skip the first line + for line in proc.stdout: + m = pattern.match(line) + if m: + defines[m.group('id')] = {k: v + for k, v in re.findall('(\w+)=(\w+)', m.group('defines'))} + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + sys.exit(-1) + + return defines + + class TestFailure(Exception): - def __init__(self, id, returncode, stdout, assert_=None): + def __init__(self, id, returncode, output, assert_=None): self.id = id self.returncode = returncode - self.stdout = stdout + self.output = output self.assert_ = assert_ def run_step(name, runner_, **args): @@ -531,7 +585,7 @@ def run_step(name, runner_, **args): pattern = re.compile('^(?:' '(?Prunning|finished|skipped) ' '(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)' - '|' '(?P[^:]+):(?P[0-9]+):(?Passert):' + '|' '(?P[^:]+):(?P\d+):(?Passert):' ' *(?P.*)' ')$') locals = th.local() # TODO use process group instead of this set? @@ -554,7 +608,7 @@ def run_step(name, runner_, **args): children.add(proc) last_id = None - last_stdout = [] + last_output = [] last_assert = None try: while True: @@ -567,7 +621,7 @@ def run_step(name, runner_, **args): raise if not line: break - last_stdout.append(line) + last_output.append(line) if args.get('verbose'): sys.stdout.write(line) @@ -577,7 +631,7 @@ def run_step(name, runner_, **args): if op == 'running': locals.seen_perms += 1 last_id = m.group('id') - last_stdout = [] + last_output = [] last_assert = None elif op == 'finished': passed_suite_perms[m.group('suite')] += 1 @@ -590,10 +644,11 @@ def run_step(name, runner_, **args): m.group('path'), int(m.group('lineno')), m.group('message')) - # TODO why is kill _so_ much faster than terminate? - proc.kill() + # go ahead and kill the process, aborting takes a while + if args.get('keep_going'): + proc.kill() except KeyboardInterrupt: - raise TestFailure(last_id, 1, last_stdout) + raise TestFailure(last_id, 1, last_output) finally: children.remove(proc) mpty.close() @@ -603,7 +658,7 @@ def run_step(name, runner_, **args): raise TestFailure( last_id, proc.returncode, - last_stdout, + last_output, last_assert) def run_job(runner, skip=None, every=None): @@ -636,7 +691,6 @@ def run_step(name, runner_, **args): else: # stop other tests for child in children: - # TODO why is kill _so_ much faster than terminate? child.kill() break @@ -759,11 +813,28 @@ def run(**args): print() # print each failure - # TODO get line, defines, path + if failures: + # get some extra info from runner + runner_paths = find_paths(runner_, **args) + runner_defines = find_defines(runner_, **args) + for failure in failures: -# print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s failed' -# # TODO this should be the suite path and lineno -# % (failure.assert + # show summary of failure + path, lineno = runner_paths[testcase(failure.id)] + defines = runner_defines[failure.id] + + print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s%s failed' + % (path, lineno, failure.id, + ' (%s)' % ', '.join( + '%s=%s' % (k, v) for k, v in defines.items()) + if defines else '')) + + if failure.output: + output = failure.output + if failure.assert_ is not None: + output = output[:-1] + for line in output[-5:]: + sys.stdout.write(line) if failure.assert_ is not None: path, lineno, message = failure.assert_ @@ -785,7 +856,8 @@ def main(**args): or args.get('list_cases') or args.get('list_paths') or args.get('list_defines') - or args.get('list_geometries')): + or args.get('list_geometries') + or args.get('list_defaults')): list_(**args) else: run(**args) @@ -816,6 +888,8 @@ if __name__ == "__main__": help="List the defines for each test permutation.") test_parser.add_argument('--list-geometries', action='store_true', help="List the disk geometries used for testing.") + test_parser.add_argument('--list-defaults', action='store_true', + help="List the default defines in this test-runner.") test_parser.add_argument('-D', '--define', action='append', help="Override a test define.") test_parser.add_argument('-G', '--geometry',