Github/littlefs
1
Fork 0
Code Actions Activity

Reintroduced test-defines into the new test_runner

Browse Source 
This moves defines entirely into the runtime of the test_runner,
simplifying thing and reducing the amount of generated code that needs
to be build, at the cost of limiting test-defines to uintmax_t types.

This is implemented using a set of index-based scopes (created by
test.py) that allow different layers to override defines from other
layers, accessible through the global `test_define` function.

layers:
1. command-line overrides
2. per-case defines
3. per-geometry defines
This commit is contained in:
Christopher Haster
2022-04-17 21:41:38 -05:00
parent 56a990336b
commit d683f1c76c
5 changed files with 542 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runners/test_runner.c
+375 -81
View File
@@ -1,71 +1,363 @@
#include "runners/test_runner.h"
#include <getopt.h>
#include "bd/lfs_testbd.h"
#include <getopt.h>
#include <sys/types.h>
// disk geometries
struct test_geometry {
const char *name;
lfs_size_t read_size;
lfs_size_t prog_size;
lfs_size_t erase_size;
lfs_size_t erase_count;
const uintmax_t *defines;
};
// Note this includes the default configuration for test pre-defines
#define TEST_GEOMETRY(name, read, prog, erase, count) \
{name, (const uintmax_t[]){ \
/* READ_SIZE */ read, \
/* PROG_SIZE */ prog, \
/* BLOCK_SIZE */ erase, \
/* BLOCK_COUNT */ count, \
/* BLOCK_CYCLES */ -1, \
/* CACHE_SIZE */ (64 % (prog) == 0) ? 64 : (prog), \
/* LOOKAHEAD_SIZE */ 16, \
/* ERASE_VALUE */ 0xff, \
/* ERASE_CYCLES */ 0, \
/* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \
}}
const struct test_geometry test_geometries[] = {
// Made up geometries that works well for testing
{"small", 16, 16, 512, (1024*1024)/512},
{"medium", 16, 16, 4096, (1024*1024)/4096},
{"big", 16, 16, 32*1024, (1024*1024)/(32*1024)},
// Made up geometry that works well for testing
TEST_GEOMETRY("small", 16, 16, 512, (1024*1024)/512),
TEST_GEOMETRY("medium", 16, 16, 4096, (1024*1024)/4096),
TEST_GEOMETRY("big", 16, 16, 32*1024, (1024*1024)/(32*1024)),
// EEPROM/NVRAM
{"eeprom", 1, 1, 512, (1024*1024)/512},
TEST_GEOMETRY("eeprom", 1, 1, 512, (1024*1024)/512),
// SD/eMMC
{"emmc", 512, 512, 512, (1024*1024)/512},
// NOR flash
{"nor", 1, 1, 4096, (1024*1024)/4096},
// NAND flash
{"nand", 4096, 4096, 32*1024, (1024*1024)/(32*1024)},
TEST_GEOMETRY("emmc", 512, 512, 512, (1024*1024)/512),
// NOR flash
TEST_GEOMETRY("nor", 1, 1, 4096, (1024*1024)/4096),
// NAND flash
TEST_GEOMETRY("nand", 4096, 4096, 32*1024, (1024*1024)/(32*1024)),
};
const size_t test_geometry_count = (
sizeof(test_geometries) / sizeof(test_geometries[0]));
// test define lookup and management
const uintmax_t *test_defines[3] = {NULL};
const bool *test_define_masks[2] = {NULL};
uintmax_t test_define(size_t define) {
if (test_define_masks[0] && test_define_masks[0][define]) {
return test_defines[0][define];
} else if (test_define_masks[1] && test_define_masks[1][define]) {
return test_defines[1][define];
} else {
return test_defines[2][define];
}
}
void test_define_geometry(const struct test_geometry *geometry) {
if (geometry) {
test_defines[2] = geometry->defines;
} else {
test_defines[2] = NULL;
}
}
void test_define_case(const struct test_case *case_, size_t perm) {
if (case_ && case_->defines) {
test_defines[1] = case_->defines[perm];
test_define_masks[1] = case_->define_mask;
} else {
test_defines[1] = NULL;
test_define_masks[1] = NULL;
}
}
struct override {
const char *name;
uintmax_t override;
};
void test_define_overrides(
const struct test_suite *suite,
const struct override *overrides,
size_t override_count) {
if (overrides && override_count > 0) {
uintmax_t *defines = malloc(suite->define_count * sizeof(uintmax_t));
memset(defines, 0, suite->define_count * sizeof(uintmax_t));
bool *define_mask = malloc(suite->define_count * sizeof(bool));
memset(define_mask, 0, suite->define_count * sizeof(bool));
// lookup each override in the suite defines, they may have a
// different index in each suite
for (size_t i = 0; i < override_count; i++) {
ssize_t index = -1;
for (size_t j = 0; j < suite->define_count; j++) {
if (strcmp(overrides[i].name, suite->define_names[j]) == 0) {
index = j;
break;
}
}
if (index >= 0) {
defines[index] = overrides[i].override;
define_mask[index] = true;
}
}
test_defines[0] = defines;
test_define_masks[0] = define_mask;
} else {
free((uintmax_t *)test_defines[0]);
test_defines[0] = NULL;
free((bool *)test_define_masks[0]);
test_define_masks[0] = NULL;
}
}
// operations we can do
void summary(
struct override *overrides,
size_t override_count) {
(void)overrides;
(void)override_count;
printf("%-36s %7s %7s %7s %7s\n",
"", "geoms", "suites", "cases", "perms");
size_t cases = 0;
size_t perms = 0;
for (size_t i = 0; i < test_suite_count; i++) {
cases += test_suites[i]->case_count;
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
perms += test_suites[i]->cases[j]->permutations;
}
}
printf("%-36s %7zu %7zu %7zu %7zu\n",
"TOTAL",
test_geometry_count,
test_suite_count,
cases,
test_geometry_count*perms);
}
void list_suites(
struct override *overrides,
size_t override_count) {
(void)overrides;
(void)override_count;
printf("%-36s %-12s %7s %7s %7s\n",
"id", "suite", "types", "cases", "perms");
for (size_t i = 0; i < test_suite_count; i++) {
size_t perms = 0;
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
perms += test_suites[i]->cases[j]->permutations;
}
printf("%-36s %-12s %7s %7zu %7zu\n",
test_suites[i]->id,
test_suites[i]->name,
"n", // TODO
test_suites[i]->case_count,
test_geometry_count*perms);
}
}
void list_cases(
struct override *overrides,
size_t override_count) {
(void)overrides;
(void)override_count;
printf("%-36s %-12s %-12s %7s %7s\n",
"id", "suite", "case", "types", "perms");
for (size_t i = 0; i < test_suite_count; i++) {
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
printf("%-36s %-12s %-12s %7s %7zu\n",
test_suites[i]->cases[j]->id,
test_suites[i]->name,
test_suites[i]->cases[j]->name,
"n", // TODO
test_geometry_count
* test_suites[i]->cases[j]->permutations);
}
}
}
void list_paths(
struct override *overrides,
size_t override_count) {
(void)overrides;
(void)override_count;
printf("%-36s %-36s\n", "id", "path");
for (size_t i = 0; i < test_suite_count; i++) {
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
printf("%-36s %-36s\n",
test_suites[i]->cases[j]->id,
test_suites[i]->cases[j]->path);
}
}
}
void list_defines(
struct override *overrides,
size_t override_count) {
(void)overrides;
(void)override_count;
// TODO
}
void list_geometries(
struct override *overrides,
size_t override_count) {
(void)overrides;
(void)override_count;
printf("%-36s %7s %7s %7s %7s %7s\n",
"name", "read", "prog", "erase", "count", "size");
for (size_t i = 0; i < test_geometry_count; i++) {
test_define_geometry(&test_geometries[i]);
printf("%-36s %7ju %7ju %7ju %7ju %7ju\n",
test_geometries[i].name,
READ_SIZE,
PROG_SIZE,
BLOCK_SIZE,
BLOCK_COUNT,
BLOCK_SIZE*BLOCK_COUNT);
}
}
void run(
struct override *overrides,
size_t override_count) {
for (size_t i = 0; i < test_suite_count; i++) {
test_define_overrides(test_suites[i], overrides, override_count);
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
for (size_t perm = 0;
perm < test_geometry_count
* test_suites[i]->cases[j]->permutations;
perm++) {
size_t case_perm = perm / test_geometry_count;
size_t geom_perm = perm % test_geometry_count;
// setup defines
test_define_geometry(&test_geometries[geom_perm]);
test_define_case(test_suites[i]->cases[j], case_perm);
// create block device and configuration
lfs_testbd_t bd;
struct lfs_config cfg = {
.context = &bd,
.read = lfs_testbd_read,
.prog = lfs_testbd_prog,
.erase = lfs_testbd_erase,
.sync = lfs_testbd_sync,
.read_size = READ_SIZE,
.prog_size = PROG_SIZE,
.block_size = BLOCK_SIZE,
.block_count = BLOCK_COUNT,
.block_cycles = BLOCK_CYCLES,
.cache_size = CACHE_SIZE,
.lookahead_size = LOOKAHEAD_SIZE,
};
struct lfs_testbd_config bdcfg = {
.erase_value = ERASE_VALUE,
.erase_cycles = ERASE_CYCLES,
.badblock_behavior = BADBLOCK_BEHAVIOR,
.power_cycles = 0,
};
lfs_testbd_createcfg(&cfg, NULL, &bdcfg) => 0;
// filter?
if (test_suites[i]->cases[j]->filter) {
bool filter = test_suites[i]->cases[j]->filter(
&cfg, case_perm);
if (!filter) {
printf("skipped %s#%zu\n",
test_suites[i]->cases[j]->id,
perm);
continue;
}
}
// run the test
printf("running %s#%zu\n", test_suites[i]->cases[j]->id, perm);
test_suites[i]->cases[j]->run(&cfg, case_perm);
printf("finished %s#%zu\n", test_suites[i]->cases[j]->id, perm);
// cleanup
lfs_testbd_destroy(&cfg) => 0;
test_define_geometry(NULL);
test_define_case(NULL, 0);
}
}
test_define_overrides(NULL, NULL, 0);
}
}
// option handling
enum opt_flags {
OPT_HELP = 'h',
OPT_LIST = 'l',
OPT_LIST_PATHS = 1,
OPT_LIST_DEFINES = 2,
OPT_LIST_GEOMETRIES = 3,
OPT_SUMMARY = 'Y',
OPT_LIST_SUITES = 1,
OPT_LIST_CASES = 'l',
OPT_LIST_PATHS = 2,
OPT_LIST_DEFINES = 3,
OPT_LIST_GEOMETRIES = 4,
OPT_DEFINE = 'D',
};
const char *short_opts = "hYlD:";
const struct option long_opts[] = {
{"help", no_argument, NULL, OPT_HELP},
{"list", no_argument, NULL, OPT_LIST},
{"list-paths", no_argument, NULL, OPT_LIST_PATHS},
{"list-defines", no_argument, NULL, OPT_LIST_DEFINES},
{"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES},
{"help", no_argument, NULL, OPT_HELP},
{"summary", no_argument, NULL, OPT_SUMMARY},
{"list-suites", no_argument, NULL, OPT_LIST_SUITES},
{"list-cases", no_argument, NULL, OPT_LIST_CASES},
{"list-paths", no_argument, NULL, OPT_LIST_PATHS},
{"list-defines", no_argument, NULL, OPT_LIST_DEFINES},
{"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES},
{"define", required_argument, NULL, OPT_DEFINE},
{NULL, 0, NULL, 0},
};
const char *const help_text[] = {
"Show this help message.",
"Show quick summary.",
"List test suites.",
"List test cases.",
"List the path for each test case.",
"List the defines for each test permutation.",
"List the disk geometries used for testing.",
"Override a test define.",
};
int main(int argc, char **argv) {
bool list = false;
bool list_paths = false;
bool list_defines = false;
bool list_geometries = false;
void (*op)(
struct override *overrides,
size_t override_count) = run;
struct override *overrides = NULL;
size_t override_count = 0;
size_t override_cap = 0;
// parse options
while (true) {
int index = 0;
int c = getopt_long(argc, argv, "hl", long_opts, &index);
int c = getopt_long(argc, argv, short_opts, long_opts, NULL);
switch (c) {
// generate help message
case OPT_HELP: {
@@ -105,19 +397,56 @@ int main(int argc, char **argv) {
printf("\n");
exit(0);
}
// list flags
case OPT_LIST:
list = true;
// summary/list flags
case OPT_SUMMARY:
op = summary;
break;
case OPT_LIST_SUITES:
op = list_suites;
break;
case OPT_LIST_CASES:
op = list_cases;
break;
case OPT_LIST_PATHS:
list_paths = true;
op = list_paths;
break;
case OPT_LIST_DEFINES:
list_defines = true;
op = list_defines;
break;
case OPT_LIST_GEOMETRIES:
list_geometries = true;
op = list_geometries;
break;
// configuration
case OPT_DEFINE: {
// realloc if necessary
override_count += 1;
if (override_count > override_cap) {
override_cap = (2*override_cap > 4) ? 2*override_cap : 4;
overrides = realloc(overrides, override_cap
* sizeof(struct override));
}
// parse into string key/uintmax_t value, cannibalizing the
// arg in the process
char *sep = strchr(optarg, '=');
char *parsed = NULL;
if (!sep) {
goto invalid_define;
}
overrides[override_count-1].override
= strtoumax(sep+1, &parsed, 0);
if (parsed == sep+1) {
goto invalid_define;
}
overrides[override_count-1].name = optarg;
*sep = '\0';
break;
invalid_define:
fprintf(stderr, "error: invalid define: %s\n", optarg);
exit(-1);
}
// done parsing
case -1:
goto getopt_done;
@@ -128,51 +457,16 @@ int main(int argc, char **argv) {
}
getopt_done:
// what do we need to do?
if (list) {
printf("%-36s %-12s %-12s %7s %7s\n",
"id", "suite", "case", "type", "perms");
for (size_t i = 0; i < test_suite_count; i++) {
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
printf("%-36s %-12s %-12s %7s %7d\n",
test_suites[i]->cases[j]->id,
test_suites[i]->name,
test_suites[i]->cases[j]->name,
"n", // TODO
test_suites[i]->cases[j]->permutations);
}
}
} else if (list_paths) {
printf("%-36s %-36s\n", "id", "path");
for (size_t i = 0; i < test_suite_count; i++) {
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
printf("%-36s %-36s\n",
test_suites[i]->cases[j]->id,
test_suites[i]->cases[j]->path);
}
}
} else if (list_defines) {
// TODO
} else if (list_geometries) {
printf("%-12s %7s %7s %7s %7s %7s\n",
"name", "read", "prog", "erase", "count", "size");
for (size_t i = 0; i < test_geometry_count; i++) {
printf("%-12s %7d %7d %7d %7d %7d\n",
test_geometries[i].name,
test_geometries[i].read_size,
test_geometries[i].prog_size,
test_geometries[i].erase_size,
test_geometries[i].erase_count,
test_geometries[i].erase_size
* test_geometries[i].erase_count);
}
} else {
printf("remaining: ");
for (int i = optind; i < argc; i++) {
printf("%s ", argv[i]);
}
printf("\n");
for (size_t i = 0; i < override_count; i++) {
printf("define: %s %ju\n", overrides[i].name, overrides[i].override);
}
// do the thing
op(
overrides,
override_count);
// cleanup (need to be done for valgrind testing)
free(overrides);
}
runners/test_runner.h
+36 -1
View File
@@ -4,11 +4,24 @@
#include "lfs.h"
// generated test configurations
enum test_type {
TEST_NORMAL = 0x1,
TEST_REENTRANT = 0x2,
TEST_VALGRIND = 0x4,
};
struct test_case {
const char *id;
const char *name;
const char *path;
uint32_t permutations;
uint8_t types;
size_t permutations;
const uintmax_t *const *defines;
const bool *define_mask;
bool (*filter)(struct lfs_config *cfg, uint32_t perm);
void (*run)(struct lfs_config *cfg, uint32_t perm);
};
@@ -16,11 +29,33 @@ struct test_suite {
const char *id;
const char *name;
const char *path;
const char *const *define_names;
size_t define_count;
const struct test_case *const *cases;
size_t case_count;
};
// TODO remove this indirection
extern const struct test_suite *test_suites[];
extern const size_t test_suite_count;
// access generated test defines
uintmax_t test_define(size_t define);
// a few preconfigured defines that control how tests run
#define READ_SIZE test_define(0)
#define PROG_SIZE test_define(1)
#define BLOCK_SIZE test_define(2)
#define BLOCK_COUNT test_define(3)
#define BLOCK_CYCLES test_define(4)
#define CACHE_SIZE test_define(5)
#define LOOKAHEAD_SIZE test_define(6)
#define ERASE_VALUE test_define(7)
#define ERASE_CYCLES test_define(8)
#define BADBLOCK_BEHAVIOR test_define(9)
#endif