Github/littlefs
1
Fork 0
Code Actions Activity

Removed concept of geometries from test/bench runners

Browse Source 
This turned out to not be all that useful.

Tests already take quite a bit to run, which is a good thing! We have a
lot of tests! 942.68s or ~15 minutes of tests at the time of writing to
be exact. But simply multiplying the number of tests by some number of
geometries is heavy handed and not a great use of testing time.

Instead, tests where different geometries are relevant can parameterize
READ_SIZE/PROG_SIZE/BLOCK_SIZE at the suite level where needed. The
geometry system was just another define parameterization layer anyways.

Testing different geometries can still be done in CI by overriding the
relevant defines anyways, and it _might_ be interesting there.
This commit is contained in:
Christopher Haster
2023-11-29 13:51:40 -06:00
parent b8d3a5ef46
commit d485795336
6 changed files with 104 additions and 602 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runners/test_runner.c
+38 -266
View File
@@ -110,11 +110,6 @@ static uintmax_t leb16_parse(const char *s, char **tail) {
// test_runner types
typedef struct test_geometry {
const char *name;
test_define_t defines[TEST_GEOMETRY_DEFINE_COUNT];
} test_geometry_t;
typedef struct test_powerloss {
const char *name;
void (*run)(
@@ -168,9 +163,9 @@ intmax_t test_define_lit(void *data, size_t i) {
#define TEST_DEFINE_MAP_OVERRIDE 0
#define TEST_DEFINE_MAP_EXPLICIT 1
#define TEST_DEFINE_MAP_CASE 2
#define TEST_DEFINE_MAP_GEOMETRY 3
#define TEST_DEFINE_MAP_IMPLICIT 4
#define TEST_DEFINE_MAP_COUNT 5
#define TEST_DEFINE_MAP_IMPLICIT 3
#define TEST_DEFINE_MAP_COUNT 4
test_define_map_t test_define_maps[TEST_DEFINE_MAP_COUNT] = {
[TEST_DEFINE_MAP_IMPLICIT] = {
@@ -187,6 +182,7 @@ test_define_map_t test_define_maps[TEST_DEFINE_MAP_COUNT] = {
#define TEST_DEFINE_NAMES_SUITE 0
#define TEST_DEFINE_NAMES_IMPLICIT 1
#define TEST_DEFINE_NAMES_COUNT 2
test_define_names_t test_define_names[TEST_DEFINE_NAMES_COUNT] = {
@@ -347,14 +343,6 @@ void test_define_case(
}
}
// geometry updates
const test_geometry_t *test_geometry = NULL;
void test_define_geometry(const test_geometry_t *geometry) {
test_define_maps[TEST_DEFINE_MAP_GEOMETRY] = (test_define_map_t){
geometry->defines, TEST_GEOMETRY_DEFINE_COUNT};
}
// override updates
typedef struct test_override {
const char *name;
@@ -464,9 +452,6 @@ void test_define_cleanup(void) {
// test state
extern const test_geometry_t *test_geometries;
extern size_t test_geometry_count;
extern const test_powerloss_t *test_powerlosses;
extern size_t test_powerloss_count;
size_t test_pls = 0;
@@ -800,36 +785,31 @@ static void case_forperm(
// define case permutation
test_define_case(suite, case_, k);
for (size_t g = 0; g < test_geometry_count; g++) {
// define geometry
test_define_geometry(&test_geometries[g]);
size_t permutations = test_define_permutationpermutations();
for (size_t p = 0; p < permutations; p++) {
// define permutation permutation
test_define_permutation(p);
size_t permutations = test_define_permutationpermutations();
for (size_t p = 0; p < permutations; p++) {
// define permutation permutation
test_define_permutation(p);
// have we seen this permutation before?
bool was_seen = test_seen_insert(&seen);
if (!(k == 0 && p == 0) && was_seen) {
continue;
}
// have we seen this permutation before?
bool was_seen = test_seen_insert(&seen);
if (!(k == 0 && g == 0 && p == 0) && was_seen) {
continue;
}
if (cycles) {
cb(data, suite, case_, &(test_powerloss_t){
.run=run_powerloss_cycles,
.cycles=cycles,
.cycle_count=cycle_count});
} else {
for (size_t p = 0; p < test_powerloss_count; p++) {
// skip non-reentrant tests when powerloss testing
if (test_powerlosses[p].run != run_powerloss_none
&& !(case_->flags & TEST_REENTRANT)) {
continue;
}
cb(data, suite, case_, &test_powerlosses[p]);
if (cycles) {
cb(data, suite, case_, &(test_powerloss_t){
.run=run_powerloss_cycles,
.cycles=cycles,
.cycle_count=cycle_count});
} else {
for (size_t p = 0; p < test_powerloss_count; p++) {
// skip non-reentrant tests when powerloss testing
if (test_powerlosses[p].run != run_powerloss_none
&& !(case_->flags & TEST_REENTRANT)) {
continue;
}
cb(data, suite, case_, &test_powerlosses[p]);
}
}
}
@@ -1211,8 +1191,6 @@ void perm_list_permutation_defines(
}
}
extern const test_geometry_t builtin_geometries[];
static void list_defines(void) {
struct list_defines_defines defines = {NULL, 0, 0};
@@ -1315,17 +1293,11 @@ static void list_implicit_defines(void) {
// yes we do need to define a suite, this does a bit of bookeeping
// such as setting up the define cache
test_define_suite(&(const struct test_suite){0});
test_define_permutation(0);
// make sure to include builtin geometries here
extern const test_geometry_t builtin_geometries[];
for (size_t g = 0; builtin_geometries[g].name; g++) {
test_define_geometry(&builtin_geometries[g]);
test_define_permutation(0);
// add implicit defines
for (size_t d = 0; d < TEST_IMPLICIT_DEFINE_COUNT; d++) {
list_defines_add(&defines, d);
}
// add implicit defines
for (size_t d = 0; d < TEST_IMPLICIT_DEFINE_COUNT; d++) {
list_defines_add(&defines, d);
}
for (size_t i = 0; i < defines.define_count; i++) {
@@ -1347,56 +1319,6 @@ static void list_implicit_defines(void) {
// geometries to test
const test_geometry_t builtin_geometries[] = {
#define TEST_GEO(name, read_size, prog_size, block_size) \
{name, { \
TEST_CONST(read_size), \
TEST_CONST(prog_size), \
TEST_CONST(block_size), \
}},
TEST_GEOMETRIES
#undef TEST_GEO
{NULL, {{0}, {0}, {0}}},
};
const test_geometry_t *test_geometries = builtin_geometries;
size_t test_geometry_count = 5;
static void list_geometries(void) {
// at least size so that names fit
unsigned name_width = 23;
for (size_t g = 0; builtin_geometries[g].name; g++) {
size_t len = strlen(builtin_geometries[g].name);
if (len > name_width) {
name_width = len;
}
}
name_width = 4*((name_width+1+4-1)/4)-1;
// yes we do need to define a suite, this does a bit of bookeeping
// such as setting up the define cache
test_define_suite(&(const struct test_suite){0});
printf("%-*s %7s %7s %7s %7s %11s\n",
name_width, "geometry", "read", "prog", "erase", "count", "size");
for (size_t g = 0; builtin_geometries[g].name; g++) {
test_define_geometry(&builtin_geometries[g]);
test_define_permutation(0);
printf("%-*s %7ju %7ju %7ju %7ju %11ju\n",
name_width,
builtin_geometries[g].name,
READ_SIZE,
PROG_SIZE,
BLOCK_SIZE,
BLOCK_COUNT,
DISK_SIZE);
}
}
// scenarios to run tests under power-loss
static void run_powerloss_none(
@@ -1998,23 +1920,21 @@ enum opt_flags {
OPT_LIST_DEFINES = 3,
OPT_LIST_PERMUTATION_DEFINES = 4,
OPT_LIST_IMPLICIT_DEFINES = 5,
OPT_LIST_GEOMETRIES = 6,
OPT_LIST_POWERLOSSES = 7,
OPT_LIST_POWERLOSSES = 6,
OPT_DEFINE = 'D',
OPT_GEOMETRY = 'G',
OPT_POWERLOSS = 'P',
OPT_STEP = 's',
OPT_DISK = 'd',
OPT_TRACE = 't',
OPT_TRACE_BACKTRACE = 8,
OPT_TRACE_PERIOD = 9,
OPT_TRACE_FREQ = 10,
OPT_READ_SLEEP = 11,
OPT_PROG_SLEEP = 12,
OPT_ERASE_SLEEP = 13,
OPT_TRACE_BACKTRACE = 7,
OPT_TRACE_PERIOD = 8,
OPT_TRACE_FREQ = 9,
OPT_READ_SLEEP = 10,
OPT_PROG_SLEEP = 11,
OPT_ERASE_SLEEP = 12,
};
const char *short_opts = "hYlLD:G:P:s:d:t:";
const char *short_opts = "hYlLD:P:s:d:t:";
const struct option long_opts[] = {
{"help", no_argument, NULL, OPT_HELP},
@@ -2028,10 +1948,8 @@ const struct option long_opts[] = {
no_argument, NULL, OPT_LIST_PERMUTATION_DEFINES},
{"list-implicit-defines",
no_argument, NULL, OPT_LIST_IMPLICIT_DEFINES},
{"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES},
{"list-powerlosses", no_argument, NULL, OPT_LIST_POWERLOSSES},
{"define", required_argument, NULL, OPT_DEFINE},
{"geometry", required_argument, NULL, OPT_GEOMETRY},
{"powerloss", required_argument, NULL, OPT_POWERLOSS},
{"step", required_argument, NULL, OPT_STEP},
{"disk", required_argument, NULL, OPT_DISK},
@@ -2055,10 +1973,8 @@ const char *const help_text[] = {
"List all defines in this test-runner.",
"List explicit defines in this test-runner.",
"List implicit defines in this test-runner.",
"List the available disk geometries.",
"List the available power-loss scenarios.",
"Override a test define.",
"Comma-separated list of disk geometries to test.",
"Comma-separated list of power-loss scenarios to test.",
"Comma-separated range of test permutations to run (start,stop,step).",
"Direct block device operations to this file.",
@@ -2075,7 +1991,6 @@ int main(int argc, char **argv) {
void (*op)(void) = run;
size_t test_override_capacity = 0;
size_t test_geometry_capacity = 0;
size_t test_powerloss_capacity = 0;
size_t test_id_capacity = 0;
@@ -2164,9 +2079,6 @@ int main(int argc, char **argv) {
case OPT_LIST_IMPLICIT_DEFINES:
op = list_implicit_defines;
break;
case OPT_LIST_GEOMETRIES:
op = list_geometries;
break;
case OPT_LIST_POWERLOSSES:
op = list_powerlosses;
break;
@@ -2320,143 +2232,6 @@ invalid_define:
fprintf(stderr, "error: invalid define: %s\n", optarg);
exit(-1);
}
case OPT_GEOMETRY: {
// reset our geometry scenarios
if (test_geometry_capacity > 0) {
free((test_geometry_t*)test_geometries);
}
test_geometries = NULL;
test_geometry_count = 0;
test_geometry_capacity = 0;
// parse the comma separated list of disk geometries
while (*optarg) {
// allocate space
test_geometry_t *geometry = mappend(
(void**)&test_geometries,
sizeof(test_geometry_t),
&test_geometry_count,
&test_geometry_capacity);
// parse the disk geometry
optarg += strspn(optarg, " ");
// named disk geometry
size_t len = strcspn(optarg, " ,");
for (size_t i = 0; builtin_geometries[i].name; i++) {
if (len == strlen(builtin_geometries[i].name)
&& memcmp(optarg,
builtin_geometries[i].name,
len) == 0) {
*geometry = builtin_geometries[i];
optarg += len;
goto geometry_next;
}
}
// comma-separated read/prog/erase
if (*optarg == '{') {
lfs_size_t sizes[3];
size_t count = 0;
char *s = optarg + 1;
while (count < 3) {
char *parsed = NULL;
sizes[count] = strtoumax(s, &parsed, 0);
count += 1;
s = parsed + strspn(parsed, " ");
if (*s == ',') {
s += 1;
continue;
} else if (*s == '}') {
s += 1;
break;
} else {
goto geometry_unknown;
}
}
// allow implicit r=p and p=e for common geometries
memset(geometry, 0, sizeof(test_geometry_t));
if (count >= 3) {
geometry->defines[READ_SIZE_i]
= TEST_LIT(sizes[0]);
geometry->defines[PROG_SIZE_i]
= TEST_LIT(sizes[1]);
geometry->defines[BLOCK_SIZE_i]
= TEST_LIT(sizes[2]);
} else if (count >= 2) {
geometry->defines[PROG_SIZE_i]
= TEST_LIT(sizes[0]);
geometry->defines[BLOCK_SIZE_i]
= TEST_LIT(sizes[1]);
} else {
geometry->defines[BLOCK_SIZE_i]
= TEST_LIT(sizes[0]);
}
optarg = s;
goto geometry_next;
}
// leb16-encoded read/prog/erase/count
if (*optarg == ':') {
lfs_size_t sizes[3];
size_t count = 0;
char *s = optarg + 1;
while (true) {
char *parsed = NULL;
uintmax_t x = leb16_parse(s, &parsed);
if (parsed == s || count >= 3) {
break;
}
sizes[count] = x;
count += 1;
s = parsed;
}
// allow implicit r=p and p=e for common geometries
memset(geometry, 0, sizeof(test_geometry_t));
if (count >= 3) {
geometry->defines[READ_SIZE_i]
= TEST_LIT(sizes[0]);
geometry->defines[PROG_SIZE_i]
= TEST_LIT(sizes[1]);
geometry->defines[BLOCK_SIZE_i]
= TEST_LIT(sizes[2]);
} else if (count >= 2) {
geometry->defines[PROG_SIZE_i]
= TEST_LIT(sizes[0]);
geometry->defines[BLOCK_SIZE_i]
= TEST_LIT(sizes[1]);
} else {
geometry->defines[BLOCK_SIZE_i]
= TEST_LIT(sizes[0]);
}
optarg = s;
goto geometry_next;
}
geometry_unknown:
// unknown scenario?
fprintf(stderr, "error: unknown disk geometry: %s\n",
optarg);
exit(-1);
geometry_next:
optarg += strspn(optarg, " ");
if (*optarg == ',') {
optarg += 1;
} else if (*optarg == '\0') {
break;
} else {
goto geometry_unknown;
}
}
break;
}
case OPT_POWERLOSS: {
// reset our powerloss scenarios
if (test_powerloss_capacity > 0) {
@@ -2822,9 +2597,6 @@ getopt_done: ;
}
free((void*)test_overrides);
}
if (test_geometry_capacity) {
free((void*)test_geometries);
}
if (test_powerloss_capacity) {
for (size_t i = 0; i < test_powerloss_count; i++) {
free((void*)test_powerlosses[i].cycles);