Reworked test_runner/bench_runner to evaluate define permutations lazily

I wondered if walking in Python 2's footsteps was going to run into the
same issues and sure enough, memory backed iterators became unweildy.

The motivation for this change is that large ranges in tests, such as
iterators over seeds or permutations, became prohibitively expensive to
compile. This meant more iteration moving into tests with more steps to
reproduce failures. This sort of defeats the purpuse of the test
framework.

The solution here is to move test permutation generation out of test.py
and into the test runner itself. The allows defines to generate their
values programmatically.

This does conflict with the test frameworks support of sets of explicit
permutations, but this is fixed by also moving these "permutation sets"
down into the test runner.

I guess it turns out the closer your representation matches your
implementation the better everythign works.

Additionally the define caching layer got a bit of tweaking. We can't
precalculate the defines because of mutual recursion, but we can
precalculate which define/permutation each define id maps to. This is
necessary as otherwise figuring out each define's define-specific
permutation would be prohibitively expensive.

runners/bench_runner.c

@@ -142,17 +142,19 @@ typedef struct bench_define_names {
     size_t count;
 } bench_define_names_t;
 
-intmax_t bench_define_lit(void *data) {
+intmax_t bench_define_lit(void *data, size_t i) {
+    (void)i;
     return (intptr_t)data;
 }
 
-#define BENCH_CONST(x) {bench_define_lit, (void*)(uintptr_t)(x)}
+#define BENCH_CONST(x) {bench_define_lit, (void*)(uintptr_t)(x), 1}
 #define BENCH_LIT(x) ((bench_define_t)BENCH_CONST(x))
 
 
 #define BENCH_DEF(k, v) \
-    intmax_t bench_define_##k(void *data) { \
+    intmax_t bench_define_##k(void *data, size_t i) { \
         (void)data; \
+        (void)i; \
         return v; \
     }
 
@@ -161,7 +163,7 @@ intmax_t bench_define_lit(void *data) {
 
 #define BENCH_DEFINE_MAP_OVERRIDE    0
 #define BENCH_DEFINE_MAP_EXPLICIT    1
-#define BENCH_DEFINE_MAP_PERMUTATION 2
+#define BENCH_DEFINE_MAP_CASE        2
 #define BENCH_DEFINE_MAP_GEOMETRY    3
 #define BENCH_DEFINE_MAP_IMPLICIT    4
 #define BENCH_DEFINE_MAP_COUNT       5
@@ -170,7 +172,7 @@ bench_define_map_t bench_define_maps[BENCH_DEFINE_MAP_COUNT] = {
     [BENCH_DEFINE_MAP_IMPLICIT] = {
         (const bench_define_t[BENCH_IMPLICIT_DEFINE_COUNT]) {
             #define BENCH_DEF(k, v) \
-                [k##_i] = {bench_define_##k, NULL},
+                [k##_i] = {bench_define_##k, NULL, 1},
 
                 BENCH_IMPLICIT_DEFINES
             #undef BENCH_DEF
@@ -196,9 +198,20 @@ bench_define_names_t bench_define_names[BENCH_DEFINE_NAMES_COUNT] = {
     },
 };
 
-intmax_t *bench_define_cache;
+size_t bench_define_count;
-size_t bench_define_cache_count;
+
-unsigned *bench_define_cache_mask;
+typedef struct bench_define_cache_entry {
+    // - >=0 => not cached
+    // - -1  => cached
+    ssize_t permutation;
+    union {
+        intmax_t value;
+        const bench_define_t *define;
+    } u;
+} bench_define_cache_entry_t;
+
+bench_define_cache_entry_t *bench_define_cache;
+size_t bench_define_cache_capacity;
 
 const char *bench_define_name(size_t define) {
     // lookup in our bench names
@@ -225,47 +238,109 @@ bool bench_define_ispermutation(size_t define) {
     return false;
 }
 
-intmax_t bench_define(size_t define) {
+size_t bench_define_permutations(size_t define) {
-    // is the define in our cache?
-    if (define < bench_define_cache_count
-            && (bench_define_cache_mask[define/(8*sizeof(unsigned))]
-                & (1 << (define%(8*sizeof(unsigned)))))) {
-        return bench_define_cache[define];
-    }
-
-    // lookup in our bench defines
     for (size_t i = 0; i < BENCH_DEFINE_MAP_COUNT; i++) {
         if (define < bench_define_maps[i].count
                 && bench_define_maps[i].defines[define].cb) {
-            intmax_t v = bench_define_maps[i].defines[define].cb(
+            return bench_define_maps[i].defines[define].permutations
-                    bench_define_maps[i].defines[define].data);
+                    ? bench_define_maps[i].defines[define].permutations
-
+                    : 1;
-            // insert into cache!
-            bench_define_cache[define] = v;
-            bench_define_cache_mask[define / (8*sizeof(unsigned))]
-                    |= 1 << (define%(8*sizeof(unsigned)));
-
-            return v;
         }
     }
 
     return 0;
-
-    // not found?
-    const char *name = bench_define_name(define);
-    fprintf(stderr, "error: undefined define %s (%zd)\n",
-            name ? name : "(unknown)",
-            define);
-    assert(false);
-    exit(-1);
 }
 
-void bench_define_flush(void) {
+size_t bench_define_permutationpermutations(void) {
-    // clear cache between permutations
+    size_t prod = 1;
-    memset(bench_define_cache_mask, 0,
+    for (size_t d = 0; d < bench_define_count; d++) {
-            sizeof(unsigned)*(
+        size_t permutations = bench_define_permutations(d);
-                (bench_define_cache_count+(8*sizeof(unsigned))-1)
+        if (permutations > 0) {
-                / (8*sizeof(unsigned))));
+            prod *= permutations;
+        }
+    }
+    return prod;
+}
+
+intmax_t bench_define(size_t define) {
+    // cached?
+    if (bench_define_cache[define].permutation == -1) {
+        return bench_define_cache[define].u.value;
+
+    // lazily defined?
+    } else if (bench_define_cache[define].u.define) {
+        // evaluate and store in cache
+        bench_define_cache[define].u.value
+                = bench_define_cache[define].u.define->cb(
+                    bench_define_cache[define].u.define->data,
+                    bench_define_cache[define].permutation);
+        bench_define_cache[define].permutation = -1;
+        return bench_define_cache[define].u.value;
+
+    // not defined?
+    } else {
+        const char *name = bench_define_name(define);
+        fprintf(stderr, "error: undefined define %s (%zd)\n",
+                name ? name : "(unknown)",
+                define);
+        assert(false);
+        exit(-1);
+    }
+}
+
+// permutation updates
+void bench_define_permutation(size_t perm) {
+    // We can't completely precompute the defines easily, since they may be
+    // mutually recursive. But we can precompute the permutations, which is
+    // expensive otherwise.
+    //
+    // Note that it's not really worth it to make define lookup completely
+    // lazy, the first thing we do is evaluate all defines for 1. deduplication
+    // and 2. logging.
+
+    if (bench_define_cache_capacity < bench_define_count) {
+        // align to power of two to avoid any superlinear growth
+        bench_define_cache_capacity = 1 << lfs_npw2(bench_define_count);
+        bench_define_cache = realloc(
+                bench_define_cache,
+                bench_define_cache_capacity*sizeof(bench_define_cache_entry_t));
+    }
+
+    for (size_t d = 0; d < bench_define_count; d++) {
+        // lookup our bench defines
+        for (size_t i = 0; i < BENCH_DEFINE_MAP_COUNT; i++) {
+            if (d < bench_define_maps[i].count
+                    && bench_define_maps[i].defines[d].cb) {
+                // note we can't precompute these due to mutual recursion
+                const bench_define_t *define = &bench_define_maps[i].defines[d];
+                bench_define_cache[d] = (bench_define_cache_entry_t){
+                    .permutation = perm % define->permutations,
+                    .u.define = define,
+                };
+                perm /= bench_define_maps[i].defines[d].permutations;
+                goto next;
+            }
+        }
+
+        // default to a null value, these should be unreachable
+        bench_define_cache[d] = (bench_define_cache_entry_t){0};
+    next:;
+    }
+}
+
+// case updates
+void bench_define_case(
+        const struct bench_suite *suite,
+        const struct bench_case *case_,
+        size_t perm) {
+    if (case_->defines) {
+        bench_define_maps[BENCH_DEFINE_MAP_CASE] = (bench_define_map_t){
+                &case_->defines[perm*suite->define_count],
+                suite->define_count};
+    } else {
+        bench_define_maps[BENCH_DEFINE_MAP_CASE] = (bench_define_map_t){
+                NULL, 0};
+    }
 }
 
 // geometry updates
@@ -279,122 +354,96 @@ void bench_define_geometry(const bench_geometry_t *geometry) {
 // override updates
 typedef struct bench_override {
     const char *name;
-    const intmax_t *defines;
+    bench_define_t define;
-    size_t permutations;
 } bench_override_t;
 
+typedef struct bench_override_value {
+    intmax_t start;
+    intmax_t stop;
+    // step == 0 indicates a single value
+    intmax_t step;
+} bench_override_value_t;
+
+typedef struct bench_override_data {
+    bench_override_value_t *values;
+    size_t value_count;
+} bench_override_data_t;
+
+intmax_t bench_override_cb(void *data, size_t i) {
+    const bench_override_data_t *data_ = data;
+    for (size_t j = 0; j < data_->value_count; j++) {
+        const bench_override_value_t *v = &data_->values[j];
+        // range?
+        if (v->step) {
+            size_t range_count;
+            if (v->step > 0) {
+                range_count = (v->stop-1 - v->start) / v->step + 1;
+            } else {
+                range_count = (v->start-1 - v->stop) / -v->step + 1;
+            }
+
+            if (i < range_count) {
+                return i*v->step + v->start;
+            }
+            i -= range_count;
+        // value?
+        } else {
+            if (i == 0) {
+                return v->start;
+            }
+            i -= 1;
+        }
+    }
+
+    // should never get here
+    assert(false);
+    __builtin_unreachable();
+}
+
 const bench_override_t *bench_overrides = NULL;
 size_t bench_override_count = 0;
 
 bench_define_t *bench_override_defines = NULL;
-size_t bench_override_define_count = 0;
-size_t bench_override_define_permutations = 1;
 size_t bench_override_define_capacity = 0;
 
 // suite/perm updates
 void bench_define_suite(const struct bench_suite *suite) {
+    // set define names
     bench_define_names[BENCH_DEFINE_NAMES_SUITE] = (bench_define_names_t){
             suite->define_names, suite->define_count};
 
-    // make sure our cache is large enough
+    // set define count
-    if (lfs_max(suite->define_count, BENCH_IMPLICIT_DEFINE_COUNT)
+    bench_define_count = suite->define_count > BENCH_IMPLICIT_DEFINE_COUNT
-            > bench_define_cache_count) {
+            ? suite->define_count
-        // align to power of two to avoid any superlinear growth
+            : BENCH_IMPLICIT_DEFINE_COUNT;
-        size_t ncount = 1 << lfs_npw2(
-                lfs_max(suite->define_count, BENCH_IMPLICIT_DEFINE_COUNT));
-        bench_define_cache = realloc(bench_define_cache, ncount*sizeof(intmax_t));
-        bench_define_cache_mask = realloc(bench_define_cache_mask,
-                sizeof(unsigned)*(
-                    (ncount+(8*sizeof(unsigned))-1)
-                    / (8*sizeof(unsigned))));
-        bench_define_cache_count = ncount;
-    }
 
     // map any overrides
     if (bench_override_count > 0) {
-        // first figure out the total size of override permutations
+        if (bench_define_count > bench_override_define_capacity) {
-        size_t count = 0;
-        size_t permutations = 1;
-        for (size_t i = 0; i < bench_override_count; i++) {
-            for (size_t d = 0;
-                    d < lfs_max(
-                        suite->define_count,
-                        BENCH_IMPLICIT_DEFINE_COUNT);
-                    d++) {
-                // define name match?
-                const char *name = bench_define_name(d);
-                if (name && strcmp(name, bench_overrides[i].name) == 0) {
-                    count = lfs_max(count, d+1);
-                    permutations *= bench_overrides[i].permutations;
-                    break;
-                }
-            }
-        }
-        bench_override_define_count = count;
-        bench_override_define_permutations = permutations;
-
-        // make sure our override arrays are big enough
-        if (count * permutations > bench_override_define_capacity) {
             // align to power of two to avoid any superlinear growth
-            size_t ncapacity = 1 << lfs_npw2(count * permutations);
+            bench_override_define_capacity = 1 << lfs_npw2(bench_define_count);
             bench_override_defines = realloc(
                     bench_override_defines,
-                    sizeof(bench_define_t)*ncapacity);
+                    bench_override_define_capacity*sizeof(bench_define_t));
-            bench_override_define_capacity = ncapacity;
         }
 
-        // zero unoverridden defines
         memset(bench_override_defines, 0,
-                sizeof(bench_define_t) * count * permutations);
+                bench_define_count*sizeof(bench_define_t));
-
-        // compute permutations
-        size_t p = 1;
         for (size_t i = 0; i < bench_override_count; i++) {
-            for (size_t d = 0;
+            for (size_t d = 0; d < bench_define_count; d++) {
-                    d < lfs_max(
+                // name match?
-                        suite->define_count,
-                        BENCH_IMPLICIT_DEFINE_COUNT);
-                    d++) {
-                // define name match?
                 const char *name = bench_define_name(d);
                 if (name && strcmp(name, bench_overrides[i].name) == 0) {
-                    // scatter the define permutations based on already
+                    bench_override_defines[d] = bench_overrides[i].define;
-                    // seen permutations
-                    for (size_t j = 0; j < permutations; j++) {
-                        bench_override_defines[j*count + d] = BENCH_LIT(
-                                bench_overrides[i].defines[(j/p)
-                                    % bench_overrides[i].permutations]);
-                    }
-
-                    // keep track of how many permutations we've seen so far
-                    p *= bench_overrides[i].permutations;
-                    break;
                 }
             }
         }
-    }
-}
 
-void bench_define_perm(
+        bench_define_maps[BENCH_DEFINE_MAP_OVERRIDE] = (bench_define_map_t){
-        const struct bench_suite *suite,
+                bench_override_defines, bench_define_count};
-        const struct bench_case *case_,
-        size_t perm) {
-    if (case_->defines) {
-        bench_define_maps[BENCH_DEFINE_MAP_PERMUTATION] = (bench_define_map_t){
-                case_->defines + perm*suite->define_count,
-                suite->define_count};
-    } else {
-        bench_define_maps[BENCH_DEFINE_MAP_PERMUTATION] = (bench_define_map_t){
-                NULL, 0};
     }
 }
 
-void bench_define_override(size_t perm) {
-    bench_define_maps[BENCH_DEFINE_MAP_OVERRIDE] = (bench_define_map_t){
-            bench_override_defines + perm*bench_override_define_count,
-            bench_override_define_count};
-}
-
 void bench_define_explicit(
         const bench_define_t *defines,
         size_t define_count) {
@@ -405,7 +454,6 @@ void bench_define_explicit(
 void bench_define_cleanup(void) {
     // bench define management can allocate a few things
     free(bench_define_cache);
-    free(bench_define_cache_mask);
     free(bench_override_defines);
 }
 
@@ -610,11 +658,7 @@ static void perm_printid(
     (void)suite;
     // case[:permutation]
     printf("%s:", case_->name);
-    for (size_t d = 0;
+    for (size_t d = 0; d < bench_define_count; d++) {
-            d < lfs_max(
-                suite->define_count,
-                BENCH_IMPLICIT_DEFINE_COUNT);
-            d++) {
         if (bench_define_ispermutation(d)) {
             leb16_print(d);
             leb16_print(BENCH_DEFINE(d));
@@ -634,19 +678,10 @@ struct bench_seen_branch {
     struct bench_seen branch;
 };
 
-bool bench_seen_insert(
+bool bench_seen_insert(bench_seen_t *seen) {
-        bench_seen_t *seen,
-        const struct bench_suite *suite,
-        const struct bench_case *case_) {
-    (void)case_;
-    bool was_seen = true;
-
     // use the currently set defines
-    for (size_t d = 0;
+    bool was_seen = true;
-            d < lfs_max(
+    for (size_t d = 0; d < bench_define_count; d++) {
-                suite->define_count,
-                BENCH_IMPLICIT_DEFINE_COUNT);
-            d++) {
         // treat unpermuted defines the same as 0
         intmax_t define = bench_define_ispermutation(d) ? BENCH_DEFINE(d) : 0;
 
@@ -699,10 +734,10 @@ static void case_forperm(
     if (defines) {
         bench_define_explicit(defines, define_count);
 
-        for (size_t v = 0; v < bench_override_define_permutations; v++) {
+        size_t permutations = bench_define_permutationpermutations();
-            // define override permutation
+        for (size_t p = 0; p < permutations; p++) {
-            bench_define_override(v);
+            // define permutation permutation
-            bench_define_flush();
+            bench_define_permutation(p);
 
             cb(data, suite, case_);
         }
@@ -710,24 +745,31 @@ static void case_forperm(
         return;
     }
 
+    // deduplicate permutations with the same defines
+    //
+    // this can easily happen when overriding multiple case permutations,
+    // we can't tell that multiple case permutations don't change defines,
+    // duplicating results
     bench_seen_t seen = {NULL, 0, 0};
 
-    for (size_t k = 0; k < case_->permutations; k++) {
+    for (size_t k = 0;
-        // define permutation
+            k < (case_->permutations ? case_->permutations : 1);
-        bench_define_perm(suite, case_, k);
+            k++) {
+        // define case permutation
+        bench_define_case(suite, case_, k);
 
-        for (size_t v = 0; v < bench_override_define_permutations; v++) {
+        for (size_t g = 0; g < bench_geometry_count; g++) {
-            // define override permutation
+            // define geometry
-            bench_define_override(v);
+            bench_define_geometry(&bench_geometries[g]);
 
-            for (size_t g = 0; g < bench_geometry_count; g++) {
+            size_t permutations = bench_define_permutationpermutations();
-                // define geometry
+            for (size_t p = 0; p < permutations; p++) {
-                bench_define_geometry(&bench_geometries[g]);
+                // define permutation permutation
-                bench_define_flush();
+                bench_define_permutation(p);
 
                 // have we seen this permutation before?
-                bool was_seen = bench_seen_insert(&seen, suite, case_);
+                bool was_seen = bench_seen_insert(&seen);
-                if (!(k == 0 && v == 0 && g == 0) && was_seen) {
+                if (!(k == 0 && g == 0 && p == 0) && was_seen) {
                     continue;
                 }
 
@@ -1070,10 +1112,7 @@ void perm_list_defines(
     (void)case_;
 
     // collect defines
-    for (size_t d = 0;
+    for (size_t d = 0; d < bench_define_count; d++) {
-            d < lfs_max(suite->define_count,
-                BENCH_IMPLICIT_DEFINE_COUNT);
-            d++) {
         if (d < BENCH_IMPLICIT_DEFINE_COUNT
                 || bench_define_ispermutation(d)) {
             list_defines_add(defines, d);
@@ -1090,10 +1129,7 @@ void perm_list_permutation_defines(
     (void)case_;
 
     // collect permutation_defines
-    for (size_t d = 0;
+    for (size_t d = 0; d < bench_define_count; d++) {
-            d < lfs_max(suite->define_count,
-                BENCH_IMPLICIT_DEFINE_COUNT);
-            d++) {
         if (bench_define_ispermutation(d)) {
             list_defines_add(defines, d);
         }
@@ -1205,7 +1241,7 @@ static void list_implicit_defines(void) {
     extern const bench_geometry_t builtin_geometries[];
     for (size_t g = 0; builtin_geometries[g].name; g++) {
         bench_define_geometry(&builtin_geometries[g]);
-        bench_define_flush();
+        bench_define_permutation(0);
 
         // add implicit defines
         for (size_t d = 0; d < BENCH_IMPLICIT_DEFINE_COUNT; d++) {
@@ -1269,7 +1305,7 @@ static void list_geometries(void) {
             name_width, "geometry", "read", "prog", "erase", "count", "size");
     for (size_t g = 0; builtin_geometries[g].name; g++) {
         bench_define_geometry(&builtin_geometries[g]);
-        bench_define_flush();
+        bench_define_permutation(0);
         printf("%-*s  %7ju %7ju %7ju %7ju %11ju\n",
                 name_width,
                 builtin_geometries[g].name,
@@ -1595,9 +1631,10 @@ int main(int argc, char **argv) {
 
                 // parse comma-separated permutations
                 {
-                    override->defines = NULL;
+                    bench_override_value_t *override_values = NULL;
-                    override->permutations = 0;
+                    size_t override_value_count = 0;
-                    size_t override_capacity = 0;
+                    size_t override_value_capacity = 0;
+                    size_t override_permutations = 0;
                     while (true) {
                         optarg += strspn(optarg, " ");
 
@@ -1660,31 +1697,43 @@ int main(int argc, char **argv) {
                             }
                             optarg += 1;
 
-                            // calculate the range of values
+                            // append range
-                            assert(step != 0);
+                            *(bench_override_value_t*)mappend(
-                            for (intmax_t i = start;
+                                    (void**)&override_values,
-                                    (step < 0)
+                                    sizeof(bench_override_value_t),
-                                        ? i > stop
+                                    &override_value_count,
-                                        : (uintmax_t)i < (uintmax_t)stop;
+                                    &override_value_capacity)
-                                    i += step) {
+                                    = (bench_override_value_t){
-                                *(intmax_t*)mappend(
+                                .start = start,
-                                        (void**)&override->defines,
+                                .stop = stop,
-                                        sizeof(intmax_t),
+                                .step = step,
-                                        &override->permutations,
+                            };
-                                        &override_capacity) = i;
+                            if (step > 0) {
+                                override_permutations += (stop-1 - start)
+                                        / step + 1;
+                            } else {
+                                override_permutations += (start-1 - stop)
+                                        / -step + 1;
                             }
                         } else if (*optarg != '\0') {
                             // single value
-                            intmax_t define = strtoimax(optarg, &parsed, 0);
+                            intmax_t define = strtoumax(optarg, &parsed, 0);
                             if (parsed == optarg) {
                                 goto invalid_define;
                             }
                             optarg = parsed + strspn(parsed, " ");
-                            *(intmax_t*)mappend(
+
-                                    (void**)&override->defines,
+                            // append value
-                                    sizeof(intmax_t),
+                            *(bench_override_value_t*)mappend(
-                                    &override->permutations,
+                                    (void**)&override_values,
-                                    &override_capacity) = define;
+                                    sizeof(bench_override_value_t),
+                                    &override_value_count,
+                                    &override_value_capacity)
+                                    = (bench_override_value_t){
+                                .start = define,
+                                .step = 0,
+                            };
+                            override_permutations += 1;
                         } else {
                             break;
                         }
@@ -1693,8 +1742,17 @@ int main(int argc, char **argv) {
                             optarg += 1;
                         }
                     }
+
+                    override->define.cb = bench_override_cb;
+                    override->define.data = malloc(
+                            sizeof(bench_override_data_t));
+                    *(bench_override_data_t*)override->define.data
+                            = (bench_override_data_t){
+                        .values = override_values,
+                        .value_count = override_value_count,
+                    };
+                    override->define.permutations = override_permutations;
                 }
-                assert(override->permutations > 0);
                 break;
 
 invalid_define:
@@ -2031,7 +2089,7 @@ getopt_done: ;
     bench_define_cleanup();
     if (bench_overrides) {
         for (size_t i = 0; i < bench_override_count; i++) {
-            free((void*)bench_overrides[i].defines);
+            free((void*)bench_overrides[i].define.data);
         }
         free((void*)bench_overrides);
     }

runners/bench_runner.h

@@ -45,17 +45,18 @@ enum bench_flags {
 typedef uint8_t bench_flags_t;
 
 typedef struct bench_define {
-    intmax_t (*cb)(void *data);
+    intmax_t (*cb)(void *data, size_t i);
     void *data;
+    size_t permutations;
 } bench_define_t;
 
 struct bench_case {
     const char *name;
     const char *path;
     bench_flags_t flags;
-    size_t permutations;
 
     const bench_define_t *defines;
+    size_t permutations;
 
     bool (*filter)(void);
     void (*run)(struct lfs_config *cfg);

runners/test_runner.c

@@ -155,17 +155,19 @@ typedef struct test_define_names {
     size_t count;
 } test_define_names_t;
 
-intmax_t test_define_lit(void *data) {
+intmax_t test_define_lit(void *data, size_t i) {
+    (void)i;
     return (intptr_t)data;
 }
 
-#define TEST_CONST(x) {test_define_lit, (void*)(uintptr_t)(x)}
+#define TEST_CONST(x) {test_define_lit, (void*)(uintptr_t)(x), 1}
 #define TEST_LIT(x) ((test_define_t)TEST_CONST(x))
 
 
 #define TEST_DEF(k, v) \
-    intmax_t test_define_##k(void *data) { \
+    intmax_t test_define_##k(void *data, size_t i) { \
         (void)data; \
+        (void)i; \
         return v; \
     }
 
@@ -174,7 +176,7 @@ intmax_t test_define_lit(void *data) {
 
 #define TEST_DEFINE_MAP_OVERRIDE    0
 #define TEST_DEFINE_MAP_EXPLICIT    1
-#define TEST_DEFINE_MAP_PERMUTATION 2
+#define TEST_DEFINE_MAP_CASE        2
 #define TEST_DEFINE_MAP_GEOMETRY    3
 #define TEST_DEFINE_MAP_IMPLICIT    4
 #define TEST_DEFINE_MAP_COUNT       5
@@ -183,7 +185,7 @@ test_define_map_t test_define_maps[TEST_DEFINE_MAP_COUNT] = {
     [TEST_DEFINE_MAP_IMPLICIT] = {
         (const test_define_t[TEST_IMPLICIT_DEFINE_COUNT]) {
             #define TEST_DEF(k, v) \
-                [k##_i] = {test_define_##k, NULL},
+                [k##_i] = {test_define_##k, NULL, 1},
 
                 TEST_IMPLICIT_DEFINES
             #undef TEST_DEF
@@ -209,9 +211,20 @@ test_define_names_t test_define_names[TEST_DEFINE_NAMES_COUNT] = {
     },
 };
 
-intmax_t *test_define_cache;
+size_t test_define_count;
-size_t test_define_cache_count;
+
-unsigned *test_define_cache_mask;
+typedef struct test_define_cache_entry {
+    // - >=0 => not cached
+    // - -1  => cached
+    ssize_t permutation;
+    union {
+        intmax_t value;
+        const test_define_t *define;
+    } u;
+} test_define_cache_entry_t;
+
+test_define_cache_entry_t *test_define_cache;
+size_t test_define_cache_capacity;
 
 const char *test_define_name(size_t define) {
     // lookup in our test names
@@ -238,47 +251,109 @@ bool test_define_ispermutation(size_t define) {
     return false;
 }
 
-intmax_t test_define(size_t define) {
+size_t test_define_permutations(size_t define) {
-    // is the define in our cache?
-    if (define < test_define_cache_count
-            && (test_define_cache_mask[define/(8*sizeof(unsigned))]
-                & (1 << (define%(8*sizeof(unsigned)))))) {
-        return test_define_cache[define];
-    }
-
-    // lookup in our test defines
     for (size_t i = 0; i < TEST_DEFINE_MAP_COUNT; i++) {
         if (define < test_define_maps[i].count
                 && test_define_maps[i].defines[define].cb) {
-            intmax_t v = test_define_maps[i].defines[define].cb(
+            return test_define_maps[i].defines[define].permutations
-                    test_define_maps[i].defines[define].data);
+                    ? test_define_maps[i].defines[define].permutations
-
+                    : 1;
-            // insert into cache!
-            test_define_cache[define] = v;
-            test_define_cache_mask[define / (8*sizeof(unsigned))]
-                    |= 1 << (define%(8*sizeof(unsigned)));
-
-            return v;
         }
     }
 
     return 0;
-
-    // not found?
-    const char *name = test_define_name(define);
-    fprintf(stderr, "error: undefined define %s (%zd)\n",
-            name ? name : "(unknown)",
-            define);
-    assert(false);
-    exit(-1);
 }
 
-void test_define_flush(void) {
+size_t test_define_permutationpermutations(void) {
-    // clear cache between permutations
+    size_t prod = 1;
-    memset(test_define_cache_mask, 0,
+    for (size_t d = 0; d < test_define_count; d++) {
-            sizeof(unsigned)*(
+        size_t permutations = test_define_permutations(d);
-                (test_define_cache_count+(8*sizeof(unsigned))-1)
+        if (permutations > 0) {
-                / (8*sizeof(unsigned))));
+            prod *= permutations;
+        }
+    }
+    return prod;
+}
+
+intmax_t test_define(size_t define) {
+    // cached?
+    if (test_define_cache[define].permutation == -1) {
+        return test_define_cache[define].u.value;
+
+    // lazily defined?
+    } else if (test_define_cache[define].u.define) {
+        // evaluate and store in cache
+        test_define_cache[define].u.value
+                = test_define_cache[define].u.define->cb(
+                    test_define_cache[define].u.define->data,
+                    test_define_cache[define].permutation);
+        test_define_cache[define].permutation = -1;
+        return test_define_cache[define].u.value;
+
+    // not defined?
+    } else {
+        const char *name = test_define_name(define);
+        fprintf(stderr, "error: undefined define %s (%zd)\n",
+                name ? name : "(unknown)",
+                define);
+        assert(false);
+        exit(-1);
+    }
+}
+
+// permutation updates
+void test_define_permutation(size_t perm) {
+    // We can't completely precompute the defines easily, since they may be
+    // mutually recursive. But we can precompute the permutations, which is
+    // expensive otherwise.
+    //
+    // Note that it's not really worth it to make define lookup completely
+    // lazy, the first thing we do is evaluate all defines for 1. deduplication
+    // and 2. logging.
+
+    if (test_define_cache_capacity < test_define_count) {
+        // align to power of two to avoid any superlinear growth
+        test_define_cache_capacity = 1 << lfs_npw2(test_define_count);
+        test_define_cache = realloc(
+                test_define_cache,
+                test_define_cache_capacity*sizeof(test_define_cache_entry_t));
+    }
+
+    for (size_t d = 0; d < test_define_count; d++) {
+        // lookup our test defines
+        for (size_t i = 0; i < TEST_DEFINE_MAP_COUNT; i++) {
+            if (d < test_define_maps[i].count
+                    && test_define_maps[i].defines[d].cb) {
+                // note we can't precompute these due to mutual recursion
+                const test_define_t *define = &test_define_maps[i].defines[d];
+                test_define_cache[d] = (test_define_cache_entry_t){
+                    .permutation = perm % define->permutations,
+                    .u.define = define,
+                };
+                perm /= test_define_maps[i].defines[d].permutations;
+                goto next;
+            }
+        }
+
+        // default to a null value, these should be unreachable
+        test_define_cache[d] = (test_define_cache_entry_t){0};
+    next:;
+    }
+}
+
+// case updates
+void test_define_case(
+        const struct test_suite *suite,
+        const struct test_case *case_,
+        size_t perm) {
+    if (case_->defines) {
+        test_define_maps[TEST_DEFINE_MAP_CASE] = (test_define_map_t){
+                &case_->defines[perm*suite->define_count],
+                suite->define_count};
+    } else {
+        test_define_maps[TEST_DEFINE_MAP_CASE] = (test_define_map_t){
+                NULL, 0};
+    }
 }
 
 // geometry updates
@@ -292,122 +367,96 @@ void test_define_geometry(const test_geometry_t *geometry) {
 // override updates
 typedef struct test_override {
     const char *name;
-    const intmax_t *defines;
+    test_define_t define;
-    size_t permutations;
 } test_override_t;
 
+typedef struct test_override_value {
+    intmax_t start;
+    intmax_t stop;
+    // step == 0 indicates a single value
+    intmax_t step;
+} test_override_value_t;
+
+typedef struct test_override_data {
+    test_override_value_t *values;
+    size_t value_count;
+} test_override_data_t;
+
+intmax_t test_override_cb(void *data, size_t i) {
+    const test_override_data_t *data_ = data;
+    for (size_t j = 0; j < data_->value_count; j++) {
+        const test_override_value_t *v = &data_->values[j];
+        // range?
+        if (v->step) {
+            size_t range_count;
+            if (v->step > 0) {
+                range_count = (v->stop-1 - v->start) / v->step + 1;
+            } else {
+                range_count = (v->start-1 - v->stop) / -v->step + 1;
+            }
+
+            if (i < range_count) {
+                return i*v->step + v->start;
+            }
+            i -= range_count;
+        // value?
+        } else {
+            if (i == 0) {
+                return v->start;
+            }
+            i -= 1;
+        }
+    }
+
+    // should never get here
+    assert(false);
+    __builtin_unreachable();
+}
+
 const test_override_t *test_overrides = NULL;
 size_t test_override_count = 0;
 
 test_define_t *test_override_defines = NULL;
-size_t test_override_define_count = 0;
-size_t test_override_define_permutations = 1;
 size_t test_override_define_capacity = 0;
 
 // suite/perm updates
 void test_define_suite(const struct test_suite *suite) {
+    // set define names
     test_define_names[TEST_DEFINE_NAMES_SUITE] = (test_define_names_t){
             suite->define_names, suite->define_count};
 
-    // make sure our cache is large enough
+    // set define count
-    if (lfs_max(suite->define_count, TEST_IMPLICIT_DEFINE_COUNT)
+    test_define_count = suite->define_count > TEST_IMPLICIT_DEFINE_COUNT
-            > test_define_cache_count) {
+            ? suite->define_count
-        // align to power of two to avoid any superlinear growth
+            : TEST_IMPLICIT_DEFINE_COUNT;
-        size_t ncount = 1 << lfs_npw2(
-                lfs_max(suite->define_count, TEST_IMPLICIT_DEFINE_COUNT));
-        test_define_cache = realloc(test_define_cache, ncount*sizeof(intmax_t));
-        test_define_cache_mask = realloc(test_define_cache_mask,
-                sizeof(unsigned)*(
-                    (ncount+(8*sizeof(unsigned))-1)
-                    / (8*sizeof(unsigned))));
-        test_define_cache_count = ncount;
-    }
 
     // map any overrides
     if (test_override_count > 0) {
-        // first figure out the total size of override permutations
+        if (test_define_count > test_override_define_capacity) {
-        size_t count = 0;
-        size_t permutations = 1;
-        for (size_t i = 0; i < test_override_count; i++) {
-            for (size_t d = 0;
-                    d < lfs_max(
-                        suite->define_count,
-                        TEST_IMPLICIT_DEFINE_COUNT);
-                    d++) {
-                // define name match?
-                const char *name = test_define_name(d);
-                if (name && strcmp(name, test_overrides[i].name) == 0) {
-                    count = lfs_max(count, d+1);
-                    permutations *= test_overrides[i].permutations;
-                    break;
-                }
-            }
-        }
-        test_override_define_count = count;
-        test_override_define_permutations = permutations;
-
-        // make sure our override arrays are big enough
-        if (count * permutations > test_override_define_capacity) {
             // align to power of two to avoid any superlinear growth
-            size_t ncapacity = 1 << lfs_npw2(count * permutations);
+            test_override_define_capacity = 1 << lfs_npw2(test_define_count);
             test_override_defines = realloc(
                     test_override_defines,
-                    sizeof(test_define_t)*ncapacity);
+                    test_override_define_capacity*sizeof(test_define_t));
-            test_override_define_capacity = ncapacity;
         }
 
-        // zero unoverridden defines
         memset(test_override_defines, 0,
-                sizeof(test_define_t) * count * permutations);
+                test_define_count*sizeof(test_define_t));
-
-        // compute permutations
-        size_t p = 1;
         for (size_t i = 0; i < test_override_count; i++) {
-            for (size_t d = 0;
+            for (size_t d = 0; d < test_define_count; d++) {
-                    d < lfs_max(
+                // name match?
-                        suite->define_count,
-                        TEST_IMPLICIT_DEFINE_COUNT);
-                    d++) {
-                // define name match?
                 const char *name = test_define_name(d);
                 if (name && strcmp(name, test_overrides[i].name) == 0) {
-                    // scatter the define permutations based on already
+                    test_override_defines[d] = test_overrides[i].define;
-                    // seen permutations
-                    for (size_t j = 0; j < permutations; j++) {
-                        test_override_defines[j*count + d] = TEST_LIT(
-                                test_overrides[i].defines[(j/p)
-                                    % test_overrides[i].permutations]);
-                    }
-
-                    // keep track of how many permutations we've seen so far
-                    p *= test_overrides[i].permutations;
-                    break;
                 }
             }
         }
-    }
-}
 
-void test_define_perm(
+        test_define_maps[TEST_DEFINE_MAP_OVERRIDE] = (test_define_map_t){
-        const struct test_suite *suite,
+                test_override_defines, test_define_count};
-        const struct test_case *case_,
-        size_t perm) {
-    if (case_->defines) {
-        test_define_maps[TEST_DEFINE_MAP_PERMUTATION] = (test_define_map_t){
-                case_->defines + perm*suite->define_count,
-                suite->define_count};
-    } else {
-        test_define_maps[TEST_DEFINE_MAP_PERMUTATION] = (test_define_map_t){
-                NULL, 0};
     }
 }
 
-void test_define_override(size_t perm) {
-    test_define_maps[TEST_DEFINE_MAP_OVERRIDE] = (test_define_map_t){
-            test_override_defines + perm*test_override_define_count,
-            test_override_define_count};
-}
-
 void test_define_explicit(
         const test_define_t *defines,
         size_t define_count) {
@@ -418,7 +467,6 @@ void test_define_explicit(
 void test_define_cleanup(void) {
     // test define management can allocate a few things
     free(test_define_cache);
-    free(test_define_cache_mask);
     free(test_override_defines);
 }
 
@@ -581,11 +629,7 @@ static void perm_printid(
     (void)suite;
     // case[:permutation[:powercycles]]
     printf("%s:", case_->name);
-    for (size_t d = 0;
+    for (size_t d = 0; d < test_define_count; d++) {
-            d < lfs_max(
-                suite->define_count,
-                TEST_IMPLICIT_DEFINE_COUNT);
-            d++) {
         if (test_define_ispermutation(d)) {
             leb16_print(d);
             leb16_print(TEST_DEFINE(d));
@@ -614,19 +658,10 @@ struct test_seen_branch {
     struct test_seen branch;
 };
 
-bool test_seen_insert(
+bool test_seen_insert(test_seen_t *seen) {
-        test_seen_t *seen,
-        const struct test_suite *suite,
-        const struct test_case *case_) {
-    (void)case_;
-    bool was_seen = true;
-
     // use the currently set defines
-    for (size_t d = 0;
+    bool was_seen = true;
-            d < lfs_max(
+    for (size_t d = 0; d < test_define_count; d++) {
-                suite->define_count,
-                TEST_IMPLICIT_DEFINE_COUNT);
-            d++) {
         // treat unpermuted defines the same as 0
         intmax_t define = test_define_ispermutation(d) ? TEST_DEFINE(d) : 0;
 
@@ -693,10 +728,10 @@ static void case_forperm(
     if (defines) {
         test_define_explicit(defines, define_count);
 
-        for (size_t v = 0; v < test_override_define_permutations; v++) {
+        size_t permutations = test_define_permutationpermutations();
-            // define override permutation
+        for (size_t p = 0; p < permutations; p++) {
-            test_define_override(v);
+            // define permutation permutation
-            test_define_flush();
+            test_define_permutation(p);
 
             // explicit powerloss cycles?
             if (cycles) {
@@ -720,24 +755,31 @@ static void case_forperm(
         return;
     }
 
+    // deduplicate permutations with the same defines
+    //
+    // this can easily happen when overriding multiple case permutations,
+    // we can't tell that multiple case permutations don't change defines,
+    // duplicating results
     test_seen_t seen = {NULL, 0, 0};
 
-    for (size_t k = 0; k < case_->permutations; k++) {
+    for (size_t k = 0;
-        // define permutation
+            k < (case_->permutations ? case_->permutations : 1);
-        test_define_perm(suite, case_, k);
+            k++) {
+        // define case permutation
+        test_define_case(suite, case_, k);
 
-        for (size_t v = 0; v < test_override_define_permutations; v++) {
+        for (size_t g = 0; g < test_geometry_count; g++) {
-            // define override permutation
+            // define geometry
-            test_define_override(v);
+            test_define_geometry(&test_geometries[g]);
 
-            for (size_t g = 0; g < test_geometry_count; g++) {
+            size_t permutations = test_define_permutationpermutations();
-                // define geometry
+            for (size_t p = 0; p < permutations; p++) {
-                test_define_geometry(&test_geometries[g]);
+                // define permutation permutation
-                test_define_flush();
+                test_define_permutation(p);
 
                 // have we seen this permutation before?
-                bool was_seen = test_seen_insert(&seen, suite, case_);
+                bool was_seen = test_seen_insert(&seen);
-                if (!(k == 0 && v == 0 && g == 0) && was_seen) {
+                if (!(k == 0 && g == 0 && p == 0) && was_seen) {
                     continue;
                 }
 
@@ -1105,10 +1147,7 @@ void perm_list_defines(
     (void)powerloss;
 
     // collect defines
-    for (size_t d = 0;
+    for (size_t d = 0; d < test_define_count; d++) {
-            d < lfs_max(suite->define_count,
-                TEST_IMPLICIT_DEFINE_COUNT);
-            d++) {
         if (d < TEST_IMPLICIT_DEFINE_COUNT
                 || test_define_ispermutation(d)) {
             list_defines_add(defines, d);
@@ -1127,10 +1166,7 @@ void perm_list_permutation_defines(
     (void)powerloss;
 
     // collect permutation_defines
-    for (size_t d = 0;
+    for (size_t d = 0; d < test_define_count; d++) {
-            d < lfs_max(suite->define_count,
-                TEST_IMPLICIT_DEFINE_COUNT);
-            d++) {
         if (test_define_ispermutation(d)) {
             list_defines_add(defines, d);
         }
@@ -1246,7 +1282,7 @@ static void list_implicit_defines(void) {
     extern const test_geometry_t builtin_geometries[];
     for (size_t g = 0; builtin_geometries[g].name; g++) {
         test_define_geometry(&builtin_geometries[g]);
-        test_define_flush();
+        test_define_permutation(0);
 
         // add implicit defines
         for (size_t d = 0; d < TEST_IMPLICIT_DEFINE_COUNT; d++) {
@@ -1310,7 +1346,7 @@ static void list_geometries(void) {
             name_width, "geometry", "read", "prog", "erase", "count", "size");
     for (size_t g = 0; builtin_geometries[g].name; g++) {
         test_define_geometry(&builtin_geometries[g]);
-        test_define_flush();
+        test_define_permutation(0);
         printf("%-*s  %7ju %7ju %7ju %7ju %11ju\n",
                 name_width,
                 builtin_geometries[g].name,
@@ -2133,9 +2169,10 @@ int main(int argc, char **argv) {
 
                 // parse comma-separated permutations
                 {
-                    override->defines = NULL;
+                    test_override_value_t *override_values = NULL;
-                    override->permutations = 0;
+                    size_t override_value_count = 0;
-                    size_t override_capacity = 0;
+                    size_t override_value_capacity = 0;
+                    size_t override_permutations = 0;
                     while (true) {
                         optarg += strspn(optarg, " ");
 
@@ -2198,31 +2235,43 @@ int main(int argc, char **argv) {
                             }
                             optarg += 1;
 
-                            // calculate the range of values
+                            // append range
-                            assert(step != 0);
+                            *(test_override_value_t*)mappend(
-                            for (intmax_t i = start;
+                                    (void**)&override_values,
-                                    (step < 0)
+                                    sizeof(test_override_value_t),
-                                        ? i > stop
+                                    &override_value_count,
-                                        : (uintmax_t)i < (uintmax_t)stop;
+                                    &override_value_capacity)
-                                    i += step) {
+                                    = (test_override_value_t){
-                                *(intmax_t*)mappend(
+                                .start = start,
-                                        (void**)&override->defines,
+                                .stop = stop,
-                                        sizeof(intmax_t),
+                                .step = step,
-                                        &override->permutations,
+                            };
-                                        &override_capacity) = i;
+                            if (step > 0) {
+                                override_permutations += (stop-1 - start)
+                                        / step + 1;
+                            } else {
+                                override_permutations += (start-1 - stop)
+                                        / -step + 1;
                             }
                         } else if (*optarg != '\0') {
                             // single value
-                            intmax_t define = strtoimax(optarg, &parsed, 0);
+                            intmax_t define = strtoumax(optarg, &parsed, 0);
                             if (parsed == optarg) {
                                 goto invalid_define;
                             }
                             optarg = parsed + strspn(parsed, " ");
-                            *(intmax_t*)mappend(
+
-                                    (void**)&override->defines,
+                            // append value
-                                    sizeof(intmax_t),
+                            *(test_override_value_t*)mappend(
-                                    &override->permutations,
+                                    (void**)&override_values,
-                                    &override_capacity) = define;
+                                    sizeof(test_override_value_t),
+                                    &override_value_count,
+                                    &override_value_capacity)
+                                    = (test_override_value_t){
+                                .start = define,
+                                .step = 0,
+                            };
+                            override_permutations += 1;
                         } else {
                             break;
                         }
@@ -2231,8 +2280,17 @@ int main(int argc, char **argv) {
                             optarg += 1;
                         }
                     }
+
+                    override->define.cb = test_override_cb;
+                    override->define.data = malloc(
+                            sizeof(test_override_data_t));
+                    *(test_override_data_t*)override->define.data
+                            = (test_override_data_t){
+                        .values = override_values,
+                        .value_count = override_value_count,
+                    };
+                    override->define.permutations = override_permutations;
                 }
-                assert(override->permutations > 0);
                 break;
 
 invalid_define:
@@ -2736,7 +2794,7 @@ getopt_done: ;
     test_define_cleanup();
     if (test_overrides) {
         for (size_t i = 0; i < test_override_count; i++) {
-            free((void*)test_overrides[i].defines);
+            free((void*)test_overrides[i].define.data);
         }
         free((void*)test_overrides);
     }

runners/test_runner.h

@@ -38,17 +38,18 @@ enum test_flags {
 typedef uint8_t test_flags_t;
 
 typedef struct test_define {
-    intmax_t (*cb)(void *data);
+    intmax_t (*cb)(void *data, size_t i);
     void *data;
+    size_t permutations;
 } test_define_t;
 
 struct test_case {
     const char *name;
     const char *path;
     test_flags_t flags;
-    size_t permutations;
 
     const test_define_t *defines;
+    size_t permutations;
 
     bool (*filter)(void);
     void (*run)(struct lfs_config *cfg);

scripts/bench.py

@@ -90,13 +90,13 @@ class BenchCase:
         def parse_define(v):
             # a define entry can be a list
             if isinstance(v, list):
-                for v_ in v:
+                return sum((parse_define(v_) for v_ in v), [])
-                    yield from parse_define(v_)
             # or a string
             elif isinstance(v, str):
                 # which can be comma-separated values, with optional
                 # range statements. This matches the runtime define parser in
                 # the runner itself.
+                vs = []
                 for v_ in csplit(v):
                     m = re.search(r'\brange\b\s*\('
                         '(?P<start>[^,\s]*)'
@@ -112,25 +112,24 @@ class BenchCase:
                             if m.group('step') else 1)
                         if m.lastindex <= 1:
                             start, stop = 0, start
-                        for x in range(start, stop, step):
+                        vs.append(range(start, stop, step))
-                            yield from parse_define('%s(%d)%s' % (
-                                v_[:m.start()], x, v_[m.end():]))
                     else:
-                        yield v_
+                        vs.append(v_)
+                return vs
             # or a literal value
             elif isinstance(v, bool):
-                yield 'true' if v else 'false'
+                return ['true' if v else 'false']
             else:
-                yield v
+                return [v]
 
         # build possible permutations
         for suite_defines_ in suite_defines:
             self.defines |= suite_defines_.keys()
             for defines_ in defines:
                 self.defines |= defines_.keys()
-                self.permutations.extend(dict(perm) for perm in it.product(*(
+                self.permutations.append({
-                    [(k, v) for v in parse_define(vs)]
+                    k: parse_define(v)
-                    for k, vs in sorted((suite_defines_ | defines_).items()))))
+                    for k, v in (suite_defines_ | defines_).items()})
 
         for k in config.keys():
             print('%swarning:%s in %s, found unused key %r' % (
@@ -308,34 +307,32 @@ def compile(bench_paths, **args):
             # the bench defines
             def write_case_functions(f, suite, case):
                     # create case define functions
-                    if case.defines:
+                    for i, permutation in enumerate(case.permutations):
-                        # deduplicate defines by value to try to reduce the
+                        for k, vs in sorted(permutation.items()):
-                        # number of functions we generate
+                            f.writeln('intmax_t __bench__%s__%s__%d('
-                        define_cbs = {}
+                                '__attribute__((unused)) void *data, '
-                        for i, defines in enumerate(case.permutations):
+                                'size_t i) {'
-                            for k, v in sorted(defines.items()):
+                                % (case.name, k, i))
-                                if v not in define_cbs:
+                            j = 0
-                                    name = ('__bench__%s__%s__%d'
+                            for v in vs:
-                                        % (case.name, k, i))
+                                # generate range
-                                    define_cbs[v] = name
+                                if isinstance(v, range):
-                                    f.writeln('intmax_t %s('
+                                    f.writeln(
-                                        '__attribute__((unused)) '
+                                        4*' '+'if (i < %d) '
-                                        'void *data) {' % name)
+                                        'return (i-%d)*%d + %d;'
-                                    f.writeln(4*' '+'return %s;' % v)
+                                        % (j+len(v), j, v.step, v.start))
-                                    f.writeln('}')
+                                    j += len(v)
-                                    f.writeln()
+                                # translate index to define
-                        f.writeln('const bench_define_t '
+                                else:
-                            '__bench__%s__defines[]['
+                                    f.writeln(
-                            'BENCH_IMPLICIT_DEFINE_COUNT+%d] = {'
+                                        4*' '+'if (i == %d) '
-                            % (case.name, len(suite.defines)))
+                                        'return %s;'
-                        for defines in case.permutations:
+                                        % (j, v))
-                            f.writeln(4*' '+'{')
+                                    j += 1;
-                            for k, v in sorted(defines.items()):
+
-                                f.writeln(8*' '+'[%-24s] = {%s, NULL},' % (
+                            f.writeln(4*' '+'__builtin_unreachable();')
-                                    k+'_i', define_cbs[v]))
+                            f.writeln('}')
-                            f.writeln(4*' '+'},')
+                            f.writeln()
-                        f.writeln('};')
-                        f.writeln()
 
                     # create case filter function
                     if suite.if_ is not None or case.if_ is not None:
@@ -389,11 +386,11 @@ def compile(bench_paths, **args):
                     if case.in_ is None:
                         write_case_functions(f, suite, case)
                     else:
-                        if case.defines:
+                        for i, permutation in enumerate(case.permutations):
-                            f.writeln('extern const bench_define_t '
+                            for k, vs in sorted(permutation.items()):
-                                '__bench__%s__defines[]['
+                                f.writeln('extern intmax_t __bench__%s__%s__%d('
-                                'BENCH_IMPLICIT_DEFINE_COUNT+%d];'
+                                    'void *data, size_t i);'
-                                % (case.name, len(suite.defines)))
+                                    % (case.name, k, i))
                         if suite.if_ is not None or case.if_ is not None:
                             f.writeln('extern bool __bench__%s__filter('
                                 'void);'
@@ -418,13 +415,14 @@ def compile(bench_paths, **args):
                 if suite.defines:
                     # create suite define names
                     f.writeln(4*' '+'.define_names = (const char *const['
-                        'BENCH_IMPLICIT_DEFINE_COUNT+%d]){' % (
+                        'BENCH_IMPLICIT_DEFINE_COUNT+%d]){'
-                        len(suite.defines)))
+                        % (len(suite.defines)))
                     for k in sorted(suite.defines):
                         f.writeln(8*' '+'[%-24s] = "%s",' % (k+'_i', k))
                     f.writeln(4*' '+'},')
                     f.writeln(4*' '+'.define_count = '
-                        'BENCH_IMPLICIT_DEFINE_COUNT+%d,' % len(suite.defines))
+                        'BENCH_IMPLICIT_DEFINE_COUNT+%d,'
+                        % len(suite.defines))
                 if suite.cases:
                     f.writeln(4*' '+'.cases = (const struct bench_case[]){')
                     for case in suite.cases:
@@ -433,12 +431,26 @@ def compile(bench_paths, **args):
                         f.writeln(12*' '+'.name = "%s",' % case.name)
                         f.writeln(12*' '+'.path = "%s",' % case.path)
                         f.writeln(12*' '+'.flags = 0,')
-                        f.writeln(12*' '+'.permutations = %d,'
-                            % len(case.permutations))
                         if case.defines:
-                            f.writeln(12*' '+'.defines '
+                            f.writeln(12*' '+'.defines = '
-                                '= (const bench_define_t*)__bench__%s__defines,'
+                                '(const bench_define_t*)(const bench_define_t[]['
-                                % (case.name))
+                                'BENCH_IMPLICIT_DEFINE_COUNT+%d]){'
+                                % (len(suite.defines)))
+                            for i, permutation in enumerate(case.permutations):
+                                f.writeln(16*' '+'{')
+                                for k, vs in sorted(permutation.items()):
+                                    f.writeln(20*' '
+                                        +'[%-24s] = {__bench__%s__%s__%d, NULL, '
+                                        '%d},'
+                                        % (k+'_i', case.name, k, i,
+                                            sum(len(v)
+                                                if isinstance(v, range)
+                                                else 1
+                                            for v in vs)))
+                                f.writeln(16*' '+'},')
+                            f.writeln(12*' '+'},')
+                            f.writeln(12*' '+'.permutations = %d,'
+                                % len(case.permutations))
                         if suite.if_ is not None or case.if_ is not None:
                             f.writeln(12*' '+'.filter = __bench__%s__filter,'
                                 % (case.name))
@@ -457,7 +469,7 @@ def compile(bench_paths, **args):
                     shutil.copyfileobj(sf, f)
                 f.writeln()
 
-                # write any internal benches
+                # write any internal benchs
                 for suite in suites:
                     for case in suite.cases:
                         if (case.in_ is not None

scripts/test.py

@@ -93,13 +93,13 @@ class TestCase:
         def parse_define(v):
             # a define entry can be a list
             if isinstance(v, list):
-                for v_ in v:
+                return sum((parse_define(v_) for v_ in v), [])
-                    yield from parse_define(v_)
             # or a string
             elif isinstance(v, str):
                 # which can be comma-separated values, with optional
                 # range statements. This matches the runtime define parser in
                 # the runner itself.
+                vs = []
                 for v_ in csplit(v):
                     m = re.search(r'\brange\b\s*\('
                         '(?P<start>[^,\s]*)'
@@ -115,25 +115,24 @@ class TestCase:
                             if m.group('step') else 1)
                         if m.lastindex <= 1:
                             start, stop = 0, start
-                        for x in range(start, stop, step):
+                        vs.append(range(start, stop, step))
-                            yield from parse_define('%s(%d)%s' % (
-                                v_[:m.start()], x, v_[m.end():]))
                     else:
-                        yield v_
+                        vs.append(v_)
+                return vs
             # or a literal value
             elif isinstance(v, bool):
-                yield 'true' if v else 'false'
+                return ['true' if v else 'false']
             else:
-                yield v
+                return [v]
 
         # build possible permutations
         for suite_defines_ in suite_defines:
             self.defines |= suite_defines_.keys()
             for defines_ in defines:
                 self.defines |= defines_.keys()
-                self.permutations.extend(dict(perm) for perm in it.product(*(
+                self.permutations.append({
-                    [(k, v) for v in parse_define(vs)]
+                    k: parse_define(v)
-                    for k, vs in sorted((suite_defines_ | defines_).items()))))
+                    for k, v in (suite_defines_ | defines_).items()})
 
         for k in config.keys():
             print('%swarning:%s in %s, found unused key %r' % (
@@ -316,34 +315,32 @@ def compile(test_paths, **args):
             # the test defines
             def write_case_functions(f, suite, case):
                     # create case define functions
-                    if case.defines:
+                    for i, permutation in enumerate(case.permutations):
-                        # deduplicate defines by value to try to reduce the
+                        for k, vs in sorted(permutation.items()):
-                        # number of functions we generate
+                            f.writeln('intmax_t __test__%s__%s__%d('
-                        define_cbs = {}
+                                '__attribute__((unused)) void *data, '
-                        for i, defines in enumerate(case.permutations):
+                                'size_t i) {'
-                            for k, v in sorted(defines.items()):
+                                % (case.name, k, i))
-                                if v not in define_cbs:
+                            j = 0
-                                    name = ('__test__%s__%s__%d'
+                            for v in vs:
-                                        % (case.name, k, i))
+                                # generate range
-                                    define_cbs[v] = name
+                                if isinstance(v, range):
-                                    f.writeln('intmax_t %s('
+                                    f.writeln(
-                                        '__attribute__((unused)) '
+                                        4*' '+'if (i < %d) '
-                                        'void *data) {' % name)
+                                        'return (i-%d)*%d + %d;'
-                                    f.writeln(4*' '+'return %s;' % v)
+                                        % (j+len(v), j, v.step, v.start))
-                                    f.writeln('}')
+                                    j += len(v)
-                                    f.writeln()
+                                # translate index to define
-                        f.writeln('const test_define_t '
+                                else:
-                            '__test__%s__defines[]['
+                                    f.writeln(
-                            'TEST_IMPLICIT_DEFINE_COUNT+%d] = {'
+                                        4*' '+'if (i == %d) '
-                            % (case.name, len(suite.defines)))
+                                        'return %s;'
-                        for defines in case.permutations:
+                                        % (j, v))
-                            f.writeln(4*' '+'{')
+                                    j += 1;
-                            for k, v in sorted(defines.items()):
+
-                                f.writeln(8*' '+'[%-24s] = {%s, NULL},' % (
+                            f.writeln(4*' '+'__builtin_unreachable();')
-                                    k+'_i', define_cbs[v]))
+                            f.writeln('}')
-                            f.writeln(4*' '+'},')
+                            f.writeln()
-                        f.writeln('};')
-                        f.writeln()
 
                     # create case filter function
                     if suite.if_ is not None or case.if_ is not None:
@@ -397,11 +394,11 @@ def compile(test_paths, **args):
                     if case.in_ is None:
                         write_case_functions(f, suite, case)
                     else:
-                        if case.defines:
+                        for i, permutation in enumerate(case.permutations):
-                            f.writeln('extern const test_define_t '
+                            for k, vs in sorted(permutation.items()):
-                                '__test__%s__defines[]['
+                                f.writeln('extern intmax_t __test__%s__%s__%d('
-                                'TEST_IMPLICIT_DEFINE_COUNT+%d];'
+                                    'void *data, size_t i);'
-                                % (case.name, len(suite.defines)))
+                                    % (case.name, k, i))
                         if suite.if_ is not None or case.if_ is not None:
                             f.writeln('extern bool __test__%s__filter('
                                 'void);'
@@ -429,13 +426,14 @@ def compile(test_paths, **args):
                 if suite.defines:
                     # create suite define names
                     f.writeln(4*' '+'.define_names = (const char *const['
-                        'TEST_IMPLICIT_DEFINE_COUNT+%d]){' % (
+                        'TEST_IMPLICIT_DEFINE_COUNT+%d]){'
-                        len(suite.defines)))
+                        % (len(suite.defines)))
                     for k in sorted(suite.defines):
                         f.writeln(8*' '+'[%-24s] = "%s",' % (k+'_i', k))
                     f.writeln(4*' '+'},')
                     f.writeln(4*' '+'.define_count = '
-                        'TEST_IMPLICIT_DEFINE_COUNT+%d,' % len(suite.defines))
+                        'TEST_IMPLICIT_DEFINE_COUNT+%d,'
+                        % len(suite.defines))
                 if suite.cases:
                     f.writeln(4*' '+'.cases = (const struct test_case[]){')
                     for case in suite.cases:
@@ -447,12 +445,26 @@ def compile(test_paths, **args):
                             % (' | '.join(filter(None, [
                                 'TEST_REENTRANT' if case.reentrant else None]))
                                 or 0))
-                        f.writeln(12*' '+'.permutations = %d,'
-                            % len(case.permutations))
                         if case.defines:
-                            f.writeln(12*' '+'.defines '
+                            f.writeln(12*' '+'.defines = '
-                                '= (const test_define_t*)__test__%s__defines,'
+                                '(const test_define_t*)(const test_define_t[]['
-                                % (case.name))
+                                'TEST_IMPLICIT_DEFINE_COUNT+%d]){'
+                                % (len(suite.defines)))
+                            for i, permutation in enumerate(case.permutations):
+                                f.writeln(16*' '+'{')
+                                for k, vs in sorted(permutation.items()):
+                                    f.writeln(20*' '
+                                        +'[%-24s] = {__test__%s__%s__%d, NULL, '
+                                        '%d},'
+                                        % (k+'_i', case.name, k, i,
+                                            sum(len(v)
+                                                if isinstance(v, range)
+                                                else 1
+                                            for v in vs)))
+                                f.writeln(16*' '+'},')
+                            f.writeln(12*' '+'},')
+                            f.writeln(12*' '+'.permutations = %d,'
+                                % len(case.permutations))
                         if suite.if_ is not None or case.if_ is not None:
                             f.writeln(12*' '+'.filter = __test__%s__filter,'
                                 % (case.name))

tests/test_btree.toml

@@ -2459,7 +2459,7 @@ code = '''
         memset(sim, 0, N);
 
         uint32_t prng = seed;
-        for (lfs_size_t i = 0; i < (N-REMAINING); i++) {
+        for (lfs_size_t i = 0; i < N; i++) {
             // choose a pseudo-random op
             uint8_t op = TEST_PRNG(&prng) % 3;
             // choose a pseudo-random id
@@ -2573,7 +2573,7 @@ code = '''
         memset(sim_weights, 0, N*sizeof(lfs_size_t));
 
         uint32_t prng = seed;
-        for (lfs_size_t i = 0; i < (N-REMAINING); i++) {
+        for (lfs_size_t i = 0; i < N; i++) {
             // choose a pseudo-random op
             uint8_t op = TEST_PRNG(&prng) % 3;
             // choose a pseudo-random id