Some more minor improvements to the test_runner

- Indirect index map instead of bitmap+sparse array
- test_define_t and test_type_t
- Added back conditional filtering
- Added suite-level defines and filtering

runners/test_runner.c

@@ -8,12 +8,12 @@
 // disk geometries
 struct test_geometry {
     const char *name;
-    const uintmax_t *defines;
+    const test_define_t *defines;
 };
 
 // Note this includes the default configuration for test pre-defines
 #define TEST_GEOMETRY(name, read, prog, erase, count) \
-    {name, (const uintmax_t[]){ \
+    {name, (const test_define_t[]){ \
         /* READ_SIZE */         read, \
         /* PROG_SIZE */         prog, \
         /* BLOCK_SIZE */        erase, \
@@ -46,14 +46,14 @@ const size_t test_geometry_count = (
 
 
 // test define lookup and management
-const uintmax_t *test_defines[3] = {NULL};
+const test_define_t *test_defines[3] = {NULL};
-const bool *test_define_masks[2] = {NULL};
+const uint8_t *test_define_maps[2] = {NULL};
 
-uintmax_t test_define(size_t define) {
+test_define_t test_define(size_t define) {
-    if (test_define_masks[0] && test_define_masks[0][define]) {
+    if (test_define_maps[0] && test_define_maps[0][define] != 0xff) {
-        return test_defines[0][define];
+        return test_defines[0][test_define_maps[0][define]];
-    } else if (test_define_masks[1] && test_define_masks[1][define]) {
+    } else if (test_define_maps[1] && test_define_maps[1][define] != 0xff) {
-        return test_defines[1][define];
+        return test_defines[1][test_define_maps[1][define]];
     } else {
         return test_defines[2][define];
     }
@@ -70,61 +70,54 @@ void test_define_geometry(const struct test_geometry *geometry) {
 void test_define_case(const struct test_case *case_, size_t perm) {
     if (case_ && case_->defines) {
         test_defines[1] = case_->defines[perm];
-        test_define_masks[1] = case_->define_mask;
+        test_define_maps[1] = case_->define_map;
     } else {
         test_defines[1] = NULL;
-        test_define_masks[1] = NULL;
+        test_define_maps[1] = NULL;
     }
 }
 
-struct override {
-    const char *name;
-    uintmax_t override;
-};
-
 void test_define_overrides(
         const struct test_suite *suite,
-        const struct override *overrides,
+        const char *const *override_names,
+        const test_define_t *override_defines,
         size_t override_count) {
-    if (overrides && override_count > 0) {
+    if (override_names && override_defines && override_count > 0) {
-        uintmax_t *defines = malloc(suite->define_count * sizeof(uintmax_t));
+        uint8_t *define_map = malloc(suite->define_count * sizeof(uint8_t));
-        memset(defines, 0, suite->define_count * sizeof(uintmax_t));
+        memset(define_map, 0xff, suite->define_count * sizeof(bool));
-        bool *define_mask = malloc(suite->define_count * sizeof(bool));
-        memset(define_mask, 0, suite->define_count * sizeof(bool));
 
         // lookup each override in the suite defines, they may have a
         // different index in each suite
         for (size_t i = 0; i < override_count; i++) {
-            ssize_t index = -1;
+            size_t j = 0;
-            for (size_t j = 0; j < suite->define_count; j++) {
+            for (; j < suite->define_count; j++) {
-                if (strcmp(overrides[i].name, suite->define_names[j]) == 0) {
+                if (strcmp(override_names[i], suite->define_names[j]) == 0) {
-                    index = j;
                     break;
                 }
             }
 
-            if (index >= 0) {
+            if (j < suite->define_count) {
-                defines[index] = overrides[i].override;
+                define_map[j] = i;
-                define_mask[index] = true;
             }
         }
 
-        test_defines[0] = defines;
+        test_defines[0] = override_defines;
-        test_define_masks[0] = define_mask;
+        test_define_maps[0] = define_map;
     } else {
-        free((uintmax_t *)test_defines[0]);
         test_defines[0] = NULL;
-        free((bool *)test_define_masks[0]);
+        free((uint8_t *)test_define_maps[0]);
-        test_define_masks[0] = NULL;
+        test_define_maps[0] = NULL;
     }
 }
 
 
 // operations we can do
 void summary(
-        struct override *overrides,
+        const char *const *override_names,
+        const test_define_t *override_defines,
         size_t override_count) {
-    (void)overrides;
+    (void)override_names;
+    (void)override_defines;
     (void)override_count;
     printf("%-36s %7s %7s %7s %7s\n",
             "", "geoms", "suites", "cases", "perms");
@@ -147,9 +140,11 @@ void summary(
 }
 
 void list_suites(
-        struct override *overrides,
+        const char *const *override_names,
+        const test_define_t *override_defines,
         size_t override_count) {
-    (void)overrides;
+    (void)override_names;
+    (void)override_defines;
     (void)override_count;
     printf("%-36s %-12s %7s %7s %7s\n",
             "id", "suite", "types", "cases", "perms");
@@ -169,9 +164,11 @@ void list_suites(
 }
 
 void list_cases(
-        struct override *overrides,
+        const char *const *override_names,
+        const test_define_t *override_defines,
         size_t override_count) {
-    (void)overrides;
+    (void)override_names;
+    (void)override_defines;
     (void)override_count;
     printf("%-36s %-12s %-12s %7s %7s\n",
             "id", "suite", "case", "types", "perms");
@@ -189,9 +186,11 @@ void list_cases(
 }
 
 void list_paths(
-        struct override *overrides,
+        const char *const *override_names,
+        const test_define_t *override_defines,
         size_t override_count) {
-    (void)overrides;
+    (void)override_names;
+    (void)override_defines;
     (void)override_count;
     printf("%-36s %-36s\n", "id", "path");
     for (size_t i = 0; i < test_suite_count; i++) {
@@ -204,17 +203,21 @@ void list_paths(
 }
 
 void list_defines(
-        struct override *overrides,
+        const char *const *override_names,
+        const test_define_t *override_defines,
         size_t override_count) {
-    (void)overrides;
+    (void)override_names;
+    (void)override_defines;
     (void)override_count;
     // TODO
 }
 
 void list_geometries(
-        struct override *overrides,
+        const char *const *override_names,
+        const test_define_t *override_defines,
         size_t override_count) {
-    (void)overrides;
+    (void)override_names;
+    (void)override_defines;
     (void)override_count;
     printf("%-36s %7s %7s %7s %7s %7s\n",
             "name", "read", "prog", "erase", "count", "size");
@@ -232,10 +235,13 @@ void list_geometries(
 }
 
 void run(
-        struct override *overrides,
+        const char *const *override_names,
+        const test_define_t *override_defines,
         size_t override_count) {
     for (size_t i = 0; i < test_suite_count; i++) {
-        test_define_overrides(test_suites[i], overrides, override_count);
+        test_define_overrides(
+                test_suites[i],
+                override_names, override_defines, override_count);
 
         for (size_t j = 0; j < test_suites[i]->case_count; j++) {
             for (size_t perm = 0;
@@ -303,7 +309,7 @@ void run(
             }
         }
 
-        test_define_overrides(NULL, NULL, 0);
+        test_define_overrides(NULL, NULL, NULL, 0);
     }
 }
 
@@ -349,9 +355,11 @@ const char *const help_text[] = {
 
 int main(int argc, char **argv) {
     void (*op)(
-            struct override *overrides,
+            const char *const *override_names,
+            const test_define_t *override_defines,
             size_t override_count) = run;
-    struct override *overrides = NULL;
+    const char **override_names = NULL;
+    test_define_t *override_defines = NULL;
     size_t override_count = 0;
     size_t override_cap = 0;
 
@@ -422,24 +430,26 @@ int main(int argc, char **argv) {
                 override_count += 1;
                 if (override_count > override_cap) {
                     override_cap = (2*override_cap > 4) ? 2*override_cap : 4;
-                    overrides = realloc(overrides, override_cap
+                    override_names = realloc(override_names, override_cap
-                            * sizeof(struct override));
+                            * sizeof(const char *));
+                    override_defines = realloc(override_defines, override_cap
+                            * sizeof(test_define_t));
                 }
 
-                // parse into string key/uintmax_t value, cannibalizing the
+                // parse into string key/test_define_t value, cannibalizing the
                 // arg in the process
                 char *sep = strchr(optarg, '=');
                 char *parsed = NULL;
                 if (!sep) {
                     goto invalid_define;
                 }
-                overrides[override_count-1].override
+                override_defines[override_count-1]
                         = strtoumax(sep+1, &parsed, 0);
                 if (parsed == sep+1) {
                     goto invalid_define;
                 }
 
-                overrides[override_count-1].name = optarg;
+                override_names[override_count-1] = optarg;
                 *sep = '\0';
                 break;
 
@@ -458,15 +468,16 @@ invalid_define:
 getopt_done:
 
     for (size_t i = 0; i < override_count; i++) {
-        printf("define: %s %ju\n", overrides[i].name, overrides[i].override);
+        printf("define: %s %ju\n", override_names[i], override_defines[i]);
     }
 
     // do the thing
     op(
-            overrides,
+            override_names,
+            override_defines,
             override_count);
 
     // cleanup (need to be done for valgrind testing)
-    free(overrides);
+    free(override_names);
-}
+    free(override_defines);}
 

runners/test_runner.h

@@ -5,21 +5,24 @@
 
 
 // generated test configurations
-enum test_type {
+enum test_types {
     TEST_NORMAL    = 0x1,
     TEST_REENTRANT = 0x2,
     TEST_VALGRIND  = 0x4,
 };
 
+typedef uint8_t test_types_t;
+typedef uintmax_t test_define_t;
+
 struct test_case {
     const char *id;
     const char *name;
     const char *path;
-    uint8_t types;
+    test_types_t types;
     size_t permutations;
 
-    const uintmax_t *const *defines;
+    const test_define_t *const *defines;
-    const bool *define_mask;
+    const uint8_t *define_map;
 
     bool (*filter)(struct lfs_config *cfg, uint32_t perm);
     void (*run)(struct lfs_config *cfg, uint32_t perm);
@@ -43,7 +46,7 @@ extern const size_t test_suite_count;
 
 
 // access generated test defines
-uintmax_t test_define(size_t define);
+test_define_t test_define(size_t define);
 
 // a few preconfigured defines that control how tests run
 #define READ_SIZE           test_define(0)

scripts/test_.py

@@ -66,23 +66,19 @@ class TestCase:
         self.path = config.pop('path')
         self.suite = config.pop('suite')
         self.lineno = config.pop('lineno', None)
+        self.if_ = config.pop('if', None)
+        if isinstance(self.if_, bool):
+            self.if_ = 'true' if self.if_ else 'false'
+        self.if_lineno = config.pop('if_lineno', None)
         self.code = config.pop('code')
         self.code_lineno = config.pop('code_lineno', None)
 
         # figure out defines and the number of resulting permutations
         self.defines = {}
-        for k, v in config.pop('defines', {}).items():
+        for k, v in (
-            try:
+                config.pop('suite_defines', {})
-                v = eval(v)
+                | config.pop('defines', {})).items():
-            except:
+            if not isinstance(v, list):
-                v = v
-
-            if not isinstance(v, str):
-                try:
-                    v = list(v)
-                except:
-                    v = [v]
-            else:
                 v = [v]
 
             self.defines[k] = v
@@ -111,14 +107,18 @@ class TestSuite:
             # find line numbers
             f.seek(0)
             case_linenos = []
+            if_linenos = []
             code_linenos = []
             for i, line in enumerate(f):
                 match = re.match(
                     '(?P<case>\[\s*cases\s*\.\s*(?P<name>\w+)\s*\])' +
-                    '|(?P<code>code\s*=\s*(?:\'\'\'|"""))',
+                    '|(?P<if>if\s*=)'
+                    '|(?P<code>code\s*=)',
                     line)
                 if match and match.group('case'):
                     case_linenos.append((i+1, match.group('name')))
+                elif match and match.group('if'):
+                    if_linenos.append(i+1)
                 elif match and match.group('code'):
                     code_linenos.append(i+2)
 
@@ -129,18 +129,33 @@ class TestSuite:
             for (lineno, name), (nlineno, _) in it.zip_longest(
                     case_linenos, case_linenos[1:],
                     fillvalue=(float('inf'), None)):
+                if_lineno = min(
+                    (l for l in if_linenos if l >= lineno and l < nlineno),
+                    default=None)
                 code_lineno = min(
                     (l for l in code_linenos if l >= lineno and l < nlineno),
                     default=None)
                 cases[name]['lineno'] = lineno
+                cases[name]['if_lineno'] = if_lineno
                 cases[name]['code_lineno'] = code_lineno
 
+            self.if_ = config.pop('if', None)
+            if isinstance(self.if_, bool):
+                self.if_ = 'true' if self.if_ else 'false'
+            self.if_lineno = min(
+                (l for l in if_linenos
+                    if not case_linenos or l < case_linenos[0][0]),
+                default=None)
+
             self.code = config.pop('code', None)
             self.code_lineno = min(
                 (l for l in code_linenos
                     if not case_linenos or l < case_linenos[0][0]),
                 default=None)
 
+            # a couple of these we just forward to all cases
+            defines = config.pop('defines', {})
+
             self.cases = []
             for name, case in sorted(cases.items(),
                     key=lambda c: c[1].get('lineno')):
@@ -149,6 +164,7 @@ class TestSuite:
                     'path': path + (':%d' % case['lineno']
                         if 'lineno' in case else ''),
                     'suite': self.name,
+                    'suite_defines': defines,
                     **case}))
 
             # combine pre-defines and per-case defines
@@ -162,8 +178,6 @@ class TestSuite:
     def id(self):
         return self.name
             
-            
-            
 
 
 def compile(**args):
@@ -208,20 +222,21 @@ def compile(**args):
                 for case in suite.cases:
                     # create case defines
                     if case.defines:
+                        sorted_defines = sorted(case.defines.items())
+
                         for perm, defines in enumerate(
                                 it.product(*(
                                     [(k, v) for v in vs]
-                                    for k, vs in case.defines.items()))):
+                                    for k, vs in sorted_defines))):
-                            f.write('const uintmax_t '
+                            f.write('const test_define_t '
                                 '__test__%s__%s__%d__defines[] = {\n'
                                 % (suite.name, case.name, perm))
-                            for k, v in sorted(defines):
+                            for k, v in defines:
-                                f.write(4*' '+'[%d] = %s,\n'
+                                f.write(4*' '+'%s,\n' % v)
-                                    % (suite.defines.index(k), v))
                             f.write('};\n')
                             f.write('\n')
 
-                        f.write('const uintmax_t *const '
+                        f.write('const test_define_t *const '
                             '__test__%s__%s__defines[] = {\n'
                             % (suite.name, case.name))
                         for perm in range(case.permutations):
@@ -230,23 +245,39 @@ def compile(**args):
                         f.write('};\n')
                         f.write('\n')
 
-                        f.write('const bool '
+                        f.write('const uint8_t '
-                            '__test__%s__%s__define_mask[] = {\n'
+                            '__test__%s__%s__define_map[] = {\n'
                             % (suite.name, case.name))
-                        for i, k in enumerate(suite.defines):
+                        for k in suite.defines:
                             f.write(4*' '+'%s,\n'
-                                % ('true' if k in case.defines else 'false'))
+                                % ([k for k, _ in sorted_defines].index(k)
+                                    if k in case.defines else '0xff'))
                         f.write('};\n')
                         f.write('\n')
 
                     # create case filter function
-                    f.write('bool __test__%s__%s__filter('
+                    if suite.if_ is not None or case.if_ is not None:
-                        '__attribute__((unused)) struct lfs_config *cfg, '
+                        f.write('bool __test__%s__%s__filter('
-                        '__attribute__((unused)) uint32_t perm) {\n'
+                            '__attribute__((unused)) struct lfs_config *cfg, '
-                        % (suite.name, case.name))
+                            '__attribute__((unused)) uint32_t perm) {\n'
-                    f.write(4*' '+'return true;\n')
+                            % (suite.name, case.name))
-                    f.write('}\n')
+                        if suite.if_ is not None:
-                    f.write('\n')
+                            f.write(4*' '+'#line %d "%s"\n'
+                                % (suite.if_lineno, suite.path))
+                            f.write(4*' '+'if (!(%s)) {\n' % suite.if_)
+                            f.write(8*' '+'return false;\n')
+                            f.write(4*' '+'}\n')
+                            f.write('\n')
+                        if case.if_ is not None:
+                            f.write(4*' '+'#line %d "%s"\n'
+                                % (case.if_lineno, suite.path))
+                            f.write(4*' '+'if (!(%s)) {\n' % case.if_)
+                            f.write(8*' '+'return false;\n')
+                            f.write(4*' '+'}\n')
+                            f.write('\n')
+                        f.write(4*' '+'return true;\n')
+                        f.write('}\n')
+                        f.write('\n')
 
                     # create case run function
                     f.write('void __test__%s__%s__run('
@@ -278,11 +309,12 @@ def compile(**args):
                     if case.defines:
                         f.write(4*' '+'.defines = __test__%s__%s__defines,\n'
                             % (suite.name, case.name))
-                        f.write(4*' '+'.define_mask = '
+                        f.write(4*' '+'.define_map = '
-                            '__test__%s__%s__define_mask,\n'
+                            '__test__%s__%s__define_map,\n'
+                            % (suite.name, case.name))
+                    if suite.if_ is not None or case.if_ is not None:
+                        f.write(4*' '+'.filter = __test__%s__%s__filter,\n'
                             % (suite.name, case.name))
-                    f.write(4*' '+'.filter = __test__%s__%s__filter,\n'
-                        % (suite.name, case.name))
                     f.write(4*' '+'.run = __test__%s__%s__run,\n'
                         % (suite.name, case.name))
                     f.write('};\n')