Created new test_runner.c and test_.py

This is to try a different design for testing, the goals are to make the
test infrastructure a bit simpler, with clear stages for building and
running, and faster, by avoiding rebuilding lfs.c n-times.

Makefile

@@ -6,6 +6,7 @@ override BUILDDIR := $(BUILDDIR)/
 $(if $(findstring n,$(MAKEFLAGS)),, $(shell mkdir -p \
 	$(BUILDDIR) \
 	$(BUILDDIR)bd \
+	$(BUILDDIR)runners \
 	$(BUILDDIR)tests))
 endif
 
@@ -25,12 +26,19 @@ NM      ?= nm
 OBJDUMP ?= objdump
 LCOV    ?= lcov
 
-SRC ?= $(wildcard *.c)
+SRC ?= $(filter-out $(wildcard *.*.c),$(wildcard *.c))
 OBJ := $(SRC:%.c=$(BUILDDIR)%.o)
 DEP := $(SRC:%.c=$(BUILDDIR)%.d)
 ASM := $(SRC:%.c=$(BUILDDIR)%.s)
 CGI := $(SRC:%.c=$(BUILDDIR)%.ci)
 
+TESTS ?= $(wildcard tests_/*.toml)
+TEST_TSRC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) \
+	$(SRC:%.c=$(BUILDDIR)%.t.c) \
+	$(BUILDDIR)runners/test_runner.t.c
+TEST_TASRC := $(TEST_TSRC:%.t.c=%.t.a.c)
+TEST_TAOBJ := $(TEST_TASRC:%.t.a.c=%.t.a.o)
+
 ifdef DEBUG
 override CFLAGS += -O0
 else
@@ -103,6 +111,9 @@ test:
 test%: tests/test$$(firstword $$(subst \#, ,%)).toml
 	./scripts/test.py $@ $(TESTFLAGS)
 
+.PHONY: test_
+test_: $(BUILDDIR)runners/test_runner
+
 .PHONY: code
 code: $(OBJ)
 	./scripts/code.py $^ -S $(CODEFLAGS)
@@ -131,6 +142,7 @@ summary: $(BUILDDIR)lfs.csv
 # rules
 -include $(DEP)
 .SUFFIXES:
+.SECONDARY:
 
 $(BUILDDIR)lfs: $(OBJ)
 	$(CC) $(CFLAGS) $^ $(LFLAGS) -o $@
@@ -147,6 +159,9 @@ $(BUILDDIR)lfs.csv: $(OBJ) $(CGI)
 		./scripts/coverage.py $(BUILDDIR)tests/*.toml.info \
 			-q -m $@ $(COVERAGEFLAGS) -o $@)
 
+$(BUILDDIR)runners/test_runner: $(TEST_TAOBJ)
+	$(CC) $(CFLAGS) $^ $(LFLAGS) -o $@
+
 $(BUILDDIR)%.o: %.c
 	$(CC) -c -MMD $(CFLAGS) $< -o $@
 
@@ -160,14 +175,30 @@ $(BUILDDIR)%.s: %.c
 $(BUILDDIR)%.ci: %.c | $(BUILDDIR)%.o
 	$(CC) -c -MMD -fcallgraph-info=su $(CFLAGS) $< -o $|
 
+$(BUILDDIR)%.a.c: %.c
+	./scripts/explode_asserts.py $< -o $@
+
+$(BUILDDIR)%.a.c: $(BUILDDIR)%.c
+	./scripts/explode_asserts.py $< -o $@
+
+$(BUILDDIR)%.t.c: %.toml
+	./scripts/test_.py -c $< -o $@
+
+$(BUILDDIR)%.t.c: %.c $(TESTS)
+	./scripts/test_.py -c $(TESTS) -s $< -o $@
+
 # clean everything
 .PHONY: clean
 clean:
 	rm -f $(BUILDDIR)lfs
 	rm -f $(BUILDDIR)lfs.a
 	rm -f $(BUILDDIR)lfs.csv
+	rm -f $(BUILDDIR)runners/test_runner
 	rm -f $(OBJ)
 	rm -f $(CGI)
 	rm -f $(DEP)
 	rm -f $(ASM)
 	rm -f $(BUILDDIR)tests/*.toml.*
+	rm -f $(TEST_TSRC)
+	rm -f $(TEST_TASRC)
+	rm -f $(TEST_TAOBJ)

runners/test_runner.c

@@ -0,0 +1,178 @@
+
+#include "runners/test_runner.h"
+#include <getopt.h>
+
+
+// disk geometries
+struct test_geometry {
+    const char *name;
+    lfs_size_t read_size;
+    lfs_size_t prog_size;
+    lfs_size_t erase_size;
+    lfs_size_t erase_count;
+};
+
+const struct test_geometry test_geometries[] = {
+    // Made up geometries that works well for testing
+    {"small",    16,   16,     512, (1024*1024)/512},
+    {"medium",   16,   16,    4096, (1024*1024)/4096},
+    {"big",      16,   16, 32*1024, (1024*1024)/(32*1024)},
+    // EEPROM/NVRAM
+    {"eeprom",    1,    1,     512, (1024*1024)/512},
+    // SD/eMMC
+    {"emmc",    512,  512,     512, (1024*1024)/512},
+    // NOR flash
+    {"nor",       1,    1,    4096, (1024*1024)/4096},
+    // NAND flash
+    {"nand",   4096, 4096, 32*1024, (1024*1024)/(32*1024)},
+};
+const size_t test_geometry_count = (
+        sizeof(test_geometries) / sizeof(test_geometries[0]));
+
+
+// option handling
+enum opt_flags {
+    OPT_HELP            = 'h',
+    OPT_LIST            = 'l',
+    OPT_LIST_PATHS      = 1,
+    OPT_LIST_DEFINES    = 2,
+    OPT_LIST_GEOMETRIES = 3,
+};
+
+const struct option long_opts[] = {
+    {"help",            no_argument, NULL, OPT_HELP},
+    {"list",            no_argument, NULL, OPT_LIST},
+    {"list-paths",      no_argument, NULL, OPT_LIST_PATHS},
+    {"list-defines",    no_argument, NULL, OPT_LIST_DEFINES},
+    {"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES},
+    {NULL, 0, NULL, 0},
+};
+
+const char *const help_text[] = {
+    "Show this help message.",
+    "List test cases.",
+    "List the path for each test case.",
+    "List the defines for each test permutation.",
+    "List the disk geometries used for testing.",
+};
+
+int main(int argc, char **argv) {
+    bool list = false;
+    bool list_paths = false;
+    bool list_defines = false;
+    bool list_geometries = false;
+
+    // parse options
+    while (true) {
+        int index = 0;
+        int c = getopt_long(argc, argv, "hl", long_opts, &index);
+        switch (c) {
+            // generate help message
+            case OPT_HELP: {
+                printf("usage: %s [options] [test_case]\n", argv[0]);
+                printf("\n");
+
+                printf("options:\n");
+                size_t i = 0;
+                while (long_opts[i].name) {
+                    size_t indent;
+                    if (long_opts[i].val >= '0' && long_opts[i].val < 'z') {
+                        printf("  -%c, --%-16s",
+                                long_opts[i].val,
+                                long_opts[i].name);
+                        indent = 8+strlen(long_opts[i].name);
+                    } else {
+                        printf("  --%-20s", long_opts[i].name);
+                        indent = 4+strlen(long_opts[i].name);
+                    }
+
+                    // a quick, hacky, byte-level method for text wrapping
+                    size_t len = strlen(help_text[i]);
+                    size_t j = 0;
+                    if (indent < 24) {
+                        printf("%.80s\n", &help_text[i][j]);
+                        j += 80;
+                    }
+
+                    while (j < len) {
+                        printf("%24s%.80s\n", "", &help_text[i][j]);
+                        j += 80;
+                    }
+
+                    i += 1;
+                }
+
+                printf("\n");
+                exit(0);
+            }
+            // list flags
+            case OPT_LIST:
+                list = true;
+                break;
+            case OPT_LIST_PATHS:
+                list_paths = true;
+                break;
+            case OPT_LIST_DEFINES:
+                list_defines = true;
+                break;
+            case OPT_LIST_GEOMETRIES:
+                list_geometries = true;
+                break;
+            // done parsing
+            case -1:
+                goto getopt_done;
+            // unknown arg, getopt prints a message for us
+            default:
+                exit(-1);
+        }
+    }
+getopt_done:
+
+    // what do we need to do?
+    if (list) {
+        printf("%-36s %-12s %-12s %7s %7s\n",
+                "id", "suite", "case", "type", "perms");
+        for (size_t i = 0; i < test_suite_count; i++) {
+            for (size_t j = 0; j < test_suites[i]->case_count; j++) {
+                printf("%-36s %-12s %-12s %7s %7d\n",
+                        test_suites[i]->cases[j]->id,
+                        test_suites[i]->name,
+                        test_suites[i]->cases[j]->name,
+                        "n", // TODO
+                        test_suites[i]->cases[j]->permutations);
+            }
+        }
+
+    } else if (list_paths) {
+        printf("%-36s %-36s\n", "id", "path");
+        for (size_t i = 0; i < test_suite_count; i++) {
+            for (size_t j = 0; j < test_suites[i]->case_count; j++) {
+                printf("%-36s %-36s\n",
+                        test_suites[i]->cases[j]->id,
+                        test_suites[i]->cases[j]->path);
+            }
+        }
+    } else if (list_defines) {
+        // TODO
+    } else if (list_geometries) {
+        printf("%-12s %7s %7s %7s %7s %7s\n",
+                "name", "read", "prog", "erase", "count", "size");
+        for (size_t i = 0; i < test_geometry_count; i++) {
+            printf("%-12s %7d %7d %7d %7d %7d\n",
+                    test_geometries[i].name,
+                    test_geometries[i].read_size,
+                    test_geometries[i].prog_size,
+                    test_geometries[i].erase_size,
+                    test_geometries[i].erase_count,
+                    test_geometries[i].erase_size
+                        * test_geometries[i].erase_count);
+        }
+    } else {
+        printf("remaining: ");
+        for (int i = optind; i < argc; i++) {
+            printf("%s ", argv[i]);
+        }
+        printf("\n");
+    }
+}
+

runners/test_runner.h

@@ -0,0 +1,26 @@
+#ifndef TEST_RUNNER_H
+#define TEST_RUNNER_H
+
+#include "lfs.h"
+
+
+struct test_case {
+    const char *id;
+    const char *name;
+    const char *path;
+    uint32_t permutations;
+    void (*run)(struct lfs_config *cfg, uint32_t perm);
+};
+
+struct test_suite {
+    const char *id;
+    const char *name;
+    const char *path;
+    const struct test_case *const *cases;
+    size_t case_count;
+};
+
+extern const struct test_suite *test_suites[];
+extern const size_t test_suite_count;
+
+#endif

scripts/test_.py

@@ -0,0 +1,269 @@
+#!/usr/bin/env python3
+#
+# Script to compile and runs tests.
+#
+
+import glob
+import itertools as it
+import os
+import re
+import shutil
+import toml
+
+TEST_PATHS = ['tests_']
+
+SUITE_PROLOGUE = """
+//////// AUTOGENERATED ////////
+#include "runners/test_runner.h"
+#include <stdio.h>
+"""
+# TODO handle indention implicity?
+# TODO change cfg to be not by value? maybe not?
+CASE_PROLOGUE = """
+    lfs_t lfs;
+    struct lfs_config cfg = *cfg_;
+"""
+CASE_EPILOGUE = """
+"""
+
+
+# TODO
+# def testpath(path):
+# def testcase(path):
+# def testperm(path):
+
+def testsuite(path):
+    name = os.path.basename(path)
+    if name.endswith('.toml'):
+        name = name[:-len('.toml')]
+    return name
+
+# TODO move this out in other files
+def openio(path, mode='r'):
+    if path == '-':
+        if 'r' in mode:
+            return os.fdopen(os.dup(sys.stdin.fileno()), 'r')
+        else:
+            return os.fdopen(os.dup(sys.stdout.fileno()), 'w')
+    else:
+        return open(path, mode)
+
+class TestCase:
+    # create a TestCase object from a config
+    def __init__(self, config, args={}):
+        self.name = config.pop('name')
+        self.path = config.pop('path')
+        self.suite = config.pop('suite')
+        self.lineno = config.pop('lineno', None)
+        self.code = config.pop('code')
+        self.code_lineno = config.pop('code_lineno', None)
+
+        self.permutations = 1
+
+        for k in config.keys():
+            print('warning: in %s, found unused key %r' % (self.id(), k),
+                file=sys.stderr)
+
+    def id(self):
+        return '%s#%s' % (self.suite, self.name)
+
+
+class TestSuite:
+    # create a TestSuite object from a toml file
+    def __init__(self, path, args={}):
+        self.name = testsuite(path)
+        self.path = path
+
+        # load toml file and parse test cases
+        with open(self.path) as f:
+            # load tests
+            config = toml.load(f)
+
+            # find line numbers
+            f.seek(0)
+            case_linenos = []
+            code_linenos = []
+            for i, line in enumerate(f):
+                match = re.match(
+                    '(?P<case>\[\s*cases\s*\.\s*(?P<name>\w+)\s*\])' +
+                    '|(?P<code>code\s*=\s*(?:\'\'\'|"""))',
+                    line)
+                if match and match.group('case'):
+                    case_linenos.append((i+1, match.group('name')))
+                elif match and match.group('code'):
+                    code_linenos.append(i+2)
+
+            # sort in case toml parsing did not retain order
+            case_linenos.sort()
+
+            cases = config.pop('cases', [])
+            for (lineno, name), (nlineno, _) in it.zip_longest(
+                    case_linenos, case_linenos[1:],
+                    fillvalue=(float('inf'), None)):
+                code_lineno = min(
+                    (l for l in code_linenos if l >= lineno and l < nlineno),
+                    default=None)
+                cases[name]['lineno'] = lineno
+                cases[name]['code_lineno'] = code_lineno
+
+            self.code = config.pop('code', None)
+            self.code_lineno = min(
+                (l for l in code_linenos
+                    if not case_linenos or l < case_linenos[0][0]),
+                default=None)
+
+            self.cases = []
+            for name, case in sorted(cases.items(),
+                    key=lambda c: c[1].get('lineno')):
+                self.cases.append(TestCase(config={
+                    'name': name,
+                    'path': path + (':%d' % case['lineno']
+                        if 'lineno' in case else ''),
+                    'suite': self.name,
+                    **case}))
+
+        for k in config.keys():
+            print('warning: in %s, found unused key %r' % (self.id(), k),
+                file=sys.stderr)
+
+    def id(self):
+        return self.name
+            
+
+
+def compile(**args):
+    # find .toml files
+    paths = []
+    for path in args['test_paths']:
+        if os.path.isdir(path):
+            path = path + '/*.toml'
+
+        for path in glob.glob(path):
+            paths.append(path)
+
+    if not paths:
+        print('no test suites found in %r?' % args['test_paths'])
+        sys.exit(-1)
+
+    if not args.get('source'):
+        if len(paths) > 1:
+            print('more than one test suite for compilation? (%r)'
+                % args['test_paths'])
+            sys.exit(-1)
+
+        # write out a test suite
+        suite = TestSuite(paths[0])
+        if 'output' in args:
+            with openio(args['output'], 'w') as f:
+                f.write(SUITE_PROLOGUE)
+                f.write('\n')
+                if suite.code is not None:
+                    if suite.code_lineno is not None:
+                        f.write('#line %d "%s"\n'
+                            % (suite.code_lineno, suite.path))
+                    f.write(suite.code)
+                    f.write('\n')
+
+                # create test functions and case structs
+                for case in suite.cases:
+                    f.write('void __test__%s__%s('
+                        '__attribute__((unused)) struct lfs_config *cfg_, '
+                        '__attribute__((unused)) uint32_t perm) {\n'
+                        % (suite.name, case.name))
+                    f.write(CASE_PROLOGUE)
+                    f.write('\n')
+                    f.write(4*' '+'// test case %s\n' % case.id())
+                    if case.code_lineno is not None:
+                        f.write(4*' '+'#line %d "%s"\n'
+                            % (case.code_lineno, suite.path))
+                    f.write(case.code)
+                    f.write('\n')
+                    f.write(CASE_EPILOGUE)
+                    f.write('}\n')
+                    f.write('\n')
+
+                    f.write('const struct test_case __test__%s__%s__case = {\n'
+                        % (suite.name, case.name))
+                    f.write(4*' '+'.id = "%s",\n' % case.id())
+                    f.write(4*' '+'.name = "%s",\n' % case.name)
+                    f.write(4*' '+'.path = "%s",\n' % case.path)
+                    f.write(4*' '+'.permutations = %d,\n' % case.permutations)
+                    f.write(4*' '+'.run = __test__%s__%s,\n'
+                        % (suite.name, case.name))
+                    f.write('};\n')
+                    f.write('\n')
+
+                # create suite struct
+                f.write('const struct test_suite __test__%s__suite = {\n'
+                    % (suite.name))
+                f.write(4*' '+'.id = "%s",\n' % suite.id())
+                f.write(4*' '+'.name = "%s",\n' % suite.name)
+                f.write(4*' '+'.path = "%s",\n' % suite.path)
+                f.write(4*' '+'.cases = (const struct test_case *const []){\n')
+                for case in suite.cases:
+                    f.write(8*' '+'&__test__%s__%s__case,\n'
+                        % (suite.name, case.name))
+                f.write(4*' '+'},\n')
+                f.write(4*' '+'.case_count = %d,\n' % len(suite.cases))
+                f.write('};\n')
+                f.write('\n')
+
+    else:
+        # load all suites
+        suites = [TestSuite(path) for path in paths]
+        suites.sort(key=lambda s: s.name)
+
+        # write out a test source
+        if 'output' in args:
+            with openio(args['output'], 'w') as f:
+                f.write(SUITE_PROLOGUE)
+                f.write('\n')
+                f.write('#line 1 "%s"\n' % args['source'])
+                with open(args['source']) as sf:
+                    shutil.copyfileobj(sf, f)
+
+                # add suite info to test_runner.c
+                if args['source'] == 'runners/test_runner.c':
+                    f.write('\n')
+                    for suite in suites:
+                        f.write('extern const struct test_suite '
+                            '__test__%s__suite;\n' % suite.name)
+                    f.write('const struct test_suite *test_suites[] = {\n')
+                    for suite in suites:
+                        f.write(4*' '+'&__test__%s__suite,\n' % suite.name)
+                    f.write('};\n')
+                    f.write('const size_t test_suite_count = %d;\n'
+                        % len(suites))
+
+def run(**args):
+    pass
+
+def main(**args):
+    if args.get('compile'):
+        compile(**args)
+    else:
+        run(**args)
+
+if __name__ == "__main__":
+    import argparse
+    import sys
+    parser = argparse.ArgumentParser(
+        description="Build and run tests.")
+    # TODO document test case/perm specifier
+    parser.add_argument('test_paths', nargs='*', default=TEST_PATHS,
+        help="Description of test(s) to run. May be a directory, a path, or \
+            test identifier. Defaults to all tests in %r." % TEST_PATHS)
+    # test flags
+    test_parser = parser.add_argument_group('test options')
+    # compilation flags
+    comp_parser = parser.add_argument_group('compilation options')
+    comp_parser.add_argument('-c', '--compile', action='store_true',
+        help="Compile a test suite or source file.")
+    comp_parser.add_argument('-s', '--source',
+        help="Source file to compile, possibly injecting internal tests.")
+    comp_parser.add_argument('-o', '--output',
+        help="Output file.")
+    # TODO apply this to other scripts?
+    sys.exit(main(**{k: v
+        for k, v in vars(parser.parse_args()).items()
+        if v is not None}))