Github/littlefs
1
Fork 0
Code Actions Activity

Created new test_runner.c and test_.py

Browse Source 
This is to try a different design for testing, the goals are to make the
test infrastructure a bit simpler, with clear stages for building and
running, and faster, by avoiding rebuilding lfs.c n-times.
This commit is contained in:
Christopher Haster
2022-04-16 13:32:35 -05:00
parent 40dba4a556
commit 56a990336b
4 changed files with 505 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+32 -1
View File
@@ -6,6 +6,7 @@ override BUILDDIR := $(BUILDDIR)/
$(if $(findstring n,$(MAKEFLAGS)),, $(shell mkdir -p \
$(BUILDDIR) \
$(BUILDDIR)bd \
$(BUILDDIR)runners \
$(BUILDDIR)tests))
endif
@@ -25,12 +26,19 @@ NM ?= nm
OBJDUMP ?= objdump
LCOV ?= lcov
SRC ?= $(wildcard *.c)
SRC ?= $(filter-out $(wildcard *.*.c),$(wildcard *.c))
OBJ := $(SRC:%.c=$(BUILDDIR)%.o)
DEP := $(SRC:%.c=$(BUILDDIR)%.d)
ASM := $(SRC:%.c=$(BUILDDIR)%.s)
CGI := $(SRC:%.c=$(BUILDDIR)%.ci)
TESTS ?= $(wildcard tests_/*.toml)
TEST_TSRC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) \
$(SRC:%.c=$(BUILDDIR)%.t.c) \
$(BUILDDIR)runners/test_runner.t.c
TEST_TASRC := $(TEST_TSRC:%.t.c=%.t.a.c)
TEST_TAOBJ := $(TEST_TASRC:%.t.a.c=%.t.a.o)
ifdef DEBUG
override CFLAGS += -O0
else
@@ -103,6 +111,9 @@ test:
test%: tests/test$$(firstword $$(subst \#, ,%)).toml
./scripts/test.py $@ $(TESTFLAGS)
.PHONY: test_
test_: $(BUILDDIR)runners/test_runner
.PHONY: code
code: $(OBJ)
./scripts/code.py $^ -S $(CODEFLAGS)
@@ -131,6 +142,7 @@ summary: $(BUILDDIR)lfs.csv
# rules
-include $(DEP)
.SUFFIXES:
.SECONDARY:
$(BUILDDIR)lfs: $(OBJ)
$(CC) $(CFLAGS) $^ $(LFLAGS) -o $@
@@ -147,6 +159,9 @@ $(BUILDDIR)lfs.csv: $(OBJ) $(CGI)
./scripts/coverage.py $(BUILDDIR)tests/*.toml.info \
-q -m $@ $(COVERAGEFLAGS) -o $@)
$(BUILDDIR)runners/test_runner: $(TEST_TAOBJ)
$(CC) $(CFLAGS) $^ $(LFLAGS) -o $@
$(BUILDDIR)%.o: %.c
$(CC) -c -MMD $(CFLAGS) $< -o $@
@@ -160,14 +175,30 @@ $(BUILDDIR)%.s: %.c
$(BUILDDIR)%.ci: %.c | $(BUILDDIR)%.o
$(CC) -c -MMD -fcallgraph-info=su $(CFLAGS) $< -o $|
$(BUILDDIR)%.a.c: %.c
./scripts/explode_asserts.py $< -o $@
$(BUILDDIR)%.a.c: $(BUILDDIR)%.c
./scripts/explode_asserts.py $< -o $@
$(BUILDDIR)%.t.c: %.toml
./scripts/test_.py -c $< -o $@
$(BUILDDIR)%.t.c: %.c $(TESTS)
./scripts/test_.py -c $(TESTS) -s $< -o $@
# clean everything
.PHONY: clean
clean:
rm -f $(BUILDDIR)lfs
rm -f $(BUILDDIR)lfs.a
rm -f $(BUILDDIR)lfs.csv
rm -f $(BUILDDIR)runners/test_runner
rm -f $(OBJ)
rm -f $(CGI)
rm -f $(DEP)
rm -f $(ASM)
rm -f $(BUILDDIR)tests/*.toml.*
rm -f $(TEST_TSRC)
rm -f $(TEST_TASRC)
rm -f $(TEST_TAOBJ)
runners/test_runner.c
+178
View File
@@ -0,0 +1,178 @@
#include "runners/test_runner.h"
#include <getopt.h>
// disk geometries
struct test_geometry {
const char *name;
lfs_size_t read_size;
lfs_size_t prog_size;
lfs_size_t erase_size;
lfs_size_t erase_count;
};
const struct test_geometry test_geometries[] = {
// Made up geometries that works well for testing
{"small", 16, 16, 512, (1024*1024)/512},
{"medium", 16, 16, 4096, (1024*1024)/4096},
{"big", 16, 16, 32*1024, (1024*1024)/(32*1024)},
// EEPROM/NVRAM
{"eeprom", 1, 1, 512, (1024*1024)/512},
// SD/eMMC
{"emmc", 512, 512, 512, (1024*1024)/512},
// NOR flash
{"nor", 1, 1, 4096, (1024*1024)/4096},
// NAND flash
{"nand", 4096, 4096, 32*1024, (1024*1024)/(32*1024)},
};
const size_t test_geometry_count = (
sizeof(test_geometries) / sizeof(test_geometries[0]));
// option handling
enum opt_flags {
OPT_HELP = 'h',
OPT_LIST = 'l',
OPT_LIST_PATHS = 1,
OPT_LIST_DEFINES = 2,
OPT_LIST_GEOMETRIES = 3,
};
const struct option long_opts[] = {
{"help", no_argument, NULL, OPT_HELP},
{"list", no_argument, NULL, OPT_LIST},
{"list-paths", no_argument, NULL, OPT_LIST_PATHS},
{"list-defines", no_argument, NULL, OPT_LIST_DEFINES},
{"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES},
{NULL, 0, NULL, 0},
};
const char *const help_text[] = {
"Show this help message.",
"List test cases.",
"List the path for each test case.",
"List the defines for each test permutation.",
"List the disk geometries used for testing.",
};
int main(int argc, char **argv) {
bool list = false;
bool list_paths = false;
bool list_defines = false;
bool list_geometries = false;
// parse options
while (true) {
int index = 0;
int c = getopt_long(argc, argv, "hl", long_opts, &index);
switch (c) {
// generate help message
case OPT_HELP: {
printf("usage: %s [options] [test_case]\n", argv[0]);
printf("\n");
printf("options:\n");
size_t i = 0;
while (long_opts[i].name) {
size_t indent;
if (long_opts[i].val >= '0' && long_opts[i].val < 'z') {
printf(" -%c, --%-16s",
long_opts[i].val,
long_opts[i].name);
indent = 8+strlen(long_opts[i].name);
} else {
printf(" --%-20s", long_opts[i].name);
indent = 4+strlen(long_opts[i].name);
}
// a quick, hacky, byte-level method for text wrapping
size_t len = strlen(help_text[i]);
size_t j = 0;
if (indent < 24) {
printf("%.80s\n", &help_text[i][j]);
j += 80;
}
while (j < len) {
printf("%24s%.80s\n", "", &help_text[i][j]);
j += 80;
}
i += 1;
}
printf("\n");
exit(0);
}
// list flags
case OPT_LIST:
list = true;
break;
case OPT_LIST_PATHS:
list_paths = true;
break;
case OPT_LIST_DEFINES:
list_defines = true;
break;
case OPT_LIST_GEOMETRIES:
list_geometries = true;
break;
// done parsing
case -1:
goto getopt_done;
// unknown arg, getopt prints a message for us
default:
exit(-1);
}
}
getopt_done:
// what do we need to do?
if (list) {
printf("%-36s %-12s %-12s %7s %7s\n",
"id", "suite", "case", "type", "perms");
for (size_t i = 0; i < test_suite_count; i++) {
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
printf("%-36s %-12s %-12s %7s %7d\n",
test_suites[i]->cases[j]->id,
test_suites[i]->name,
test_suites[i]->cases[j]->name,
"n", // TODO
test_suites[i]->cases[j]->permutations);
}
}
} else if (list_paths) {
printf("%-36s %-36s\n", "id", "path");
for (size_t i = 0; i < test_suite_count; i++) {
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
printf("%-36s %-36s\n",
test_suites[i]->cases[j]->id,
test_suites[i]->cases[j]->path);
}
}
} else if (list_defines) {
// TODO
} else if (list_geometries) {
printf("%-12s %7s %7s %7s %7s %7s\n",
"name", "read", "prog", "erase", "count", "size");
for (size_t i = 0; i < test_geometry_count; i++) {
printf("%-12s %7d %7d %7d %7d %7d\n",
test_geometries[i].name,
test_geometries[i].read_size,
test_geometries[i].prog_size,
test_geometries[i].erase_size,
test_geometries[i].erase_count,
test_geometries[i].erase_size
* test_geometries[i].erase_count);
}
} else {
printf("remaining: ");
for (int i = optind; i < argc; i++) {
printf("%s ", argv[i]);
}
printf("\n");
}
}
runners/test_runner.h
+26
View File
@@ -0,0 +1,26 @@
#ifndef TEST_RUNNER_H
#define TEST_RUNNER_H
#include "lfs.h"
struct test_case {
const char *id;
const char *name;
const char *path;
uint32_t permutations;
void (*run)(struct lfs_config *cfg, uint32_t perm);
};
struct test_suite {
const char *id;
const char *name;
const char *path;
const struct test_case *const *cases;
size_t case_count;
};
extern const struct test_suite *test_suites[];
extern const size_t test_suite_count;
#endif
scripts/test_.py
Executable
+269
View File
@@ -0,0 +1,269 @@
#!/usr/bin/env python3
#
# Script to compile and runs tests.
#
import glob
import itertools as it
import os
import re
import shutil
import toml
TEST_PATHS = ['tests_']
SUITE_PROLOGUE = """
//////// AUTOGENERATED ////////
#include "runners/test_runner.h"
#include <stdio.h>
"""
# TODO handle indention implicity?
# TODO change cfg to be not by value? maybe not?
CASE_PROLOGUE = """
lfs_t lfs;
struct lfs_config cfg = *cfg_;
"""
CASE_EPILOGUE = """
"""
# TODO
# def testpath(path):
# def testcase(path):
# def testperm(path):
def testsuite(path):
name = os.path.basename(path)
if name.endswith('.toml'):
name = name[:-len('.toml')]
return name
# TODO move this out in other files
def openio(path, mode='r'):
if path == '-':
if 'r' in mode:
return os.fdopen(os.dup(sys.stdin.fileno()), 'r')
else:
return os.fdopen(os.dup(sys.stdout.fileno()), 'w')
else:
return open(path, mode)
class TestCase:
# create a TestCase object from a config
def __init__(self, config, args={}):
self.name = config.pop('name')
self.path = config.pop('path')
self.suite = config.pop('suite')
self.lineno = config.pop('lineno', None)
self.code = config.pop('code')
self.code_lineno = config.pop('code_lineno', None)
self.permutations = 1
for k in config.keys():
print('warning: in %s, found unused key %r' % (self.id(), k),
file=sys.stderr)
def id(self):
return '%s#%s' % (self.suite, self.name)
class TestSuite:
# create a TestSuite object from a toml file
def __init__(self, path, args={}):
self.name = testsuite(path)
self.path = path
# load toml file and parse test cases
with open(self.path) as f:
# load tests
config = toml.load(f)
# find line numbers
f.seek(0)
case_linenos = []
code_linenos = []
for i, line in enumerate(f):
match = re.match(
'(?P<case>\[\s*cases\s*\.\s*(?P<name>\w+)\s*\])' +
'|(?P<code>code\s*=\s*(?:\'\'\'|"""))',
line)
if match and match.group('case'):
case_linenos.append((i+1, match.group('name')))
elif match and match.group('code'):
code_linenos.append(i+2)
# sort in case toml parsing did not retain order
case_linenos.sort()
cases = config.pop('cases', [])
for (lineno, name), (nlineno, _) in it.zip_longest(
case_linenos, case_linenos[1:],
fillvalue=(float('inf'), None)):
code_lineno = min(
(l for l in code_linenos if l >= lineno and l < nlineno),
default=None)
cases[name]['lineno'] = lineno
cases[name]['code_lineno'] = code_lineno
self.code = config.pop('code', None)
self.code_lineno = min(
(l for l in code_linenos
if not case_linenos or l < case_linenos[0][0]),
default=None)
self.cases = []
for name, case in sorted(cases.items(),
key=lambda c: c[1].get('lineno')):
self.cases.append(TestCase(config={
'name': name,
'path': path + (':%d' % case['lineno']
if 'lineno' in case else ''),
'suite': self.name,
**case}))
for k in config.keys():
print('warning: in %s, found unused key %r' % (self.id(), k),
file=sys.stderr)
def id(self):
return self.name
def compile(**args):
# find .toml files
paths = []
for path in args['test_paths']:
if os.path.isdir(path):
path = path + '/*.toml'
for path in glob.glob(path):
paths.append(path)
if not paths:
print('no test suites found in %r?' % args['test_paths'])
sys.exit(-1)
if not args.get('source'):
if len(paths) > 1:
print('more than one test suite for compilation? (%r)'
% args['test_paths'])
sys.exit(-1)
# write out a test suite
suite = TestSuite(paths[0])
if 'output' in args:
with openio(args['output'], 'w') as f:
f.write(SUITE_PROLOGUE)
f.write('\n')
if suite.code is not None:
if suite.code_lineno is not None:
f.write('#line %d "%s"\n'
% (suite.code_lineno, suite.path))
f.write(suite.code)
f.write('\n')
# create test functions and case structs
for case in suite.cases:
f.write('void __test__%s__%s('
'__attribute__((unused)) struct lfs_config *cfg_, '
'__attribute__((unused)) uint32_t perm) {\n'
% (suite.name, case.name))
f.write(CASE_PROLOGUE)
f.write('\n')
f.write(4*' '+'// test case %s\n' % case.id())
if case.code_lineno is not None:
f.write(4*' '+'#line %d "%s"\n'
% (case.code_lineno, suite.path))
f.write(case.code)
f.write('\n')
f.write(CASE_EPILOGUE)
f.write('}\n')
f.write('\n')
f.write('const struct test_case __test__%s__%s__case = {\n'
% (suite.name, case.name))
f.write(4*' '+'.id = "%s",\n' % case.id())
f.write(4*' '+'.name = "%s",\n' % case.name)
f.write(4*' '+'.path = "%s",\n' % case.path)
f.write(4*' '+'.permutations = %d,\n' % case.permutations)
f.write(4*' '+'.run = __test__%s__%s,\n'
% (suite.name, case.name))
f.write('};\n')
f.write('\n')
# create suite struct
f.write('const struct test_suite __test__%s__suite = {\n'
% (suite.name))
f.write(4*' '+'.id = "%s",\n' % suite.id())
f.write(4*' '+'.name = "%s",\n' % suite.name)
f.write(4*' '+'.path = "%s",\n' % suite.path)
f.write(4*' '+'.cases = (const struct test_case *const []){\n')
for case in suite.cases:
f.write(8*' '+'&__test__%s__%s__case,\n'
% (suite.name, case.name))
f.write(4*' '+'},\n')
f.write(4*' '+'.case_count = %d,\n' % len(suite.cases))
f.write('};\n')
f.write('\n')
else:
# load all suites
suites = [TestSuite(path) for path in paths]
suites.sort(key=lambda s: s.name)
# write out a test source
if 'output' in args:
with openio(args['output'], 'w') as f:
f.write(SUITE_PROLOGUE)
f.write('\n')
f.write('#line 1 "%s"\n' % args['source'])
with open(args['source']) as sf:
shutil.copyfileobj(sf, f)
# add suite info to test_runner.c
if args['source'] == 'runners/test_runner.c':
f.write('\n')
for suite in suites:
f.write('extern const struct test_suite '
'__test__%s__suite;\n' % suite.name)
f.write('const struct test_suite *test_suites[] = {\n')
for suite in suites:
f.write(4*' '+'&__test__%s__suite,\n' % suite.name)
f.write('};\n')
f.write('const size_t test_suite_count = %d;\n'
% len(suites))
def run(**args):
pass
def main(**args):
if args.get('compile'):
compile(**args)
else:
run(**args)
if __name__ == "__main__":
import argparse
import sys
parser = argparse.ArgumentParser(
description="Build and run tests.")
# TODO document test case/perm specifier
parser.add_argument('test_paths', nargs='*', default=TEST_PATHS,
help="Description of test(s) to run. May be a directory, a path, or \
test identifier. Defaults to all tests in %r." % TEST_PATHS)
# test flags
test_parser = parser.add_argument_group('test options')
# compilation flags
comp_parser = parser.add_argument_group('compilation options')
comp_parser.add_argument('-c', '--compile', action='store_true',
help="Compile a test suite or source file.")
comp_parser.add_argument('-s', '--source',
help="Source file to compile, possibly injecting internal tests.")
comp_parser.add_argument('-o', '--output',
help="Output file.")
# TODO apply this to other scripts?
sys.exit(main(**{k: v
for k, v in vars(parser.parse_args()).items()
if v is not None}))