Christopher Haster
a124ee54e7
Motivation: - Debuggability. Accessing the current test/bench defines from inside gdb was basically impossible for some dumb macro-debug-info reason I can't figure out. In theory, GCC provides a .debug_macro section when compiled with -g3. I can see this section with objdump --dwarf=macro, but somehow gdb can't seem to find any definitions? I'm guess the #line source remapping is causing things to break somehow... Though even if macro-debugging gets fixed, which would be valuable, accessing defines in the current test/bench runner can trigger quite a bit of hidden machinery. This risks side-effects, which is never great when debugging. All of this is quite annoying because the test/bench defines is usually the most important piece of information when debugging! This replaces the previous hidden define machinery with simple global variables, which gdb can access no problem. - Also when debugging we no longer awkwardly step into the test_define function all the time! - In theory, global variables, being a simple memory access, should be quite a bit faster than the hidden define machinery. This does matter because running tests _is_ a dev bottleneck. In practice though, any performance benefit is below the noise floor, which isn't too surprising (~630s +-~20s). - Using global variables for defines simplifies the test/bench runner quite a bit. Though some of the previous complexity was due to a whole internal define caching system, which was supposed to lazily evaluate test defines to avoid evaluating defines we don't use. This all proved to be useless because the first thing we do when running each test is evaluate all defines to generate the test id (lol). So now, instead of lazily evaluating and caching defines, we just generate global variables during compilation and evaluate all defines for each test permutation immediately before running. This relies heavily on __attribute__((weak)) symbols, and lets the linker really shine. As a funny perk this also effectively interns all test/bench defines by the address of the resulting global variable. So we don't even need to do string comparisons when mapping suite-level defines to the runner-level defines. --- Perhaps the more interesting thing to note, is the change in strategy in how we actually evaluate the test defines. This ends up being a surprisingly tricky problem, due to the potential of mutual recursion between our defines. Previously, because our define machinery was lazy, we could just evaluate each define on demand. If a define required another define, it would lazily trigger another evaluation, implicitly recursing through C's stack. If cyclic, this would eventually lead to a stack overflow, but that's ok because it's a user error to let this happen. The "correct" way, at least in terms of being computationally optimal, would be to topologically sort the defines and evaluate the resulting tree from the leaves up. But I ain't got time for that, so the solution here is equal parts hacky, simple, and effective. Basically, we just evaluate the defines repeatedly until they stop changing: - Initially, mutually recursive defines may read the uninitialized values of their dependencies, and end up with some arbitrarily wrong result. But as the defines are repeatedly evaluated, assuming no cycles, the correct results should eventually bubble up the tree until all defines converge to the correct value. - This is O(n*e) vs O(n+e), but our define graph is usually quite shallow. - To prevent non-halting, we error after an arbitrary 1000 iterations. If you hit this, it's likely because there is a cycle in the define graph. This is runtime configurable via the new --define-depth flag. - To keep things consistent and reproducible, we zero initialize all defines before the first evaluation. I don't think this is strictly necessary, but it's important for the test runner to have the exact same results on every run. No one wants a "works on my machine" situation when the tests are involved. Experimentation shows we only need an evaluation depth of 2 to successfully evaluate the current set of defines: $ ./runners/test_runner --list-defines --define-depth=2 And any performance impact is negligible (~630s +-~20s).
143 lines
4.4 KiB
C
143 lines
4.4 KiB
C
/*
|
|
* Runner for littlefs tests
|
|
*
|
|
* Copyright (c) 2022, The littlefs authors.
|
|
* SPDX-License-Identifier: BSD-3-Clause
|
|
*/
|
|
#ifndef TEST_RUNNER_H
|
|
#define TEST_RUNNER_H
|
|
|
|
|
|
// override LFS_TRACE
|
|
void test_trace(const char *fmt, ...);
|
|
|
|
#define LFS_TRACE_(fmt, ...) \
|
|
test_trace("%s:%d:trace: " fmt "%s\n", \
|
|
__FILE__, \
|
|
__LINE__, \
|
|
__VA_ARGS__)
|
|
#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")
|
|
#define LFS_EMUBD_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")
|
|
|
|
|
|
// note these are indirectly included in any generated files
|
|
#include "bd/lfs_emubd.h"
|
|
#include <stdio.h>
|
|
|
|
// give source a chance to define feature macros
|
|
#undef _FEATURES_H
|
|
#undef _STDIO_H
|
|
|
|
|
|
// generated test configurations
|
|
struct lfs_config;
|
|
|
|
enum test_flags {
|
|
TEST_INTERNAL = 0x1,
|
|
TEST_REENTRANT = 0x2,
|
|
};
|
|
typedef uint8_t test_flags_t;
|
|
|
|
typedef struct test_define {
|
|
const char *name;
|
|
intmax_t *define;
|
|
intmax_t (*cb)(void *data, size_t i);
|
|
void *data;
|
|
size_t permutations;
|
|
} test_define_t;
|
|
|
|
struct test_case {
|
|
const char *name;
|
|
const char *path;
|
|
test_flags_t flags;
|
|
|
|
const test_define_t *defines;
|
|
size_t permutations;
|
|
|
|
bool (*if_)(void);
|
|
void (*run)(struct lfs_config *cfg);
|
|
};
|
|
|
|
struct test_suite {
|
|
const char *name;
|
|
const char *path;
|
|
test_flags_t flags;
|
|
|
|
const test_define_t *defines;
|
|
size_t define_count;
|
|
|
|
const struct test_case *cases;
|
|
size_t case_count;
|
|
};
|
|
|
|
extern const struct test_suite *const test_suites[];
|
|
extern const size_t test_suite_count;
|
|
|
|
|
|
// this variable tracks the number of powerlosses triggered during the
|
|
// current test permutation, this is useful for both tests and debugging
|
|
extern volatile size_t TEST_PLS;
|
|
|
|
// deterministic prng for pseudo-randomness in tests
|
|
uint32_t test_prng(uint32_t *state);
|
|
|
|
#define TEST_PRNG(state) test_prng(state)
|
|
|
|
// generation of specific permutations of an array for exhaustive testing
|
|
size_t test_factorial(size_t x);
|
|
void test_permutation(size_t i, uint32_t *buffer, size_t size);
|
|
|
|
#define TEST_FACTORIAL(x) test_factorial(x)
|
|
#define TEST_PERMUTATION(i, buffer, size) test_permutation(i, buffer, size)
|
|
|
|
|
|
// a few preconfigured defines that control how tests run
|
|
#define TEST_IMPLICIT_DEFINES \
|
|
/* name value (overridable) */ \
|
|
TEST_DEFINE(READ_SIZE, 1 ) \
|
|
TEST_DEFINE(PROG_SIZE, 1 ) \
|
|
TEST_DEFINE(BLOCK_SIZE, 4096 ) \
|
|
TEST_DEFINE(BLOCK_COUNT, DISK_SIZE/BLOCK_SIZE ) \
|
|
TEST_DEFINE(DISK_SIZE, 1024*1024 ) \
|
|
TEST_DEFINE(CACHE_SIZE, \
|
|
lfs_max(16, lfs_max(READ_SIZE, PROG_SIZE)) ) \
|
|
TEST_DEFINE(INLINE_SIZE, BLOCK_SIZE/4 ) \
|
|
TEST_DEFINE(SHRUB_SIZE, INLINE_SIZE ) \
|
|
TEST_DEFINE(FRAGMENT_SIZE, CACHE_SIZE ) \
|
|
TEST_DEFINE(CRYSTAL_THRESH, BLOCK_SIZE/8 ) \
|
|
TEST_DEFINE(LOOKAHEAD_SIZE, 16 ) \
|
|
TEST_DEFINE(BLOCK_CYCLES, -1 ) \
|
|
TEST_DEFINE(ERASE_VALUE, 0xff ) \
|
|
TEST_DEFINE(ERASE_CYCLES, 0 ) \
|
|
TEST_DEFINE(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR ) \
|
|
TEST_DEFINE(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP )
|
|
|
|
// declare defines as global intmax_ts
|
|
#define TEST_DEFINE(k, v) \
|
|
extern intmax_t k;
|
|
|
|
TEST_IMPLICIT_DEFINES
|
|
#undef TEST_DEFINE
|
|
|
|
// map defines to cfg struct fields
|
|
#define TEST_CFG \
|
|
.read_size = READ_SIZE, \
|
|
.prog_size = PROG_SIZE, \
|
|
.block_size = BLOCK_SIZE, \
|
|
.block_count = BLOCK_COUNT, \
|
|
.block_cycles = BLOCK_CYCLES, \
|
|
.cache_size = CACHE_SIZE, \
|
|
.inline_size = INLINE_SIZE, \
|
|
.shrub_size = SHRUB_SIZE, \
|
|
.fragment_size = FRAGMENT_SIZE, \
|
|
.crystal_thresh = CRYSTAL_THRESH, \
|
|
.lookahead_size = LOOKAHEAD_SIZE,
|
|
|
|
#define TEST_BDCFG \
|
|
.erase_value = ERASE_VALUE, \
|
|
.erase_cycles = ERASE_CYCLES, \
|
|
.badblock_behavior = BADBLOCK_BEHAVIOR,
|
|
|
|
|
|
#endif
|