Christopher Haster
a124ee54e7
Motivation: - Debuggability. Accessing the current test/bench defines from inside gdb was basically impossible for some dumb macro-debug-info reason I can't figure out. In theory, GCC provides a .debug_macro section when compiled with -g3. I can see this section with objdump --dwarf=macro, but somehow gdb can't seem to find any definitions? I'm guess the #line source remapping is causing things to break somehow... Though even if macro-debugging gets fixed, which would be valuable, accessing defines in the current test/bench runner can trigger quite a bit of hidden machinery. This risks side-effects, which is never great when debugging. All of this is quite annoying because the test/bench defines is usually the most important piece of information when debugging! This replaces the previous hidden define machinery with simple global variables, which gdb can access no problem. - Also when debugging we no longer awkwardly step into the test_define function all the time! - In theory, global variables, being a simple memory access, should be quite a bit faster than the hidden define machinery. This does matter because running tests _is_ a dev bottleneck. In practice though, any performance benefit is below the noise floor, which isn't too surprising (~630s +-~20s). - Using global variables for defines simplifies the test/bench runner quite a bit. Though some of the previous complexity was due to a whole internal define caching system, which was supposed to lazily evaluate test defines to avoid evaluating defines we don't use. This all proved to be useless because the first thing we do when running each test is evaluate all defines to generate the test id (lol). So now, instead of lazily evaluating and caching defines, we just generate global variables during compilation and evaluate all defines for each test permutation immediately before running. This relies heavily on __attribute__((weak)) symbols, and lets the linker really shine. As a funny perk this also effectively interns all test/bench defines by the address of the resulting global variable. So we don't even need to do string comparisons when mapping suite-level defines to the runner-level defines. --- Perhaps the more interesting thing to note, is the change in strategy in how we actually evaluate the test defines. This ends up being a surprisingly tricky problem, due to the potential of mutual recursion between our defines. Previously, because our define machinery was lazy, we could just evaluate each define on demand. If a define required another define, it would lazily trigger another evaluation, implicitly recursing through C's stack. If cyclic, this would eventually lead to a stack overflow, but that's ok because it's a user error to let this happen. The "correct" way, at least in terms of being computationally optimal, would be to topologically sort the defines and evaluate the resulting tree from the leaves up. But I ain't got time for that, so the solution here is equal parts hacky, simple, and effective. Basically, we just evaluate the defines repeatedly until they stop changing: - Initially, mutually recursive defines may read the uninitialized values of their dependencies, and end up with some arbitrarily wrong result. But as the defines are repeatedly evaluated, assuming no cycles, the correct results should eventually bubble up the tree until all defines converge to the correct value. - This is O(n*e) vs O(n+e), but our define graph is usually quite shallow. - To prevent non-halting, we error after an arbitrary 1000 iterations. If you hit this, it's likely because there is a cycle in the define graph. This is runtime configurable via the new --define-depth flag. - To keep things consistent and reproducible, we zero initialize all defines before the first evaluation. I don't think this is strictly necessary, but it's important for the test runner to have the exact same results on every run. No one wants a "works on my machine" situation when the tests are involved. Experimentation shows we only need an evaluation depth of 2 to successfully evaluate the current set of defines: $ ./runners/test_runner --list-defines --define-depth=2 And any performance impact is negligible (~630s +-~20s).
159 lines
4.9 KiB
C
159 lines
4.9 KiB
C
/*
|
|
* Runner for littlefs benchmarks
|
|
*
|
|
* Copyright (c) 2022, The littlefs authors.
|
|
* SPDX-License-Identifier: BSD-3-Clause
|
|
*/
|
|
#ifndef BENCH_RUNNER_H
|
|
#define BENCH_RUNNER_H
|
|
|
|
|
|
// override LFS_TRACE
|
|
void bench_trace(const char *fmt, ...);
|
|
|
|
#define LFS_TRACE_(fmt, ...) \
|
|
bench_trace("%s:%d:trace: " fmt "%s\n", \
|
|
__FILE__, \
|
|
__LINE__, \
|
|
__VA_ARGS__)
|
|
#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")
|
|
#define LFS_EMUBD_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")
|
|
|
|
// BENCH_START/BENCH_STOP macros measure readed/proged/erased bytes
|
|
// through emubd
|
|
void bench_start(const char *meas, uintmax_t iter, uintmax_t size);
|
|
void bench_stop(const char *meas);
|
|
|
|
#define BENCH_START(meas, iter, size) \
|
|
bench_start(meas, iter, size)
|
|
#define BENCH_STOP(meas) \
|
|
bench_stop(meas)
|
|
|
|
// BENCH_RESULT/BENCH_FRESULT allow for explicit non-io measurements
|
|
void bench_result(const char *meas, uintmax_t iter, uintmax_t size,
|
|
uintmax_t result);
|
|
void bench_fresult(const char *meas, uintmax_t iter, uintmax_t size,
|
|
double result);
|
|
|
|
#define BENCH_RESULT(meas, iter, size, result) \
|
|
bench_result(meas, iter, size, result)
|
|
#define BENCH_FRESULT(meas, iter, size, result) \
|
|
bench_fresult(meas, iter, size, result)
|
|
|
|
|
|
// note these are indirectly included in any generated files
|
|
#include "bd/lfs_emubd.h"
|
|
#include <stdio.h>
|
|
|
|
// give source a chance to define feature macros
|
|
#undef _FEATURES_H
|
|
#undef _STDIO_H
|
|
|
|
|
|
// generated bench configurations
|
|
struct lfs_config;
|
|
|
|
enum bench_flags {
|
|
BENCH_INTERNAL = 0x1,
|
|
};
|
|
typedef uint8_t bench_flags_t;
|
|
|
|
typedef struct bench_define {
|
|
const char *name;
|
|
intmax_t *define;
|
|
intmax_t (*cb)(void *data, size_t i);
|
|
void *data;
|
|
size_t permutations;
|
|
} bench_define_t;
|
|
|
|
struct bench_case {
|
|
const char *name;
|
|
const char *path;
|
|
bench_flags_t flags;
|
|
|
|
const bench_define_t *defines;
|
|
size_t permutations;
|
|
|
|
bool (*if_)(void);
|
|
void (*run)(struct lfs_config *cfg);
|
|
};
|
|
|
|
struct bench_suite {
|
|
const char *name;
|
|
const char *path;
|
|
bench_flags_t flags;
|
|
|
|
const bench_define_t *defines;
|
|
size_t define_count;
|
|
|
|
const struct bench_case *cases;
|
|
size_t case_count;
|
|
};
|
|
|
|
extern const struct bench_suite *const bench_suites[];
|
|
extern const size_t bench_suite_count;
|
|
|
|
|
|
// deterministic prng for pseudo-randomness in benches
|
|
uint32_t bench_prng(uint32_t *state);
|
|
|
|
#define BENCH_PRNG(state) bench_prng(state)
|
|
|
|
// generation of specific permutations of an array for exhaustive benching
|
|
size_t bench_factorial(size_t x);
|
|
void bench_permutation(size_t i, uint32_t *buffer, size_t size);
|
|
|
|
#define BENCH_FACTORIAL(x) bench_factorial(x)
|
|
#define BENCH_PERMUTATION(i, buffer, size) bench_permutation(i, buffer, size)
|
|
|
|
|
|
// a few preconfigured defines that control how benches run
|
|
#define BENCH_IMPLICIT_DEFINES \
|
|
/* name value (overridable) */ \
|
|
BENCH_DEFINE(READ_SIZE, 1 ) \
|
|
BENCH_DEFINE(PROG_SIZE, 1 ) \
|
|
BENCH_DEFINE(BLOCK_SIZE, 4096 ) \
|
|
BENCH_DEFINE(BLOCK_COUNT, DISK_SIZE/BLOCK_SIZE ) \
|
|
BENCH_DEFINE(DISK_SIZE, 1024*1024 ) \
|
|
BENCH_DEFINE(CACHE_SIZE, \
|
|
lfs_max(16, lfs_max(READ_SIZE, PROG_SIZE))) \
|
|
BENCH_DEFINE(INLINE_SIZE, BLOCK_SIZE/4 ) \
|
|
BENCH_DEFINE(SHRUB_SIZE, INLINE_SIZE ) \
|
|
BENCH_DEFINE(FRAGMENT_SIZE, CACHE_SIZE ) \
|
|
BENCH_DEFINE(CRYSTAL_THRESH, BLOCK_SIZE/8 ) \
|
|
BENCH_DEFINE(LOOKAHEAD_SIZE, 16 ) \
|
|
BENCH_DEFINE(BLOCK_CYCLES, -1 ) \
|
|
BENCH_DEFINE(ERASE_VALUE, 0xff ) \
|
|
BENCH_DEFINE(ERASE_CYCLES, 0 ) \
|
|
BENCH_DEFINE(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR ) \
|
|
BENCH_DEFINE(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP )
|
|
|
|
// declare defines as global intmax_ts
|
|
#define BENCH_DEFINE(k, v) \
|
|
extern intmax_t k;
|
|
|
|
BENCH_IMPLICIT_DEFINES
|
|
#undef BENCH_DEFINE
|
|
|
|
// map defines to cfg struct fields
|
|
#define BENCH_CFG \
|
|
.read_size = READ_SIZE, \
|
|
.prog_size = PROG_SIZE, \
|
|
.block_size = BLOCK_SIZE, \
|
|
.block_count = BLOCK_COUNT, \
|
|
.block_cycles = BLOCK_CYCLES, \
|
|
.cache_size = CACHE_SIZE, \
|
|
.inline_size = INLINE_SIZE, \
|
|
.shrub_size = SHRUB_SIZE, \
|
|
.fragment_size = FRAGMENT_SIZE, \
|
|
.crystal_thresh = CRYSTAL_THRESH, \
|
|
.lookahead_size = LOOKAHEAD_SIZE,
|
|
|
|
#define BENCH_BDCFG \
|
|
.erase_value = ERASE_VALUE, \
|
|
.erase_cycles = ERASE_CYCLES, \
|
|
.badblock_behavior = BADBLOCK_BEHAVIOR,
|
|
|
|
|
|
#endif
|