Github/littlefs
1
Fork 0
Code Actions Activity

runners: bench: Added flags to control reading from bench probes

Browse Source 
- -S/--probe         - Specify a probe to sample.
- -x/--probe-step    - Sample probes every n steps.
- --probe-runfreq    - Sample probes at this frequency in hz.
- -X/--probe-simfreq - Sample probes at this frequency in simulated hz.

Also:

- --trace-simfreq    - Sample trace output at this frequency in
                       simulated hz.

These give finer grain control over which probes we measure during
benching, and how we measure them.

These also introduce several exciting bench features:

- -S/--probe provides the ability to easily filter which probes you're
  interested in at runtime.

  This should replace the growing use of MASK defines in the benches.

- -x/--probe-step makes it easy to relax sampling rate when the amount
  of data overwhelms later scripts.

  This should replace the growing use of STEP defines in the benches.

- The additional concept of simfreq, which allows perf-esque sampling in
  simtime. This provides another option for intuitively relaxing probe
  sampling rate without sacrificing reproducibility.

  (runfreq depends on wall time, so good bye reproducibility, though may
  still be useful in interactive contexts.)

Note -S/--probe and -x/--probe-step replace MASK/STEP defines, which
have already proved their usefulness, but required reimplementation in
every bench case. An obvious contender to move into the bench_runner!

---

Note note that -S/--probe also supports some simple sample expressions,
allowing flexible step/simfreq/runfreq at the per-probe level:

- -Swrite=100    - Sample probe "write" every 100 steps
- -Swrite=100rhz - Sample probe "write" 100 times a runtime second
- -Swrite=100shz - Sample probe "write" 100 times a simulated second

Though I wonder how long it will take before I forget this feature
exists.
This commit is contained in:
Christopher Haster
2026-02-09 13:43:55 -06:00
parent 81d681cab2
commit 4af4cf3212
8 changed files with 773 additions and 378 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benches/bench_rt.toml
+7 -7
View File
@@ -56,7 +56,7 @@ code = '''
lfs3_file_write(&lfs3, &file, wbuf, CHUNK) => CHUNK;
// taking too long?
if (SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME) {
if (SIM_TIME && BENCH_SIMTIME() >= SIM_TIME) {
return;
}
}
@@ -71,7 +71,7 @@ code = '''
lfs3_off_t size = 0;
uint64_t readed = 0;
while (!(SIM_SIZE && readed >= (uint64_t)SIM_SIZE)
&& !(SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME)) {
&& !(SIM_TIME && BENCH_SIMTIME() >= SIM_TIME)) {
// read from the file
uint8_t rbuf[CHUNK];
lfs3_file_read(&lfs3, &file, rbuf, CHUNK) => CHUNK;
@@ -123,7 +123,7 @@ code = '''
lfs3_file_write(&lfs3, &file, wbuf, CHUNK) => CHUNK;
// taking too long?
if (SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME) {
if (SIM_TIME && BENCH_SIMTIME() >= SIM_TIME) {
return;
}
}
@@ -137,7 +137,7 @@ code = '''
lfs3_file_open(&lfs3, &file, "bench_random", LFS3_O_RDONLY) => 0;
uint64_t readed = 0;
while (!(SIM_SIZE && readed >= (uint64_t)SIM_SIZE)
&& !(SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME)) {
&& !(SIM_TIME && BENCH_SIMTIME() >= SIM_TIME)) {
// seek to a random location
lfs3_off_t pos = BENCH_PRNG(&prng) % SIZE;
lfs3_file_seek(&lfs3, &file, pos, LFS3_SEEK_SET) => pos;
@@ -191,7 +191,7 @@ code = '''
lfs3_file_write(&lfs3, &file, wbuf, d) => d;
// taking too long?
if (SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME) {
if (SIM_TIME && BENCH_SIMTIME() >= SIM_TIME) {
return;
}
}
@@ -205,7 +205,7 @@ code = '''
BENCH_START("read");
uint64_t readed = 0;
while (!(SIM_SIZE && readed >= (uint64_t)SIM_SIZE)
&& !(SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME)) {
&& !(SIM_TIME && BENCH_SIMTIME() >= SIM_TIME)) {
// choose a random filename
lfs3_off_t pos = BENCH_PRNG(&prng) % ((SIZE+(CHUNK-1))/CHUNK);
char name[256];
@@ -224,7 +224,7 @@ code = '''
readed += d;
// taking too long?
if (SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME) {
if (SIM_TIME && BENCH_SIMTIME() >= SIM_TIME) {
break;
}
}
benches/bench_wt.toml
+5 -5
View File
@@ -54,7 +54,7 @@ code = '''
// ok, one of these needs to be non-zero
LFS3_ASSERT(SIM_TIME > 0 || SIM_SIZE > 0);
while (!(SIM_SIZE && written >= (uint64_t)SIM_SIZE)
&& !(SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME)) {
&& !(SIM_TIME && BENCH_SIMTIME() >= SIM_TIME)) {
// arguably we should just rewind and continue writing to the
// front of the file when we hit the end, but this overly
// penalizes littlefs2, so instead we truncate
@@ -122,7 +122,7 @@ code = '''
lfs3_off_t size = 0;
uint64_t written = 0;
while (!(SIM_SIZE && written >= (uint64_t)SIM_SIZE)
&& !(SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME)) {
&& !(SIM_TIME && BENCH_SIMTIME() >= SIM_TIME)) {
// seek to a random location
lfs3_off_t pos = BENCH_PRNG(&prng) % SIZE;
lfs3_file_seek(&lfs3, &file, pos, LFS3_SEEK_SET) => pos;
@@ -189,7 +189,7 @@ code = '''
// ok, one of these needs to be non-zero
LFS3_ASSERT(SIM_TIME > 0 || SIM_SIZE > 0);
while (!(SIM_SIZE && written >= (uint64_t)SIM_SIZE)
&& !(SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME)) {
&& !(SIM_TIME && BENCH_SIMTIME() >= SIM_TIME)) {
// append to log
uint8_t wbuf[CHUNK];
for (lfs3_size_t j = 0; j < CHUNK; j++) {
@@ -265,7 +265,7 @@ code = '''
// ok, one of these needs to be non-zero
LFS3_ASSERT(SIM_TIME > 0 || SIM_SIZE > 0);
while (!(SIM_SIZE && written >= (uint64_t)SIM_SIZE)
&& !(SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME)) {
&& !(SIM_TIME && BENCH_SIMTIME() >= SIM_TIME)) {
// choose a random filename
lfs3_off_t pos = BENCH_PRNG(&prng) % FILE_COUNT;
char name[256];
@@ -287,7 +287,7 @@ code = '''
written += d;
// taking too long?
if (SIM_TIME && BENCH_SIMTIME() >= (bench_ns_t)SIM_TIME) {
if (SIM_TIME && BENCH_SIMTIME() >= SIM_TIME) {
break;
}
}
runners/bench_runner.c
+582 -245
View File
File diff suppressed because it is too large Load Diff
runners/bench_runner.h
+8 -1
View File
@@ -156,7 +156,7 @@ void bench_fresult(const char *probe, uintmax_t n, double result);
// extra hooks to get the current simtime, pause readed/progged/erased
// counters, etc
bench_ns_t bench_simtime(void);
bench_sns_t bench_simtime(void);
void bench_simreset(void);
void bench_simpause(void);
void bench_simresume(void);
@@ -185,6 +185,13 @@ void bench_permutation(size_t i, uint32_t *buffer, size_t size);
#define BENCH_FACTORIAL(x) bench_factorial(x)
#define BENCH_PERMUTATION(i, buffer, size) bench_permutation(i, buffer, size)
// option to pause trace output
void bench_trace_pause(void);
void bench_trace_resume(void);
#define BENCH_TRACE_PAUSE() bench_trace_pause()
#define BENCH_TRACE_RESUME() bench_trace_resume()
#ifdef BENCH_STACK
// get the maximum/current stack usage for this run
extern size_t bench_stack_watermark;
runners/test_runner.c
+133 -114
View File
@@ -176,6 +176,7 @@ ssize_t *test_suite_define_map = NULL;
test_define_t *test_override_defines = NULL;
size_t test_override_define_count = 0;
size_t test_override_define_capacity = 0;
size_t test_define_depth = 1000;
@@ -432,23 +433,25 @@ const test_id_t *test_ids = (const test_id_t[]) {
{NULL, NULL, 0, {NULL, NULL, NULL, 0}},
};
size_t test_id_count = 1;
size_t test_id_capacity = 0;
size_t test_step_start = 0;
size_t test_step_stop = -1;
size_t test_step_step = 1;
size_t test_step = 0; // incremented every permutation
size_t test_steps = 0; // incremented every permutation
bool test_force = false;
test_flags_t test_mask = 0;
const char *test_disk_path = NULL;
const char *test_trace_path = NULL;
bool test_trace_backtrace = false;
uint32_t test_trace_step = 0;
uint32_t test_trace_runfreq = 0;
size_t test_trace_step = 0;
double test_trace_runfreq = 0;
uint32_t test_trace_paused = false;
FILE *test_trace_file = NULL;
uint32_t test_trace_cycles = 0;
uint64_t test_trace_time = 0;
uint64_t test_trace_open_time = 0;
size_t test_trace_steps = 0;
test_ns_t test_trace_runtime = 0;
test_ns_t test_trace_open_runtime = 0;
test_ns_t test_read_sleep = 0.0;
test_ns_t test_prog_sleep = 0.0;
test_ns_t test_erase_sleep = 0.0;
@@ -469,104 +472,123 @@ void *test_trace_backtrace_buffer[
// trace printing
void test_trace(const char *fmt, ...) {
if (test_trace_path) {
// sample at a specific step?
if (test_trace_step) {
if (test_trace_cycles % test_trace_step != 0) {
test_trace_cycles += 1;
goto done;
}
test_trace_cycles += 1;
}
// sample at a specific frequency?
if (test_trace_runfreq) {
struct timespec t;
clock_gettime(CLOCK_MONOTONIC, &t);
uint64_t now = (uint64_t)t.tv_sec*1000*1000*1000
+ (uint64_t)t.tv_nsec;
if (now - test_trace_time
< (1000*1000*1000) / test_trace_runfreq) {
goto done;
}
test_trace_time = now;
}
if (!test_trace_file) {
// Tracing output is heavy and trying to open every trace
// call is slow, so we only try to open the trace file every
// so often. Note this doesn't affect successfully opened files
struct timespec t;
clock_gettime(CLOCK_MONOTONIC, &t);
uint64_t now = (uint64_t)t.tv_sec*1000*1000*1000
+ (uint64_t)t.tv_nsec;
if (now - test_trace_open_time < 100*1000*1000) {
goto done;
}
test_trace_open_time = now;
// try to open the trace file
int fd;
if (strcmp(test_trace_path, "-") == 0) {
fd = dup(1);
if (fd < 0) {
goto done;
}
} else {
fd = open(
test_trace_path,
O_WRONLY | O_CREAT | O_APPEND | O_NONBLOCK,
0666);
if (fd < 0) {
goto done;
}
int err = fcntl(fd, F_SETFL, O_WRONLY | O_CREAT | O_APPEND);
assert(!err);
}
FILE *f = fdopen(fd, "a");
assert(f);
int err = setvbuf(f, NULL, _IOFBF,
TEST_TRACE_BACKTRACE_BUFFER_SIZE);
assert(!err);
test_trace_file = f;
}
// print trace
va_list va;
va_start(va, fmt);
int res = vfprintf(test_trace_file, fmt, va);
va_end(va);
if (res < 0) {
fclose(test_trace_file);
test_trace_file = NULL;
goto done;
}
if (test_trace_backtrace) {
// print backtrace
size_t count = backtrace(
test_trace_backtrace_buffer,
TEST_TRACE_BACKTRACE_BUFFER_SIZE);
// note we skip our own stack frame
for (size_t i = 1; i < count; i++) {
res = fprintf(test_trace_file, "\tat %p\n",
test_trace_backtrace_buffer[i]);
if (res < 0) {
fclose(test_trace_file);
test_trace_file = NULL;
goto done;
}
}
}
// flush immediately
fflush(test_trace_file);
if (!test_trace_path || test_trace_paused) {
goto done;
}
// prevent accidental recursion
TEST_TRACE_PAUSE();
// sample at a specific step?
if (test_trace_step) {
if (test_trace_steps % test_trace_step != 0) {
test_trace_steps += 1;
goto done_;
}
test_trace_steps += 1;
}
// sample at a specific frequency?
if (test_trace_runfreq) {
struct timespec t;
clock_gettime(CLOCK_MONOTONIC, &t);
test_ns_t now = (test_ns_t)t.tv_sec*1000*1000*1000
+ (test_ns_t)t.tv_nsec;
if (now - test_trace_runtime
< (test_ns_t)((1000.0*1000.0*1000.0)
/ test_trace_runfreq)) {
goto done_;
}
test_trace_runtime = now;
}
if (!test_trace_file) {
// Tracing output is heavy and trying to open every trace
// call is slow, so we only try to open the trace file every
// so often. Note this doesn't affect successfully opened files
struct timespec t;
clock_gettime(CLOCK_MONOTONIC, &t);
test_ns_t now = (test_ns_t)t.tv_sec*1000*1000*1000
+ (test_ns_t)t.tv_nsec;
if (now - test_trace_open_runtime < 100*1000*1000) {
goto done_;
}
test_trace_open_runtime = now;
// try to open the trace file
int fd;
if (strcmp(test_trace_path, "-") == 0) {
fd = dup(1);
if (fd < 0) {
goto done_;
}
} else {
fd = open(
test_trace_path,
O_WRONLY | O_CREAT | O_APPEND | O_NONBLOCK,
0666);
if (fd < 0) {
goto done_;
}
int err = fcntl(fd, F_SETFL, O_WRONLY | O_CREAT | O_APPEND);
assert(!err);
}
FILE *f = fdopen(fd, "a");
assert(f);
int err = setvbuf(f, NULL, _IOFBF,
TEST_TRACE_BACKTRACE_BUFFER_SIZE);
assert(!err);
test_trace_file = f;
}
// print trace
va_list va;
va_start(va, fmt);
int res = vfprintf(test_trace_file, fmt, va);
va_end(va);
if (res < 0) {
fclose(test_trace_file);
test_trace_file = NULL;
goto done_;
}
if (test_trace_backtrace) {
// print backtrace
size_t count = backtrace(
test_trace_backtrace_buffer,
TEST_TRACE_BACKTRACE_BUFFER_SIZE);
// note we skip our own stack frame
for (size_t i = 1; i < count; i++) {
res = fprintf(test_trace_file, "\tat %p\n",
test_trace_backtrace_buffer[i]);
if (res < 0) {
fclose(test_trace_file);
test_trace_file = NULL;
goto done_;
}
}
}
// flush immediately
fflush(test_trace_file);
done_:;
TEST_TRACE_RESUME();
done:;
}
void test_trace_pause(void) {
test_trace_paused += 1;
}
void test_trace_resume(void) {
assert(test_trace_paused);
test_trace_paused -= 1;
}
// test prng
uint32_t test_prng(uint32_t *state) {
// A simple xorshift32 generator, easily reproducible. Keep in mind
@@ -822,13 +844,13 @@ void perm_count(
}
// skip this step?
if (!(test_step >= test_step_start
&& test_step < test_step_stop
&& (test_step-test_step_start) % test_step_step == 0)) {
test_step += 1;
if (!(test_steps >= test_step_start
&& test_steps < test_step_stop
&& (test_steps-test_step_start) % test_step_step == 0)) {
test_steps += 1;
return;
}
test_step += 1;
test_steps += 1;
state->total += 1;
@@ -1868,6 +1890,7 @@ size_t test_powerloss_count = 2;
#else
size_t test_powerloss_count = 1;
#endif
size_t test_powerloss_capacity = 0;
static void list_powerlosses(void) {
// at least size so that names fit
@@ -1913,13 +1936,13 @@ void perm_run(
}
// skip this step?
if (!(test_step >= test_step_start
&& test_step < test_step_stop
&& (test_step-test_step_start) % test_step_step == 0)) {
test_step += 1;
if (!(test_steps >= test_step_start
&& test_steps < test_step_stop
&& (test_steps-test_step_start) % test_step_step == 0)) {
test_steps += 1;
return;
}
test_step += 1;
test_steps += 1;
// set pls to 1 if running under powerloss so it useful for if predicates
TEST_PLS = (powerloss->run != run_powerloss_none);
@@ -2062,10 +2085,6 @@ const char *const help_text[] = {
int main(int argc, char **argv) {
void (*op)(void) = run;
size_t test_override_define_capacity = 0;
size_t test_powerloss_capacity = 0;
size_t test_id_capacity = 0;
// parse options
while (true) {
int c = getopt_long(argc, argv, short_opts, long_opts, NULL);
@@ -2582,7 +2601,7 @@ int main(int argc, char **argv) {
case OPT_TRACE_RUNFREQ:;
parsed = NULL;
test_trace_runfreq = strtoumax(optarg, &parsed, 0);
test_trace_runfreq = strtod(optarg, &parsed);
if (parsed == optarg) {
fprintf(stderr, "error: invalid trace-runfreq: %s\n", optarg);
exit(-1);
runners/test_runner.h
+7
View File
@@ -153,6 +153,13 @@ void test_permutation(size_t i, uint32_t *buffer, size_t size);
#define TEST_FACTORIAL(x) test_factorial(x)
#define TEST_PERMUTATION(i, buffer, size) test_permutation(i, buffer, size)
// option to pause trace output
void test_trace_pause(void);
void test_trace_resume(void);
#define TEST_TRACE_PAUSE() test_trace_pause()
#define TEST_TRACE_RESUME() test_trace_resume()
// declare implicit defines as global intmax_ts
#define TEST_DEFINE(k, v) \
scripts/bench.py
+28 -3
View File
@@ -822,6 +822,15 @@ def find_runner(runner, id=None, main=True, **args):
# other context
if args.get('define_depth'):
cmd.append('--define-depth=%s' % args['define_depth'])
if args.get('probe'):
for probe in args['probe']:
cmd.append('-S%s' % probe)
if args.get('probe_step'):
cmd.append('-x%s' % args['probe_step'])
if args.get('probe_runfreq'):
cmd.append('--probe-runfreq=%s' % args['probe_runfreq'])
if args.get('probe_simfreq'):
cmd.append('-X%s' % args['probe_simfreq'])
if args.get('force'):
cmd.append('--force')
if args.get('no_internal'):
@@ -1203,7 +1212,7 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
last_defines = None # fetched on demand
last_stdout = co.deque(maxlen=args.get('context', 5) + 1)
last_assert = None
last_time = time.time()
last_runtime = time.time()
try:
while True:
# parse a line for state changes
@@ -1234,7 +1243,7 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
last_defines = None
last_stdout.clear()
last_assert = None
last_time = time.time()
last_runtime = time.time()
elif op == 'finished':
# force a failure
if args.get('fail'):
@@ -1296,7 +1305,7 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
'bench_erased': erased_,
'bench_simtime': simtime_,
'bench_runtime': '%.6f' % (
time.time() - last_time)})
time.time() - last_runtime)})
# keep track of total for summary
readed += readed_
progged += progged_
@@ -1758,6 +1767,19 @@ if __name__ == "__main__":
bench_parser.add_argument(
'--define-depth',
help="How deep to evaluate recursive defines before erroring.")
bench_parser.add_argument(
'-S', '--probe',
action='append',
help="Specify a probe to sample.")
bench_parser.add_argument(
'-x', '--probe-step',
help="Sample probes every n steps.")
bench_parser.add_argument(
'--probe-runfreq',
help="Sample probes at this frequency in hz.")
bench_parser.add_argument(
'-X', '--probe-simfreq',
help="Sample probes at this frequency in simulated hz.")
bench_parser.add_argument(
'--force',
action='store_true',
@@ -1786,6 +1808,9 @@ if __name__ == "__main__":
bench_parser.add_argument(
'--trace-runfreq',
help="Sample trace output at this frequency in hz.")
bench_parser.add_argument(
'--trace-simfreq',
help="Sample trace output at this frequency in simulated hz.")
bench_parser.add_argument(
'-O', '--stdout',
help="Direct stdout to this file. Note stderr is already merged "
scripts/test.py
+3 -3
View File
@@ -1179,7 +1179,7 @@ def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args):
last_id = None
last_stdout = co.deque(maxlen=args.get('context', 5) + 1)
last_assert = None
last_time = time.time()
last_runtime = time.time()
try:
while True:
# parse a line for state changes
@@ -1207,7 +1207,7 @@ def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args):
last_id = m.group('id')
last_stdout.clear()
last_assert = None
last_time = time.time()
last_runtime = time.time()
elif op == 'powerloss':
last_id = m.group('id')
powerlosses += 1
@@ -1232,7 +1232,7 @@ def run_stage(name, runner, test_ids, stdout_, trace_, output_, **args):
**defines,
'test_passed': '1/1',
'test_runtime': '%.6f' % (
time.time() - last_time)})
time.time() - last_runtime)})
elif op == 'skipped':
locals.seen_perms += 1
elif op == 'assert':