Github/littlefs
1
Fork 0
Code Actions Activity

Reworked bench.py/bench_runner/how bench measurements are recorded

Browse Source 
This is based on how bench.py/bench_runners have actually been used in
practice. The main changes have been to make the output of bench.py more
readibly consumable by plot.py/plotmpl.py without needing a bunch of
hacky intermediary scripts.

Now instead of a single per-bench BENCH_START/BENCH_STOP, benches can
have multiple named BENCH_START/BENCH_STOP invocations to measure
multiple things in one run:

  BENCH_START("fetch", i, STEP);
  lfsr_rbyd_fetch(&lfs, &rbyd_, rbyd.block, CFG->block_size) => 0;
  BENCH_STOP("fetch");

Benches can also now report explicit results, for non-io measurements:

  BENCH_RESULT("usage", i, STEP, rbyd.eoff);

The extra iter/size parameters to BENCH_START/BENCH_RESULT also allow
some extra information to be calculated post-bench. This infomation gets
tagged with an extra bench_agg field to help organize results in
plot.py/plotmpl.py:

  - bench_meas=<meas>+amor, bench_agg=raw - amortized results
  - bench_meas=<meas>+div,  bench_agg=raw - per-byte results
  - bench_meas=<meas>+avg,  bench_agg=avg - average over BENCH_SEED
  - bench_meas=<meas>+min,  bench_agg=min - minimum over BENCH_SEED
  - bench_meas=<meas>+max,  bench_agg=max - maximum over BENCH_SEED

---

Also removed all bench.tomls for now. This may seem counterproductive in
a commit to improve benchmarking, but I'm not sure there's actual value
to keeping bench cases committed in tree.

These were alway quick to fall out of date (at the time of this commit
most of the low-level bench.tomls, rbyd, btree, etc, no longer
compiled), and most benchmarks were one-off collections of scripts/data
with results too large/cumbersome to commit and keep updated in tree.

I think the better way to approach benchmarking is a seperate repo
(multiple repos?) with all related scripts/state/code and results
committed into a hopefully reproducible snapshot. Keeping the
bench.tomls in that repo makes more sense in this model.

There may be some value to having benchmarks in CI in the future, but
for that to make sense they would need to actually fail on performance
regression. How to do that isn't so clear. Anyways we can always address
this in the future rather than now.
This commit is contained in:
Christopher Haster
2023-11-03 10:27:17 -05:00
parent 4069cf5701
commit e8bdd4d381
11 changed files with 272 additions and 1399 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benches/bench_rbyd.toml
-597
View File
@@ -1,597 +0,0 @@
# Bench our low-level rbyd data-structure
# set block_size to the full size of disk so we can test arbitrarily
# large rbyd trees, we don't really care about block sizes at this
# abstraction level
defines.BLOCK_SIZE = 'DISK_SIZE'
[cases.bench_rbyd_attr_commit]
# 0 = in-order
# 1 = reversed-order
# 2 = random-order
defines.ORDER = [0, 1, 2]
# 0 = 1 commit
# 1 = N commits
defines.COMMIT = [0, 1]
defines.N = [8, 16, 32, 64, 128, 256]
in = 'lfs.c'
if = 'COMMIT == 0 || PROG_SIZE*N <= BLOCK_SIZE'
code = '''
lfs_t lfs;
lfs_init(&lfs, CFG) => 0;
lfsr_rbyd_t rbyd = {
.block = 0,
.off = 0,
.crc = 0,
.trunk = 0,
.weight = 0,
};
lfs_bd_erase(&lfs, rbyd.block) => 0;
// build the attribute list for the current permutations
//
// NOTE we only have 256 user attributes, so this benchmark is
// a bit limited
uint32_t prng = 42;
BENCH_START();
if (COMMIT == 0) {
struct lfsr_attr attrs[N];
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (N-1-i)
: BENCH_PRNG(&prng) % N;
attrs[i] = LFSR_ATTR(-1, UATTR(i_ & 0xff), 0,
"\xaa\xaa\xaa\xaa", 4);
}
lfsr_rbyd_commit(&lfs, &rbyd, attrs, N) => 0;
} else {
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (N-1-i)
: BENCH_PRNG(&prng) % N;
lfsr_rbyd_commit(&lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(i_ & 0xff), 0,
"\xaa\xaa\xaa\xaa", 4))) => 0;
}
}
BENCH_STOP();
lfsr_rbyd_fetch(&lfs, &rbyd, rbyd.block, CFG->block_size) => 0;
'''
[cases.bench_rbyd_attr_fetch]
# 0 = in-order
# 1 = reversed-order
# 2 = random-order
defines.ORDER = [0, 1, 2]
# 0 = 1 commit
# 1 = N commits
defines.COMMIT = [0, 1]
defines.N = [8, 16, 32, 64, 128, 256]
in = 'lfs.c'
if = 'COMMIT == 0 || PROG_SIZE*N <= BLOCK_SIZE'
code = '''
lfs_t lfs;
lfs_init(&lfs, CFG) => 0;
lfsr_rbyd_t rbyd = {
.block = 0,
.off = 0,
.crc = 0,
.trunk = 0,
.weight = 0,
};
lfs_bd_erase(&lfs, rbyd.block) => 0;
// build the attribute list for the current permutations
//
// NOTE we only have 256 user attributes, so this benchmark is
// a bit limited
uint32_t prng = 42;
if (COMMIT == 0) {
struct lfsr_attr attrs[N];
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (N-1-i)
: BENCH_PRNG(&prng) % N;
attrs[i] = LFSR_ATTR(-1, UATTR(i_ & 0xff), 0,
"\xaa\xaa\xaa\xaa", 4);
}
lfsr_rbyd_commit(&lfs, &rbyd, attrs, N) => 0;
} else {
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (N-1-i)
: BENCH_PRNG(&prng) % N;
lfsr_rbyd_commit(&lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(i_ & 0xff), 0,
"\xaa\xaa\xaa\xaa", 4))) => 0;
}
}
BENCH_START();
lfsr_rbyd_fetch(&lfs, &rbyd, rbyd.block, CFG->block_size) => 0;
BENCH_STOP();
'''
[cases.bench_rbyd_attr_lookup]
# 0 = in-order
# 1 = reversed-order
# 2 = random-order
defines.ORDER = [0, 1, 2]
# 0 = 1 commit
# 1 = N commits
defines.COMMIT = [0, 1]
defines.N = [8, 16, 32, 64, 128, 256]
in = 'lfs.c'
if = 'COMMIT == 0 || PROG_SIZE*N <= BLOCK_SIZE'
code = '''
lfs_t lfs;
lfs_init(&lfs, CFG) => 0;
lfsr_rbyd_t rbyd = {
.block = 0,
.off = 0,
.crc = 0,
.trunk = 0,
.weight = 0,
};
lfs_bd_erase(&lfs, rbyd.block) => 0;
// build the attribute list for the current permutations
//
// NOTE we only have 256 user attributes, so this benchmark is
// a bit limited
uint32_t prng = 42;
if (COMMIT == 0) {
struct lfsr_attr attrs[N];
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (N-1-i)
: BENCH_PRNG(&prng) % N;
attrs[i] = LFSR_ATTR(-1, UATTR(i_ & 0xff), 0,
"\xaa\xaa\xaa\xaa", 4);
}
lfsr_rbyd_commit(&lfs, &rbyd, attrs, N) => 0;
} else {
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (N-1-i)
: BENCH_PRNG(&prng) % N;
lfsr_rbyd_commit(&lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(i_ & 0xff), 0,
"\xaa\xaa\xaa\xaa", 4))) => 0;
}
}
lfsr_rbyd_fetch(&lfs, &rbyd, rbyd.block, CFG->block_size) => 0;
BENCH_START();
lfs_off_t i_ = BENCH_PRNG(&prng) % N;
lfsr_data_t data_;
int err = lfsr_rbyd_lookup(&lfs, &rbyd, -1, LFSR_TAG_UATTR(i_ & 0xff),
NULL, &data_);
// note that random order may have some collisions
assert(!err || err == LFS_ERR_NOENT);
BENCH_STOP();
'''
[cases.bench_rbyd_attr_append]
# 0 = in-order
# 1 = reversed-order
# 2 = random-order
defines.ORDER = [0, 1, 2]
# 0 = 1 commit
# 1 = N commits
defines.COMMIT = [0, 1]
defines.N = [8, 16, 32, 64, 128, 256]
in = 'lfs.c'
if = 'COMMIT == 0 || PROG_SIZE*(N+1) <= BLOCK_SIZE'
code = '''
lfs_t lfs;
lfs_init(&lfs, CFG) => 0;
lfsr_rbyd_t rbyd = {
.block = 0,
.off = 0,
.crc = 0,
.trunk = 0,
.weight = 0,
};
lfs_bd_erase(&lfs, rbyd.block) => 0;
// build the attribute list for the current permutations
//
// NOTE we only have 256 user attributes, so this benchmark is
// a bit limited
uint32_t prng = 42;
if (COMMIT == 0) {
struct lfsr_attr attrs[N];
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (N-1-i)
: BENCH_PRNG(&prng) % N;
attrs[i] = LFSR_ATTR(-1, UATTR(i_ & 0xff), 0,
"\xaa\xaa\xaa\xaa", 4);
}
lfsr_rbyd_commit(&lfs, &rbyd, attrs, N) => 0;
} else {
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (N-1-i)
: BENCH_PRNG(&prng) % N;
lfsr_rbyd_commit(&lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(i_ & 0xff), 0,
"\xaa\xaa\xaa\xaa", 4))) => 0;
}
}
lfsr_rbyd_fetch(&lfs, &rbyd, rbyd.block, CFG->block_size) => 0;
BENCH_START();
lfs_off_t i_ = BENCH_PRNG(&prng) % N;
lfsr_rbyd_commit(&lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(i_ & 0xff), 0,
"\xbb\xbb\xbb\xbb", 4))) => 0;
BENCH_STOP();
uint8_t buffer[4];
lfsr_rbyd_get(&lfs, &rbyd, -1, LFSR_TAG_UATTR(i_ & 0xff), buffer, 4) => 4;
assert(memcmp(buffer, "\xbb\xbb\xbb\xbb", 4) == 0);
'''
[cases.bench_rbyd_attr_remove]
# 0 = in-order
# 1 = reversed-order
# 2 = random-order
defines.ORDER = [0, 1, 2]
# 0 = 1 commit
# 1 = N commits
defines.COMMIT = [0, 1]
defines.N = [8, 16, 32, 64, 128, 256]
in = 'lfs.c'
if = 'COMMIT == 0 || PROG_SIZE*(N+1) <= BLOCK_SIZE'
code = '''
lfs_t lfs;
lfs_init(&lfs, CFG) => 0;
lfsr_rbyd_t rbyd = {
.block = 0,
.off = 0,
.crc = 0,
.trunk = 0,
.weight = 0,
};
lfs_bd_erase(&lfs, rbyd.block) => 0;
// build the attribute list for the current permutations
//
// NOTE we only have 256 user attributes, so this benchmark is
// a bit limited
uint32_t prng = 42;
if (COMMIT == 0) {
struct lfsr_attr attrs[N];
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (N-1-i)
: BENCH_PRNG(&prng) % N;
attrs[i] = LFSR_ATTR(-1, UATTR(i_ & 0xff), 0,
"\xaa\xaa\xaa\xaa", 4);
}
lfsr_rbyd_commit(&lfs, &rbyd, attrs, N) => 0;
} else {
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? (N-1-i)
: BENCH_PRNG(&prng) % N;
lfsr_rbyd_commit(&lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(-1, UATTR(i_ & 0xff), 0,
"\xaa\xaa\xaa\xaa", 4))) => 0;
}
}
lfsr_rbyd_fetch(&lfs, &rbyd, rbyd.block, CFG->block_size) => 0;
BENCH_START();
lfs_off_t i_ = BENCH_PRNG(&prng) % N;
lfsr_rbyd_commit(&lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(-1, RMUATTR(i_ & 0xff), 0, NULL, 0))) => 0;
BENCH_STOP();
uint8_t buffer[4];
lfsr_rbyd_get(&lfs, &rbyd, -1, LFSR_TAG_UATTR(i_ & 0xff), buffer, 4)
=> LFS_ERR_NOENT;
'''
[cases.bench_rbyd_id_commit]
# 0 = in-order
# 1 = reversed-order
# 2 = random-order
defines.ORDER = [0, 1, 2]
# 0 = 1 commit
# 1 = N commits
defines.COMMIT = [0, 1]
defines.N = [8, 16, 32, 64, 128, 256, 1024, 2048, 4096]
in = 'lfs.c'
if = 'COMMIT == 0 || PROG_SIZE*N <= BLOCK_SIZE'
code = '''
lfs_t lfs;
lfs_init(&lfs, CFG) => 0;
lfsr_rbyd_t rbyd = {
.block = 0,
.off = 0,
.crc = 0,
.trunk = 0,
.weight = 0,
};
lfs_bd_erase(&lfs, rbyd.block) => 0;
// create commits, note we need to take care to generate
// indexes within a valid range as the rbyd grows
uint32_t prng = 42;
BENCH_START();
if (COMMIT == 0) {
struct lfsr_attr attrs[N];
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? 0
: BENCH_PRNG(&prng) % (rbyd.weight+1);
attrs[i] = LFSR_ATTR(i_, REG, +1, "\xaa\xaa\xaa\xaa", 4);
}
lfsr_rbyd_commit(&lfs, &rbyd, attrs, N) => 0;
} else {
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? 0
: BENCH_PRNG(&prng) % (rbyd.weight+1);
lfsr_rbyd_commit(&lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(i_, REG, +1, "\xaa\xaa\xaa\xaa", 4))) => 0;
}
}
BENCH_STOP();
lfsr_rbyd_fetch(&lfs, &rbyd, rbyd.block, CFG->block_size) => 0;
'''
[cases.bench_rbyd_id_fetch]
# 0 = in-order
# 1 = reversed-order
# 2 = random-order
defines.ORDER = [0, 1, 2]
# 0 = 1 commit
# 1 = N commits
defines.COMMIT = [0, 1]
defines.N = [8, 16, 32, 64, 128, 256, 1024, 2048, 4096]
in = 'lfs.c'
if = 'COMMIT == 0 || PROG_SIZE*N <= BLOCK_SIZE'
code = '''
lfs_t lfs;
lfs_init(&lfs, CFG) => 0;
lfsr_rbyd_t rbyd = {
.block = 0,
.off = 0,
.crc = 0,
.trunk = 0,
.weight = 0,
};
lfs_bd_erase(&lfs, rbyd.block) => 0;
// create commits, note we need to take care to generate
// indexes within a valid range as the rbyd grows
uint32_t prng = 42;
if (COMMIT == 0) {
struct lfsr_attr attrs[N];
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? 0
: BENCH_PRNG(&prng) % (rbyd.weight+1);
attrs[i] = LFSR_ATTR(i_, REG, +1, "\xaa\xaa\xaa\xaa", 4);
}
lfsr_rbyd_commit(&lfs, &rbyd, attrs, N) => 0;
} else {
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? 0
: BENCH_PRNG(&prng) % (rbyd.weight+1);
lfsr_rbyd_commit(&lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(i_, REG, +1, "\xaa\xaa\xaa\xaa", 4))) => 0;
}
}
BENCH_START();
lfsr_rbyd_fetch(&lfs, &rbyd, rbyd.block, CFG->block_size) => 0;
BENCH_STOP();
'''
[cases.bench_rbyd_id_lookup]
# 0 = in-order
# 1 = reversed-order
# 2 = random-order
defines.ORDER = [0, 1, 2]
# 0 = 1 commit
# 1 = N commits
defines.COMMIT = [0, 1]
defines.N = [8, 16, 32, 64, 128, 256, 1024, 2048, 4096]
in = 'lfs.c'
if = 'COMMIT == 0 || PROG_SIZE*N <= BLOCK_SIZE'
code = '''
lfs_t lfs;
lfs_init(&lfs, CFG) => 0;
lfsr_rbyd_t rbyd = {
.block = 0,
.off = 0,
.crc = 0,
.trunk = 0,
.weight = 0,
};
lfs_bd_erase(&lfs, rbyd.block) => 0;
// create commits, note we need to take care to generate
// indexes within a valid range as the rbyd grows
uint32_t prng = 42;
if (COMMIT == 0) {
struct lfsr_attr attrs[N];
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? 0
: BENCH_PRNG(&prng) % (rbyd.weight+1);
attrs[i] = LFSR_ATTR(i_, REG, +1, "\xaa\xaa\xaa\xaa", 4);
}
lfsr_rbyd_commit(&lfs, &rbyd, attrs, N) => 0;
} else {
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? 0
: BENCH_PRNG(&prng) % (rbyd.weight+1);
lfsr_rbyd_commit(&lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(i_, REG, +1, "\xaa\xaa\xaa\xaa", 4))) => 0;
}
}
lfsr_rbyd_fetch(&lfs, &rbyd, rbyd.block, CFG->block_size) => 0;
BENCH_START();
lfs_off_t i_ = BENCH_PRNG(&prng) % N;
lfsr_data_t data_;
lfsr_rbyd_lookup(&lfs, &rbyd, i_, LFSR_TAG_REG,
NULL, &data_) => 0;
BENCH_STOP();
'''
[cases.bench_rbyd_id_create]
# 0 = in-order
# 1 = reversed-order
# 2 = random-order
defines.ORDER = [0, 1, 2]
# 0 = 1 commit
# 1 = N commits
defines.COMMIT = [0, 1]
defines.N = [8, 16, 32, 64, 128, 256, 1024, 2048, 4096]
in = 'lfs.c'
if = 'COMMIT == 0 || PROG_SIZE*(N+1) <= BLOCK_SIZE'
code = '''
lfs_t lfs;
lfs_init(&lfs, CFG) => 0;
lfsr_rbyd_t rbyd = {
.block = 0,
.off = 0,
.crc = 0,
.trunk = 0,
.weight = 0,
};
lfs_bd_erase(&lfs, rbyd.block) => 0;
// create commits, note we need to take care to generate
// indexes within a valid range as the rbyd grows
uint32_t prng = 42;
if (COMMIT == 0) {
struct lfsr_attr attrs[N];
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? 0
: BENCH_PRNG(&prng) % (rbyd.weight+1);
attrs[i] = LFSR_ATTR(i_, REG, +1, "\xaa\xaa\xaa\xaa", 4);
}
lfsr_rbyd_commit(&lfs, &rbyd, attrs, N) => 0;
} else {
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? 0
: BENCH_PRNG(&prng) % (rbyd.weight+1);
lfsr_rbyd_commit(&lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(i_, REG, +1, "\xaa\xaa\xaa\xaa", 4))) => 0;
}
}
lfsr_rbyd_fetch(&lfs, &rbyd, rbyd.block, CFG->block_size) => 0;
BENCH_START();
lfs_off_t i_ = BENCH_PRNG(&prng) % (N+1);
lfsr_rbyd_commit(&lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(i_, REG, +1, "\xbb\xbb\xbb\xbb", 4))) => 0;
BENCH_STOP();
uint8_t buffer[4];
lfsr_rbyd_get(&lfs, &rbyd, i_, LFSR_TAG_REG, buffer, 4) => 4;
assert(memcmp(buffer, "\xbb\xbb\xbb\xbb", 4) == 0);
'''
[cases.bench_rbyd_id_delete]
# 0 = in-order
# 1 = reversed-order
# 2 = random-order
defines.ORDER = [0, 1, 2]
# 0 = 1 commit
# 1 = N commits
defines.COMMIT = [0, 1]
defines.N = [8, 16, 32, 64, 128, 256, 1024, 2048, 4096]
in = 'lfs.c'
if = 'COMMIT == 0 || PROG_SIZE*(N+1) <= BLOCK_SIZE'
code = '''
lfs_t lfs;
lfs_init(&lfs, CFG) => 0;
lfsr_rbyd_t rbyd = {
.block = 0,
.off = 0,
.crc = 0,
.trunk = 0,
.weight = 0,
};
lfs_bd_erase(&lfs, rbyd.block) => 0;
// create commits, note we need to take care to generate
// indexes within a valid range as the rbyd grows
uint32_t prng = 42;
if (COMMIT == 0) {
struct lfsr_attr attrs[N];
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? 0
: BENCH_PRNG(&prng) % (rbyd.weight+1);
attrs[i] = LFSR_ATTR(i_, REG, +1, "\xaa\xaa\xaa\xaa", 4);
}
lfsr_rbyd_commit(&lfs, &rbyd, attrs, N) => 0;
} else {
for (lfs_size_t i = 0; i < N; i++) {
lfs_off_t i_
= (ORDER == 0) ? i
: (ORDER == 1) ? 0
: BENCH_PRNG(&prng) % (rbyd.weight+1);
lfsr_rbyd_commit(&lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(i_, REG, +1, "\xaa\xaa\xaa\xaa", 4))) => 0;
}
}
lfsr_rbyd_fetch(&lfs, &rbyd, rbyd.block, CFG->block_size) => 0;
BENCH_START();
lfs_off_t i_ = BENCH_PRNG(&prng) % N;
lfsr_rbyd_commit(&lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(i_, UNR, -1, NULL, 0))) => 0;
BENCH_STOP();
'''