Moved post-bench amor/avg analysis out into amor.py and avg.py
1. Being able to inspect results before benchmarks complete was useful to track their status. It also allows some analysis even if a benchmark fails. 2. Moving these scripts out of bench.py allows them to be a bit more flexible, at the cost of CSV parsing/structuring overhead. 3. Writing benchmark measurements immediately avoids RAM buildup as we store intermediate measurements for each bench permutation. This may increase the IO bottleneck, but we end up writing the same number of lines, so not sure... I realize avg.py has quite a bit of overlap with summary.py, but I don't want to entangle them further. summary.py is already trying to do too much as is...
This commit is contained in:
Christopher Haster
+36
-120
@@ -942,54 +942,6 @@ class BenchOutput:
|
||||
for row in self.rows:
|
||||
self.writer.writerow(row)
|
||||
|
||||
def avg(self):
|
||||
# compute min/max/avg
|
||||
ops = ['bench_readed', 'bench_proged', 'bench_erased']
|
||||
results = co.defaultdict(lambda: {
|
||||
'sums': {op: 0 for op in ops},
|
||||
'mins': {op: +m.inf for op in ops},
|
||||
'maxs': {op: -m.inf for op in ops},
|
||||
'count': 0})
|
||||
|
||||
for row in self.rows:
|
||||
# we only care about results with a BENCH_SEED entry
|
||||
if 'BENCH_SEED' not in row:
|
||||
continue
|
||||
|
||||
# figure our a key for each row, this is everything but the bench
|
||||
# results/seed reencoded as a big tuple-tuple for hashability
|
||||
key = (row['bench_meas'], tuple(sorted(
|
||||
(k, v) for k, v in row.items()
|
||||
if k != 'BENCH_SEED'
|
||||
and k != 'bench_meas'
|
||||
and k != 'bench_agg'
|
||||
and k not in ops)))
|
||||
# find sum/min/max/etc
|
||||
result = results[key]
|
||||
for op in ops:
|
||||
result['sums'][op] += row[op]
|
||||
result['mins'][op] = min(result['mins'][op], row[op])
|
||||
result['maxs'][op] = max(result['maxs'][op], row[op])
|
||||
result['count'] += 1
|
||||
|
||||
# append results to output
|
||||
for (meas, key), result in results.items():
|
||||
self.writerow({
|
||||
'bench_meas': meas+'+avg',
|
||||
'bench_agg': 'avg',
|
||||
**{k: v for k, v in key},
|
||||
**{op: result['sums'][op] / result['count'] for op in ops}})
|
||||
self.writerow({
|
||||
'bench_meas': meas+'+min',
|
||||
'bench_agg': 'bnd',
|
||||
**{k: v for k, v in key},
|
||||
**{op: result['mins'][op] for op in ops}})
|
||||
self.writerow({
|
||||
'bench_meas': meas+'+max',
|
||||
'bench_agg': 'bnd',
|
||||
**{k: v for k, v in key},
|
||||
**{op: result['maxs'][op] for op in ops}})
|
||||
|
||||
# A bench failure
|
||||
class BenchFailure(Exception):
|
||||
def __init__(self, id, returncode, stdout, assert_=None):
|
||||
@@ -998,34 +950,6 @@ class BenchFailure(Exception):
|
||||
self.stdout = stdout
|
||||
self.assert_ = assert_
|
||||
|
||||
# computer extra result stuff, this includes averages and amortized results
|
||||
def bench_results(results):
|
||||
ops = ['readed', 'proged', 'erased']
|
||||
|
||||
# first compute amortized results
|
||||
amors = {}
|
||||
for meas in set(meas for meas, _ in results.keys()):
|
||||
# keep a running sum
|
||||
sums = {op: 0 for op in ops}
|
||||
size = 0
|
||||
for i, (iter, result) in enumerate(sorted(
|
||||
(iter, result) for (meas_, iter), result in results.items()
|
||||
if meas_ == meas)):
|
||||
for op in ops:
|
||||
sums[op] += result.get(op, 0)
|
||||
size += result.get('size', 1)
|
||||
|
||||
# find amortized results
|
||||
amors[meas+'+amor', iter] = {
|
||||
'size': result.get('size', 1),
|
||||
**{op: sums[op] / (i+1) for op in ops}}
|
||||
# also find per-byte results
|
||||
amors[meas+'+div', iter] = {
|
||||
'size': result.get('size', 1),
|
||||
**{op: result.get(op, 0) / size for op in ops}}
|
||||
|
||||
return results | amors
|
||||
|
||||
|
||||
def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
|
||||
# get expected suite/case/perm counts
|
||||
@@ -1081,10 +1005,11 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
|
||||
mpty = os.fdopen(mpty, 'r', 1)
|
||||
|
||||
last_id = None
|
||||
last_case = None
|
||||
last_suite = None
|
||||
last_defines = None # fetched on demand
|
||||
last_stdout = co.deque(maxlen=args.get('context', 5) + 1)
|
||||
last_assert = None
|
||||
if output_:
|
||||
last_results = {}
|
||||
try:
|
||||
while True:
|
||||
# parse a line for state changes
|
||||
@@ -1110,35 +1035,17 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
|
||||
if op == 'running':
|
||||
locals.seen_perms += 1
|
||||
last_id = m.group('id')
|
||||
last_case = m.group('case')
|
||||
last_suite = case_suites[last_case]
|
||||
last_defines = None
|
||||
last_stdout.clear()
|
||||
last_assert = None
|
||||
if output_:
|
||||
last_results = {}
|
||||
elif op == 'finished':
|
||||
case = m.group('case')
|
||||
suite = case_suites[case]
|
||||
passed_suite_perms[suite] += 1
|
||||
passed_case_perms[case] += 1
|
||||
passed_perms += 1
|
||||
if output_:
|
||||
# get defines and write to csv
|
||||
defines = find_defines(
|
||||
runner, m.group('id'), **args)
|
||||
# compute extra measurements here
|
||||
last_results = bench_results(last_results)
|
||||
for (meas, iter), result in (
|
||||
last_results.items()):
|
||||
output_.writerow({
|
||||
'suite': suite,
|
||||
'case': case,
|
||||
**defines,
|
||||
'bench_meas': meas,
|
||||
'bench_agg': 'raw',
|
||||
'bench_iter': iter,
|
||||
'bench_size': result['size'],
|
||||
'bench_readed': result['readed'],
|
||||
'bench_proged': result['proged'],
|
||||
'bench_erased': result['erased']})
|
||||
elif op == 'skipped':
|
||||
locals.seen_perms += 1
|
||||
elif op == 'assert':
|
||||
@@ -1153,28 +1060,39 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
|
||||
meas = m.group('meas')
|
||||
iter = int(m.group('iter'))
|
||||
size = int(m.group('size'))
|
||||
result = {'size': size}
|
||||
for op in ['readed', 'proged', 'erased']:
|
||||
if m.group(op) is None:
|
||||
result[op] = 0
|
||||
elif '.' in m.group(op):
|
||||
result[op] = float(m.group(op))
|
||||
# parse measurements
|
||||
def dat(v):
|
||||
if v is None:
|
||||
return 0
|
||||
elif '.' in v:
|
||||
return float(v)
|
||||
else:
|
||||
result[op] = int(m.group(op))
|
||||
# keep track of per-perm results
|
||||
return int(v)
|
||||
readed_ = dat(m.group('readed'))
|
||||
proged_ = dat(m.group('proged'))
|
||||
erased_ = dat(m.group('erased'))
|
||||
if output_:
|
||||
# if we've already seen this measurement, sum
|
||||
result_ = last_results.get((meas, iter))
|
||||
if result_ is not None:
|
||||
result['readed'] += result_['readed']
|
||||
result['proged'] += result_['proged']
|
||||
result['erased'] += result_['erased']
|
||||
result['size'] += result_['size']
|
||||
last_results[meas, iter] = result
|
||||
# fetch defines if needed, only do this at most
|
||||
# once per perm
|
||||
if last_defines is None:
|
||||
last_defines = find_defines(
|
||||
runner, last_id, **args)
|
||||
# write measurements immediately, this allows
|
||||
# analysis of partial results
|
||||
output_.writerow({
|
||||
'suite': last_suite,
|
||||
'case': last_case,
|
||||
**last_defines,
|
||||
'bench_meas': meas,
|
||||
'bench_iter': iter,
|
||||
'bench_size': size,
|
||||
'bench_readed': readed_,
|
||||
'bench_proged': proged_,
|
||||
'bench_erased': erased_})
|
||||
# keep track of total for summary
|
||||
readed += result['readed']
|
||||
proged += result['proged']
|
||||
erased += result['erased']
|
||||
readed += readed_
|
||||
proged += proged_
|
||||
erased += erased_
|
||||
except KeyboardInterrupt:
|
||||
raise BenchFailure(last_id, 1, list(last_stdout))
|
||||
finally:
|
||||
@@ -1388,8 +1306,6 @@ def run(runner, bench_ids=[], **args):
|
||||
except BrokenPipeError:
|
||||
pass
|
||||
if output:
|
||||
# computer averages?
|
||||
output.avg()
|
||||
output.close()
|
||||
|
||||
# show summary
|
||||
|
||||
Reference in New Issue
Block a user