Christopher Haster
48c2e7784b
Mainly to avoid conflicts with match results m, this frees up the single letter variables m for other purposes. Choosing a two letter alias was surprisingly difficult, but mt is nice in that it somewhat matches it (for itertools) and ft (for functools).
313 lines
9.6 KiB
Python
Executable File
313 lines
9.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Compute averages/etc of benchmark measurements
|
|
#
|
|
|
|
import collections as co
|
|
import csv
|
|
import itertools as it
|
|
import math as mt
|
|
import os
|
|
|
|
|
|
def openio(path, mode='r', buffering=-1):
|
|
# allow '-' for stdin/stdout
|
|
if path == '-':
|
|
if 'r' in mode:
|
|
return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering)
|
|
else:
|
|
return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering)
|
|
else:
|
|
return open(path, mode, buffering)
|
|
|
|
# parse different data representations
|
|
def dat(x):
|
|
# allow the first part of an a/b fraction
|
|
if '/' in x:
|
|
x, _ = x.split('/', 1)
|
|
|
|
# first try as int
|
|
try:
|
|
return int(x, 0)
|
|
except ValueError:
|
|
pass
|
|
|
|
# then try as float
|
|
try:
|
|
return float(x)
|
|
# just don't allow infinity or nan
|
|
if mt.isinf(x) or mt.isnan(x):
|
|
raise ValueError("invalid dat %r" % x)
|
|
except ValueError:
|
|
pass
|
|
|
|
# else give up
|
|
raise ValueError("invalid dat %r" % x)
|
|
|
|
def collect(csv_paths, renames=[], defines=[]):
|
|
# collect results from CSV files
|
|
fields = []
|
|
results = []
|
|
for path in csv_paths:
|
|
try:
|
|
with openio(path) as f:
|
|
reader = csv.DictReader(f, restval='')
|
|
fields.extend(
|
|
k for k in reader.fieldnames
|
|
if k not in fields)
|
|
for r in reader:
|
|
# apply any renames
|
|
if renames:
|
|
# make a copy so renames can overlap
|
|
r_ = {}
|
|
for new_k, old_k in renames:
|
|
if old_k in r:
|
|
r_[new_k] = r[old_k]
|
|
r.update(r_)
|
|
|
|
# filter by matching defines
|
|
if not all(k in r and r[k] in vs for k, vs in defines):
|
|
continue
|
|
|
|
results.append(r)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
return fields, results
|
|
|
|
def main(csv_paths, output, *,
|
|
sum=False,
|
|
prod=False,
|
|
min=False,
|
|
max=False,
|
|
bnd=False,
|
|
avg=False,
|
|
stddev=False,
|
|
gmean=False,
|
|
gstddev=False,
|
|
by=None,
|
|
meas=None,
|
|
seeds=None,
|
|
fields=None,
|
|
defines=[]):
|
|
sum_, sum = sum, __builtins__.sum
|
|
min_, min = min, __builtins__.min
|
|
max_, max = max, __builtins__.max
|
|
|
|
# default to averaging
|
|
if (not sum_
|
|
and not prod
|
|
and not min_
|
|
and not max_
|
|
and not bnd
|
|
and not avg
|
|
and not stddev
|
|
and not gmean
|
|
and not gstddev):
|
|
avg = True
|
|
|
|
# separate out renames
|
|
renames = list(it.chain.from_iterable(
|
|
((k, v) for v in vs)
|
|
for k, vs in it.chain(by or [], seeds or [], fields or [])))
|
|
if by is not None:
|
|
by = [k for k, _ in by]
|
|
if seeds is not None:
|
|
seeds = [k for k, _ in seeds]
|
|
if fields is not None:
|
|
fields = [k for k, _ in fields]
|
|
|
|
if by is None and fields is None:
|
|
print("error: needs --by or --fields to figure out fields",
|
|
file=sys.stderr)
|
|
sys.exit(-1)
|
|
|
|
# collect results from csv files
|
|
fields_, results = collect(csv_paths, renames, defines)
|
|
|
|
# if by not specified, guess it's anything not in
|
|
# seeds/fields/renames/defines
|
|
if by is None:
|
|
by = [
|
|
k for k in fields_
|
|
if k not in (seeds or [])
|
|
and k not in (fields or [])
|
|
and not any(k == old_k for _, old_k in renames)
|
|
and not any(k == k_ for k_, _ in defines)]
|
|
|
|
# if fields not specified, guess it's anything not in
|
|
# by/seeds/renames/defines
|
|
if fields is None:
|
|
fields = [
|
|
k for k in fields_
|
|
if k not in (by or [])
|
|
and k not in (seeds or [])
|
|
and not any(k == old_k for _, old_k in renames)
|
|
and not any(k == k_ for k_, _ in defines)]
|
|
|
|
# add meas to by if it isn't already present
|
|
if meas is not None and meas not in by:
|
|
by.append(meas)
|
|
|
|
# convert fields to ints/floats
|
|
for r in results:
|
|
for k in fields:
|
|
if k in r and isinstance(r[k], str):
|
|
r[k] = dat(r[k]) if r[k].strip() else 0
|
|
|
|
# organize by 'by' values
|
|
results_ = co.defaultdict(lambda: [])
|
|
for r in results:
|
|
key = tuple(r.get(k, '') for k in by)
|
|
results_[key].append(r)
|
|
results = results_
|
|
|
|
# for each key calculate the avgs/etc
|
|
avgs = []
|
|
for key, rs in results.items():
|
|
vs = {f: [] for f in fields}
|
|
meas__ = None
|
|
for r in rs:
|
|
for f in fields:
|
|
vs[f].append(r.get(f, 0))
|
|
if meas is not None and meas in r:
|
|
meas__ = r[meas]
|
|
|
|
def append(meas_, f_):
|
|
avgs.append(
|
|
{k: v for k, v in zip(by, key)}
|
|
| {f: f_(vs_) for f, vs_ in vs.items()}
|
|
| ({} if meas is None
|
|
else {meas: meas_} if meas__ is None
|
|
else {meas: meas__+'+'+meas_}))
|
|
|
|
if sum_: append('sum', lambda vs: sum(vs))
|
|
if prod: append('prod', lambda vs: mt.prod(vs))
|
|
if min_: append('min', lambda vs: min(vs, default=0))
|
|
if max_: append('max', lambda vs: max(vs, default=0))
|
|
if bnd: append('bnd', lambda vs: min(vs, default=0))
|
|
if bnd: append('bnd', lambda vs: max(vs, default=0))
|
|
if avg: append('avg', lambda vs: sum(vs) / max(len(vs), 1))
|
|
if stddev: append('stddev', lambda vs: (
|
|
lambda avg: mt.sqrt(
|
|
sum((v - avg)**2 for v in vs) / max(len(vs), 1))
|
|
)(sum(vs) / max(len(vs), 1)))
|
|
if gmean: append('gmean', lambda vs:
|
|
mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1)))
|
|
if gstddev: append('gstddev', lambda vs: (
|
|
lambda gmean: mt.exp(mt.sqrt(
|
|
sum(mt.log(v/gmean)**2 for v in vs) / max(len(vs), 1)))
|
|
if gmean else mt.inf
|
|
)(mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1))))
|
|
|
|
# write results to CSVS
|
|
with openio(output, 'w') as f:
|
|
writer = csv.DictWriter(f, by + fields)
|
|
writer.writeheader()
|
|
for r in avgs:
|
|
writer.writerow(r)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
import sys
|
|
parser = argparse.ArgumentParser(
|
|
description="Compute averages/etc of benchmark measurements.",
|
|
allow_abbrev=False)
|
|
parser.add_argument(
|
|
'csv_paths',
|
|
nargs='*',
|
|
help="Input *.csv files.")
|
|
parser.add_argument(
|
|
'-o', '--output',
|
|
required=True,
|
|
help="*.csv file to write amortized measurements to.")
|
|
parser.add_argument(
|
|
'--sum',
|
|
action='store_true',
|
|
help="Compute the sum.")
|
|
parser.add_argument(
|
|
'--prod',
|
|
action='store_true',
|
|
help="Compute the product.")
|
|
parser.add_argument(
|
|
'--min',
|
|
action='store_true',
|
|
help="Compute the min.")
|
|
parser.add_argument(
|
|
'--max',
|
|
action='store_true',
|
|
help="Compute the max.")
|
|
parser.add_argument(
|
|
'--bnd',
|
|
action='store_true',
|
|
help="Compute the bounds (min+max concatenated).")
|
|
parser.add_argument(
|
|
'--avg', '--mean',
|
|
action='store_true',
|
|
help="Compute the average (the default).")
|
|
parser.add_argument(
|
|
'--stddev',
|
|
action='store_true',
|
|
help="Compute the standard deviation.")
|
|
parser.add_argument(
|
|
'--gmean',
|
|
action='store_true',
|
|
help="Compute the geometric mean.")
|
|
parser.add_argument(
|
|
'--gstddev',
|
|
action='store_true',
|
|
help="Compute the geometric standard deviation.")
|
|
parser.add_argument(
|
|
'-b', '--by',
|
|
action='append',
|
|
type=lambda x: (
|
|
lambda k, vs=None: (
|
|
k.strip(),
|
|
tuple(v.strip() for v in vs.split(','))
|
|
if vs is not None else ())
|
|
)(*x.split('=', 1)),
|
|
help="Group by this field. Can rename fields with new_name=old_name.")
|
|
parser.add_argument(
|
|
'-m', '--meas',
|
|
help="Optional name of measurement name field. If provided, the name "
|
|
"will be modified with +amor or +per.")
|
|
parser.add_argument(
|
|
'-s', '--seed',
|
|
dest='seeds',
|
|
action='append',
|
|
type=lambda x: (
|
|
lambda k, vs=None: (
|
|
k.strip(),
|
|
tuple(v.strip() for v in vs.split(','))
|
|
if vs is not None else ())
|
|
)(*x.split('=', 1)),
|
|
help="Field to ignore when averaging. Can rename fields with "
|
|
"new_name=old_name.")
|
|
parser.add_argument(
|
|
'-f', '--field',
|
|
dest='fields',
|
|
action='append',
|
|
type=lambda x: (
|
|
lambda k, vs=None: (
|
|
k.strip(),
|
|
tuple(v.strip() for v in vs.split(','))
|
|
if vs is not None else ())
|
|
)(*x.split('=', 1)),
|
|
help="Field to amortize. Can rename fields with new_name=old_name.")
|
|
parser.add_argument(
|
|
'-D', '--define',
|
|
dest='defines',
|
|
action='append',
|
|
type=lambda x: (
|
|
lambda k, vs: (
|
|
k.strip(),
|
|
{v.strip() for v in vs.split(',')})
|
|
)(*x.split('=', 1)),
|
|
help="Only include results where this field is this value. May include "
|
|
"comma-separated options.")
|
|
sys.exit(main(**{k: v
|
|
for k, v in vars(parser.parse_intermixed_args()).items()
|
|
if v is not None}))
|
|
|