Christopher Haster
7cfcc1af1d
This seems like a more fitting name now that this script has evolved
into more of a general purpose high-level CSV tool.
Unfortunately this does conflict with the standard csv module in Python,
breaking every script that imports csv (which is most of them).
Fortunately, Python is flexible enough to let us remove the current
directory before imports with a bit of an ugly hack:
# prevent local imports
__import__('sys').path.pop(0)
These scripts are intended to be standalone anyways, so this is probably
a good pattern to adopt.
315 lines
10 KiB
Python
Executable File
315 lines
10 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Compute averages/etc of benchmark measurements
|
|
#
|
|
|
|
# prevent local imports
|
|
__import__('sys').path.pop(0)
|
|
|
|
import collections as co
|
|
import csv
|
|
import itertools as it
|
|
import math as mt
|
|
import os
|
|
|
|
|
|
def openio(path, mode='r', buffering=-1):
|
|
# allow '-' for stdin/stdout
|
|
if path == '-':
|
|
if 'r' in mode:
|
|
return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering)
|
|
else:
|
|
return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering)
|
|
else:
|
|
return open(path, mode, buffering)
|
|
|
|
# parse different data representations
|
|
def dat(x):
|
|
# allow the first part of an a/b fraction
|
|
if '/' in x:
|
|
x, _ = x.split('/', 1)
|
|
|
|
# first try as int
|
|
try:
|
|
return int(x, 0)
|
|
except ValueError:
|
|
pass
|
|
|
|
# then try as float
|
|
try:
|
|
return float(x)
|
|
# just don't allow infinity or nan
|
|
if mt.isinf(x) or mt.isnan(x):
|
|
raise ValueError("invalid dat %r" % x)
|
|
except ValueError:
|
|
pass
|
|
|
|
# else give up
|
|
raise ValueError("invalid dat %r" % x)
|
|
|
|
def collect(csv_paths, renames=[], defines=[]):
|
|
# collect results from CSV files
|
|
fields = []
|
|
results = []
|
|
for path in csv_paths:
|
|
try:
|
|
with openio(path) as f:
|
|
reader = csv.DictReader(f, restval='')
|
|
fields.extend(
|
|
k for k in reader.fieldnames
|
|
if k not in fields)
|
|
for r in reader:
|
|
# apply any renames
|
|
if renames:
|
|
# make a copy so renames can overlap
|
|
r_ = {}
|
|
for new_k, old_k in renames:
|
|
if old_k in r:
|
|
r_[new_k] = r[old_k]
|
|
r.update(r_)
|
|
|
|
# filter by matching defines
|
|
if not all(k in r and r[k] in vs for k, vs in defines):
|
|
continue
|
|
|
|
results.append(r)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
return fields, results
|
|
|
|
def main(csv_paths, output, *,
|
|
sum=False,
|
|
prod=False,
|
|
min=False,
|
|
max=False,
|
|
bnd=False,
|
|
avg=False,
|
|
stddev=False,
|
|
gmean=False,
|
|
gstddev=False,
|
|
by=None,
|
|
meas=None,
|
|
seeds=None,
|
|
fields=None,
|
|
defines=[]):
|
|
sum_, sum = sum, __builtins__.sum
|
|
min_, min = min, __builtins__.min
|
|
max_, max = max, __builtins__.max
|
|
|
|
# default to averaging
|
|
if (not sum_
|
|
and not prod
|
|
and not min_
|
|
and not max_
|
|
and not bnd
|
|
and not avg
|
|
and not stddev
|
|
and not gmean
|
|
and not gstddev):
|
|
avg = True
|
|
|
|
# separate out renames
|
|
renames = list(it.chain.from_iterable(
|
|
((k, v) for v in vs)
|
|
for k, vs in it.chain(by or [], seeds or [], fields or [])))
|
|
if by is not None:
|
|
by = [k for k, _ in by]
|
|
if seeds is not None:
|
|
seeds = [k for k, _ in seeds]
|
|
if fields is not None:
|
|
fields = [k for k, _ in fields]
|
|
|
|
if by is None and fields is None:
|
|
print("error: needs --by or --fields to figure out fields",
|
|
file=sys.stderr)
|
|
sys.exit(-1)
|
|
|
|
# collect results from csv files
|
|
fields_, results = collect(csv_paths, renames, defines)
|
|
|
|
# if by not specified, guess it's anything not in
|
|
# seeds/fields/renames/defines
|
|
if by is None:
|
|
by = [k for k in fields_
|
|
if k not in (seeds or [])
|
|
and k not in (fields or [])
|
|
and not any(k == old_k for _, old_k in renames)
|
|
and not any(k == k_ for k_, _ in defines)]
|
|
|
|
# if fields not specified, guess it's anything not in
|
|
# by/seeds/renames/defines
|
|
if fields is None:
|
|
fields = [k for k in fields_
|
|
if k not in (by or [])
|
|
and k not in (seeds or [])
|
|
and not any(k == old_k for _, old_k in renames)
|
|
and not any(k == k_ for k_, _ in defines)]
|
|
|
|
# add meas to by if it isn't already present
|
|
if meas is not None and meas not in by:
|
|
by.append(meas)
|
|
|
|
# convert fields to ints/floats
|
|
for r in results:
|
|
for k in fields:
|
|
if k in r and isinstance(r[k], str):
|
|
r[k] = dat(r[k]) if r[k].strip() else 0
|
|
|
|
# organize by 'by' values
|
|
results_ = co.defaultdict(lambda: [])
|
|
for r in results:
|
|
key = tuple(r.get(k, '') for k in by)
|
|
results_[key].append(r)
|
|
results = results_
|
|
|
|
# for each key calculate the avgs/etc
|
|
avgs = []
|
|
for key, rs in results.items():
|
|
vs = {f: [] for f in fields}
|
|
meas__ = None
|
|
for r in rs:
|
|
for f in fields:
|
|
vs[f].append(r.get(f, 0))
|
|
if meas is not None and meas in r:
|
|
meas__ = r[meas]
|
|
|
|
def append(meas_, f_):
|
|
avgs.append({k: v for k, v in zip(by, key)}
|
|
| {f: f_(vs_) for f, vs_ in vs.items()}
|
|
| ({} if meas is None
|
|
else {meas: meas_} if meas__ is None
|
|
else {meas: meas__+'+'+meas_}))
|
|
|
|
if sum_: append('sum', lambda vs: sum(vs))
|
|
if prod: append('prod', lambda vs: mt.prod(vs))
|
|
if min_: append('min', lambda vs: min(vs, default=0))
|
|
if max_: append('max', lambda vs: max(vs, default=0))
|
|
if bnd: append('bnd', lambda vs: min(vs, default=0))
|
|
if bnd: append('bnd', lambda vs: max(vs, default=0))
|
|
if avg: append('avg', lambda vs: sum(vs) / max(len(vs), 1))
|
|
if stddev: append('stddev', lambda vs: (
|
|
lambda avg: mt.sqrt(
|
|
sum((v - avg)**2 for v in vs) / max(len(vs), 1))
|
|
)(sum(vs) / max(len(vs), 1)))
|
|
if gmean: append('gmean', lambda vs:
|
|
mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1)))
|
|
if gstddev: append('gstddev', lambda vs: (
|
|
lambda gmean: mt.exp(mt.sqrt(
|
|
sum(mt.log(v/gmean)**2 for v in vs) / max(len(vs), 1)))
|
|
if gmean else mt.inf
|
|
)(mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1))))
|
|
|
|
# write results to CSVS
|
|
with openio(output, 'w') as f:
|
|
writer = csv.DictWriter(f, by + fields)
|
|
writer.writeheader()
|
|
for r in avgs:
|
|
writer.writerow(r)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
import sys
|
|
parser = argparse.ArgumentParser(
|
|
description="Compute averages/etc of benchmark measurements.",
|
|
allow_abbrev=False)
|
|
parser.add_argument(
|
|
'csv_paths',
|
|
nargs='*',
|
|
help="Input *.csv files.")
|
|
parser.add_argument(
|
|
'-o', '--output',
|
|
required=True,
|
|
help="*.csv file to write amortized measurements to.")
|
|
parser.add_argument(
|
|
'--sum',
|
|
action='store_true',
|
|
help="Compute the sum.")
|
|
parser.add_argument(
|
|
'--prod',
|
|
action='store_true',
|
|
help="Compute the product.")
|
|
parser.add_argument(
|
|
'--min',
|
|
action='store_true',
|
|
help="Compute the min.")
|
|
parser.add_argument(
|
|
'--max',
|
|
action='store_true',
|
|
help="Compute the max.")
|
|
parser.add_argument(
|
|
'--bnd',
|
|
action='store_true',
|
|
help="Compute the bounds (min+max concatenated).")
|
|
parser.add_argument(
|
|
'--avg', '--mean',
|
|
action='store_true',
|
|
help="Compute the average (the default).")
|
|
parser.add_argument(
|
|
'--stddev',
|
|
action='store_true',
|
|
help="Compute the standard deviation.")
|
|
parser.add_argument(
|
|
'--gmean',
|
|
action='store_true',
|
|
help="Compute the geometric mean.")
|
|
parser.add_argument(
|
|
'--gstddev',
|
|
action='store_true',
|
|
help="Compute the geometric standard deviation.")
|
|
parser.add_argument(
|
|
'-b', '--by',
|
|
action='append',
|
|
type=lambda x: (
|
|
lambda k, vs=None: (
|
|
k.strip(),
|
|
tuple(v.strip() for v in vs.split(','))
|
|
if vs is not None else ())
|
|
)(*x.split('=', 1)),
|
|
help="Group by this field. Can rename fields with "
|
|
"new_name=old_name.")
|
|
parser.add_argument(
|
|
'-m', '--meas',
|
|
help="Optional name of measurement name field. If provided, the "
|
|
"name will be modified with +amor or +per.")
|
|
parser.add_argument(
|
|
'-s', '--seed',
|
|
dest='seeds',
|
|
action='append',
|
|
type=lambda x: (
|
|
lambda k, vs=None: (
|
|
k.strip(),
|
|
tuple(v.strip() for v in vs.split(','))
|
|
if vs is not None else ())
|
|
)(*x.split('=', 1)),
|
|
help="Field to ignore when averaging. Can rename fields with "
|
|
"new_name=old_name.")
|
|
parser.add_argument(
|
|
'-f', '--field',
|
|
dest='fields',
|
|
action='append',
|
|
type=lambda x: (
|
|
lambda k, vs=None: (
|
|
k.strip(),
|
|
tuple(v.strip() for v in vs.split(','))
|
|
if vs is not None else ())
|
|
)(*x.split('=', 1)),
|
|
help="Field to amortize. Can rename fields with "
|
|
"new_name=old_name.")
|
|
parser.add_argument(
|
|
'-D', '--define',
|
|
dest='defines',
|
|
action='append',
|
|
type=lambda x: (
|
|
lambda k, vs: (
|
|
k.strip(),
|
|
{v.strip() for v in vs.split(',')})
|
|
)(*x.split('=', 1)),
|
|
help="Only include results where this field is this value. May "
|
|
"include comma-separated options.")
|
|
sys.exit(main(**{k: v
|
|
for k, v in vars(parser.parse_intermixed_args()).items()
|
|
if v is not None}))
|
|
|