scripts: Dropped amor.py and avg.py
Now that bench.py includes cumulative measurements, these scripts can be
entirely replaced by csv.py.
Replacement for amor.py:
$ ./scripts/csv.py bench.csv -q -o bench.amor.csv \
-bsuite -bcase -Dm=write -bn -bREWRITE -bSEED \
-bm='"write+amor"' \
-freaded='float(creaded) / float(n)' \
-fproged='float(cproged) / float(n)' \
-ferased='float(cerased) / float(n)'
$ ./scripts/csv.py bench.csv -q -o bench.per.csv \
-bsuite -bcase -Dm=usage -bn -bREWRITE -bSEED \
-bm='"usage+per"' \
-freaded='float(readed) / float(REWRITE ? SIZE : n)' \
-fproged='float(proged) / float(REWRITE ? SIZE : n)' \
-ferased='float(erased) / float(REWRITE ? SIZE : n)'
Replacement for avg.py:
$ ./scripts/csv.py bench.csv bench.amor.csv bench.per.csv \
-q -o bench.avg.csv \
-bsuite -bcase -bm -bn -bREWRITE \
-freaded_avg='avg(readed)' \
-fproged_avg='avg(proged)' \
-ferased_avg='avg(erased)' \
-freaded_min='min(readed)' \
-fproged_min='min(proged)' \
-ferased_min='min(erased)' \
-freaded_max='max(readed)' \
-fproged_max='max(proged)' \
-ferased_max='max(erased)'
This avoids the need to maintain two more scripts, while also increasing
flexibility. Win win!
This commit is contained in:
Christopher Haster
-223
@@ -1,223 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Amortize benchmark measurements
|
||||
#
|
||||
|
||||
# prevent local imports
|
||||
__import__('sys').path.pop(0)
|
||||
|
||||
import collections as co
|
||||
import csv
|
||||
import itertools as it
|
||||
import math as mt
|
||||
import os
|
||||
|
||||
|
||||
def openio(path, mode='r', buffering=-1):
|
||||
# allow '-' for stdin/stdout
|
||||
if path == '-':
|
||||
if 'r' in mode:
|
||||
return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering)
|
||||
else:
|
||||
return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering)
|
||||
else:
|
||||
return open(path, mode, buffering)
|
||||
|
||||
# parse different data representations
|
||||
def dat(x):
|
||||
# allow the first part of an a/b fraction
|
||||
if '/' in x:
|
||||
x, _ = x.split('/', 1)
|
||||
|
||||
# first try as int
|
||||
try:
|
||||
return int(x, 0)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# then try as float
|
||||
try:
|
||||
return float(x)
|
||||
# just don't allow infinity or nan
|
||||
if mt.isinf(x) or mt.isnan(x):
|
||||
raise ValueError("invalid dat %r" % x)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# else give up
|
||||
raise ValueError("invalid dat %r" % x)
|
||||
|
||||
def collect(csv_paths, defines=[]):
|
||||
# collect results from CSV files
|
||||
fields = []
|
||||
results = []
|
||||
for path in csv_paths:
|
||||
try:
|
||||
with openio(path) as f:
|
||||
reader = csv.DictReader(f, restval='')
|
||||
fields.extend(
|
||||
k for k in reader.fieldnames
|
||||
if k not in fields)
|
||||
for r in reader:
|
||||
# filter by matching defines
|
||||
if not all(k in r and r[k] in vs for k, vs in defines):
|
||||
continue
|
||||
|
||||
results.append(r)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
return fields, results
|
||||
|
||||
def main(csv_paths, output, *,
|
||||
amor=False,
|
||||
per=False,
|
||||
by=None,
|
||||
meas=None,
|
||||
iter=None,
|
||||
size=None,
|
||||
fields=None,
|
||||
defines=[]):
|
||||
# default to amortizing
|
||||
if not amor and not per:
|
||||
amor = True
|
||||
|
||||
if by is None and fields is None:
|
||||
print("error: needs --by or --fields to figure out fields",
|
||||
file=sys.stderr)
|
||||
sys.exit(-1)
|
||||
|
||||
# collect results from csv files
|
||||
fields_, results = collect(csv_paths, defines)
|
||||
|
||||
# if by not specified, guess it's anything not in
|
||||
# iter/size/fields/defines
|
||||
if by is None:
|
||||
by = [k for k in fields_
|
||||
if k != iter
|
||||
and k != size
|
||||
and k not in (fields or [])
|
||||
and not any(k == k_ for k_, _ in defines)]
|
||||
|
||||
# if fields not specified, guess it's anything not in
|
||||
# by/iter/size/defines
|
||||
if fields is None:
|
||||
fields = [k for k in fields_
|
||||
if k not in (by or [])
|
||||
and k != iter
|
||||
and k != size
|
||||
and not any(k == k_ for k_, _ in defines)]
|
||||
|
||||
# add meas to by if it isn't already present
|
||||
if meas is not None and meas not in by:
|
||||
by.append(meas)
|
||||
|
||||
# convert iter/fields to ints/floats
|
||||
for r in results:
|
||||
for k in it.chain([iter], [size] if size is not None else [], fields):
|
||||
if k in r and isinstance(r[k], str):
|
||||
r[k] = dat(r[k]) if r[k].strip() else 0
|
||||
|
||||
# organize by 'by' values
|
||||
results_ = co.defaultdict(lambda: [])
|
||||
for r in results:
|
||||
key = tuple(r.get(k, '') for k in by)
|
||||
results_[key].append(r)
|
||||
results = results_
|
||||
|
||||
# for each key compute the amortized results
|
||||
amors = []
|
||||
for key, rs in results.items():
|
||||
# keep a running sum for each field
|
||||
sums = {f: 0 for f in fields}
|
||||
size_ = 0
|
||||
for j, (i, r) in enumerate(sorted(
|
||||
((r.get(iter, 0), r) for r in rs),
|
||||
key=lambda p: p[0])):
|
||||
# update sums
|
||||
for f in fields:
|
||||
sums[f] += r.get(f, 0)
|
||||
size_ += r.get(size, 1)
|
||||
|
||||
# find amortized results
|
||||
if amor:
|
||||
amors.append(r
|
||||
| {f: sums[f] / size_ for f in fields}
|
||||
| ({} if meas is None
|
||||
else {meas: r[meas]+'+amor'} if meas in r
|
||||
else {meas: 'amor'}))
|
||||
|
||||
# also find per-byte results
|
||||
if per:
|
||||
amors.append(r
|
||||
| {f: r.get(f, 0) / size_ for f in fields}
|
||||
| ({} if meas is None
|
||||
else {meas: r[meas]+'+per'} if meas in r
|
||||
else {meas: 'per'}))
|
||||
|
||||
# write results to CSV
|
||||
with openio(output, 'w') as f:
|
||||
writer = csv.DictWriter(f,
|
||||
by + [iter] + ([size] if size is not None else []) + fields)
|
||||
writer.writeheader()
|
||||
for r in amors:
|
||||
writer.writerow(r)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
import sys
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Amortize benchmark measurements.",
|
||||
allow_abbrev=False)
|
||||
parser.add_argument(
|
||||
'csv_paths',
|
||||
nargs='*',
|
||||
help="Input *.csv files.")
|
||||
parser.add_argument(
|
||||
'-o', '--output',
|
||||
required=True,
|
||||
help="*.csv file to write amortized measurements to.")
|
||||
parser.add_argument(
|
||||
'--amor',
|
||||
action='store_true',
|
||||
help="Compute amortized results.")
|
||||
parser.add_argument(
|
||||
'--per',
|
||||
action='store_true',
|
||||
help="Compute per-byte results.")
|
||||
parser.add_argument(
|
||||
'-b', '--by',
|
||||
action='append',
|
||||
help="Group by this field.")
|
||||
parser.add_argument(
|
||||
'-m', '--meas',
|
||||
help="Optional name of measurement name field. If provided, the "
|
||||
"name will be modified with +amor or +per.")
|
||||
parser.add_argument(
|
||||
'-i', '--iter',
|
||||
required=True,
|
||||
help="Name of iteration field.")
|
||||
parser.add_argument(
|
||||
'-n', '--size',
|
||||
help="Optional name of size field.")
|
||||
parser.add_argument(
|
||||
'-f', '--field',
|
||||
dest='fields',
|
||||
action='append',
|
||||
help="Field to amortize.")
|
||||
parser.add_argument(
|
||||
'-D', '--define',
|
||||
dest='defines',
|
||||
action='append',
|
||||
type=lambda x: (
|
||||
lambda k, vs: (
|
||||
k.strip(),
|
||||
{v.strip() for v in vs.split(',')})
|
||||
)(*x.split('=', 1)),
|
||||
help="Only include results where this field is this value. May "
|
||||
"include comma-separated options.")
|
||||
sys.exit(main(**{k: v
|
||||
for k, v in vars(parser.parse_intermixed_args()).items()
|
||||
if v is not None}))
|
||||
|
||||
-271
@@ -1,271 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Compute averages/etc of benchmark measurements
|
||||
#
|
||||
|
||||
# prevent local imports
|
||||
__import__('sys').path.pop(0)
|
||||
|
||||
import collections as co
|
||||
import csv
|
||||
import itertools as it
|
||||
import math as mt
|
||||
import os
|
||||
|
||||
|
||||
def openio(path, mode='r', buffering=-1):
|
||||
# allow '-' for stdin/stdout
|
||||
if path == '-':
|
||||
if 'r' in mode:
|
||||
return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering)
|
||||
else:
|
||||
return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering)
|
||||
else:
|
||||
return open(path, mode, buffering)
|
||||
|
||||
# parse different data representations
|
||||
def dat(x):
|
||||
# allow the first part of an a/b fraction
|
||||
if '/' in x:
|
||||
x, _ = x.split('/', 1)
|
||||
|
||||
# first try as int
|
||||
try:
|
||||
return int(x, 0)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# then try as float
|
||||
try:
|
||||
return float(x)
|
||||
# just don't allow infinity or nan
|
||||
if mt.isinf(x) or mt.isnan(x):
|
||||
raise ValueError("invalid dat %r" % x)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# else give up
|
||||
raise ValueError("invalid dat %r" % x)
|
||||
|
||||
def collect(csv_paths, defines=[]):
|
||||
# collect results from CSV files
|
||||
fields = []
|
||||
results = []
|
||||
for path in csv_paths:
|
||||
try:
|
||||
with openio(path) as f:
|
||||
reader = csv.DictReader(f, restval='')
|
||||
fields.extend(
|
||||
k for k in reader.fieldnames
|
||||
if k not in fields)
|
||||
for r in reader:
|
||||
# filter by matching defines
|
||||
if not all(k in r and r[k] in vs for k, vs in defines):
|
||||
continue
|
||||
|
||||
results.append(r)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
return fields, results
|
||||
|
||||
def main(csv_paths, output, *,
|
||||
sum=False,
|
||||
prod=False,
|
||||
min=False,
|
||||
max=False,
|
||||
bnd=False,
|
||||
avg=False,
|
||||
stddev=False,
|
||||
gmean=False,
|
||||
gstddev=False,
|
||||
by=None,
|
||||
meas=None,
|
||||
seeds=None,
|
||||
fields=None,
|
||||
defines=[]):
|
||||
sum_, sum = sum, __builtins__.sum
|
||||
min_, min = min, __builtins__.min
|
||||
max_, max = max, __builtins__.max
|
||||
|
||||
# default to averaging
|
||||
if (not sum_
|
||||
and not prod
|
||||
and not min_
|
||||
and not max_
|
||||
and not bnd
|
||||
and not avg
|
||||
and not stddev
|
||||
and not gmean
|
||||
and not gstddev):
|
||||
avg = True
|
||||
|
||||
if by is None and fields is None:
|
||||
print("error: needs --by or --fields to figure out fields",
|
||||
file=sys.stderr)
|
||||
sys.exit(-1)
|
||||
|
||||
# collect results from csv files
|
||||
fields_, results = collect(csv_paths, defines)
|
||||
|
||||
# if by not specified, guess it's anything not in
|
||||
# seeds/fields/defines
|
||||
if by is None:
|
||||
by = [k for k in fields_
|
||||
if k not in (seeds or [])
|
||||
and k not in (fields or [])
|
||||
and not any(k == k_ for k_, _ in defines)]
|
||||
|
||||
# if fields not specified, guess it's anything not in
|
||||
# by/seeds/defines
|
||||
if fields is None:
|
||||
fields = [k for k in fields_
|
||||
if k not in (by or [])
|
||||
and k not in (seeds or [])
|
||||
and not any(k == k_ for k_, _ in defines)]
|
||||
|
||||
# add meas to by if it isn't already present
|
||||
if meas is not None and meas not in by:
|
||||
by.append(meas)
|
||||
|
||||
# convert fields to ints/floats
|
||||
for r in results:
|
||||
for k in fields:
|
||||
if k in r and isinstance(r[k], str):
|
||||
r[k] = dat(r[k]) if r[k].strip() else 0
|
||||
|
||||
# organize by 'by' values
|
||||
results_ = co.defaultdict(lambda: [])
|
||||
for r in results:
|
||||
key = tuple(r.get(k, '') for k in by)
|
||||
results_[key].append(r)
|
||||
results = results_
|
||||
|
||||
# for each key calculate the avgs/etc
|
||||
avgs = []
|
||||
for key, rs in results.items():
|
||||
vs = {f: [] for f in fields}
|
||||
meas__ = None
|
||||
for r in rs:
|
||||
for f in fields:
|
||||
vs[f].append(r.get(f, 0))
|
||||
if meas is not None and meas in r:
|
||||
meas__ = r[meas]
|
||||
|
||||
def append(meas_, f_):
|
||||
avgs.append({k: v for k, v in zip(by, key)}
|
||||
| {f: f_(vs_) for f, vs_ in vs.items()}
|
||||
| ({} if meas is None
|
||||
else {meas: meas_} if meas__ is None
|
||||
else {meas: meas__+'+'+meas_}))
|
||||
|
||||
if sum_: append('sum', lambda vs: sum(vs))
|
||||
if prod: append('prod', lambda vs: mt.prod(vs))
|
||||
if min_: append('min', lambda vs: min(vs, default=0))
|
||||
if max_: append('max', lambda vs: max(vs, default=0))
|
||||
if bnd: append('bnd', lambda vs: min(vs, default=0))
|
||||
if bnd: append('bnd', lambda vs: max(vs, default=0))
|
||||
if avg: append('avg', lambda vs: sum(vs) / max(len(vs), 1))
|
||||
if stddev: append('stddev', lambda vs: (
|
||||
lambda avg: mt.sqrt(
|
||||
sum((v - avg)**2 for v in vs) / max(len(vs), 1))
|
||||
)(sum(vs) / max(len(vs), 1)))
|
||||
if gmean: append('gmean', lambda vs:
|
||||
mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1)))
|
||||
if gstddev: append('gstddev', lambda vs: (
|
||||
lambda gmean: mt.exp(mt.sqrt(
|
||||
sum(mt.log(v/gmean)**2 for v in vs) / max(len(vs), 1)))
|
||||
if gmean else mt.inf
|
||||
)(mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1))))
|
||||
|
||||
# write results to CSVS
|
||||
with openio(output, 'w') as f:
|
||||
writer = csv.DictWriter(f, by + fields)
|
||||
writer.writeheader()
|
||||
for r in avgs:
|
||||
writer.writerow(r)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
import sys
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Compute averages/etc of benchmark measurements.",
|
||||
allow_abbrev=False)
|
||||
parser.add_argument(
|
||||
'csv_paths',
|
||||
nargs='*',
|
||||
help="Input *.csv files.")
|
||||
parser.add_argument(
|
||||
'-o', '--output',
|
||||
required=True,
|
||||
help="*.csv file to write amortized measurements to.")
|
||||
parser.add_argument(
|
||||
'--sum',
|
||||
action='store_true',
|
||||
help="Compute the sum.")
|
||||
parser.add_argument(
|
||||
'--prod',
|
||||
action='store_true',
|
||||
help="Compute the product.")
|
||||
parser.add_argument(
|
||||
'--min',
|
||||
action='store_true',
|
||||
help="Compute the min.")
|
||||
parser.add_argument(
|
||||
'--max',
|
||||
action='store_true',
|
||||
help="Compute the max.")
|
||||
parser.add_argument(
|
||||
'--bnd',
|
||||
action='store_true',
|
||||
help="Compute the bounds (min+max concatenated).")
|
||||
parser.add_argument(
|
||||
'--avg', '--mean',
|
||||
action='store_true',
|
||||
help="Compute the average (the default).")
|
||||
parser.add_argument(
|
||||
'--stddev',
|
||||
action='store_true',
|
||||
help="Compute the standard deviation.")
|
||||
parser.add_argument(
|
||||
'--gmean',
|
||||
action='store_true',
|
||||
help="Compute the geometric mean.")
|
||||
parser.add_argument(
|
||||
'--gstddev',
|
||||
action='store_true',
|
||||
help="Compute the geometric standard deviation.")
|
||||
parser.add_argument(
|
||||
'-b', '--by',
|
||||
action='append',
|
||||
help="Group by this field.")
|
||||
parser.add_argument(
|
||||
'-m', '--meas',
|
||||
help="Optional name of measurement name field. If provided, the "
|
||||
"name will be modified with +amor or +per.")
|
||||
parser.add_argument(
|
||||
'-s', '--seed',
|
||||
dest='seeds',
|
||||
action='append',
|
||||
help="Field to ignore when averaging.")
|
||||
parser.add_argument(
|
||||
'-f', '--field',
|
||||
dest='fields',
|
||||
action='append',
|
||||
help="Field to amortize.")
|
||||
parser.add_argument(
|
||||
'-D', '--define',
|
||||
dest='defines',
|
||||
action='append',
|
||||
type=lambda x: (
|
||||
lambda k, vs: (
|
||||
k.strip(),
|
||||
{v.strip() for v in vs.split(',')})
|
||||
)(*x.split('=', 1)),
|
||||
help="Only include results where this field is this value. May "
|
||||
"include comma-separated options.")
|
||||
sys.exit(main(**{k: v
|
||||
for k, v in vars(parser.parse_intermixed_args()).items()
|
||||
if v is not None}))
|
||||
|
||||
Reference in New Issue
Block a user