From bac59d49286fc5cab502e317bf4a4a4d069004d7 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 13 Nov 2024 15:30:43 -0600 Subject: [PATCH] scripts: Dropped amor.py and avg.py Now that bench.py includes cumulative measurements, these scripts can be entirely replaced by csv.py. Replacement for amor.py: $ ./scripts/csv.py bench.csv -q -o bench.amor.csv \ -bsuite -bcase -Dm=write -bn -bREWRITE -bSEED \ -bm='"write+amor"' \ -freaded='float(creaded) / float(n)' \ -fproged='float(cproged) / float(n)' \ -ferased='float(cerased) / float(n)' $ ./scripts/csv.py bench.csv -q -o bench.per.csv \ -bsuite -bcase -Dm=usage -bn -bREWRITE -bSEED \ -bm='"usage+per"' \ -freaded='float(readed) / float(REWRITE ? SIZE : n)' \ -fproged='float(proged) / float(REWRITE ? SIZE : n)' \ -ferased='float(erased) / float(REWRITE ? SIZE : n)' Replacement for avg.py: $ ./scripts/csv.py bench.csv bench.amor.csv bench.per.csv \ -q -o bench.avg.csv \ -bsuite -bcase -bm -bn -bREWRITE \ -freaded_avg='avg(readed)' \ -fproged_avg='avg(proged)' \ -ferased_avg='avg(erased)' \ -freaded_min='min(readed)' \ -fproged_min='min(proged)' \ -ferased_min='min(erased)' \ -freaded_max='max(readed)' \ -fproged_max='max(proged)' \ -ferased_max='max(erased)' This avoids the need to maintain two more scripts, while also increasing flexibility. Win win! --- scripts/amor.py | 223 --------------------------------------- scripts/avg.py | 271 ------------------------------------------------ 2 files changed, 494 deletions(-) delete mode 100755 scripts/amor.py delete mode 100755 scripts/avg.py diff --git a/scripts/amor.py b/scripts/amor.py deleted file mode 100755 index c5f4503a..00000000 --- a/scripts/amor.py +++ /dev/null @@ -1,223 +0,0 @@ -#!/usr/bin/env python3 -# -# Amortize benchmark measurements -# - -# prevent local imports -__import__('sys').path.pop(0) - -import collections as co -import csv -import itertools as it -import math as mt -import os - - -def openio(path, mode='r', buffering=-1): - # allow '-' for stdin/stdout - if path == '-': - if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering) - else: - return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering) - else: - return open(path, mode, buffering) - -# parse different data representations -def dat(x): - # allow the first part of an a/b fraction - if '/' in x: - x, _ = x.split('/', 1) - - # first try as int - try: - return int(x, 0) - except ValueError: - pass - - # then try as float - try: - return float(x) - # just don't allow infinity or nan - if mt.isinf(x) or mt.isnan(x): - raise ValueError("invalid dat %r" % x) - except ValueError: - pass - - # else give up - raise ValueError("invalid dat %r" % x) - -def collect(csv_paths, defines=[]): - # collect results from CSV files - fields = [] - results = [] - for path in csv_paths: - try: - with openio(path) as f: - reader = csv.DictReader(f, restval='') - fields.extend( - k for k in reader.fieldnames - if k not in fields) - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - results.append(r) - except FileNotFoundError: - pass - - return fields, results - -def main(csv_paths, output, *, - amor=False, - per=False, - by=None, - meas=None, - iter=None, - size=None, - fields=None, - defines=[]): - # default to amortizing - if not amor and not per: - amor = True - - if by is None and fields is None: - print("error: needs --by or --fields to figure out fields", - file=sys.stderr) - sys.exit(-1) - - # collect results from csv files - fields_, results = collect(csv_paths, defines) - - # if by not specified, guess it's anything not in - # iter/size/fields/defines - if by is None: - by = [k for k in fields_ - if k != iter - and k != size - and k not in (fields or []) - and not any(k == k_ for k_, _ in defines)] - - # if fields not specified, guess it's anything not in - # by/iter/size/defines - if fields is None: - fields = [k for k in fields_ - if k not in (by or []) - and k != iter - and k != size - and not any(k == k_ for k_, _ in defines)] - - # add meas to by if it isn't already present - if meas is not None and meas not in by: - by.append(meas) - - # convert iter/fields to ints/floats - for r in results: - for k in it.chain([iter], [size] if size is not None else [], fields): - if k in r and isinstance(r[k], str): - r[k] = dat(r[k]) if r[k].strip() else 0 - - # organize by 'by' values - results_ = co.defaultdict(lambda: []) - for r in results: - key = tuple(r.get(k, '') for k in by) - results_[key].append(r) - results = results_ - - # for each key compute the amortized results - amors = [] - for key, rs in results.items(): - # keep a running sum for each field - sums = {f: 0 for f in fields} - size_ = 0 - for j, (i, r) in enumerate(sorted( - ((r.get(iter, 0), r) for r in rs), - key=lambda p: p[0])): - # update sums - for f in fields: - sums[f] += r.get(f, 0) - size_ += r.get(size, 1) - - # find amortized results - if amor: - amors.append(r - | {f: sums[f] / size_ for f in fields} - | ({} if meas is None - else {meas: r[meas]+'+amor'} if meas in r - else {meas: 'amor'})) - - # also find per-byte results - if per: - amors.append(r - | {f: r.get(f, 0) / size_ for f in fields} - | ({} if meas is None - else {meas: r[meas]+'+per'} if meas in r - else {meas: 'per'})) - - # write results to CSV - with openio(output, 'w') as f: - writer = csv.DictWriter(f, - by + [iter] + ([size] if size is not None else []) + fields) - writer.writeheader() - for r in amors: - writer.writerow(r) - - -if __name__ == "__main__": - import argparse - import sys - parser = argparse.ArgumentParser( - description="Amortize benchmark measurements.", - allow_abbrev=False) - parser.add_argument( - 'csv_paths', - nargs='*', - help="Input *.csv files.") - parser.add_argument( - '-o', '--output', - required=True, - help="*.csv file to write amortized measurements to.") - parser.add_argument( - '--amor', - action='store_true', - help="Compute amortized results.") - parser.add_argument( - '--per', - action='store_true', - help="Compute per-byte results.") - parser.add_argument( - '-b', '--by', - action='append', - help="Group by this field.") - parser.add_argument( - '-m', '--meas', - help="Optional name of measurement name field. If provided, the " - "name will be modified with +amor or +per.") - parser.add_argument( - '-i', '--iter', - required=True, - help="Name of iteration field.") - parser.add_argument( - '-n', '--size', - help="Optional name of size field.") - parser.add_argument( - '-f', '--field', - dest='fields', - action='append', - help="Field to amortize.") - parser.add_argument( - '-D', '--define', - dest='defines', - action='append', - type=lambda x: ( - lambda k, vs: ( - k.strip(), - {v.strip() for v in vs.split(',')}) - )(*x.split('=', 1)), - help="Only include results where this field is this value. May " - "include comma-separated options.") - sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) - diff --git a/scripts/avg.py b/scripts/avg.py deleted file mode 100755 index 3af45a1d..00000000 --- a/scripts/avg.py +++ /dev/null @@ -1,271 +0,0 @@ -#!/usr/bin/env python3 -# -# Compute averages/etc of benchmark measurements -# - -# prevent local imports -__import__('sys').path.pop(0) - -import collections as co -import csv -import itertools as it -import math as mt -import os - - -def openio(path, mode='r', buffering=-1): - # allow '-' for stdin/stdout - if path == '-': - if 'r' in mode: - return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering) - else: - return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering) - else: - return open(path, mode, buffering) - -# parse different data representations -def dat(x): - # allow the first part of an a/b fraction - if '/' in x: - x, _ = x.split('/', 1) - - # first try as int - try: - return int(x, 0) - except ValueError: - pass - - # then try as float - try: - return float(x) - # just don't allow infinity or nan - if mt.isinf(x) or mt.isnan(x): - raise ValueError("invalid dat %r" % x) - except ValueError: - pass - - # else give up - raise ValueError("invalid dat %r" % x) - -def collect(csv_paths, defines=[]): - # collect results from CSV files - fields = [] - results = [] - for path in csv_paths: - try: - with openio(path) as f: - reader = csv.DictReader(f, restval='') - fields.extend( - k for k in reader.fieldnames - if k not in fields) - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - results.append(r) - except FileNotFoundError: - pass - - return fields, results - -def main(csv_paths, output, *, - sum=False, - prod=False, - min=False, - max=False, - bnd=False, - avg=False, - stddev=False, - gmean=False, - gstddev=False, - by=None, - meas=None, - seeds=None, - fields=None, - defines=[]): - sum_, sum = sum, __builtins__.sum - min_, min = min, __builtins__.min - max_, max = max, __builtins__.max - - # default to averaging - if (not sum_ - and not prod - and not min_ - and not max_ - and not bnd - and not avg - and not stddev - and not gmean - and not gstddev): - avg = True - - if by is None and fields is None: - print("error: needs --by or --fields to figure out fields", - file=sys.stderr) - sys.exit(-1) - - # collect results from csv files - fields_, results = collect(csv_paths, defines) - - # if by not specified, guess it's anything not in - # seeds/fields/defines - if by is None: - by = [k for k in fields_ - if k not in (seeds or []) - and k not in (fields or []) - and not any(k == k_ for k_, _ in defines)] - - # if fields not specified, guess it's anything not in - # by/seeds/defines - if fields is None: - fields = [k for k in fields_ - if k not in (by or []) - and k not in (seeds or []) - and not any(k == k_ for k_, _ in defines)] - - # add meas to by if it isn't already present - if meas is not None and meas not in by: - by.append(meas) - - # convert fields to ints/floats - for r in results: - for k in fields: - if k in r and isinstance(r[k], str): - r[k] = dat(r[k]) if r[k].strip() else 0 - - # organize by 'by' values - results_ = co.defaultdict(lambda: []) - for r in results: - key = tuple(r.get(k, '') for k in by) - results_[key].append(r) - results = results_ - - # for each key calculate the avgs/etc - avgs = [] - for key, rs in results.items(): - vs = {f: [] for f in fields} - meas__ = None - for r in rs: - for f in fields: - vs[f].append(r.get(f, 0)) - if meas is not None and meas in r: - meas__ = r[meas] - - def append(meas_, f_): - avgs.append({k: v for k, v in zip(by, key)} - | {f: f_(vs_) for f, vs_ in vs.items()} - | ({} if meas is None - else {meas: meas_} if meas__ is None - else {meas: meas__+'+'+meas_})) - - if sum_: append('sum', lambda vs: sum(vs)) - if prod: append('prod', lambda vs: mt.prod(vs)) - if min_: append('min', lambda vs: min(vs, default=0)) - if max_: append('max', lambda vs: max(vs, default=0)) - if bnd: append('bnd', lambda vs: min(vs, default=0)) - if bnd: append('bnd', lambda vs: max(vs, default=0)) - if avg: append('avg', lambda vs: sum(vs) / max(len(vs), 1)) - if stddev: append('stddev', lambda vs: ( - lambda avg: mt.sqrt( - sum((v - avg)**2 for v in vs) / max(len(vs), 1)) - )(sum(vs) / max(len(vs), 1))) - if gmean: append('gmean', lambda vs: - mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1))) - if gstddev: append('gstddev', lambda vs: ( - lambda gmean: mt.exp(mt.sqrt( - sum(mt.log(v/gmean)**2 for v in vs) / max(len(vs), 1))) - if gmean else mt.inf - )(mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1)))) - - # write results to CSVS - with openio(output, 'w') as f: - writer = csv.DictWriter(f, by + fields) - writer.writeheader() - for r in avgs: - writer.writerow(r) - - -if __name__ == "__main__": - import argparse - import sys - parser = argparse.ArgumentParser( - description="Compute averages/etc of benchmark measurements.", - allow_abbrev=False) - parser.add_argument( - 'csv_paths', - nargs='*', - help="Input *.csv files.") - parser.add_argument( - '-o', '--output', - required=True, - help="*.csv file to write amortized measurements to.") - parser.add_argument( - '--sum', - action='store_true', - help="Compute the sum.") - parser.add_argument( - '--prod', - action='store_true', - help="Compute the product.") - parser.add_argument( - '--min', - action='store_true', - help="Compute the min.") - parser.add_argument( - '--max', - action='store_true', - help="Compute the max.") - parser.add_argument( - '--bnd', - action='store_true', - help="Compute the bounds (min+max concatenated).") - parser.add_argument( - '--avg', '--mean', - action='store_true', - help="Compute the average (the default).") - parser.add_argument( - '--stddev', - action='store_true', - help="Compute the standard deviation.") - parser.add_argument( - '--gmean', - action='store_true', - help="Compute the geometric mean.") - parser.add_argument( - '--gstddev', - action='store_true', - help="Compute the geometric standard deviation.") - parser.add_argument( - '-b', '--by', - action='append', - help="Group by this field.") - parser.add_argument( - '-m', '--meas', - help="Optional name of measurement name field. If provided, the " - "name will be modified with +amor or +per.") - parser.add_argument( - '-s', '--seed', - dest='seeds', - action='append', - help="Field to ignore when averaging.") - parser.add_argument( - '-f', '--field', - dest='fields', - action='append', - help="Field to amortize.") - parser.add_argument( - '-D', '--define', - dest='defines', - action='append', - type=lambda x: ( - lambda k, vs: ( - k.strip(), - {v.strip() for v in vs.split(',')}) - )(*x.split('=', 1)), - help="Only include results where this field is this value. May " - "include comma-separated options.") - sys.exit(main(**{k: v - for k, v in vars(parser.parse_intermixed_args()).items() - if v is not None})) -