Github/littlefs
1
Fork 0
Code Actions Activity

scripts: Dropped amor.py and avg.py

Browse Source 
Now that bench.py includes cumulative measurements, these scripts can be
entirely replaced by csv.py.

Replacement for amor.py:

  $ ./scripts/csv.py bench.csv -q -o bench.amor.csv \
          -bsuite -bcase -Dm=write -bn -bREWRITE -bSEED \
          -bm='"write+amor"' \
          -freaded='float(creaded) / float(n)' \
              -fproged='float(cproged) / float(n)' \
              -ferased='float(cerased) / float(n)'

  $ ./scripts/csv.py bench.csv -q -o bench.per.csv \
          -bsuite -bcase -Dm=usage -bn -bREWRITE -bSEED \
          -bm='"usage+per"' \
          -freaded='float(readed) / float(REWRITE ? SIZE : n)' \
              -fproged='float(proged) / float(REWRITE ? SIZE : n)' \
              -ferased='float(erased) / float(REWRITE ? SIZE : n)'

Replacement for avg.py:

  $ ./scripts/csv.py bench.csv bench.amor.csv bench.per.csv \
          -q -o bench.avg.csv \
          -bsuite -bcase -bm -bn -bREWRITE \
          -freaded_avg='avg(readed)' \
              -fproged_avg='avg(proged)' \
              -ferased_avg='avg(erased)' \
          -freaded_min='min(readed)' \
              -fproged_min='min(proged)' \
              -ferased_min='min(erased)' \
          -freaded_max='max(readed)' \
              -fproged_max='max(proged)' \
              -ferased_max='max(erased)'

This avoids the need to maintain two more scripts, while also increasing
flexibility. Win win!
This commit is contained in:
Christopher Haster
2024-11-13 15:30:43 -06:00
parent f385f8f778
commit bac59d4928
2 changed files with 0 additions and 494 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/amor.py
-223
View File
@@ -1,223 +0,0 @@
#!/usr/bin/env python3
#
# Amortize benchmark measurements
#
# prevent local imports
__import__('sys').path.pop(0)
import collections as co
import csv
import itertools as it
import math as mt
import os
def openio(path, mode='r', buffering=-1):
# allow '-' for stdin/stdout
if path == '-':
if 'r' in mode:
return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering)
else:
return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering)
else:
return open(path, mode, buffering)
# parse different data representations
def dat(x):
# allow the first part of an a/b fraction
if '/' in x:
x, _ = x.split('/', 1)
# first try as int
try:
return int(x, 0)
except ValueError:
pass
# then try as float
try:
return float(x)
# just don't allow infinity or nan
if mt.isinf(x) or mt.isnan(x):
raise ValueError("invalid dat %r" % x)
except ValueError:
pass
# else give up
raise ValueError("invalid dat %r" % x)
def collect(csv_paths, defines=[]):
# collect results from CSV files
fields = []
results = []
for path in csv_paths:
try:
with openio(path) as f:
reader = csv.DictReader(f, restval='')
fields.extend(
k for k in reader.fieldnames
if k not in fields)
for r in reader:
# filter by matching defines
if not all(k in r and r[k] in vs for k, vs in defines):
continue
results.append(r)
except FileNotFoundError:
pass
return fields, results
def main(csv_paths, output, *,
amor=False,
per=False,
by=None,
meas=None,
iter=None,
size=None,
fields=None,
defines=[]):
# default to amortizing
if not amor and not per:
amor = True
if by is None and fields is None:
print("error: needs --by or --fields to figure out fields",
file=sys.stderr)
sys.exit(-1)
# collect results from csv files
fields_, results = collect(csv_paths, defines)
# if by not specified, guess it's anything not in
# iter/size/fields/defines
if by is None:
by = [k for k in fields_
if k != iter
and k != size
and k not in (fields or [])
and not any(k == k_ for k_, _ in defines)]
# if fields not specified, guess it's anything not in
# by/iter/size/defines
if fields is None:
fields = [k for k in fields_
if k not in (by or [])
and k != iter
and k != size
and not any(k == k_ for k_, _ in defines)]
# add meas to by if it isn't already present
if meas is not None and meas not in by:
by.append(meas)
# convert iter/fields to ints/floats
for r in results:
for k in it.chain([iter], [size] if size is not None else [], fields):
if k in r and isinstance(r[k], str):
r[k] = dat(r[k]) if r[k].strip() else 0
# organize by 'by' values
results_ = co.defaultdict(lambda: [])
for r in results:
key = tuple(r.get(k, '') for k in by)
results_[key].append(r)
results = results_
# for each key compute the amortized results
amors = []
for key, rs in results.items():
# keep a running sum for each field
sums = {f: 0 for f in fields}
size_ = 0
for j, (i, r) in enumerate(sorted(
((r.get(iter, 0), r) for r in rs),
key=lambda p: p[0])):
# update sums
for f in fields:
sums[f] += r.get(f, 0)
size_ += r.get(size, 1)
# find amortized results
if amor:
amors.append(r
| {f: sums[f] / size_ for f in fields}
| ({} if meas is None
else {meas: r[meas]+'+amor'} if meas in r
else {meas: 'amor'}))
# also find per-byte results
if per:
amors.append(r
| {f: r.get(f, 0) / size_ for f in fields}
| ({} if meas is None
else {meas: r[meas]+'+per'} if meas in r
else {meas: 'per'}))
# write results to CSV
with openio(output, 'w') as f:
writer = csv.DictWriter(f,
by + [iter] + ([size] if size is not None else []) + fields)
writer.writeheader()
for r in amors:
writer.writerow(r)
if __name__ == "__main__":
import argparse
import sys
parser = argparse.ArgumentParser(
description="Amortize benchmark measurements.",
allow_abbrev=False)
parser.add_argument(
'csv_paths',
nargs='*',
help="Input *.csv files.")
parser.add_argument(
'-o', '--output',
required=True,
help="*.csv file to write amortized measurements to.")
parser.add_argument(
'--amor',
action='store_true',
help="Compute amortized results.")
parser.add_argument(
'--per',
action='store_true',
help="Compute per-byte results.")
parser.add_argument(
'-b', '--by',
action='append',
help="Group by this field.")
parser.add_argument(
'-m', '--meas',
help="Optional name of measurement name field. If provided, the "
"name will be modified with +amor or +per.")
parser.add_argument(
'-i', '--iter',
required=True,
help="Name of iteration field.")
parser.add_argument(
'-n', '--size',
help="Optional name of size field.")
parser.add_argument(
'-f', '--field',
dest='fields',
action='append',
help="Field to amortize.")
parser.add_argument(
'-D', '--define',
dest='defines',
action='append',
type=lambda x: (
lambda k, vs: (
k.strip(),
{v.strip() for v in vs.split(',')})
)(*x.split('=', 1)),
help="Only include results where this field is this value. May "
"include comma-separated options.")
sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None}))
scripts/avg.py
-271
View File
@@ -1,271 +0,0 @@
#!/usr/bin/env python3
#
# Compute averages/etc of benchmark measurements
#
# prevent local imports
__import__('sys').path.pop(0)
import collections as co
import csv
import itertools as it
import math as mt
import os
def openio(path, mode='r', buffering=-1):
# allow '-' for stdin/stdout
if path == '-':
if 'r' in mode:
return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering)
else:
return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering)
else:
return open(path, mode, buffering)
# parse different data representations
def dat(x):
# allow the first part of an a/b fraction
if '/' in x:
x, _ = x.split('/', 1)
# first try as int
try:
return int(x, 0)
except ValueError:
pass
# then try as float
try:
return float(x)
# just don't allow infinity or nan
if mt.isinf(x) or mt.isnan(x):
raise ValueError("invalid dat %r" % x)
except ValueError:
pass
# else give up
raise ValueError("invalid dat %r" % x)
def collect(csv_paths, defines=[]):
# collect results from CSV files
fields = []
results = []
for path in csv_paths:
try:
with openio(path) as f:
reader = csv.DictReader(f, restval='')
fields.extend(
k for k in reader.fieldnames
if k not in fields)
for r in reader:
# filter by matching defines
if not all(k in r and r[k] in vs for k, vs in defines):
continue
results.append(r)
except FileNotFoundError:
pass
return fields, results
def main(csv_paths, output, *,
sum=False,
prod=False,
min=False,
max=False,
bnd=False,
avg=False,
stddev=False,
gmean=False,
gstddev=False,
by=None,
meas=None,
seeds=None,
fields=None,
defines=[]):
sum_, sum = sum, __builtins__.sum
min_, min = min, __builtins__.min
max_, max = max, __builtins__.max
# default to averaging
if (not sum_
and not prod
and not min_
and not max_
and not bnd
and not avg
and not stddev
and not gmean
and not gstddev):
avg = True
if by is None and fields is None:
print("error: needs --by or --fields to figure out fields",
file=sys.stderr)
sys.exit(-1)
# collect results from csv files
fields_, results = collect(csv_paths, defines)
# if by not specified, guess it's anything not in
# seeds/fields/defines
if by is None:
by = [k for k in fields_
if k not in (seeds or [])
and k not in (fields or [])
and not any(k == k_ for k_, _ in defines)]
# if fields not specified, guess it's anything not in
# by/seeds/defines
if fields is None:
fields = [k for k in fields_
if k not in (by or [])
and k not in (seeds or [])
and not any(k == k_ for k_, _ in defines)]
# add meas to by if it isn't already present
if meas is not None and meas not in by:
by.append(meas)
# convert fields to ints/floats
for r in results:
for k in fields:
if k in r and isinstance(r[k], str):
r[k] = dat(r[k]) if r[k].strip() else 0
# organize by 'by' values
results_ = co.defaultdict(lambda: [])
for r in results:
key = tuple(r.get(k, '') for k in by)
results_[key].append(r)
results = results_
# for each key calculate the avgs/etc
avgs = []
for key, rs in results.items():
vs = {f: [] for f in fields}
meas__ = None
for r in rs:
for f in fields:
vs[f].append(r.get(f, 0))
if meas is not None and meas in r:
meas__ = r[meas]
def append(meas_, f_):
avgs.append({k: v for k, v in zip(by, key)}
| {f: f_(vs_) for f, vs_ in vs.items()}
| ({} if meas is None
else {meas: meas_} if meas__ is None
else {meas: meas__+'+'+meas_}))
if sum_: append('sum', lambda vs: sum(vs))
if prod: append('prod', lambda vs: mt.prod(vs))
if min_: append('min', lambda vs: min(vs, default=0))
if max_: append('max', lambda vs: max(vs, default=0))
if bnd: append('bnd', lambda vs: min(vs, default=0))
if bnd: append('bnd', lambda vs: max(vs, default=0))
if avg: append('avg', lambda vs: sum(vs) / max(len(vs), 1))
if stddev: append('stddev', lambda vs: (
lambda avg: mt.sqrt(
sum((v - avg)**2 for v in vs) / max(len(vs), 1))
)(sum(vs) / max(len(vs), 1)))
if gmean: append('gmean', lambda vs:
mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1)))
if gstddev: append('gstddev', lambda vs: (
lambda gmean: mt.exp(mt.sqrt(
sum(mt.log(v/gmean)**2 for v in vs) / max(len(vs), 1)))
if gmean else mt.inf
)(mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1))))
# write results to CSVS
with openio(output, 'w') as f:
writer = csv.DictWriter(f, by + fields)
writer.writeheader()
for r in avgs:
writer.writerow(r)
if __name__ == "__main__":
import argparse
import sys
parser = argparse.ArgumentParser(
description="Compute averages/etc of benchmark measurements.",
allow_abbrev=False)
parser.add_argument(
'csv_paths',
nargs='*',
help="Input *.csv files.")
parser.add_argument(
'-o', '--output',
required=True,
help="*.csv file to write amortized measurements to.")
parser.add_argument(
'--sum',
action='store_true',
help="Compute the sum.")
parser.add_argument(
'--prod',
action='store_true',
help="Compute the product.")
parser.add_argument(
'--min',
action='store_true',
help="Compute the min.")
parser.add_argument(
'--max',
action='store_true',
help="Compute the max.")
parser.add_argument(
'--bnd',
action='store_true',
help="Compute the bounds (min+max concatenated).")
parser.add_argument(
'--avg', '--mean',
action='store_true',
help="Compute the average (the default).")
parser.add_argument(
'--stddev',
action='store_true',
help="Compute the standard deviation.")
parser.add_argument(
'--gmean',
action='store_true',
help="Compute the geometric mean.")
parser.add_argument(
'--gstddev',
action='store_true',
help="Compute the geometric standard deviation.")
parser.add_argument(
'-b', '--by',
action='append',
help="Group by this field.")
parser.add_argument(
'-m', '--meas',
help="Optional name of measurement name field. If provided, the "
"name will be modified with +amor or +per.")
parser.add_argument(
'-s', '--seed',
dest='seeds',
action='append',
help="Field to ignore when averaging.")
parser.add_argument(
'-f', '--field',
dest='fields',
action='append',
help="Field to amortize.")
parser.add_argument(
'-D', '--define',
dest='defines',
action='append',
type=lambda x: (
lambda k, vs: (
k.strip(),
{v.strip() for v in vs.split(',')})
)(*x.split('=', 1)),
help="Only include results where this field is this value. May "
"include comma-separated options.")
sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None}))